Hugging Face's logo

Spaces:
FoodDesert
/
Prompt_Squirrel_RAG
Running

App Files Community
Food Desert commited on
Commit
30bedf0
·
1 Parent(s): 5188881

Consolidate pending pipeline, structural, and analysis updates

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. app.py +55 -9
  2. data/analysis/category_centroid_review.csv +0 -0
  3. data/analysis/category_centroid_summary.json +1071 -0
  4. data/analysis/category_expansion_coverage.json +73 -0
  5. data/analysis/category_expansion_proposal.csv +121 -0
  6. data/analysis/category_registry.csv +0 -0
  7. data/analysis/category_seed_overrides.csv +4 -0
  8. data/analysis/category_tag_group_map.csv +11 -0
  9. data/analysis/guided_facet_assignments.csv +0 -0
  10. data/analysis/guided_facet_summary.json +35 -0
  11. data/analysis/probe_informativeness.csv +0 -0
  12. data/analysis/probe_informativeness_summary.json +1867 -0
  13. data/analysis/probe_policy_simulation.csv +49 -0
  14. data/analysis/probe_policy_simulation_summary.json +122 -0
  15. data/analysis/probe_reliability_n100.csv +36 -0
  16. data/analysis/probe_reliability_n100.json +422 -0
  17. data/analysis/probe_reliability_sanity10.csv +36 -0
  18. data/analysis/probe_reliability_sanity10.json +422 -0
  19. data/analysis/remaining_tag_centroid_simulation.json +2572 -0
  20. data/analysis/simplified_probe_tags.csv +36 -0
  21. data/analysis/simplified_probe_tags_summary.json +129 -0
  22. data/analysis/tag_group_coverage_ranked.csv +47 -0
  23. data/analysis/tag_group_coverage_ranked_baseline200.csv +47 -0
  24. data/analysis/tag_group_coverage_ranked_combined200.csv +74 -0
  25. data/analysis/tag_group_coverage_summary.json +1042 -0
  26. data/analysis/tag_group_coverage_summary_baseline200.json +1045 -0
  27. data/analysis/tag_group_coverage_summary_combined200.json +853 -0
  28. data/analysis/tag_group_uncovered_after_topn.csv +121 -0
  29. data/analysis/tag_group_uncovered_after_topn_baseline200.csv +121 -0
  30. data/analysis/tag_group_uncovered_after_topn_combined200.csv +81 -0
  31. data/analysis/tfidf_category_insights.json +795 -0
  32. data/analysis/tfidf_uncategorized_soft_assignments.csv +0 -0
  33. data/eval_results/eval_caption_cogvlm_n10_bootstrap_baseline_explicit.jsonl +11 -0
  34. data/eval_results/eval_caption_cogvlm_n10_bootstrap_baseline_explicit_skiprewrite.jsonl +11 -0
  35. data/eval_results/eval_caption_cogvlm_n10_bootstrap_enabled_explicit.jsonl +11 -0
  36. data/eval_results/eval_caption_cogvlm_n10_bootstrap_enabled_explicit_skiprewrite.jsonl +11 -0
  37. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p3.jsonl +11 -0
  38. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p5.jsonl +11 -0
  39. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p7.jsonl +11 -0
  40. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p3.jsonl +11 -0
  41. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p5.jsonl +11 -0
  42. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p7.jsonl +11 -0
  43. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p3.jsonl +11 -0
  44. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p5.jsonl +11 -0
  45. data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p7.jsonl +11 -0
  46. data/eval_results/eval_caption_cogvlm_n10_e2e_structimp_baseline_default.jsonl +11 -0
  47. data/eval_results/eval_caption_cogvlm_n10_e2e_structimp_bootstrap_k3_cw0p5.jsonl +11 -0
  48. data/eval_results/eval_caption_cogvlm_n10_seed42_20260220_124618.jsonl +11 -0
  49. data/eval_results/eval_caption_cogvlm_n10_seed42_20260220_220531.jsonl +11 -0
  50. data/eval_results/eval_caption_cogvlm_n10_seed42_20260221_045137.jsonl +11 -0
app.py CHANGED
@@ -4,11 +4,12 @@ import logging
4
  from PIL import Image
5
  from pathlib import Path
6
  from typing import List
 
7
 
8
  from psq_rag.pipeline.preproc import extract_user_provided_tags_upto_3_words
9
  from psq_rag.llm.rewrite import llm_rewrite_prompt
10
  from psq_rag.retrieval.psq_retrieval import psq_candidates_from_rewrite_phrases, _norm_tag_for_lookup
11
- from psq_rag.llm.select import llm_select_indices, llm_infer_structural_tags
12
  from psq_rag.retrieval.state import expand_tags_via_implications
13
 
14
 
@@ -35,6 +36,28 @@ def compose_final_prompt(rewritten_prompt: str, selected_tags: List[str]) -> str
35
  return ", ".join(out)
36
 
37
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
  # Set up logging
39
  # Minimal prod logging: warnings+ to stderr, no file by default
40
  import os, logging
@@ -95,6 +118,7 @@ allow_nsfw_tags = False
95
  verbose_retrieval = True
96
  verbose_retrieval_all = False
97
  verbose_retrieval_limit = 20
 
98
 
99
  css = """
100
  .scrollable-content{
@@ -145,8 +169,17 @@ def rag_pipeline_ui(user_prompt: str):
145
  log("(none)")
146
  log("")
147
 
148
- log("Step 1: LLM rewrite")
149
- rewritten = llm_rewrite_prompt(prompt_in, log)
 
 
 
 
 
 
 
 
 
150
  log("Rewrite:")
151
  log(rewritten if rewritten else "(empty)")
152
  log("")
@@ -159,10 +192,12 @@ def rag_pipeline_ui(user_prompt: str):
159
 
160
  log("Step 2: Prompt Squirrel retrieval (hidden)")
161
  try:
 
162
  rewrite_phrases = [p.strip() for p in (rewrite_for_retrieval or "").split(",") if p.strip()]
163
  retrieval_result = psq_candidates_from_rewrite_phrases(
164
  rewrite_phrases=rewrite_phrases,
165
  allow_nsfw_tags=allow_nsfw_tags,
 
166
  global_k=300,
167
  verbose=verbose_retrieval,
168
  )
@@ -213,19 +248,22 @@ def rag_pipeline_ui(user_prompt: str):
213
  log(f"Retrieval fallback: {type(e).__name__}: {e}")
214
  candidates = []
215
 
216
- log("Step 3: LLM index selection")
217
- # We pass the original 'prompt_in' as the description for the LLM to match against
 
 
 
 
 
218
  picked_indices = llm_select_indices(
219
- query_text=prompt_in,
220
  candidates=candidates,
221
  max_pick=0,
222
- log=log
223
  )
224
 
225
  selected_tags = [candidates[i].tag for i in picked_indices] if picked_indices else []
226
 
227
- log("Step 3b: Structural tag inference (solo/duo/gender/body plan)")
228
- structural_tags = llm_infer_structural_tags(prompt_in, log=log)
229
  if structural_tags:
230
  # Add structural tags that aren't already selected
231
  existing = {t for t in selected_tags}
@@ -235,6 +273,14 @@ def rag_pipeline_ui(user_prompt: str):
235
  else:
236
  log(" No structural tags inferred")
237
 
 
 
 
 
 
 
 
 
238
  log("Step 3c: Expand via tag implications")
239
  tag_set = set(selected_tags)
240
  expanded, implied_only = expand_tags_via_implications(tag_set)
 
4
  from PIL import Image
5
  from pathlib import Path
6
  from typing import List
7
+ from concurrent.futures import ThreadPoolExecutor
8
 
9
  from psq_rag.pipeline.preproc import extract_user_provided_tags_upto_3_words
10
  from psq_rag.llm.rewrite import llm_rewrite_prompt
11
  from psq_rag.retrieval.psq_retrieval import psq_candidates_from_rewrite_phrases, _norm_tag_for_lookup
12
+ from psq_rag.llm.select import llm_select_indices, llm_infer_structural_tags, llm_infer_probe_tags
13
  from psq_rag.retrieval.state import expand_tags_via_implications
14
 
15
 
 
36
  return ", ".join(out)
37
 
38
 
39
+ def _build_selection_query(
40
+ prompt_in: str,
41
+ rewritten: str,
42
+ structural_tags: List[str],
43
+ probe_tags: List[str],
44
+ ) -> str:
45
+ lines = [f"IMAGE DESCRIPTION: {prompt_in.strip()}"]
46
+ if rewritten and rewritten.strip():
47
+ lines.append(f"REWRITE PHRASES: {rewritten.strip()}")
48
+ hint_tags = []
49
+ if structural_tags:
50
+ hint_tags.extend(structural_tags)
51
+ if probe_tags:
52
+ hint_tags.extend(probe_tags)
53
+ if hint_tags:
54
+ # Keep hints as context only; selection still must choose by candidate indices.
55
+ lines.append(
56
+ "INFERRED TAG HINTS (context only): " + ", ".join(sorted(set(hint_tags)))
57
+ )
58
+ return "\n".join(lines)
59
+
60
+
61
  # Set up logging
62
  # Minimal prod logging: warnings+ to stderr, no file by default
63
  import os, logging
 
118
  verbose_retrieval = True
119
  verbose_retrieval_all = False
120
  verbose_retrieval_limit = 20
121
+ enable_probe_tags = os.environ.get("PSQ_ENABLE_PROBE", "1").strip() not in {"0", "false", "False"}
122
 
123
  css = """
124
  .scrollable-content{
 
169
  log("(none)")
170
  log("")
171
 
172
+ log("Step 1: LLM rewrite + structural inference + probe (concurrent)")
173
+ max_workers = 3 if enable_probe_tags else 2
174
+ with ThreadPoolExecutor(max_workers=max_workers) as ex:
175
+ fut_rewrite = ex.submit(llm_rewrite_prompt, prompt_in, log)
176
+ fut_struct = ex.submit(llm_infer_structural_tags, prompt_in, log=log)
177
+ fut_probe = ex.submit(llm_infer_probe_tags, prompt_in, log=log) if enable_probe_tags else None
178
+
179
+ rewritten = fut_rewrite.result()
180
+ structural_tags = fut_struct.result()
181
+ probe_tags = fut_probe.result() if fut_probe else []
182
+
183
  log("Rewrite:")
184
  log(rewritten if rewritten else "(empty)")
185
  log("")
 
192
 
193
  log("Step 2: Prompt Squirrel retrieval (hidden)")
194
  try:
195
+ retrieval_context_tags = list(dict.fromkeys((structural_tags or []) + (probe_tags or [])))
196
  rewrite_phrases = [p.strip() for p in (rewrite_for_retrieval or "").split(",") if p.strip()]
197
  retrieval_result = psq_candidates_from_rewrite_phrases(
198
  rewrite_phrases=rewrite_phrases,
199
  allow_nsfw_tags=allow_nsfw_tags,
200
+ context_tags=retrieval_context_tags,
201
  global_k=300,
202
  verbose=verbose_retrieval,
203
  )
 
248
  log(f"Retrieval fallback: {type(e).__name__}: {e}")
249
  candidates = []
250
 
251
+ log("Step 3: LLM index selection (uses rewrite + structural/probe context)")
252
+ selection_query = _build_selection_query(
253
+ prompt_in=prompt_in,
254
+ rewritten=rewritten,
255
+ structural_tags=structural_tags,
256
+ probe_tags=probe_tags,
257
+ )
258
  picked_indices = llm_select_indices(
259
+ query_text=selection_query,
260
  candidates=candidates,
261
  max_pick=0,
262
+ log=log,
263
  )
264
 
265
  selected_tags = [candidates[i].tag for i in picked_indices] if picked_indices else []
266
 
 
 
267
  if structural_tags:
268
  # Add structural tags that aren't already selected
269
  existing = {t for t in selected_tags}
 
273
  else:
274
  log(" No structural tags inferred")
275
 
276
+ if probe_tags:
277
+ existing = {t for t in selected_tags}
278
+ new_probe = [t for t in probe_tags if t not in existing]
279
+ selected_tags.extend(new_probe)
280
+ log(f" Added {len(new_probe)} probe tags: {', '.join(new_probe)}")
281
+ elif enable_probe_tags:
282
+ log(" No probe tags inferred")
283
+
284
  log("Step 3c: Expand via tag implications")
285
  tag_set = set(selected_tags)
286
  expanded, implied_only = expand_tags_via_implications(tag_set)
data/analysis/category_centroid_review.csv ADDED
The diff for this file is too large to render. See raw diff
 
data/analysis/category_centroid_summary.json ADDED
@@ -0,0 +1,1071 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "registry_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_registry.csv",
3
+ "checklist_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\tagging_checklist.txt",
4
+ "seed_override_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_seed_overrides.csv",
5
+ "thresholds": {
6
+ "auto_sim_min": 0.78,
7
+ "auto_margin_min": 0.06,
8
+ "review_sim_min": 0.65,
9
+ "review_margin_min": 0.03
10
+ },
11
+ "n_centroids": 32,
12
+ "tag_group_seed_count": 649,
13
+ "ignored_wiki_groups": [
14
+ "ambiguous_tags",
15
+ "disambiguations",
16
+ "e621",
17
+ "multiple_tags",
18
+ "sound_effects",
19
+ "unknown_tags"
20
+ ],
21
+ "tag_groups_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\tag_groups.json",
22
+ "tag_group_map_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_tag_group_map.csv",
23
+ "seed_sizes": {
24
+ "body_type": 7,
25
+ "species": 13,
26
+ "gender": 7,
27
+ "count": 5,
28
+ "clothing": 8,
29
+ "location": 5,
30
+ "perspective": 7,
31
+ "general_activity_if_any": 7,
32
+ "posture": 10,
33
+ "body_decor": 9,
34
+ "fur_style": 3,
35
+ "hair": 3,
36
+ "breasts": 7,
37
+ "limbs": 8,
38
+ "gaze": 4,
39
+ "expression": 9,
40
+ "quality": 6,
41
+ "organization": 8,
42
+ "style": 3,
43
+ "text": 7,
44
+ "information": 4,
45
+ "requests": 3,
46
+ "resolution": 4,
47
+ "anatomy_features": 12,
48
+ "clothing_detail": 14,
49
+ "color_markings": 12,
50
+ "expression_detail": 77,
51
+ "franchise_series": 12,
52
+ "background_composition": 12,
53
+ "objects_props": 152,
54
+ "pose_action_detail": 155,
55
+ "gaze_detail": 12
56
+ },
57
+ "n_candidates": 6261,
58
+ "bucket_counts": {
59
+ "hold": 4500,
60
+ "auto_accept": 162,
61
+ "needs_review": 427
62
+ },
63
+ "high_overlap_centroid_pairs": [
64
+ {
65
+ "category_a": "limbs",
66
+ "category_b": "pose_action_detail",
67
+ "centroid_sim": 0.9933
68
+ },
69
+ {
70
+ "category_a": "limbs",
71
+ "category_b": "posture",
72
+ "centroid_sim": 0.9862
73
+ },
74
+ {
75
+ "category_a": "pose_action_detail",
76
+ "category_b": "posture",
77
+ "centroid_sim": 0.9846
78
+ },
79
+ {
80
+ "category_a": "anatomy_features",
81
+ "category_b": "gaze",
82
+ "centroid_sim": 0.9799
83
+ },
84
+ {
85
+ "category_a": "gaze_detail",
86
+ "category_b": "pose_action_detail",
87
+ "centroid_sim": 0.9789
88
+ },
89
+ {
90
+ "category_a": "gaze_detail",
91
+ "category_b": "perspective",
92
+ "centroid_sim": 0.9781
93
+ },
94
+ {
95
+ "category_a": "anatomy_features",
96
+ "category_b": "posture",
97
+ "centroid_sim": 0.9779
98
+ },
99
+ {
100
+ "category_a": "gaze",
101
+ "category_b": "posture",
102
+ "centroid_sim": 0.9774
103
+ },
104
+ {
105
+ "category_a": "expression",
106
+ "category_b": "gaze",
107
+ "centroid_sim": 0.9745
108
+ },
109
+ {
110
+ "category_a": "expression",
111
+ "category_b": "expression_detail",
112
+ "centroid_sim": 0.9739
113
+ },
114
+ {
115
+ "category_a": "expression_detail",
116
+ "category_b": "gaze_detail",
117
+ "centroid_sim": 0.9713
118
+ },
119
+ {
120
+ "category_a": "color_markings",
121
+ "category_b": "gaze_detail",
122
+ "centroid_sim": 0.9712
123
+ },
124
+ {
125
+ "category_a": "gaze_detail",
126
+ "category_b": "limbs",
127
+ "centroid_sim": 0.9709
128
+ },
129
+ {
130
+ "category_a": "gaze",
131
+ "category_b": "limbs",
132
+ "centroid_sim": 0.9701
133
+ },
134
+ {
135
+ "category_a": "anatomy_features",
136
+ "category_b": "limbs",
137
+ "centroid_sim": 0.969
138
+ },
139
+ {
140
+ "category_a": "color_markings",
141
+ "category_b": "pose_action_detail",
142
+ "centroid_sim": 0.9684
143
+ },
144
+ {
145
+ "category_a": "expression",
146
+ "category_b": "pose_action_detail",
147
+ "centroid_sim": 0.9677
148
+ },
149
+ {
150
+ "category_a": "color_markings",
151
+ "category_b": "limbs",
152
+ "centroid_sim": 0.9676
153
+ },
154
+ {
155
+ "category_a": "anatomy_features",
156
+ "category_b": "pose_action_detail",
157
+ "centroid_sim": 0.9672
158
+ },
159
+ {
160
+ "category_a": "gaze",
161
+ "category_b": "pose_action_detail",
162
+ "centroid_sim": 0.9668
163
+ },
164
+ {
165
+ "category_a": "gaze",
166
+ "category_b": "location",
167
+ "centroid_sim": 0.9664
168
+ },
169
+ {
170
+ "category_a": "perspective",
171
+ "category_b": "pose_action_detail",
172
+ "centroid_sim": 0.9647
173
+ },
174
+ {
175
+ "category_a": "count",
176
+ "category_b": "resolution",
177
+ "centroid_sim": 0.9646
178
+ },
179
+ {
180
+ "category_a": "expression",
181
+ "category_b": "limbs",
182
+ "centroid_sim": 0.9636
183
+ },
184
+ {
185
+ "category_a": "anatomy_features",
186
+ "category_b": "expression",
187
+ "centroid_sim": 0.9629
188
+ },
189
+ {
190
+ "category_a": "gaze_detail",
191
+ "category_b": "posture",
192
+ "centroid_sim": 0.9627
193
+ },
194
+ {
195
+ "category_a": "background_composition",
196
+ "category_b": "location",
197
+ "centroid_sim": 0.9625
198
+ },
199
+ {
200
+ "category_a": "gaze",
201
+ "category_b": "gaze_detail",
202
+ "centroid_sim": 0.9624
203
+ },
204
+ {
205
+ "category_a": "anatomy_features",
206
+ "category_b": "organization",
207
+ "centroid_sim": 0.9623
208
+ },
209
+ {
210
+ "category_a": "background_composition",
211
+ "category_b": "general_activity_if_any",
212
+ "centroid_sim": 0.9622
213
+ },
214
+ {
215
+ "category_a": "expression_detail",
216
+ "category_b": "pose_action_detail",
217
+ "centroid_sim": 0.9619
218
+ },
219
+ {
220
+ "category_a": "clothing",
221
+ "category_b": "clothing_detail",
222
+ "centroid_sim": 0.9615
223
+ },
224
+ {
225
+ "category_a": "expression",
226
+ "category_b": "gaze_detail",
227
+ "centroid_sim": 0.9608
228
+ },
229
+ {
230
+ "category_a": "anatomy_features",
231
+ "category_b": "gaze_detail",
232
+ "centroid_sim": 0.9604
233
+ },
234
+ {
235
+ "category_a": "background_composition",
236
+ "category_b": "gaze",
237
+ "centroid_sim": 0.9604
238
+ },
239
+ {
240
+ "category_a": "anatomy_features",
241
+ "category_b": "location",
242
+ "centroid_sim": 0.9602
243
+ },
244
+ {
245
+ "category_a": "anatomy_features",
246
+ "category_b": "background_composition",
247
+ "centroid_sim": 0.9601
248
+ },
249
+ {
250
+ "category_a": "expression",
251
+ "category_b": "posture",
252
+ "centroid_sim": 0.9601
253
+ },
254
+ {
255
+ "category_a": "background_composition",
256
+ "category_b": "organization",
257
+ "centroid_sim": 0.9599
258
+ },
259
+ {
260
+ "category_a": "gaze",
261
+ "category_b": "resolution",
262
+ "centroid_sim": 0.9594
263
+ }
264
+ ],
265
+ "bridge_tags_low_margin_high_sim": [
266
+ {
267
+ "tag": "mammal",
268
+ "fluffyrock_count": "2971571",
269
+ "best_category": "resolution",
270
+ "best_sim": "0.966316",
271
+ "second_category": "gaze",
272
+ "second_sim": "0.962334",
273
+ "margin": "0.003981",
274
+ "decision": "hold"
275
+ },
276
+ {
277
+ "tag": "fur",
278
+ "fluffyrock_count": "1417091",
279
+ "best_category": "color_markings",
280
+ "best_sim": "0.974221",
281
+ "second_category": "expression",
282
+ "second_sim": "0.971658",
283
+ "margin": "0.002563",
284
+ "decision": "hold"
285
+ },
286
+ {
287
+ "tag": "canid",
288
+ "fluffyrock_count": "1141448",
289
+ "best_category": "color_markings",
290
+ "best_sim": "0.902822",
291
+ "second_category": "limbs",
292
+ "second_sim": "0.887986",
293
+ "margin": "0.014836",
294
+ "decision": "hold"
295
+ },
296
+ {
297
+ "tag": "text",
298
+ "fluffyrock_count": "926878",
299
+ "best_category": "count",
300
+ "best_sim": "0.967529",
301
+ "second_category": "expression",
302
+ "second_sim": "0.956083",
303
+ "margin": "0.011446",
304
+ "decision": "hold"
305
+ },
306
+ {
307
+ "tag": "felid",
308
+ "fluffyrock_count": "598698",
309
+ "best_category": "clothing",
310
+ "best_sim": "0.874730",
311
+ "second_category": "color_markings",
312
+ "second_sim": "0.859014",
313
+ "margin": "0.015716",
314
+ "decision": "hold"
315
+ },
316
+ {
317
+ "tag": "canis",
318
+ "fluffyrock_count": "587772",
319
+ "best_category": "color_markings",
320
+ "best_sim": "0.842697",
321
+ "second_category": "limbs",
322
+ "second_sim": "0.826552",
323
+ "margin": "0.016145",
324
+ "decision": "hold"
325
+ },
326
+ {
327
+ "tag": "blue_eyes",
328
+ "fluffyrock_count": "451508",
329
+ "best_category": "gaze",
330
+ "best_sim": "0.967488",
331
+ "second_category": "anatomy_features",
332
+ "second_sim": "0.956279",
333
+ "margin": "0.011209",
334
+ "decision": "hold"
335
+ },
336
+ {
337
+ "tag": "feet",
338
+ "fluffyrock_count": "448730",
339
+ "best_category": "pose_action_detail",
340
+ "best_sim": "0.960693",
341
+ "second_category": "limbs",
342
+ "second_sim": "0.957081",
343
+ "margin": "0.003612",
344
+ "decision": "hold"
345
+ },
346
+ {
347
+ "tag": "biped",
348
+ "fluffyrock_count": "372051",
349
+ "best_category": "color_markings",
350
+ "best_sim": "0.976403",
351
+ "second_category": "pose_action_detail",
352
+ "second_sim": "0.963554",
353
+ "margin": "0.012849",
354
+ "decision": "hold"
355
+ },
356
+ {
357
+ "tag": "fox",
358
+ "fluffyrock_count": "337215",
359
+ "best_category": "quality",
360
+ "best_sim": "0.746687",
361
+ "second_category": "species",
362
+ "second_sim": "0.739898",
363
+ "margin": "0.006789",
364
+ "decision": "hold"
365
+ },
366
+ {
367
+ "tag": "domestic_dog",
368
+ "fluffyrock_count": "300258",
369
+ "best_category": "color_markings",
370
+ "best_sim": "0.725407",
371
+ "second_category": "limbs",
372
+ "second_sim": "0.709594",
373
+ "margin": "0.015814",
374
+ "decision": "hold"
375
+ },
376
+ {
377
+ "tag": "green_eyes",
378
+ "fluffyrock_count": "287579",
379
+ "best_category": "gaze",
380
+ "best_sim": "0.937881",
381
+ "second_category": "resolution",
382
+ "second_sim": "0.928477",
383
+ "margin": "0.009404",
384
+ "decision": "hold"
385
+ },
386
+ {
387
+ "tag": "wolf",
388
+ "fluffyrock_count": "273053",
389
+ "best_category": "color_markings",
390
+ "best_sim": "0.831498",
391
+ "second_category": "fur_style",
392
+ "second_sim": "0.824364",
393
+ "margin": "0.007134",
394
+ "decision": "hold"
395
+ },
396
+ {
397
+ "tag": "eyewear",
398
+ "fluffyrock_count": "251234",
399
+ "best_category": "expression",
400
+ "best_sim": "0.934410",
401
+ "second_category": "clothing_detail",
402
+ "second_sim": "0.932123",
403
+ "margin": "0.002288",
404
+ "decision": "hold"
405
+ },
406
+ {
407
+ "tag": "sweat",
408
+ "fluffyrock_count": "250842",
409
+ "best_category": "expression",
410
+ "best_sim": "0.935591",
411
+ "second_category": "posture",
412
+ "second_sim": "0.920475",
413
+ "margin": "0.015116",
414
+ "decision": "hold"
415
+ },
416
+ {
417
+ "tag": "felis",
418
+ "fluffyrock_count": "241656",
419
+ "best_category": "clothing",
420
+ "best_sim": "0.808244",
421
+ "second_category": "pose_action_detail",
422
+ "second_sim": "0.804629",
423
+ "margin": "0.003616",
424
+ "decision": "hold"
425
+ },
426
+ {
427
+ "tag": "domestic_cat",
428
+ "fluffyrock_count": "236967",
429
+ "best_category": "clothing",
430
+ "best_sim": "0.806201",
431
+ "second_category": "pose_action_detail",
432
+ "second_sim": "0.802073",
433
+ "margin": "0.004127",
434
+ "decision": "hold"
435
+ },
436
+ {
437
+ "tag": "narrowed_eyes",
438
+ "fluffyrock_count": "231816",
439
+ "best_category": "gaze",
440
+ "best_sim": "0.974548",
441
+ "second_category": "gaze_detail",
442
+ "second_sim": "0.964485",
443
+ "margin": "0.010064",
444
+ "decision": "hold"
445
+ },
446
+ {
447
+ "tag": "red_eyes",
448
+ "fluffyrock_count": "213545",
449
+ "best_category": "expression",
450
+ "best_sim": "0.907432",
451
+ "second_category": "anatomy_features",
452
+ "second_sim": "0.899259",
453
+ "margin": "0.008172",
454
+ "decision": "hold"
455
+ },
456
+ {
457
+ "tag": "signature",
458
+ "fluffyrock_count": "195111",
459
+ "best_category": "gaze_detail",
460
+ "best_sim": "0.971716",
461
+ "second_category": "color_markings",
462
+ "second_sim": "0.967869",
463
+ "margin": "0.003847",
464
+ "decision": "hold"
465
+ },
466
+ {
467
+ "tag": "brown_hair",
468
+ "fluffyrock_count": "183486",
469
+ "best_category": "clothing",
470
+ "best_sim": "0.937711",
471
+ "second_category": "pose_action_detail",
472
+ "second_sim": "0.934854",
473
+ "margin": "0.002857",
474
+ "decision": "hold"
475
+ },
476
+ {
477
+ "tag": "black_hair",
478
+ "fluffyrock_count": "180936",
479
+ "best_category": "body_decor",
480
+ "best_sim": "0.937858",
481
+ "second_category": "hair",
482
+ "second_sim": "0.937190",
483
+ "margin": "0.000668",
484
+ "decision": "hold"
485
+ },
486
+ {
487
+ "tag": "eyebrows",
488
+ "fluffyrock_count": "170102",
489
+ "best_category": "gaze_detail",
490
+ "best_sim": "0.952595",
491
+ "second_category": "color_markings",
492
+ "second_sim": "0.938983",
493
+ "margin": "0.013612",
494
+ "decision": "hold"
495
+ },
496
+ {
497
+ "tag": "yellow_eyes",
498
+ "fluffyrock_count": "168780",
499
+ "best_category": "anatomy_features",
500
+ "best_sim": "0.924401",
501
+ "second_category": "background_composition",
502
+ "second_sim": "0.918780",
503
+ "margin": "0.005621",
504
+ "decision": "hold"
505
+ },
506
+ {
507
+ "tag": "half-closed_eyes",
508
+ "fluffyrock_count": "158388",
509
+ "best_category": "gaze",
510
+ "best_sim": "0.967035",
511
+ "second_category": "gaze_detail",
512
+ "second_sim": "0.951442",
513
+ "margin": "0.015593",
514
+ "decision": "hold"
515
+ },
516
+ {
517
+ "tag": "barefoot",
518
+ "fluffyrock_count": "148056",
519
+ "best_category": "pose_action_detail",
520
+ "best_sim": "0.924109",
521
+ "second_category": "limbs",
522
+ "second_sim": "0.913093",
523
+ "margin": "0.011016",
524
+ "decision": "hold"
525
+ },
526
+ {
527
+ "tag": "panties",
528
+ "fluffyrock_count": "142486",
529
+ "best_category": "clothing_detail",
530
+ "best_sim": "0.808561",
531
+ "second_category": "clothing",
532
+ "second_sim": "0.797045",
533
+ "margin": "0.011516",
534
+ "decision": "hold"
535
+ },
536
+ {
537
+ "tag": "white_hair",
538
+ "fluffyrock_count": "139416",
539
+ "best_category": "hair",
540
+ "best_sim": "0.912607",
541
+ "second_category": "body_decor",
542
+ "second_sim": "0.898145",
543
+ "margin": "0.014462",
544
+ "decision": "hold"
545
+ },
546
+ {
547
+ "tag": "blue_hair",
548
+ "fluffyrock_count": "138674",
549
+ "best_category": "hair",
550
+ "best_sim": "0.914754",
551
+ "second_category": "resolution",
552
+ "second_sim": "0.908004",
553
+ "margin": "0.006750",
554
+ "decision": "hold"
555
+ },
556
+ {
557
+ "tag": "hybrid",
558
+ "fluffyrock_count": "134372",
559
+ "best_category": "anatomy_features",
560
+ "best_sim": "0.903589",
561
+ "second_category": "color_markings",
562
+ "second_sim": "0.892934",
563
+ "margin": "0.010656",
564
+ "decision": "hold"
565
+ },
566
+ {
567
+ "tag": "stripes",
568
+ "fluffyrock_count": "132789",
569
+ "best_category": "color_markings",
570
+ "best_sim": "0.885430",
571
+ "second_category": "pose_action_detail",
572
+ "second_sim": "0.869481",
573
+ "margin": "0.015949",
574
+ "decision": "hold"
575
+ },
576
+ {
577
+ "tag": "purple_eyes",
578
+ "fluffyrock_count": "131683",
579
+ "best_category": "resolution",
580
+ "best_sim": "0.850555",
581
+ "second_category": "body_type",
582
+ "second_sim": "0.834105",
583
+ "margin": "0.016450",
584
+ "decision": "hold"
585
+ },
586
+ {
587
+ "tag": "pose",
588
+ "fluffyrock_count": "130171",
589
+ "best_category": "body_decor",
590
+ "best_sim": "0.932085",
591
+ "second_category": "clothing",
592
+ "second_sim": "0.926449",
593
+ "margin": "0.005637",
594
+ "decision": "hold"
595
+ },
596
+ {
597
+ "tag": "girly",
598
+ "fluffyrock_count": "120948",
599
+ "best_category": "posture",
600
+ "best_sim": "0.906607",
601
+ "second_category": "limbs",
602
+ "second_sim": "0.900758",
603
+ "margin": "0.005850",
604
+ "decision": "hold"
605
+ },
606
+ {
607
+ "tag": "fan_character",
608
+ "fluffyrock_count": "120684",
609
+ "best_category": "resolution",
610
+ "best_sim": "0.805026",
611
+ "second_category": "body_type",
612
+ "second_sim": "0.798139",
613
+ "margin": "0.006888",
614
+ "decision": "hold"
615
+ },
616
+ {
617
+ "tag": "slightly_chubby",
618
+ "fluffyrock_count": "118826",
619
+ "best_category": "clothing",
620
+ "best_sim": "0.891814",
621
+ "second_category": "expression",
622
+ "second_sim": "0.875092",
623
+ "margin": "0.016721",
624
+ "decision": "hold"
625
+ },
626
+ {
627
+ "tag": "cleavage",
628
+ "fluffyrock_count": "116981",
629
+ "best_category": "breasts",
630
+ "best_sim": "0.893953",
631
+ "second_category": "hair",
632
+ "second_sim": "0.887062",
633
+ "margin": "0.006892",
634
+ "decision": "hold"
635
+ },
636
+ {
637
+ "tag": "red_hair",
638
+ "fluffyrock_count": "115770",
639
+ "best_category": "body_decor",
640
+ "best_sim": "0.933220",
641
+ "second_category": "hair",
642
+ "second_sim": "0.923598",
643
+ "margin": "0.009622",
644
+ "decision": "hold"
645
+ },
646
+ {
647
+ "tag": "accessory",
648
+ "fluffyrock_count": "114975",
649
+ "best_category": "hair",
650
+ "best_sim": "0.922713",
651
+ "second_category": "clothing_detail",
652
+ "second_sim": "0.915335",
653
+ "margin": "0.007378",
654
+ "decision": "hold"
655
+ },
656
+ {
657
+ "tag": "one_eye_closed",
658
+ "fluffyrock_count": "113898",
659
+ "best_category": "expression",
660
+ "best_sim": "0.978476",
661
+ "second_category": "gaze",
662
+ "second_sim": "0.958936",
663
+ "margin": "0.019540",
664
+ "decision": "hold"
665
+ },
666
+ {
667
+ "tag": "membrane_(anatomy)",
668
+ "fluffyrock_count": "107123",
669
+ "best_category": "anatomy_features",
670
+ "best_sim": "0.804104",
671
+ "second_category": "background_composition",
672
+ "second_sim": "0.784603",
673
+ "margin": "0.019501",
674
+ "decision": "hold"
675
+ },
676
+ {
677
+ "tag": "digital_drawing_(artwork)",
678
+ "fluffyrock_count": "105828",
679
+ "best_category": "gaze_detail",
680
+ "best_sim": "0.928962",
681
+ "second_category": "expression_detail",
682
+ "second_sim": "0.923921",
683
+ "margin": "0.005042",
684
+ "decision": "hold"
685
+ },
686
+ {
687
+ "tag": "thigh_highs",
688
+ "fluffyrock_count": "104261",
689
+ "best_category": "hair",
690
+ "best_sim": "0.825228",
691
+ "second_category": "breasts",
692
+ "second_sim": "0.810663",
693
+ "margin": "0.014565",
694
+ "decision": "hold"
695
+ },
696
+ {
697
+ "tag": "seductive",
698
+ "fluffyrock_count": "103923",
699
+ "best_category": "gaze",
700
+ "best_sim": "0.956244",
701
+ "second_category": "limbs",
702
+ "second_sim": "0.941192",
703
+ "margin": "0.015052",
704
+ "decision": "hold"
705
+ },
706
+ {
707
+ "tag": "purple_hair",
708
+ "fluffyrock_count": "101470",
709
+ "best_category": "resolution",
710
+ "best_sim": "0.776883",
711
+ "second_category": "body_type",
712
+ "second_sim": "0.766707",
713
+ "margin": "0.010176",
714
+ "decision": "hold"
715
+ },
716
+ {
717
+ "tag": "pink_hair",
718
+ "fluffyrock_count": "99350",
719
+ "best_category": "resolution",
720
+ "best_sim": "0.795603",
721
+ "second_category": "hair",
722
+ "second_sim": "0.779317",
723
+ "margin": "0.016286",
724
+ "decision": "hold"
725
+ },
726
+ {
727
+ "tag": "mostly_nude",
728
+ "fluffyrock_count": "99006",
729
+ "best_category": "pose_action_detail",
730
+ "best_sim": "0.937643",
731
+ "second_category": "posture",
732
+ "second_sim": "0.930551",
733
+ "margin": "0.007092",
734
+ "decision": "hold"
735
+ },
736
+ {
737
+ "tag": "url",
738
+ "fluffyrock_count": "98800",
739
+ "best_category": "gaze",
740
+ "best_sim": "0.886256",
741
+ "second_category": "location",
742
+ "second_sim": "0.884944",
743
+ "margin": "0.001313",
744
+ "decision": "hold"
745
+ },
746
+ {
747
+ "tag": "stockings",
748
+ "fluffyrock_count": "96677",
749
+ "best_category": "hair",
750
+ "best_sim": "0.863455",
751
+ "second_category": "breasts",
752
+ "second_sim": "0.847350",
753
+ "margin": "0.016105",
754
+ "decision": "hold"
755
+ },
756
+ {
757
+ "tag": "tears",
758
+ "fluffyrock_count": "90853",
759
+ "best_category": "count",
760
+ "best_sim": "0.842997",
761
+ "second_category": "expression_detail",
762
+ "second_sim": "0.841432",
763
+ "margin": "0.001565",
764
+ "decision": "hold"
765
+ },
766
+ {
767
+ "tag": "female/female",
768
+ "fluffyrock_count": "89943",
769
+ "best_category": "resolution",
770
+ "best_sim": "0.879301",
771
+ "second_category": "gaze",
772
+ "second_sim": "0.870119",
773
+ "margin": "0.009182",
774
+ "decision": "hold"
775
+ },
776
+ {
777
+ "tag": "digitigrade",
778
+ "fluffyrock_count": "87950",
779
+ "best_category": "color_markings",
780
+ "best_sim": "0.880638",
781
+ "second_category": "limbs",
782
+ "second_sim": "0.873191",
783
+ "margin": "0.007447",
784
+ "decision": "hold"
785
+ },
786
+ {
787
+ "tag": "brown_eyes",
788
+ "fluffyrock_count": "82369",
789
+ "best_category": "color_markings",
790
+ "best_sim": "0.936662",
791
+ "second_category": "pose_action_detail",
792
+ "second_sim": "0.932875",
793
+ "margin": "0.003788",
794
+ "decision": "hold"
795
+ },
796
+ {
797
+ "tag": "tiger",
798
+ "fluffyrock_count": "79829",
799
+ "best_category": "fur_style",
800
+ "best_sim": "0.702155",
801
+ "second_category": "clothing",
802
+ "second_sim": "0.687733",
803
+ "margin": "0.014423",
804
+ "decision": "hold"
805
+ },
806
+ {
807
+ "tag": "plantigrade",
808
+ "fluffyrock_count": "78753",
809
+ "best_category": "pose_action_detail",
810
+ "best_sim": "0.850908",
811
+ "second_category": "limbs",
812
+ "second_sim": "0.845585",
813
+ "margin": "0.005323",
814
+ "decision": "hold"
815
+ },
816
+ {
817
+ "tag": "border",
818
+ "fluffyrock_count": "77664",
819
+ "best_category": "expression_detail",
820
+ "best_sim": "0.946144",
821
+ "second_category": "organization",
822
+ "second_sim": "0.942520",
823
+ "margin": "0.003624",
824
+ "decision": "hold"
825
+ },
826
+ {
827
+ "tag": "glistening",
828
+ "fluffyrock_count": "77473",
829
+ "best_category": "perspective",
830
+ "best_sim": "0.888259",
831
+ "second_category": "gaze_detail",
832
+ "second_sim": "0.886374",
833
+ "margin": "0.001885",
834
+ "decision": "hold"
835
+ },
836
+ {
837
+ "tag": "spots",
838
+ "fluffyrock_count": "75219",
839
+ "best_category": "color_markings",
840
+ "best_sim": "0.893942",
841
+ "second_category": "limbs",
842
+ "second_sim": "0.880259",
843
+ "margin": "0.013683",
844
+ "decision": "hold"
845
+ },
846
+ {
847
+ "tag": "mythology",
848
+ "fluffyrock_count": "70578",
849
+ "best_category": "resolution",
850
+ "best_sim": "0.782674",
851
+ "second_category": "body_type",
852
+ "second_sim": "0.780435",
853
+ "margin": "0.002238",
854
+ "decision": "hold"
855
+ },
856
+ {
857
+ "tag": "pupils",
858
+ "fluffyrock_count": "69130",
859
+ "best_category": "gaze_detail",
860
+ "best_sim": "0.912714",
861
+ "second_category": "perspective",
862
+ "second_sim": "0.900226",
863
+ "margin": "0.012489",
864
+ "decision": "hold"
865
+ },
866
+ {
867
+ "tag": "soles",
868
+ "fluffyrock_count": "68647",
869
+ "best_category": "limbs",
870
+ "best_sim": "0.803874",
871
+ "second_category": "pose_action_detail",
872
+ "second_sim": "0.794181",
873
+ "margin": "0.009693",
874
+ "decision": "hold"
875
+ },
876
+ {
877
+ "tag": "deer",
878
+ "fluffyrock_count": "67106",
879
+ "best_category": "species",
880
+ "best_sim": "0.768303",
881
+ "second_category": "limbs",
882
+ "second_sim": "0.757657",
883
+ "margin": "0.010647",
884
+ "decision": "hold"
885
+ },
886
+ {
887
+ "tag": "facial_tuft",
888
+ "fluffyrock_count": "66479",
889
+ "best_category": "color_markings",
890
+ "best_sim": "0.887860",
891
+ "second_category": "gaze_detail",
892
+ "second_sim": "0.881925",
893
+ "margin": "0.005935",
894
+ "decision": "hold"
895
+ },
896
+ {
897
+ "tag": "armwear",
898
+ "fluffyrock_count": "65958",
899
+ "best_category": "hair",
900
+ "best_sim": "0.806044",
901
+ "second_category": "breasts",
902
+ "second_sim": "0.803883",
903
+ "margin": "0.002161",
904
+ "decision": "hold"
905
+ },
906
+ {
907
+ "tag": "hindpaw",
908
+ "fluffyrock_count": "65867",
909
+ "best_category": "limbs",
910
+ "best_sim": "0.908189",
911
+ "second_category": "color_markings",
912
+ "second_sim": "0.898025",
913
+ "margin": "0.010164",
914
+ "decision": "hold"
915
+ },
916
+ {
917
+ "tag": "pink_nose",
918
+ "fluffyrock_count": "65693",
919
+ "best_category": "color_markings",
920
+ "best_sim": "0.905979",
921
+ "second_category": "gaze_detail",
922
+ "second_sim": "0.892451",
923
+ "margin": "0.013528",
924
+ "decision": "hold"
925
+ },
926
+ {
927
+ "tag": "cheek_tuft",
928
+ "fluffyrock_count": "63157",
929
+ "best_category": "color_markings",
930
+ "best_sim": "0.888716",
931
+ "second_category": "gaze_detail",
932
+ "second_sim": "0.880596",
933
+ "margin": "0.008120",
934
+ "decision": "hold"
935
+ },
936
+ {
937
+ "tag": "traditional_media_(artwork)",
938
+ "fluffyrock_count": "62691",
939
+ "best_category": "quality",
940
+ "best_sim": "0.810561",
941
+ "second_category": "style",
942
+ "second_sim": "0.806736",
943
+ "margin": "0.003825",
944
+ "decision": "hold"
945
+ },
946
+ {
947
+ "tag": "holidays",
948
+ "fluffyrock_count": "60499",
949
+ "best_category": "clothing_detail",
950
+ "best_sim": "0.922842",
951
+ "second_category": "expression",
952
+ "second_sim": "0.917368",
953
+ "margin": "0.005473",
954
+ "decision": "hold"
955
+ },
956
+ {
957
+ "tag": "wet",
958
+ "fluffyrock_count": "57922",
959
+ "best_category": "clothing",
960
+ "best_sim": "0.918274",
961
+ "second_category": "expression",
962
+ "second_sim": "0.916525",
963
+ "margin": "0.001748",
964
+ "decision": "hold"
965
+ },
966
+ {
967
+ "tag": "bra",
968
+ "fluffyrock_count": "57003",
969
+ "best_category": "clothing_detail",
970
+ "best_sim": "0.803868",
971
+ "second_category": "clothing",
972
+ "second_sim": "0.790065",
973
+ "margin": "0.013802",
974
+ "decision": "hold"
975
+ },
976
+ {
977
+ "tag": "full-length_portrait",
978
+ "fluffyrock_count": "56605",
979
+ "best_category": "gaze_detail",
980
+ "best_sim": "0.873990",
981
+ "second_category": "color_markings",
982
+ "second_sim": "0.863915",
983
+ "margin": "0.010075",
984
+ "decision": "hold"
985
+ },
986
+ {
987
+ "tag": "ribbons",
988
+ "fluffyrock_count": "54689",
989
+ "best_category": "hair",
990
+ "best_sim": "0.813789",
991
+ "second_category": "objects_props",
992
+ "second_sim": "0.795981",
993
+ "margin": "0.017808",
994
+ "decision": "hold"
995
+ },
996
+ {
997
+ "tag": "crossgender",
998
+ "fluffyrock_count": "54149",
999
+ "best_category": "gender",
1000
+ "best_sim": "0.741670",
1001
+ "second_category": "breasts",
1002
+ "second_sim": "0.728958",
1003
+ "margin": "0.012712",
1004
+ "decision": "hold"
1005
+ },
1006
+ {
1007
+ "tag": "black_sclera",
1008
+ "fluffyrock_count": "51667",
1009
+ "best_category": "anatomy_features",
1010
+ "best_sim": "0.777686",
1011
+ "second_category": "color_markings",
1012
+ "second_sim": "0.773014",
1013
+ "margin": "0.004672",
1014
+ "decision": "hold"
1015
+ },
1016
+ {
1017
+ "tag": "black_eyes",
1018
+ "fluffyrock_count": "50462",
1019
+ "best_category": "style",
1020
+ "best_sim": "0.813997",
1021
+ "second_category": "expression_detail",
1022
+ "second_sim": "0.812944",
1023
+ "margin": "0.001053",
1024
+ "decision": "hold"
1025
+ },
1026
+ {
1027
+ "tag": "ponytail",
1028
+ "fluffyrock_count": "50191",
1029
+ "best_category": "hair",
1030
+ "best_sim": "0.936813",
1031
+ "second_category": "clothing_detail",
1032
+ "second_sim": "0.920436",
1033
+ "margin": "0.016377",
1034
+ "decision": "hold"
1035
+ },
1036
+ {
1037
+ "tag": "larger_female",
1038
+ "fluffyrock_count": "50146",
1039
+ "best_category": "posture",
1040
+ "best_sim": "0.850431",
1041
+ "second_category": "perspective",
1042
+ "second_sim": "0.837875",
1043
+ "margin": "0.012556",
1044
+ "decision": "hold"
1045
+ },
1046
+ {
1047
+ "tag": "lizard",
1048
+ "fluffyrock_count": "46938",
1049
+ "best_category": "species",
1050
+ "best_sim": "0.734055",
1051
+ "second_category": "anatomy_features",
1052
+ "second_sim": "0.734000",
1053
+ "margin": "0.000055",
1054
+ "decision": "hold"
1055
+ },
1056
+ {
1057
+ "tag": "orange_eyes",
1058
+ "fluffyrock_count": "46900",
1059
+ "best_category": "color_markings",
1060
+ "best_sim": "0.928687",
1061
+ "second_category": "gaze_detail",
1062
+ "second_sim": "0.928468",
1063
+ "margin": "0.000220",
1064
+ "decision": "hold"
1065
+ }
1066
+ ],
1067
+ "outputs": {
1068
+ "review_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_centroid_review.csv",
1069
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_centroid_summary.json"
1070
+ }
1071
+ }
data/analysis/category_expansion_coverage.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "inputs": {
3
+ "min_count": 200,
4
+ "top_n_groups": 15,
5
+ "sample_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\eval_samples\\e621_sfw_sample_1000_seed123_buffer10000.jsonl",
6
+ "proposal_source_uncovered": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_uncovered_after_topn_combined200.csv"
7
+ },
8
+ "proposal_counts": {
9
+ "merge_existing": 37,
10
+ "deprioritize": 7,
11
+ "new_category": 31,
12
+ "needs_review": 45
13
+ },
14
+ "art_tags_in_proposal": [
15
+ {
16
+ "tag": "kemono",
17
+ "fluffyrock_count": "103919",
18
+ "sample_occurrences": "33",
19
+ "proposed_action": "needs_review",
20
+ "target_category": "uncategorized_review",
21
+ "in_art_tag_group": "1",
22
+ "reason": "high-frequency uncovered tag needing manual judgment"
23
+ }
24
+ ],
25
+ "coverage_baseline": {
26
+ "n_groups": 78,
27
+ "unique_covered_pct": 38.33,
28
+ "top15_greedy_cumulative_pct": 59.93,
29
+ "top15_groups": [
30
+ "cat:clothing_detail",
31
+ "cat:anatomy_features",
32
+ "species",
33
+ "cat:color_markings",
34
+ "genders",
35
+ "cat:background_composition",
36
+ "body_types",
37
+ "cat:count",
38
+ "facial_expressions",
39
+ "cat:objects_props",
40
+ "pose",
41
+ "narrative_elements",
42
+ "cat:hair",
43
+ "cat:franchise_series",
44
+ "cat:body_decor"
45
+ ]
46
+ },
47
+ "coverage_projected_with_proposal": {
48
+ "n_groups": 82,
49
+ "unique_covered_pct": 40.35,
50
+ "top15_greedy_cumulative_pct": 70.77,
51
+ "top15_groups": [
52
+ "cat:clothing_detail",
53
+ "cat:anatomy_features",
54
+ "cat:color_markings",
55
+ "species",
56
+ "cat:species_specific",
57
+ "genders",
58
+ "cat:background_composition",
59
+ "body_types",
60
+ "cat:count",
61
+ "facial_expressions",
62
+ "cat:objects_props",
63
+ "cat:fur_style",
64
+ "pose",
65
+ "cat:hair",
66
+ "narrative_elements"
67
+ ]
68
+ },
69
+ "outputs": {
70
+ "proposal_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_expansion_proposal.csv",
71
+ "coverage_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\category_expansion_coverage.json"
72
+ }
73
+ }
data/analysis/category_expansion_proposal.csv ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,fluffyrock_count,sample_occurrences,proposed_action,target_category,in_art_tag_group,reason
2
+ fur,1417091,422,merge_existing,fur_style,0,fur/covering style detail
3
+ bodily_fluids,1225999,49,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
4
+ canid,1141448,304,new_category,species_specific,0,taxonomy/detail species cluster
5
+ nude,1103510,57,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
6
+ nipples,1079208,17,merge_existing,anatomy_features,0,anatomy/body-part trait
7
+ butt,828096,25,merge_existing,anatomy_features,0,anatomy/body-part trait
8
+ felid,598698,158,new_category,species_specific,0,taxonomy/detail species cluster
9
+ canis,587772,159,new_category,species_specific,0,taxonomy/detail species cluster
10
+ male/female,551565,12,new_category,relationship_pairing,0,relationship/pairing semantics shown best together
11
+ blue_eyes,451508,144,merge_existing,color_markings,0,color-region/attribute tag
12
+ feet,448730,113,merge_existing,anatomy_features,0,anatomy/body-part trait
13
+ male/male,427119,26,new_category,relationship_pairing,0,relationship/pairing semantics shown best together
14
+ equid,399764,97,new_category,species_specific,0,taxonomy/detail species cluster
15
+ navel,373746,45,merge_existing,anatomy_features,0,anatomy/body-part trait
16
+ biped,372051,95,merge_existing,pose_action_detail,0,pose/action detail
17
+ muscular,358573,56,new_category,body_build,0,body-shape alternatives useful side-by-side
18
+ thick_thighs,339009,25,new_category,body_build,0,body-shape alternatives useful side-by-side
19
+ fox,337215,105,new_category,species_specific,0,taxonomy/detail species cluster
20
+ tuft,335534,105,merge_existing,fur_style,0,fur/covering style detail
21
+ areola,320191,1,merge_existing,anatomy_features,0,anatomy/body-part trait
22
+ dragon,309478,84,new_category,species_specific,0,taxonomy/detail species cluster
23
+ domestic_dog,300258,77,new_category,species_specific,0,taxonomy/detail species cluster
24
+ <3,293372,50,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
25
+ green_eyes,287579,78,merge_existing,color_markings,0,color-region/attribute tag
26
+ wolf,273053,79,new_category,species_specific,0,taxonomy/detail species cluster
27
+ plant,265700,97,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
28
+ size_difference,265067,34,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
29
+ big_butt,262825,11,new_category,body_build,0,body-shape alternatives useful side-by-side
30
+ interspecies,261649,4,new_category,relationship_pairing,0,relationship/pairing semantics shown best together
31
+ huge_breasts,261003,13,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
32
+ eyewear,251234,93,merge_existing,clothing_detail,0,attire/accessory detail
33
+ sweat,250842,20,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
34
+ felis,241656,64,new_category,species_specific,0,taxonomy/detail species cluster
35
+ young,238777,40,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
36
+ domestic_cat,236967,64,new_category,species_specific,0,taxonomy/detail species cluster
37
+ wide_hips,235877,30,new_category,body_build,0,body-shape alternatives useful side-by-side
38
+ black_nose,216679,80,merge_existing,color_markings,0,color-region/attribute tag
39
+ red_eyes,213545,53,merge_existing,color_markings,0,color-region/attribute tag
40
+ reptile,206968,47,new_category,species_specific,0,taxonomy/detail species cluster
41
+ leporid,204404,73,new_category,species_specific,0,taxonomy/detail species cluster
42
+ rabbit,199906,73,new_category,species_specific,0,taxonomy/detail species cluster
43
+ belly,194310,31,merge_existing,anatomy_features,0,anatomy/body-part trait
44
+ eyelashes,193540,63,merge_existing,expression_detail,0,eye/expression detail
45
+ horse,193042,40,new_category,species_specific,0,taxonomy/detail species cluster
46
+ feathers,192797,76,merge_existing,fur_style,0,fur/covering style detail
47
+ solo_focus,191284,18,new_category,character_count,0,mutually exclusive count-like options
48
+ saliva,187352,10,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
49
+ jewelry,185593,53,merge_existing,clothing_detail,0,attire/accessory detail
50
+ brown_hair,183486,48,merge_existing,color_markings,0,color-region/attribute tag
51
+ blonde_hair,183468,47,merge_existing,hair,0,hair style/color detail
52
+ pantherine,181019,45,new_category,species_specific,0,taxonomy/detail species cluster
53
+ black_hair,180936,45,merge_existing,color_markings,0,color-region/attribute tag
54
+ overweight,176776,36,new_category,body_build,0,body-shape alternatives useful side-by-side
55
+ abs,171761,23,merge_existing,anatomy_features,0,anatomy/body-part trait
56
+ eyebrows,170102,50,merge_existing,expression_detail,0,eye/expression detail
57
+ vein,168934,5,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
58
+ yellow_eyes,168780,45,merge_existing,color_markings,0,color-region/attribute tag
59
+ muscular_anthro,164876,30,new_category,body_build,0,body-shape alternatives useful side-by-side
60
+ half-closed_eyes,158388,19,merge_existing,expression_detail,0,eye/expression detail
61
+ pecs,158385,23,merge_existing,anatomy_features,0,anatomy/body-part trait
62
+ bovid,156674,35,new_category,species_specific,0,taxonomy/detail species cluster
63
+ barefoot,148056,46,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
64
+ cub,147547,21,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
65
+ bound,143669,3,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
66
+ panties,142486,13,merge_existing,clothing_detail,0,attire/accessory detail
67
+ anthrofied,139874,11,merge_existing,fur_style,0,fur/covering style detail
68
+ white_hair,139416,37,merge_existing,color_markings,0,color-region/attribute tag
69
+ animal_humanoid,139252,31,new_category,species_specific,0,taxonomy/detail species cluster
70
+ blue_hair,138674,48,merge_existing,color_markings,0,color-region/attribute tag
71
+ hyper,137712,2,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
72
+ hybrid,134372,30,new_category,species_specific,0,taxonomy/detail species cluster
73
+ stripes,132789,32,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
74
+ bottomless,132767,7,deprioritize,none,0,sensitive/noisy for default non-explicit-centric UX
75
+ purple_eyes,131683,34,merge_existing,color_markings,0,color-region/attribute tag
76
+ pose,130171,25,merge_existing,pose_action_detail,0,pose/action detail
77
+ rear_view,128927,7,merge_existing,pose_action_detail,0,pose/action detail
78
+ licking,126818,9,merge_existing,pose_action_detail,0,pose/action detail
79
+ pony,122813,35,new_category,species_specific,0,taxonomy/detail species cluster
80
+ slightly_chubby,118826,21,new_category,body_build,0,body-shape alternatives useful side-by-side
81
+ scales,118407,36,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
82
+ dominant,118222,2,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
83
+ cleavage,116981,41,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
84
+ red_hair,115770,24,merge_existing,color_markings,0,color-region/attribute tag
85
+ bird,115422,42,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
86
+ topless,115349,45,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
87
+ accessory,114975,29,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
88
+ chest_tuft,114724,37,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
89
+ fluffy,113761,35,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
90
+ unicorn,112328,39,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
91
+ bear,111640,38,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
92
+ partially_clothed,110046,3,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
93
+ membrane_(anatomy),107123,34,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
94
+ curvy_figure,106779,7,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
95
+ biceps,106240,22,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
96
+ looking_at_another,105719,19,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
97
+ thigh_highs,104261,16,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
98
+ cutie_mark,104225,29,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
99
+ seductive,103923,7,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
100
+ kemono,103919,33,needs_review,uncategorized_review,1,high-frequency uncovered tag needing manual judgment
101
+ submissive,102144,1,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
102
+ purple_hair,101470,27,merge_existing,color_markings,0,color-region/attribute tag
103
+ front_view,101036,29,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
104
+ pink_hair,99350,25,merge_existing,hair,0,hair style/color detail
105
+ mostly_nude,99006,9,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
106
+ stockings,96677,6,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
107
+ ring_piercing,96328,21,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
108
+ dripping,94534,1,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
109
+ makeup,93493,14,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
110
+ mammal_humanoid,91171,19,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
111
+ machine,90867,34,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
112
+ female/female,89943,6,new_category,relationship_pairing,0,relationship/pairing semantics shown best together
113
+ digitigrade,87950,23,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
114
+ humanoid_hands,87880,28,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
115
+ huge_butt,87380,2,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
116
+ drooling,86754,5,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
117
+ demon,84604,16,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
118
+ faceless_character,83520,2,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
119
+ brown_eyes,82369,24,merge_existing,color_markings,0,color-region/attribute tag
120
+ larger_male,81280,5,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
121
+ smaller_male,81264,4,needs_review,uncategorized_review,0,high-frequency uncovered tag needing manual judgment
data/analysis/category_registry.csv ADDED
The diff for this file is too large to render. See raw diff
 
data/analysis/category_seed_overrides.csv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ category_name,tag,enabled,seed_note
2
+ objects_props,bed,1,example manual seed
3
+ background_composition,indoors,1,example manual seed
4
+ pose_action_detail,stretching,1,example manual seed
data/analysis/category_tag_group_map.csv ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ category_name,tag_group,enabled,seed_note
2
+ clothing_detail,clothes,1,e621 wiki tag group
3
+ expression_detail,facial_expressions,1,e621 wiki tag group
4
+ objects_props,food,1,e621 wiki tag group
5
+ pose_action_detail,pose,1,e621 wiki tag group
6
+ ignored_nonsemantic,ambiguous_tags,1,exclude non-semantic maintenance group from modeling
7
+ ignored_nonsemantic,unknown_tags,1,exclude non-semantic maintenance group from modeling
8
+ ignored_nonsemantic,disambiguations,1,exclude non-semantic maintenance group from modeling
9
+ ignored_nonsemantic,multiple_tags,1,exclude non-semantic maintenance group from modeling
10
+ ignored_nonsemantic,e621,1,exclude non-semantic maintenance group from modeling
11
+ ignored_nonsemantic,sound_effects,1,exclude non-semantic maintenance group from modeling
data/analysis/guided_facet_assignments.csv ADDED
The diff for this file is too large to render. See raw diff
 
data/analysis/guided_facet_summary.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "min_count": 200,
3
+ "n_unique_tags_considered": 3068,
4
+ "n_uncovered_before_guided_facets": 1830,
5
+ "facet_names": [
6
+ "character_traits",
7
+ "clothing_coverage",
8
+ "fluids_explicit_sensitive",
9
+ "species_taxonomy",
10
+ "symbol_text_misc"
11
+ ],
12
+ "decision_counts": {
13
+ "auto_assign": 276,
14
+ "review": 1524
15
+ },
16
+ "auto_assign_counts_by_facet": {
17
+ "fluids_explicit_sensitive": 63,
18
+ "character_traits": 12,
19
+ "clothing_coverage": 154,
20
+ "species_taxonomy": 32,
21
+ "symbol_text_misc": 15
22
+ },
23
+ "coverage": {
24
+ "baseline_unique_pct": 40.35,
25
+ "baseline_top15_pct": 70.77,
26
+ "projected_unique_pct_with_explicit_facet": 49.35,
27
+ "projected_top15_pct_with_explicit_facet": 73.77,
28
+ "projected_unique_pct_without_explicit_facet": 47.29,
29
+ "projected_top15_pct_without_explicit_facet": 73.77
30
+ },
31
+ "outputs": {
32
+ "assignments_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\guided_facet_assignments.csv",
33
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\guided_facet_summary.json"
34
+ }
35
+ }
data/analysis/probe_informativeness.csv ADDED
The diff for this file is too large to render. See raw diff
 
data/analysis/probe_informativeness_summary.json ADDED
@@ -0,0 +1,1867 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "min_count": 200,
4
+ "min_probe_images": 5,
5
+ "min_group_images": 20,
6
+ "softmax_tau": 0.15,
7
+ "mmr_lambda": 0.35,
8
+ "mmr_top_pool": 120,
9
+ "mmr_k": 15
10
+ },
11
+ "n_images": 1000,
12
+ "n_candidate_probes": 796,
13
+ "n_active_groups": 53,
14
+ "excluded_wiki_groups": [
15
+ "ambiguous_tags",
16
+ "disambiguations",
17
+ "e621",
18
+ "multiple_tags",
19
+ "sound_effects",
20
+ "unknown_tags"
21
+ ],
22
+ "top_probes_by_combined_score": [
23
+ {
24
+ "tag": "clothing",
25
+ "sample_occurrences": "574",
26
+ "fluffyrock_count": "1738206",
27
+ "prevalence": "0.574000",
28
+ "ig_sum_bits": "2.622723",
29
+ "ig_mean_bits": "0.049485",
30
+ "delta_top5_mass": "0.300307",
31
+ "mean_abs_log2_lift": "0.239534",
32
+ "semantic_top_group": "cat:clothing",
33
+ "semantic_margin": "0.001094",
34
+ "semantic_entropy_norm": "0.965190",
35
+ "strongest_group_by_lift": "cat:clothing",
36
+ "strongest_group_lift": "1.742160",
37
+ "suggested_probe_bundle": "clothing_state",
38
+ "needs_glossary": "0",
39
+ "combined_score": "0.671033",
40
+ "actionable_score": "0.347207"
41
+ },
42
+ {
43
+ "tag": "hair",
44
+ "sample_occurrences": "373",
45
+ "fluffyrock_count": "1495810",
46
+ "prevalence": "0.373000",
47
+ "ig_sum_bits": "2.199276",
48
+ "ig_mean_bits": "0.041496",
49
+ "delta_top5_mass": "0.314080",
50
+ "mean_abs_log2_lift": "0.261228",
51
+ "semantic_top_group": "cat:hair",
52
+ "semantic_margin": "0.031478",
53
+ "semantic_entropy_norm": "0.966277",
54
+ "strongest_group_by_lift": "cat:hair",
55
+ "strongest_group_lift": "2.680965",
56
+ "suggested_probe_bundle": "other",
57
+ "needs_glossary": "0",
58
+ "combined_score": "0.549720",
59
+ "actionable_score": "0.266338"
60
+ },
61
+ {
62
+ "tag": "clothed",
63
+ "sample_occurrences": "337",
64
+ "fluffyrock_count": "962955",
65
+ "prevalence": "0.337000",
66
+ "ig_sum_bits": "2.016235",
67
+ "ig_mean_bits": "0.038042",
68
+ "delta_top5_mass": "0.364653",
69
+ "mean_abs_log2_lift": "0.381805",
70
+ "semantic_top_group": "cat:clothing",
71
+ "semantic_margin": "0.026471",
72
+ "semantic_entropy_norm": "0.962221",
73
+ "strongest_group_by_lift": "cat:clothing",
74
+ "strongest_group_lift": "2.653736",
75
+ "suggested_probe_bundle": "clothing_state",
76
+ "needs_glossary": "0",
77
+ "combined_score": "0.494524",
78
+ "actionable_score": "0.257132"
79
+ },
80
+ {
81
+ "tag": "simple_background",
82
+ "sample_occurrences": "345",
83
+ "fluffyrock_count": "1081480",
84
+ "prevalence": "0.345000",
85
+ "ig_sum_bits": "1.854364",
86
+ "ig_mean_bits": "0.034988",
87
+ "delta_top5_mass": "0.328348",
88
+ "mean_abs_log2_lift": "0.354546",
89
+ "semantic_top_group": "cat:expression",
90
+ "semantic_margin": "0.015478",
91
+ "semantic_entropy_norm": "0.967783",
92
+ "strongest_group_by_lift": "wiki:settings",
93
+ "strongest_group_lift": "0.115942",
94
+ "suggested_probe_bundle": "scene_pose",
95
+ "needs_glossary": "0",
96
+ "combined_score": "0.454953",
97
+ "actionable_score": "0.216830"
98
+ },
99
+ {
100
+ "tag": "text",
101
+ "sample_occurrences": "275",
102
+ "fluffyrock_count": "926878",
103
+ "prevalence": "0.275000",
104
+ "ig_sum_bits": "1.836555",
105
+ "ig_mean_bits": "0.034652",
106
+ "delta_top5_mass": "0.176727",
107
+ "mean_abs_log2_lift": "0.303566",
108
+ "semantic_top_group": "cat:count",
109
+ "semantic_margin": "0.008246",
110
+ "semantic_entropy_norm": "0.968373",
111
+ "strongest_group_by_lift": "wiki:language",
112
+ "strongest_group_lift": "3.597122",
113
+ "suggested_probe_bundle": "text_symbols",
114
+ "needs_glossary": "0",
115
+ "combined_score": "0.422992",
116
+ "actionable_score": "0.092079"
117
+ },
118
+ {
119
+ "tag": "smile",
120
+ "sample_occurrences": "230",
121
+ "fluffyrock_count": "869210",
122
+ "prevalence": "0.230000",
123
+ "ig_sum_bits": "1.862420",
124
+ "ig_mean_bits": "0.035140",
125
+ "delta_top5_mass": "0.393565",
126
+ "mean_abs_log2_lift": "0.410319",
127
+ "semantic_top_group": "cat:expression",
128
+ "semantic_margin": "0.005678",
129
+ "semantic_entropy_norm": "0.965243",
130
+ "strongest_group_by_lift": "cat:expression",
131
+ "strongest_group_lift": "2.865330",
132
+ "suggested_probe_bundle": "gaze_expression",
133
+ "needs_glossary": "0",
134
+ "combined_score": "0.405504",
135
+ "actionable_score": "0.174446"
136
+ },
137
+ {
138
+ "tag": "looking_at_viewer",
139
+ "sample_occurrences": "155",
140
+ "fluffyrock_count": "782895",
141
+ "prevalence": "0.155000",
142
+ "ig_sum_bits": "1.263787",
143
+ "ig_mean_bits": "0.023845",
144
+ "delta_top5_mass": "0.359484",
145
+ "mean_abs_log2_lift": "0.480707",
146
+ "semantic_top_group": "cat:gaze",
147
+ "semantic_margin": "0.006208",
148
+ "semantic_entropy_norm": "0.965049",
149
+ "strongest_group_by_lift": "cat:gaze",
150
+ "strongest_group_lift": "4.273504",
151
+ "suggested_probe_bundle": "gaze_expression",
152
+ "needs_glossary": "0",
153
+ "combined_score": "0.236678",
154
+ "actionable_score": "0.072879"
155
+ },
156
+ {
157
+ "tag": "anthro",
158
+ "sample_occurrences": "623",
159
+ "fluffyrock_count": "2754606",
160
+ "prevalence": "0.623000",
161
+ "ig_sum_bits": "0.918052",
162
+ "ig_mean_bits": "0.017322",
163
+ "delta_top5_mass": "0.313246",
164
+ "mean_abs_log2_lift": "0.622990",
165
+ "semantic_top_group": "cat:expression",
166
+ "semantic_margin": "0.000763",
167
+ "semantic_entropy_norm": "0.967098",
168
+ "strongest_group_by_lift": "cat:information",
169
+ "strongest_group_lift": "0.000000",
170
+ "suggested_probe_bundle": "body_type_presence",
171
+ "needs_glossary": "1",
172
+ "combined_score": "0.229779",
173
+ "actionable_score": "0.111983"
174
+ },
175
+ {
176
+ "tag": "breasts",
177
+ "sample_occurrences": "155",
178
+ "fluffyrock_count": "1503887",
179
+ "prevalence": "0.155000",
180
+ "ig_sum_bits": "1.108796",
181
+ "ig_mean_bits": "0.020921",
182
+ "delta_top5_mass": "0.391742",
183
+ "mean_abs_log2_lift": "0.465885",
184
+ "semantic_top_group": "cat:breasts",
185
+ "semantic_margin": "0.073815",
186
+ "semantic_entropy_norm": "0.963840",
187
+ "strongest_group_by_lift": "cat:breasts",
188
+ "strongest_group_lift": "6.451613",
189
+ "suggested_probe_bundle": "body_shape_breasts",
190
+ "needs_glossary": "0",
191
+ "combined_score": "0.207894",
192
+ "actionable_score": "0.069761"
193
+ },
194
+ {
195
+ "tag": "standing",
196
+ "sample_occurrences": "130",
197
+ "fluffyrock_count": "378001",
198
+ "prevalence": "0.130000",
199
+ "ig_sum_bits": "1.101691",
200
+ "ig_mean_bits": "0.020787",
201
+ "delta_top5_mass": "0.377846",
202
+ "mean_abs_log2_lift": "0.538594",
203
+ "semantic_top_group": "cat:pose_action_detail",
204
+ "semantic_margin": "0.003747",
205
+ "semantic_entropy_norm": "0.963547",
206
+ "strongest_group_by_lift": "cat:posture",
207
+ "strongest_group_lift": "4.424779",
208
+ "suggested_probe_bundle": "scene_pose",
209
+ "needs_glossary": "0",
210
+ "combined_score": "0.192004",
211
+ "actionable_score": "0.056088"
212
+ },
213
+ {
214
+ "tag": "dialogue",
215
+ "sample_occurrences": "112",
216
+ "fluffyrock_count": "395931",
217
+ "prevalence": "0.112000",
218
+ "ig_sum_bits": "1.001955",
219
+ "ig_mean_bits": "0.018905",
220
+ "delta_top5_mass": "0.281143",
221
+ "mean_abs_log2_lift": "0.424887",
222
+ "semantic_top_group": "wiki:language",
223
+ "semantic_margin": "0.016065",
224
+ "semantic_entropy_norm": "0.966402",
225
+ "strongest_group_by_lift": "cat:text",
226
+ "strongest_group_lift": "8.547008",
227
+ "suggested_probe_bundle": "text_symbols",
228
+ "needs_glossary": "0",
229
+ "combined_score": "0.163300",
230
+ "actionable_score": "0.032610"
231
+ },
232
+ {
233
+ "tag": "female",
234
+ "sample_occurrences": "463",
235
+ "fluffyrock_count": "2237115",
236
+ "prevalence": "0.463000",
237
+ "ig_sum_bits": "0.609249",
238
+ "ig_mean_bits": "0.011495",
239
+ "delta_top5_mass": "0.220976",
240
+ "mean_abs_log2_lift": "0.212905",
241
+ "semantic_top_group": "cat:gaze",
242
+ "semantic_margin": "0.011359",
243
+ "semantic_entropy_norm": "0.964751",
244
+ "strongest_group_by_lift": "cat:breasts",
245
+ "strongest_group_lift": "2.145893",
246
+ "suggested_probe_bundle": "other",
247
+ "needs_glossary": "0",
248
+ "combined_score": "0.157249",
249
+ "actionable_score": "0.064531"
250
+ },
251
+ {
252
+ "tag": "solo",
253
+ "sample_occurrences": "572",
254
+ "fluffyrock_count": "2021510",
255
+ "prevalence": "0.572000",
256
+ "ig_sum_bits": "0.547367",
257
+ "ig_mean_bits": "0.010328",
258
+ "delta_top5_mass": "0.053371",
259
+ "mean_abs_log2_lift": "0.308072",
260
+ "semantic_top_group": "cat:expression",
261
+ "semantic_margin": "0.018101",
262
+ "semantic_entropy_norm": "0.967242",
263
+ "strongest_group_by_lift": "wiki:character_relations",
264
+ "strongest_group_lift": "0.142714",
265
+ "suggested_probe_bundle": "count_cardinality",
266
+ "needs_glossary": "1",
267
+ "combined_score": "0.139851",
268
+ "actionable_score": "0.012913"
269
+ },
270
+ {
271
+ "tag": "male",
272
+ "sample_occurrences": "460",
273
+ "fluffyrock_count": "2124589",
274
+ "prevalence": "0.460000",
275
+ "ig_sum_bits": "0.517323",
276
+ "ig_mean_bits": "0.009761",
277
+ "delta_top5_mass": "0.191391",
278
+ "mean_abs_log2_lift": "0.221169",
279
+ "semantic_top_group": "cat:expression",
280
+ "semantic_margin": "0.003431",
281
+ "semantic_entropy_norm": "0.966133",
282
+ "strongest_group_by_lift": "cat:breasts",
283
+ "strongest_group_lift": "0.224404",
284
+ "suggested_probe_bundle": "other",
285
+ "needs_glossary": "0",
286
+ "combined_score": "0.133282",
287
+ "actionable_score": "0.047088"
288
+ },
289
+ {
290
+ "tag": "open_mouth",
291
+ "sample_occurrences": "179",
292
+ "fluffyrock_count": "804479",
293
+ "prevalence": "0.179000",
294
+ "ig_sum_bits": "0.649886",
295
+ "ig_mean_bits": "0.012262",
296
+ "delta_top5_mass": "0.267575",
297
+ "mean_abs_log2_lift": "0.331517",
298
+ "semantic_top_group": "cat:gaze",
299
+ "semantic_margin": "0.007682",
300
+ "semantic_entropy_norm": "0.967067",
301
+ "strongest_group_by_lift": "cat:expression_detail",
302
+ "strongest_group_lift": "2.283105",
303
+ "suggested_probe_bundle": "gaze_expression",
304
+ "needs_glossary": "0",
305
+ "combined_score": "0.128670",
306
+ "actionable_score": "0.032152"
307
+ },
308
+ {
309
+ "tag": "mammal",
310
+ "sample_occurrences": "762",
311
+ "fluffyrock_count": "2971571",
312
+ "prevalence": "0.762000",
313
+ "ig_sum_bits": "0.557876",
314
+ "ig_mean_bits": "0.010526",
315
+ "delta_top5_mass": "0.173344",
316
+ "mean_abs_log2_lift": "0.111463",
317
+ "semantic_top_group": "cat:gaze",
318
+ "semantic_margin": "0.001567",
319
+ "semantic_entropy_norm": "0.966179",
320
+ "strongest_group_by_lift": "cat:information",
321
+ "strongest_group_lift": "0.481190",
322
+ "suggested_probe_bundle": "species_taxonomy",
323
+ "needs_glossary": "0",
324
+ "combined_score": "0.122806",
325
+ "actionable_score": "0.023794"
326
+ },
327
+ {
328
+ "tag": "feral",
329
+ "sample_occurrences": "219",
330
+ "fluffyrock_count": "540877",
331
+ "prevalence": "0.219000",
332
+ "ig_sum_bits": "0.560104",
333
+ "ig_mean_bits": "0.010568",
334
+ "delta_top5_mass": "0.127196",
335
+ "mean_abs_log2_lift": "0.943429",
336
+ "semantic_top_group": "cat:body_type",
337
+ "semantic_margin": "0.040560",
338
+ "semantic_entropy_norm": "0.950280",
339
+ "strongest_group_by_lift": "cat:information",
340
+ "strongest_group_lift": "0.000000",
341
+ "suggested_probe_bundle": "body_type_presence",
342
+ "needs_glossary": "1",
343
+ "combined_score": "0.121579",
344
+ "actionable_score": "0.016378"
345
+ },
346
+ {
347
+ "tag": "biped",
348
+ "sample_occurrences": "95",
349
+ "fluffyrock_count": "372051",
350
+ "prevalence": "0.095000",
351
+ "ig_sum_bits": "0.784029",
352
+ "ig_mean_bits": "0.014793",
353
+ "delta_top5_mass": "0.339789",
354
+ "mean_abs_log2_lift": "0.560825",
355
+ "semantic_top_group": "cat:posture",
356
+ "semantic_margin": "0.012711",
357
+ "semantic_entropy_norm": "0.964497",
358
+ "strongest_group_by_lift": "wiki:feral_anatomy",
359
+ "strongest_group_lift": "9.090909",
360
+ "suggested_probe_bundle": "body_type_presence",
361
+ "needs_glossary": "0",
362
+ "combined_score": "0.119025",
363
+ "actionable_score": "0.026207"
364
+ },
365
+ {
366
+ "tag": "blush",
367
+ "sample_occurrences": "110",
368
+ "fluffyrock_count": "867903",
369
+ "prevalence": "0.110000",
370
+ "ig_sum_bits": "0.700939",
371
+ "ig_mean_bits": "0.013225",
372
+ "delta_top5_mass": "0.360364",
373
+ "mean_abs_log2_lift": "0.385709",
374
+ "semantic_top_group": "cat:expression",
375
+ "semantic_margin": "0.018136",
376
+ "semantic_entropy_norm": "0.968239",
377
+ "strongest_group_by_lift": "cat:expression",
378
+ "strongest_group_lift": "2.865330",
379
+ "suggested_probe_bundle": "gaze_expression",
380
+ "needs_glossary": "0",
381
+ "combined_score": "0.113141",
382
+ "actionable_score": "0.028668"
383
+ },
384
+ {
385
+ "tag": "topwear",
386
+ "sample_occurrences": "183",
387
+ "fluffyrock_count": "412942",
388
+ "prevalence": "0.183000",
389
+ "ig_sum_bits": "0.553210",
390
+ "ig_mean_bits": "0.010438",
391
+ "delta_top5_mass": "0.303781",
392
+ "mean_abs_log2_lift": "0.378171",
393
+ "semantic_top_group": "cat:clothing",
394
+ "semantic_margin": "0.008267",
395
+ "semantic_entropy_norm": "0.957582",
396
+ "strongest_group_by_lift": "cat:limbs",
397
+ "strongest_group_lift": "2.185792",
398
+ "suggested_probe_bundle": "clothing_state",
399
+ "needs_glossary": "1",
400
+ "combined_score": "0.111491",
401
+ "actionable_score": "0.032059"
402
+ },
403
+ {
404
+ "tag": "fur",
405
+ "sample_occurrences": "422",
406
+ "fluffyrock_count": "1417091",
407
+ "prevalence": "0.422000",
408
+ "ig_sum_bits": "0.401071",
409
+ "ig_mean_bits": "0.007567",
410
+ "delta_top5_mass": "0.085137",
411
+ "mean_abs_log2_lift": "0.232401",
412
+ "semantic_top_group": "cat:expression",
413
+ "semantic_margin": "0.005617",
414
+ "semantic_entropy_norm": "0.964398",
415
+ "strongest_group_by_lift": "cat:information",
416
+ "strongest_group_lift": "0.078989",
417
+ "suggested_probe_bundle": "other",
418
+ "needs_glossary": "0",
419
+ "combined_score": "0.102566",
420
+ "actionable_score": "0.014923"
421
+ },
422
+ {
423
+ "tag": "pokemon_(species)",
424
+ "sample_occurrences": "98",
425
+ "fluffyrock_count": "446455",
426
+ "prevalence": "0.098000",
427
+ "ig_sum_bits": "0.604839",
428
+ "ig_mean_bits": "0.011412",
429
+ "delta_top5_mass": "0.036245",
430
+ "mean_abs_log2_lift": "0.499532",
431
+ "semantic_top_group": "cat:franchise_series",
432
+ "semantic_margin": "0.169156",
433
+ "semantic_entropy_norm": "0.925702",
434
+ "strongest_group_by_lift": "cat:franchise_series",
435
+ "strongest_group_lift": "9.433962",
436
+ "suggested_probe_bundle": "species_taxonomy",
437
+ "needs_glossary": "1",
438
+ "combined_score": "0.096594",
439
+ "actionable_score": "0.002308"
440
+ },
441
+ {
442
+ "tag": "outside",
443
+ "sample_occurrences": "93",
444
+ "fluffyrock_count": "253364",
445
+ "prevalence": "0.093000",
446
+ "ig_sum_bits": "0.603079",
447
+ "ig_mean_bits": "0.011379",
448
+ "delta_top5_mass": "0.230452",
449
+ "mean_abs_log2_lift": "0.436412",
450
+ "semantic_top_group": "cat:location",
451
+ "semantic_margin": "0.013880",
452
+ "semantic_entropy_norm": "0.962489",
453
+ "strongest_group_by_lift": "cat:location",
454
+ "strongest_group_lift": "7.575758",
455
+ "suggested_probe_bundle": "scene_pose",
456
+ "needs_glossary": "0",
457
+ "combined_score": "0.090862",
458
+ "actionable_score": "0.013410"
459
+ },
460
+ {
461
+ "tag": "white_background",
462
+ "sample_occurrences": "115",
463
+ "fluffyrock_count": "310041",
464
+ "prevalence": "0.115000",
465
+ "ig_sum_bits": "0.534987",
466
+ "ig_mean_bits": "0.010094",
467
+ "delta_top5_mass": "0.363130",
468
+ "mean_abs_log2_lift": "1.386211",
469
+ "semantic_top_group": "cat:expression",
470
+ "semantic_margin": "0.014214",
471
+ "semantic_entropy_norm": "0.966449",
472
+ "strongest_group_by_lift": "cat:location",
473
+ "strongest_group_lift": "0.000000",
474
+ "suggested_probe_bundle": "scene_pose",
475
+ "needs_glossary": "0",
476
+ "combined_score": "0.088199",
477
+ "actionable_score": "0.023091"
478
+ },
479
+ {
480
+ "tag": "fingers",
481
+ "sample_occurrences": "146",
482
+ "fluffyrock_count": "383191",
483
+ "prevalence": "0.146000",
484
+ "ig_sum_bits": "0.451153",
485
+ "ig_mean_bits": "0.008512",
486
+ "delta_top5_mass": "0.280164",
487
+ "mean_abs_log2_lift": "0.393787",
488
+ "semantic_top_group": "cat:limbs",
489
+ "semantic_margin": "0.003078",
490
+ "semantic_entropy_norm": "0.961188",
491
+ "strongest_group_by_lift": "cat:fur_style",
492
+ "strongest_group_lift": "2.185952",
493
+ "suggested_probe_bundle": "other",
494
+ "needs_glossary": "0",
495
+ "combined_score": "0.082744",
496
+ "actionable_score": "0.019170"
497
+ }
498
+ ],
499
+ "top_probes_by_actionable_score": [
500
+ {
501
+ "tag": "clothing",
502
+ "sample_occurrences": "574",
503
+ "fluffyrock_count": "1738206",
504
+ "prevalence": "0.574000",
505
+ "ig_sum_bits": "2.622723",
506
+ "ig_mean_bits": "0.049485",
507
+ "delta_top5_mass": "0.300307",
508
+ "mean_abs_log2_lift": "0.239534",
509
+ "semantic_top_group": "cat:clothing",
510
+ "semantic_margin": "0.001094",
511
+ "semantic_entropy_norm": "0.965190",
512
+ "strongest_group_by_lift": "cat:clothing",
513
+ "strongest_group_lift": "1.742160",
514
+ "suggested_probe_bundle": "clothing_state",
515
+ "needs_glossary": "0",
516
+ "combined_score": "0.671033",
517
+ "actionable_score": "0.347207"
518
+ },
519
+ {
520
+ "tag": "hair",
521
+ "sample_occurrences": "373",
522
+ "fluffyrock_count": "1495810",
523
+ "prevalence": "0.373000",
524
+ "ig_sum_bits": "2.199276",
525
+ "ig_mean_bits": "0.041496",
526
+ "delta_top5_mass": "0.314080",
527
+ "mean_abs_log2_lift": "0.261228",
528
+ "semantic_top_group": "cat:hair",
529
+ "semantic_margin": "0.031478",
530
+ "semantic_entropy_norm": "0.966277",
531
+ "strongest_group_by_lift": "cat:hair",
532
+ "strongest_group_lift": "2.680965",
533
+ "suggested_probe_bundle": "other",
534
+ "needs_glossary": "0",
535
+ "combined_score": "0.549720",
536
+ "actionable_score": "0.266338"
537
+ },
538
+ {
539
+ "tag": "clothed",
540
+ "sample_occurrences": "337",
541
+ "fluffyrock_count": "962955",
542
+ "prevalence": "0.337000",
543
+ "ig_sum_bits": "2.016235",
544
+ "ig_mean_bits": "0.038042",
545
+ "delta_top5_mass": "0.364653",
546
+ "mean_abs_log2_lift": "0.381805",
547
+ "semantic_top_group": "cat:clothing",
548
+ "semantic_margin": "0.026471",
549
+ "semantic_entropy_norm": "0.962221",
550
+ "strongest_group_by_lift": "cat:clothing",
551
+ "strongest_group_lift": "2.653736",
552
+ "suggested_probe_bundle": "clothing_state",
553
+ "needs_glossary": "0",
554
+ "combined_score": "0.494524",
555
+ "actionable_score": "0.257132"
556
+ },
557
+ {
558
+ "tag": "simple_background",
559
+ "sample_occurrences": "345",
560
+ "fluffyrock_count": "1081480",
561
+ "prevalence": "0.345000",
562
+ "ig_sum_bits": "1.854364",
563
+ "ig_mean_bits": "0.034988",
564
+ "delta_top5_mass": "0.328348",
565
+ "mean_abs_log2_lift": "0.354546",
566
+ "semantic_top_group": "cat:expression",
567
+ "semantic_margin": "0.015478",
568
+ "semantic_entropy_norm": "0.967783",
569
+ "strongest_group_by_lift": "wiki:settings",
570
+ "strongest_group_lift": "0.115942",
571
+ "suggested_probe_bundle": "scene_pose",
572
+ "needs_glossary": "0",
573
+ "combined_score": "0.454953",
574
+ "actionable_score": "0.216830"
575
+ },
576
+ {
577
+ "tag": "smile",
578
+ "sample_occurrences": "230",
579
+ "fluffyrock_count": "869210",
580
+ "prevalence": "0.230000",
581
+ "ig_sum_bits": "1.862420",
582
+ "ig_mean_bits": "0.035140",
583
+ "delta_top5_mass": "0.393565",
584
+ "mean_abs_log2_lift": "0.410319",
585
+ "semantic_top_group": "cat:expression",
586
+ "semantic_margin": "0.005678",
587
+ "semantic_entropy_norm": "0.965243",
588
+ "strongest_group_by_lift": "cat:expression",
589
+ "strongest_group_lift": "2.865330",
590
+ "suggested_probe_bundle": "gaze_expression",
591
+ "needs_glossary": "0",
592
+ "combined_score": "0.405504",
593
+ "actionable_score": "0.174446"
594
+ },
595
+ {
596
+ "tag": "anthro",
597
+ "sample_occurrences": "623",
598
+ "fluffyrock_count": "2754606",
599
+ "prevalence": "0.623000",
600
+ "ig_sum_bits": "0.918052",
601
+ "ig_mean_bits": "0.017322",
602
+ "delta_top5_mass": "0.313246",
603
+ "mean_abs_log2_lift": "0.622990",
604
+ "semantic_top_group": "cat:expression",
605
+ "semantic_margin": "0.000763",
606
+ "semantic_entropy_norm": "0.967098",
607
+ "strongest_group_by_lift": "cat:information",
608
+ "strongest_group_lift": "0.000000",
609
+ "suggested_probe_bundle": "body_type_presence",
610
+ "needs_glossary": "1",
611
+ "combined_score": "0.229779",
612
+ "actionable_score": "0.111983"
613
+ },
614
+ {
615
+ "tag": "text",
616
+ "sample_occurrences": "275",
617
+ "fluffyrock_count": "926878",
618
+ "prevalence": "0.275000",
619
+ "ig_sum_bits": "1.836555",
620
+ "ig_mean_bits": "0.034652",
621
+ "delta_top5_mass": "0.176727",
622
+ "mean_abs_log2_lift": "0.303566",
623
+ "semantic_top_group": "cat:count",
624
+ "semantic_margin": "0.008246",
625
+ "semantic_entropy_norm": "0.968373",
626
+ "strongest_group_by_lift": "wiki:language",
627
+ "strongest_group_lift": "3.597122",
628
+ "suggested_probe_bundle": "text_symbols",
629
+ "needs_glossary": "0",
630
+ "combined_score": "0.422992",
631
+ "actionable_score": "0.092079"
632
+ },
633
+ {
634
+ "tag": "looking_at_viewer",
635
+ "sample_occurrences": "155",
636
+ "fluffyrock_count": "782895",
637
+ "prevalence": "0.155000",
638
+ "ig_sum_bits": "1.263787",
639
+ "ig_mean_bits": "0.023845",
640
+ "delta_top5_mass": "0.359484",
641
+ "mean_abs_log2_lift": "0.480707",
642
+ "semantic_top_group": "cat:gaze",
643
+ "semantic_margin": "0.006208",
644
+ "semantic_entropy_norm": "0.965049",
645
+ "strongest_group_by_lift": "cat:gaze",
646
+ "strongest_group_lift": "4.273504",
647
+ "suggested_probe_bundle": "gaze_expression",
648
+ "needs_glossary": "0",
649
+ "combined_score": "0.236678",
650
+ "actionable_score": "0.072879"
651
+ },
652
+ {
653
+ "tag": "breasts",
654
+ "sample_occurrences": "155",
655
+ "fluffyrock_count": "1503887",
656
+ "prevalence": "0.155000",
657
+ "ig_sum_bits": "1.108796",
658
+ "ig_mean_bits": "0.020921",
659
+ "delta_top5_mass": "0.391742",
660
+ "mean_abs_log2_lift": "0.465885",
661
+ "semantic_top_group": "cat:breasts",
662
+ "semantic_margin": "0.073815",
663
+ "semantic_entropy_norm": "0.963840",
664
+ "strongest_group_by_lift": "cat:breasts",
665
+ "strongest_group_lift": "6.451613",
666
+ "suggested_probe_bundle": "body_shape_breasts",
667
+ "needs_glossary": "0",
668
+ "combined_score": "0.207894",
669
+ "actionable_score": "0.069761"
670
+ },
671
+ {
672
+ "tag": "female",
673
+ "sample_occurrences": "463",
674
+ "fluffyrock_count": "2237115",
675
+ "prevalence": "0.463000",
676
+ "ig_sum_bits": "0.609249",
677
+ "ig_mean_bits": "0.011495",
678
+ "delta_top5_mass": "0.220976",
679
+ "mean_abs_log2_lift": "0.212905",
680
+ "semantic_top_group": "cat:gaze",
681
+ "semantic_margin": "0.011359",
682
+ "semantic_entropy_norm": "0.964751",
683
+ "strongest_group_by_lift": "cat:breasts",
684
+ "strongest_group_lift": "2.145893",
685
+ "suggested_probe_bundle": "other",
686
+ "needs_glossary": "0",
687
+ "combined_score": "0.157249",
688
+ "actionable_score": "0.064531"
689
+ },
690
+ {
691
+ "tag": "standing",
692
+ "sample_occurrences": "130",
693
+ "fluffyrock_count": "378001",
694
+ "prevalence": "0.130000",
695
+ "ig_sum_bits": "1.101691",
696
+ "ig_mean_bits": "0.020787",
697
+ "delta_top5_mass": "0.377846",
698
+ "mean_abs_log2_lift": "0.538594",
699
+ "semantic_top_group": "cat:pose_action_detail",
700
+ "semantic_margin": "0.003747",
701
+ "semantic_entropy_norm": "0.963547",
702
+ "strongest_group_by_lift": "cat:posture",
703
+ "strongest_group_lift": "4.424779",
704
+ "suggested_probe_bundle": "scene_pose",
705
+ "needs_glossary": "0",
706
+ "combined_score": "0.192004",
707
+ "actionable_score": "0.056088"
708
+ },
709
+ {
710
+ "tag": "male",
711
+ "sample_occurrences": "460",
712
+ "fluffyrock_count": "2124589",
713
+ "prevalence": "0.460000",
714
+ "ig_sum_bits": "0.517323",
715
+ "ig_mean_bits": "0.009761",
716
+ "delta_top5_mass": "0.191391",
717
+ "mean_abs_log2_lift": "0.221169",
718
+ "semantic_top_group": "cat:expression",
719
+ "semantic_margin": "0.003431",
720
+ "semantic_entropy_norm": "0.966133",
721
+ "strongest_group_by_lift": "cat:breasts",
722
+ "strongest_group_lift": "0.224404",
723
+ "suggested_probe_bundle": "other",
724
+ "needs_glossary": "0",
725
+ "combined_score": "0.133282",
726
+ "actionable_score": "0.047088"
727
+ },
728
+ {
729
+ "tag": "dialogue",
730
+ "sample_occurrences": "112",
731
+ "fluffyrock_count": "395931",
732
+ "prevalence": "0.112000",
733
+ "ig_sum_bits": "1.001955",
734
+ "ig_mean_bits": "0.018905",
735
+ "delta_top5_mass": "0.281143",
736
+ "mean_abs_log2_lift": "0.424887",
737
+ "semantic_top_group": "wiki:language",
738
+ "semantic_margin": "0.016065",
739
+ "semantic_entropy_norm": "0.966402",
740
+ "strongest_group_by_lift": "cat:text",
741
+ "strongest_group_lift": "8.547008",
742
+ "suggested_probe_bundle": "text_symbols",
743
+ "needs_glossary": "0",
744
+ "combined_score": "0.163300",
745
+ "actionable_score": "0.032610"
746
+ },
747
+ {
748
+ "tag": "open_mouth",
749
+ "sample_occurrences": "179",
750
+ "fluffyrock_count": "804479",
751
+ "prevalence": "0.179000",
752
+ "ig_sum_bits": "0.649886",
753
+ "ig_mean_bits": "0.012262",
754
+ "delta_top5_mass": "0.267575",
755
+ "mean_abs_log2_lift": "0.331517",
756
+ "semantic_top_group": "cat:gaze",
757
+ "semantic_margin": "0.007682",
758
+ "semantic_entropy_norm": "0.967067",
759
+ "strongest_group_by_lift": "cat:expression_detail",
760
+ "strongest_group_lift": "2.283105",
761
+ "suggested_probe_bundle": "gaze_expression",
762
+ "needs_glossary": "0",
763
+ "combined_score": "0.128670",
764
+ "actionable_score": "0.032152"
765
+ },
766
+ {
767
+ "tag": "topwear",
768
+ "sample_occurrences": "183",
769
+ "fluffyrock_count": "412942",
770
+ "prevalence": "0.183000",
771
+ "ig_sum_bits": "0.553210",
772
+ "ig_mean_bits": "0.010438",
773
+ "delta_top5_mass": "0.303781",
774
+ "mean_abs_log2_lift": "0.378171",
775
+ "semantic_top_group": "cat:clothing",
776
+ "semantic_margin": "0.008267",
777
+ "semantic_entropy_norm": "0.957582",
778
+ "strongest_group_by_lift": "cat:limbs",
779
+ "strongest_group_lift": "2.185792",
780
+ "suggested_probe_bundle": "clothing_state",
781
+ "needs_glossary": "1",
782
+ "combined_score": "0.111491",
783
+ "actionable_score": "0.032059"
784
+ },
785
+ {
786
+ "tag": "blush",
787
+ "sample_occurrences": "110",
788
+ "fluffyrock_count": "867903",
789
+ "prevalence": "0.110000",
790
+ "ig_sum_bits": "0.700939",
791
+ "ig_mean_bits": "0.013225",
792
+ "delta_top5_mass": "0.360364",
793
+ "mean_abs_log2_lift": "0.385709",
794
+ "semantic_top_group": "cat:expression",
795
+ "semantic_margin": "0.018136",
796
+ "semantic_entropy_norm": "0.968239",
797
+ "strongest_group_by_lift": "cat:expression",
798
+ "strongest_group_lift": "2.865330",
799
+ "suggested_probe_bundle": "gaze_expression",
800
+ "needs_glossary": "0",
801
+ "combined_score": "0.113141",
802
+ "actionable_score": "0.028668"
803
+ },
804
+ {
805
+ "tag": "canine",
806
+ "sample_occurrences": "296",
807
+ "fluffyrock_count": "1102119",
808
+ "prevalence": "0.296000",
809
+ "ig_sum_bits": "0.346189",
810
+ "ig_mean_bits": "0.006532",
811
+ "delta_top5_mass": "0.268595",
812
+ "mean_abs_log2_lift": "0.639208",
813
+ "semantic_top_group": "cat:limbs",
814
+ "semantic_margin": "0.004140",
815
+ "semantic_entropy_norm": "0.968229",
816
+ "strongest_group_by_lift": "cat:information",
817
+ "strongest_group_lift": "0.000000",
818
+ "suggested_probe_bundle": "other",
819
+ "needs_glossary": "1",
820
+ "combined_score": "0.081526",
821
+ "actionable_score": "0.028398"
822
+ },
823
+ {
824
+ "tag": "biped",
825
+ "sample_occurrences": "95",
826
+ "fluffyrock_count": "372051",
827
+ "prevalence": "0.095000",
828
+ "ig_sum_bits": "0.784029",
829
+ "ig_mean_bits": "0.014793",
830
+ "delta_top5_mass": "0.339789",
831
+ "mean_abs_log2_lift": "0.560825",
832
+ "semantic_top_group": "cat:posture",
833
+ "semantic_margin": "0.012711",
834
+ "semantic_entropy_norm": "0.964497",
835
+ "strongest_group_by_lift": "wiki:feral_anatomy",
836
+ "strongest_group_lift": "9.090909",
837
+ "suggested_probe_bundle": "body_type_presence",
838
+ "needs_glossary": "0",
839
+ "combined_score": "0.119025",
840
+ "actionable_score": "0.026207"
841
+ },
842
+ {
843
+ "tag": "canid",
844
+ "sample_occurrences": "304",
845
+ "fluffyrock_count": "1141448",
846
+ "prevalence": "0.304000",
847
+ "ig_sum_bits": "0.321648",
848
+ "ig_mean_bits": "0.006069",
849
+ "delta_top5_mass": "0.252947",
850
+ "mean_abs_log2_lift": "0.637451",
851
+ "semantic_top_group": "cat:limbs",
852
+ "semantic_margin": "0.002600",
853
+ "semantic_entropy_norm": "0.968264",
854
+ "strongest_group_by_lift": "cat:information",
855
+ "strongest_group_lift": "0.000000",
856
+ "suggested_probe_bundle": "species_taxonomy",
857
+ "needs_glossary": "1",
858
+ "combined_score": "0.076324",
859
+ "actionable_score": "0.025518"
860
+ },
861
+ {
862
+ "tag": "mammal",
863
+ "sample_occurrences": "762",
864
+ "fluffyrock_count": "2971571",
865
+ "prevalence": "0.762000",
866
+ "ig_sum_bits": "0.557876",
867
+ "ig_mean_bits": "0.010526",
868
+ "delta_top5_mass": "0.173344",
869
+ "mean_abs_log2_lift": "0.111463",
870
+ "semantic_top_group": "cat:gaze",
871
+ "semantic_margin": "0.001567",
872
+ "semantic_entropy_norm": "0.966179",
873
+ "strongest_group_by_lift": "cat:information",
874
+ "strongest_group_lift": "0.481190",
875
+ "suggested_probe_bundle": "species_taxonomy",
876
+ "needs_glossary": "0",
877
+ "combined_score": "0.122806",
878
+ "actionable_score": "0.023794"
879
+ },
880
+ {
881
+ "tag": "white_background",
882
+ "sample_occurrences": "115",
883
+ "fluffyrock_count": "310041",
884
+ "prevalence": "0.115000",
885
+ "ig_sum_bits": "0.534987",
886
+ "ig_mean_bits": "0.010094",
887
+ "delta_top5_mass": "0.363130",
888
+ "mean_abs_log2_lift": "1.386211",
889
+ "semantic_top_group": "cat:expression",
890
+ "semantic_margin": "0.014214",
891
+ "semantic_entropy_norm": "0.966449",
892
+ "strongest_group_by_lift": "cat:location",
893
+ "strongest_group_lift": "0.000000",
894
+ "suggested_probe_bundle": "scene_pose",
895
+ "needs_glossary": "0",
896
+ "combined_score": "0.088199",
897
+ "actionable_score": "0.023091"
898
+ },
899
+ {
900
+ "tag": "bottomwear",
901
+ "sample_occurrences": "135",
902
+ "fluffyrock_count": "316556",
903
+ "prevalence": "0.135000",
904
+ "ig_sum_bits": "0.454086",
905
+ "ig_mean_bits": "0.008568",
906
+ "delta_top5_mass": "0.305481",
907
+ "mean_abs_log2_lift": "0.412372",
908
+ "semantic_top_group": "cat:clothing",
909
+ "semantic_margin": "0.020139",
910
+ "semantic_entropy_norm": "0.955038",
911
+ "strongest_group_by_lift": "cat:limbs",
912
+ "strongest_group_lift": "2.592593",
913
+ "suggested_probe_bundle": "clothing_state",
914
+ "needs_glossary": "1",
915
+ "combined_score": "0.081074",
916
+ "actionable_score": "0.019568"
917
+ },
918
+ {
919
+ "tag": "fingers",
920
+ "sample_occurrences": "146",
921
+ "fluffyrock_count": "383191",
922
+ "prevalence": "0.146000",
923
+ "ig_sum_bits": "0.451153",
924
+ "ig_mean_bits": "0.008512",
925
+ "delta_top5_mass": "0.280164",
926
+ "mean_abs_log2_lift": "0.393787",
927
+ "semantic_top_group": "cat:limbs",
928
+ "semantic_margin": "0.003078",
929
+ "semantic_entropy_norm": "0.961188",
930
+ "strongest_group_by_lift": "cat:fur_style",
931
+ "strongest_group_lift": "2.185952",
932
+ "suggested_probe_bundle": "other",
933
+ "needs_glossary": "0",
934
+ "combined_score": "0.082744",
935
+ "actionable_score": "0.019170"
936
+ },
937
+ {
938
+ "tag": "feral",
939
+ "sample_occurrences": "219",
940
+ "fluffyrock_count": "540877",
941
+ "prevalence": "0.219000",
942
+ "ig_sum_bits": "0.560104",
943
+ "ig_mean_bits": "0.010568",
944
+ "delta_top5_mass": "0.127196",
945
+ "mean_abs_log2_lift": "0.943429",
946
+ "semantic_top_group": "cat:body_type",
947
+ "semantic_margin": "0.040560",
948
+ "semantic_entropy_norm": "0.950280",
949
+ "strongest_group_by_lift": "cat:information",
950
+ "strongest_group_lift": "0.000000",
951
+ "suggested_probe_bundle": "body_type_presence",
952
+ "needs_glossary": "1",
953
+ "combined_score": "0.121579",
954
+ "actionable_score": "0.016378"
955
+ },
956
+ {
957
+ "tag": "fur",
958
+ "sample_occurrences": "422",
959
+ "fluffyrock_count": "1417091",
960
+ "prevalence": "0.422000",
961
+ "ig_sum_bits": "0.401071",
962
+ "ig_mean_bits": "0.007567",
963
+ "delta_top5_mass": "0.085137",
964
+ "mean_abs_log2_lift": "0.232401",
965
+ "semantic_top_group": "cat:expression",
966
+ "semantic_margin": "0.005617",
967
+ "semantic_entropy_norm": "0.964398",
968
+ "strongest_group_by_lift": "cat:information",
969
+ "strongest_group_lift": "0.078989",
970
+ "suggested_probe_bundle": "other",
971
+ "needs_glossary": "0",
972
+ "combined_score": "0.102566",
973
+ "actionable_score": "0.014923"
974
+ }
975
+ ],
976
+ "top_actionable_mid_prevalence_for_manual_review": [
977
+ {
978
+ "tag": "clothed",
979
+ "sample_occurrences": "337",
980
+ "fluffyrock_count": "962955",
981
+ "prevalence": "0.337000",
982
+ "ig_sum_bits": "2.016235",
983
+ "ig_mean_bits": "0.038042",
984
+ "delta_top5_mass": "0.364653",
985
+ "mean_abs_log2_lift": "0.381805",
986
+ "semantic_top_group": "cat:clothing",
987
+ "semantic_margin": "0.026471",
988
+ "semantic_entropy_norm": "0.962221",
989
+ "strongest_group_by_lift": "cat:clothing",
990
+ "strongest_group_lift": "2.653736",
991
+ "suggested_probe_bundle": "clothing_state",
992
+ "needs_glossary": "0",
993
+ "combined_score": "0.494524",
994
+ "actionable_score": "0.257132"
995
+ },
996
+ {
997
+ "tag": "simple_background",
998
+ "sample_occurrences": "345",
999
+ "fluffyrock_count": "1081480",
1000
+ "prevalence": "0.345000",
1001
+ "ig_sum_bits": "1.854364",
1002
+ "ig_mean_bits": "0.034988",
1003
+ "delta_top5_mass": "0.328348",
1004
+ "mean_abs_log2_lift": "0.354546",
1005
+ "semantic_top_group": "cat:expression",
1006
+ "semantic_margin": "0.015478",
1007
+ "semantic_entropy_norm": "0.967783",
1008
+ "strongest_group_by_lift": "wiki:settings",
1009
+ "strongest_group_lift": "0.115942",
1010
+ "suggested_probe_bundle": "scene_pose",
1011
+ "needs_glossary": "0",
1012
+ "combined_score": "0.454953",
1013
+ "actionable_score": "0.216830"
1014
+ },
1015
+ {
1016
+ "tag": "smile",
1017
+ "sample_occurrences": "230",
1018
+ "fluffyrock_count": "869210",
1019
+ "prevalence": "0.230000",
1020
+ "ig_sum_bits": "1.862420",
1021
+ "ig_mean_bits": "0.035140",
1022
+ "delta_top5_mass": "0.393565",
1023
+ "mean_abs_log2_lift": "0.410319",
1024
+ "semantic_top_group": "cat:expression",
1025
+ "semantic_margin": "0.005678",
1026
+ "semantic_entropy_norm": "0.965243",
1027
+ "strongest_group_by_lift": "cat:expression",
1028
+ "strongest_group_lift": "2.865330",
1029
+ "suggested_probe_bundle": "gaze_expression",
1030
+ "needs_glossary": "0",
1031
+ "combined_score": "0.405504",
1032
+ "actionable_score": "0.174446"
1033
+ },
1034
+ {
1035
+ "tag": "text",
1036
+ "sample_occurrences": "275",
1037
+ "fluffyrock_count": "926878",
1038
+ "prevalence": "0.275000",
1039
+ "ig_sum_bits": "1.836555",
1040
+ "ig_mean_bits": "0.034652",
1041
+ "delta_top5_mass": "0.176727",
1042
+ "mean_abs_log2_lift": "0.303566",
1043
+ "semantic_top_group": "cat:count",
1044
+ "semantic_margin": "0.008246",
1045
+ "semantic_entropy_norm": "0.968373",
1046
+ "strongest_group_by_lift": "wiki:language",
1047
+ "strongest_group_lift": "3.597122",
1048
+ "suggested_probe_bundle": "text_symbols",
1049
+ "needs_glossary": "0",
1050
+ "combined_score": "0.422992",
1051
+ "actionable_score": "0.092079"
1052
+ },
1053
+ {
1054
+ "tag": "looking_at_viewer",
1055
+ "sample_occurrences": "155",
1056
+ "fluffyrock_count": "782895",
1057
+ "prevalence": "0.155000",
1058
+ "ig_sum_bits": "1.263787",
1059
+ "ig_mean_bits": "0.023845",
1060
+ "delta_top5_mass": "0.359484",
1061
+ "mean_abs_log2_lift": "0.480707",
1062
+ "semantic_top_group": "cat:gaze",
1063
+ "semantic_margin": "0.006208",
1064
+ "semantic_entropy_norm": "0.965049",
1065
+ "strongest_group_by_lift": "cat:gaze",
1066
+ "strongest_group_lift": "4.273504",
1067
+ "suggested_probe_bundle": "gaze_expression",
1068
+ "needs_glossary": "0",
1069
+ "combined_score": "0.236678",
1070
+ "actionable_score": "0.072879"
1071
+ },
1072
+ {
1073
+ "tag": "breasts",
1074
+ "sample_occurrences": "155",
1075
+ "fluffyrock_count": "1503887",
1076
+ "prevalence": "0.155000",
1077
+ "ig_sum_bits": "1.108796",
1078
+ "ig_mean_bits": "0.020921",
1079
+ "delta_top5_mass": "0.391742",
1080
+ "mean_abs_log2_lift": "0.465885",
1081
+ "semantic_top_group": "cat:breasts",
1082
+ "semantic_margin": "0.073815",
1083
+ "semantic_entropy_norm": "0.963840",
1084
+ "strongest_group_by_lift": "cat:breasts",
1085
+ "strongest_group_lift": "6.451613",
1086
+ "suggested_probe_bundle": "body_shape_breasts",
1087
+ "needs_glossary": "0",
1088
+ "combined_score": "0.207894",
1089
+ "actionable_score": "0.069761"
1090
+ },
1091
+ {
1092
+ "tag": "standing",
1093
+ "sample_occurrences": "130",
1094
+ "fluffyrock_count": "378001",
1095
+ "prevalence": "0.130000",
1096
+ "ig_sum_bits": "1.101691",
1097
+ "ig_mean_bits": "0.020787",
1098
+ "delta_top5_mass": "0.377846",
1099
+ "mean_abs_log2_lift": "0.538594",
1100
+ "semantic_top_group": "cat:pose_action_detail",
1101
+ "semantic_margin": "0.003747",
1102
+ "semantic_entropy_norm": "0.963547",
1103
+ "strongest_group_by_lift": "cat:posture",
1104
+ "strongest_group_lift": "4.424779",
1105
+ "suggested_probe_bundle": "scene_pose",
1106
+ "needs_glossary": "0",
1107
+ "combined_score": "0.192004",
1108
+ "actionable_score": "0.056088"
1109
+ },
1110
+ {
1111
+ "tag": "dialogue",
1112
+ "sample_occurrences": "112",
1113
+ "fluffyrock_count": "395931",
1114
+ "prevalence": "0.112000",
1115
+ "ig_sum_bits": "1.001955",
1116
+ "ig_mean_bits": "0.018905",
1117
+ "delta_top5_mass": "0.281143",
1118
+ "mean_abs_log2_lift": "0.424887",
1119
+ "semantic_top_group": "wiki:language",
1120
+ "semantic_margin": "0.016065",
1121
+ "semantic_entropy_norm": "0.966402",
1122
+ "strongest_group_by_lift": "cat:text",
1123
+ "strongest_group_lift": "8.547008",
1124
+ "suggested_probe_bundle": "text_symbols",
1125
+ "needs_glossary": "0",
1126
+ "combined_score": "0.163300",
1127
+ "actionable_score": "0.032610"
1128
+ },
1129
+ {
1130
+ "tag": "open_mouth",
1131
+ "sample_occurrences": "179",
1132
+ "fluffyrock_count": "804479",
1133
+ "prevalence": "0.179000",
1134
+ "ig_sum_bits": "0.649886",
1135
+ "ig_mean_bits": "0.012262",
1136
+ "delta_top5_mass": "0.267575",
1137
+ "mean_abs_log2_lift": "0.331517",
1138
+ "semantic_top_group": "cat:gaze",
1139
+ "semantic_margin": "0.007682",
1140
+ "semantic_entropy_norm": "0.967067",
1141
+ "strongest_group_by_lift": "cat:expression_detail",
1142
+ "strongest_group_lift": "2.283105",
1143
+ "suggested_probe_bundle": "gaze_expression",
1144
+ "needs_glossary": "0",
1145
+ "combined_score": "0.128670",
1146
+ "actionable_score": "0.032152"
1147
+ },
1148
+ {
1149
+ "tag": "topwear",
1150
+ "sample_occurrences": "183",
1151
+ "fluffyrock_count": "412942",
1152
+ "prevalence": "0.183000",
1153
+ "ig_sum_bits": "0.553210",
1154
+ "ig_mean_bits": "0.010438",
1155
+ "delta_top5_mass": "0.303781",
1156
+ "mean_abs_log2_lift": "0.378171",
1157
+ "semantic_top_group": "cat:clothing",
1158
+ "semantic_margin": "0.008267",
1159
+ "semantic_entropy_norm": "0.957582",
1160
+ "strongest_group_by_lift": "cat:limbs",
1161
+ "strongest_group_lift": "2.185792",
1162
+ "suggested_probe_bundle": "clothing_state",
1163
+ "needs_glossary": "1",
1164
+ "combined_score": "0.111491",
1165
+ "actionable_score": "0.032059"
1166
+ },
1167
+ {
1168
+ "tag": "blush",
1169
+ "sample_occurrences": "110",
1170
+ "fluffyrock_count": "867903",
1171
+ "prevalence": "0.110000",
1172
+ "ig_sum_bits": "0.700939",
1173
+ "ig_mean_bits": "0.013225",
1174
+ "delta_top5_mass": "0.360364",
1175
+ "mean_abs_log2_lift": "0.385709",
1176
+ "semantic_top_group": "cat:expression",
1177
+ "semantic_margin": "0.018136",
1178
+ "semantic_entropy_norm": "0.968239",
1179
+ "strongest_group_by_lift": "cat:expression",
1180
+ "strongest_group_lift": "2.865330",
1181
+ "suggested_probe_bundle": "gaze_expression",
1182
+ "needs_glossary": "0",
1183
+ "combined_score": "0.113141",
1184
+ "actionable_score": "0.028668"
1185
+ },
1186
+ {
1187
+ "tag": "canine",
1188
+ "sample_occurrences": "296",
1189
+ "fluffyrock_count": "1102119",
1190
+ "prevalence": "0.296000",
1191
+ "ig_sum_bits": "0.346189",
1192
+ "ig_mean_bits": "0.006532",
1193
+ "delta_top5_mass": "0.268595",
1194
+ "mean_abs_log2_lift": "0.639208",
1195
+ "semantic_top_group": "cat:limbs",
1196
+ "semantic_margin": "0.004140",
1197
+ "semantic_entropy_norm": "0.968229",
1198
+ "strongest_group_by_lift": "cat:information",
1199
+ "strongest_group_lift": "0.000000",
1200
+ "suggested_probe_bundle": "other",
1201
+ "needs_glossary": "1",
1202
+ "combined_score": "0.081526",
1203
+ "actionable_score": "0.028398"
1204
+ },
1205
+ {
1206
+ "tag": "biped",
1207
+ "sample_occurrences": "95",
1208
+ "fluffyrock_count": "372051",
1209
+ "prevalence": "0.095000",
1210
+ "ig_sum_bits": "0.784029",
1211
+ "ig_mean_bits": "0.014793",
1212
+ "delta_top5_mass": "0.339789",
1213
+ "mean_abs_log2_lift": "0.560825",
1214
+ "semantic_top_group": "cat:posture",
1215
+ "semantic_margin": "0.012711",
1216
+ "semantic_entropy_norm": "0.964497",
1217
+ "strongest_group_by_lift": "wiki:feral_anatomy",
1218
+ "strongest_group_lift": "9.090909",
1219
+ "suggested_probe_bundle": "body_type_presence",
1220
+ "needs_glossary": "0",
1221
+ "combined_score": "0.119025",
1222
+ "actionable_score": "0.026207"
1223
+ },
1224
+ {
1225
+ "tag": "canid",
1226
+ "sample_occurrences": "304",
1227
+ "fluffyrock_count": "1141448",
1228
+ "prevalence": "0.304000",
1229
+ "ig_sum_bits": "0.321648",
1230
+ "ig_mean_bits": "0.006069",
1231
+ "delta_top5_mass": "0.252947",
1232
+ "mean_abs_log2_lift": "0.637451",
1233
+ "semantic_top_group": "cat:limbs",
1234
+ "semantic_margin": "0.002600",
1235
+ "semantic_entropy_norm": "0.968264",
1236
+ "strongest_group_by_lift": "cat:information",
1237
+ "strongest_group_lift": "0.000000",
1238
+ "suggested_probe_bundle": "species_taxonomy",
1239
+ "needs_glossary": "1",
1240
+ "combined_score": "0.076324",
1241
+ "actionable_score": "0.025518"
1242
+ },
1243
+ {
1244
+ "tag": "white_background",
1245
+ "sample_occurrences": "115",
1246
+ "fluffyrock_count": "310041",
1247
+ "prevalence": "0.115000",
1248
+ "ig_sum_bits": "0.534987",
1249
+ "ig_mean_bits": "0.010094",
1250
+ "delta_top5_mass": "0.363130",
1251
+ "mean_abs_log2_lift": "1.386211",
1252
+ "semantic_top_group": "cat:expression",
1253
+ "semantic_margin": "0.014214",
1254
+ "semantic_entropy_norm": "0.966449",
1255
+ "strongest_group_by_lift": "cat:location",
1256
+ "strongest_group_lift": "0.000000",
1257
+ "suggested_probe_bundle": "scene_pose",
1258
+ "needs_glossary": "0",
1259
+ "combined_score": "0.088199",
1260
+ "actionable_score": "0.023091"
1261
+ },
1262
+ {
1263
+ "tag": "bottomwear",
1264
+ "sample_occurrences": "135",
1265
+ "fluffyrock_count": "316556",
1266
+ "prevalence": "0.135000",
1267
+ "ig_sum_bits": "0.454086",
1268
+ "ig_mean_bits": "0.008568",
1269
+ "delta_top5_mass": "0.305481",
1270
+ "mean_abs_log2_lift": "0.412372",
1271
+ "semantic_top_group": "cat:clothing",
1272
+ "semantic_margin": "0.020139",
1273
+ "semantic_entropy_norm": "0.955038",
1274
+ "strongest_group_by_lift": "cat:limbs",
1275
+ "strongest_group_lift": "2.592593",
1276
+ "suggested_probe_bundle": "clothing_state",
1277
+ "needs_glossary": "1",
1278
+ "combined_score": "0.081074",
1279
+ "actionable_score": "0.019568"
1280
+ },
1281
+ {
1282
+ "tag": "fingers",
1283
+ "sample_occurrences": "146",
1284
+ "fluffyrock_count": "383191",
1285
+ "prevalence": "0.146000",
1286
+ "ig_sum_bits": "0.451153",
1287
+ "ig_mean_bits": "0.008512",
1288
+ "delta_top5_mass": "0.280164",
1289
+ "mean_abs_log2_lift": "0.393787",
1290
+ "semantic_top_group": "cat:limbs",
1291
+ "semantic_margin": "0.003078",
1292
+ "semantic_entropy_norm": "0.961188",
1293
+ "strongest_group_by_lift": "cat:fur_style",
1294
+ "strongest_group_lift": "2.185952",
1295
+ "suggested_probe_bundle": "other",
1296
+ "needs_glossary": "0",
1297
+ "combined_score": "0.082744",
1298
+ "actionable_score": "0.019170"
1299
+ },
1300
+ {
1301
+ "tag": "feral",
1302
+ "sample_occurrences": "219",
1303
+ "fluffyrock_count": "540877",
1304
+ "prevalence": "0.219000",
1305
+ "ig_sum_bits": "0.560104",
1306
+ "ig_mean_bits": "0.010568",
1307
+ "delta_top5_mass": "0.127196",
1308
+ "mean_abs_log2_lift": "0.943429",
1309
+ "semantic_top_group": "cat:body_type",
1310
+ "semantic_margin": "0.040560",
1311
+ "semantic_entropy_norm": "0.950280",
1312
+ "strongest_group_by_lift": "cat:information",
1313
+ "strongest_group_lift": "0.000000",
1314
+ "suggested_probe_bundle": "body_type_presence",
1315
+ "needs_glossary": "1",
1316
+ "combined_score": "0.121579",
1317
+ "actionable_score": "0.016378"
1318
+ },
1319
+ {
1320
+ "tag": "shirt",
1321
+ "sample_occurrences": "103",
1322
+ "fluffyrock_count": "253553",
1323
+ "prevalence": "0.103000",
1324
+ "ig_sum_bits": "0.352709",
1325
+ "ig_mean_bits": "0.006655",
1326
+ "delta_top5_mass": "0.356039",
1327
+ "mean_abs_log2_lift": "0.428911",
1328
+ "semantic_top_group": "cat:clothing",
1329
+ "semantic_margin": "0.023719",
1330
+ "semantic_entropy_norm": "0.956312",
1331
+ "strongest_group_by_lift": "cat:limbs",
1332
+ "strongest_group_lift": "2.427184",
1333
+ "suggested_probe_bundle": "clothing_state",
1334
+ "needs_glossary": "0",
1335
+ "combined_score": "0.055946",
1336
+ "actionable_score": "0.013500"
1337
+ },
1338
+ {
1339
+ "tag": "outside",
1340
+ "sample_occurrences": "93",
1341
+ "fluffyrock_count": "253364",
1342
+ "prevalence": "0.093000",
1343
+ "ig_sum_bits": "0.603079",
1344
+ "ig_mean_bits": "0.011379",
1345
+ "delta_top5_mass": "0.230452",
1346
+ "mean_abs_log2_lift": "0.436412",
1347
+ "semantic_top_group": "cat:location",
1348
+ "semantic_margin": "0.013880",
1349
+ "semantic_entropy_norm": "0.962489",
1350
+ "strongest_group_by_lift": "cat:location",
1351
+ "strongest_group_lift": "7.575758",
1352
+ "suggested_probe_bundle": "scene_pose",
1353
+ "needs_glossary": "0",
1354
+ "combined_score": "0.090862",
1355
+ "actionable_score": "0.013410"
1356
+ },
1357
+ {
1358
+ "tag": "sitting",
1359
+ "sample_occurrences": "74",
1360
+ "fluffyrock_count": "266905",
1361
+ "prevalence": "0.074000",
1362
+ "ig_sum_bits": "0.480552",
1363
+ "ig_mean_bits": "0.009067",
1364
+ "delta_top5_mass": "0.342919",
1365
+ "mean_abs_log2_lift": "0.844222",
1366
+ "semantic_top_group": "cat:posture",
1367
+ "semantic_margin": "0.002072",
1368
+ "semantic_entropy_norm": "0.964281",
1369
+ "strongest_group_by_lift": "cat:organization",
1370
+ "strongest_group_lift": "0.000000",
1371
+ "suggested_probe_bundle": "scene_pose",
1372
+ "needs_glossary": "0",
1373
+ "combined_score": "0.065144",
1374
+ "actionable_score": "0.012630"
1375
+ },
1376
+ {
1377
+ "tag": "holding_object",
1378
+ "sample_occurrences": "84",
1379
+ "fluffyrock_count": "137895",
1380
+ "prevalence": "0.084000",
1381
+ "ig_sum_bits": "0.483752",
1382
+ "ig_mean_bits": "0.009127",
1383
+ "delta_top5_mass": "0.293048",
1384
+ "mean_abs_log2_lift": "0.916054",
1385
+ "semantic_top_group": "cat:objects_props",
1386
+ "semantic_margin": "0.004765",
1387
+ "semantic_entropy_norm": "0.953519",
1388
+ "strongest_group_by_lift": "cat:organization",
1389
+ "strongest_group_lift": "0.000000",
1390
+ "suggested_probe_bundle": "other",
1391
+ "needs_glossary": "0",
1392
+ "combined_score": "0.070212",
1393
+ "actionable_score": "0.012462"
1394
+ },
1395
+ {
1396
+ "tag": "eyes_closed",
1397
+ "sample_occurrences": "68",
1398
+ "fluffyrock_count": "307350",
1399
+ "prevalence": "0.068000",
1400
+ "ig_sum_bits": "0.493717",
1401
+ "ig_mean_bits": "0.009315",
1402
+ "delta_top5_mass": "0.350471",
1403
+ "mean_abs_log2_lift": "0.975645",
1404
+ "semantic_top_group": "cat:gaze",
1405
+ "semantic_margin": "0.005281",
1406
+ "semantic_entropy_norm": "0.964097",
1407
+ "strongest_group_by_lift": "cat:information",
1408
+ "strongest_group_lift": "0.000000",
1409
+ "suggested_probe_bundle": "gaze_expression",
1410
+ "needs_glossary": "0",
1411
+ "combined_score": "0.064377",
1412
+ "actionable_score": "0.012189"
1413
+ },
1414
+ {
1415
+ "tag": "white_body",
1416
+ "sample_occurrences": "171",
1417
+ "fluffyrock_count": "541452",
1418
+ "prevalence": "0.171000",
1419
+ "ig_sum_bits": "0.346391",
1420
+ "ig_mean_bits": "0.006536",
1421
+ "delta_top5_mass": "0.184234",
1422
+ "mean_abs_log2_lift": "0.357198",
1423
+ "semantic_top_group": "cat:posture",
1424
+ "semantic_margin": "0.008722",
1425
+ "semantic_entropy_norm": "0.964478",
1426
+ "strongest_group_by_lift": "cat:information",
1427
+ "strongest_group_lift": "0.194932",
1428
+ "suggested_probe_bundle": "other",
1429
+ "needs_glossary": "0",
1430
+ "combined_score": "0.067526",
1431
+ "actionable_score": "0.011300"
1432
+ },
1433
+ {
1434
+ "tag": "5_fingers",
1435
+ "sample_occurrences": "88",
1436
+ "fluffyrock_count": "220474",
1437
+ "prevalence": "0.088000",
1438
+ "ig_sum_bits": "0.317778",
1439
+ "ig_mean_bits": "0.005996",
1440
+ "delta_top5_mass": "0.344455",
1441
+ "mean_abs_log2_lift": "0.430931",
1442
+ "semantic_top_group": "cat:limbs",
1443
+ "semantic_margin": "0.004720",
1444
+ "semantic_entropy_norm": "0.961458",
1445
+ "strongest_group_by_lift": "cat:clothing",
1446
+ "strongest_group_lift": "2.217295",
1447
+ "suggested_probe_bundle": "clothing_state",
1448
+ "needs_glossary": "1",
1449
+ "combined_score": "0.046747",
1450
+ "actionable_score": "0.010004"
1451
+ },
1452
+ {
1453
+ "tag": "tail",
1454
+ "sample_occurrences": "110",
1455
+ "fluffyrock_count": "686059",
1456
+ "prevalence": "0.110000",
1457
+ "ig_sum_bits": "0.372369",
1458
+ "ig_mean_bits": "0.007026",
1459
+ "delta_top5_mass": "0.224000",
1460
+ "mean_abs_log2_lift": "0.956348",
1461
+ "semantic_top_group": "cat:posture",
1462
+ "semantic_margin": "0.009191",
1463
+ "semantic_entropy_norm": "0.965507",
1464
+ "strongest_group_by_lift": "cat:information",
1465
+ "strongest_group_lift": "0.000000",
1466
+ "suggested_probe_bundle": "other",
1467
+ "needs_glossary": "0",
1468
+ "combined_score": "0.060265",
1469
+ "actionable_score": "0.009492"
1470
+ },
1471
+ {
1472
+ "tag": "hat",
1473
+ "sample_occurrences": "67",
1474
+ "fluffyrock_count": "166176",
1475
+ "prevalence": "0.067000",
1476
+ "ig_sum_bits": "0.343140",
1477
+ "ig_mean_bits": "0.006474",
1478
+ "delta_top5_mass": "0.394149",
1479
+ "mean_abs_log2_lift": "0.425289",
1480
+ "semantic_top_group": "cat:general_activity_if_any",
1481
+ "semantic_margin": "0.001455",
1482
+ "semantic_entropy_norm": "0.964603",
1483
+ "strongest_group_by_lift": "cat:body_decor",
1484
+ "strongest_group_lift": "4.694836",
1485
+ "suggested_probe_bundle": "other",
1486
+ "needs_glossary": "0",
1487
+ "combined_score": "0.044415",
1488
+ "actionable_score": "0.009382"
1489
+ },
1490
+ {
1491
+ "tag": "headwear",
1492
+ "sample_occurrences": "69",
1493
+ "fluffyrock_count": "177119",
1494
+ "prevalence": "0.069000",
1495
+ "ig_sum_bits": "0.344774",
1496
+ "ig_mean_bits": "0.006505",
1497
+ "delta_top5_mass": "0.366029",
1498
+ "mean_abs_log2_lift": "0.416601",
1499
+ "semantic_top_group": "cat:expression",
1500
+ "semantic_margin": "0.000055",
1501
+ "semantic_entropy_norm": "0.964899",
1502
+ "strongest_group_by_lift": "cat:body_decor",
1503
+ "strongest_group_lift": "4.694836",
1504
+ "suggested_probe_bundle": "other",
1505
+ "needs_glossary": "1",
1506
+ "combined_score": "0.045226",
1507
+ "actionable_score": "0.009013"
1508
+ },
1509
+ {
1510
+ "tag": "ambiguous_gender",
1511
+ "sample_occurrences": "166",
1512
+ "fluffyrock_count": "256378",
1513
+ "prevalence": "0.166000",
1514
+ "ig_sum_bits": "0.422252",
1515
+ "ig_mean_bits": "0.007967",
1516
+ "delta_top5_mass": "0.122795",
1517
+ "mean_abs_log2_lift": "0.556217",
1518
+ "semantic_top_group": "cat:gender",
1519
+ "semantic_margin": "0.055826",
1520
+ "semantic_entropy_norm": "0.960351",
1521
+ "strongest_group_by_lift": "cat:breasts",
1522
+ "strongest_group_lift": "0.116595",
1523
+ "suggested_probe_bundle": "other",
1524
+ "needs_glossary": "0",
1525
+ "combined_score": "0.081671",
1526
+ "actionable_score": "0.008948"
1527
+ },
1528
+ {
1529
+ "tag": "piercing",
1530
+ "sample_occurrences": "56",
1531
+ "fluffyrock_count": "321077",
1532
+ "prevalence": "0.056000",
1533
+ "ig_sum_bits": "0.419859",
1534
+ "ig_mean_bits": "0.007922",
1535
+ "delta_top5_mass": "0.334714",
1536
+ "mean_abs_log2_lift": "1.412372",
1537
+ "semantic_top_group": "cat:body_decor",
1538
+ "semantic_margin": "0.031519",
1539
+ "semantic_entropy_norm": "0.962481",
1540
+ "strongest_group_by_lift": "cat:information",
1541
+ "strongest_group_lift": "0.000000",
1542
+ "suggested_probe_bundle": "other",
1543
+ "needs_glossary": "0",
1544
+ "combined_score": "0.050078",
1545
+ "actionable_score": "0.008165"
1546
+ },
1547
+ {
1548
+ "tag": "topless",
1549
+ "sample_occurrences": "45",
1550
+ "fluffyrock_count": "115349",
1551
+ "prevalence": "0.045000",
1552
+ "ig_sum_bits": "0.398750",
1553
+ "ig_mean_bits": "0.007524",
1554
+ "delta_top5_mass": "0.424000",
1555
+ "mean_abs_log2_lift": "1.554900",
1556
+ "semantic_top_group": "cat:clothing",
1557
+ "semantic_margin": "0.071466",
1558
+ "semantic_entropy_norm": "0.956919",
1559
+ "strongest_group_by_lift": "cat:franchise_series",
1560
+ "strongest_group_lift": "0.000000",
1561
+ "suggested_probe_bundle": "clothing_state",
1562
+ "needs_glossary": "0",
1563
+ "combined_score": "0.043112",
1564
+ "actionable_score": "0.007936"
1565
+ },
1566
+ {
1567
+ "tag": "pants",
1568
+ "sample_occurrences": "76",
1569
+ "fluffyrock_count": "135351",
1570
+ "prevalence": "0.076000",
1571
+ "ig_sum_bits": "0.276703",
1572
+ "ig_mean_bits": "0.005221",
1573
+ "delta_top5_mass": "0.358211",
1574
+ "mean_abs_log2_lift": "0.417602",
1575
+ "semantic_top_group": "cat:clothing",
1576
+ "semantic_margin": "0.003835",
1577
+ "semantic_entropy_norm": "0.956157",
1578
+ "strongest_group_by_lift": "cat:limbs",
1579
+ "strongest_group_lift": "3.289474",
1580
+ "suggested_probe_bundle": "clothing_state",
1581
+ "needs_glossary": "0",
1582
+ "combined_score": "0.038270",
1583
+ "actionable_score": "0.007863"
1584
+ },
1585
+ {
1586
+ "tag": "ear_piercing",
1587
+ "sample_occurrences": "44",
1588
+ "fluffyrock_count": "198778",
1589
+ "prevalence": "0.044000",
1590
+ "ig_sum_bits": "0.402942",
1591
+ "ig_mean_bits": "0.007603",
1592
+ "delta_top5_mass": "0.424000",
1593
+ "mean_abs_log2_lift": "1.520698",
1594
+ "semantic_top_group": "cat:body_decor",
1595
+ "semantic_margin": "0.058825",
1596
+ "semantic_entropy_norm": "0.963835",
1597
+ "strongest_group_by_lift": "cat:information",
1598
+ "strongest_group_lift": "0.000000",
1599
+ "suggested_probe_bundle": "other",
1600
+ "needs_glossary": "0",
1601
+ "combined_score": "0.042815",
1602
+ "actionable_score": "0.007789"
1603
+ },
1604
+ {
1605
+ "tag": "detailed_background",
1606
+ "sample_occurrences": "67",
1607
+ "fluffyrock_count": "171158",
1608
+ "prevalence": "0.067000",
1609
+ "ig_sum_bits": "0.416617",
1610
+ "ig_mean_bits": "0.007861",
1611
+ "delta_top5_mass": "0.259821",
1612
+ "mean_abs_log2_lift": "0.622179",
1613
+ "semantic_top_group": "cat:background_composition",
1614
+ "semantic_margin": "0.004520",
1615
+ "semantic_entropy_norm": "0.963134",
1616
+ "strongest_group_by_lift": "wiki:art_style",
1617
+ "strongest_group_lift": "0.213220",
1618
+ "suggested_probe_bundle": "scene_pose",
1619
+ "needs_glossary": "0",
1620
+ "combined_score": "0.054002",
1621
+ "actionable_score": "0.007520"
1622
+ },
1623
+ {
1624
+ "tag": "canis",
1625
+ "sample_occurrences": "159",
1626
+ "fluffyrock_count": "587772",
1627
+ "prevalence": "0.159000",
1628
+ "ig_sum_bits": "0.188810",
1629
+ "ig_mean_bits": "0.003562",
1630
+ "delta_top5_mass": "0.241610",
1631
+ "mean_abs_log2_lift": "0.743852",
1632
+ "semantic_top_group": "cat:limbs",
1633
+ "semantic_margin": "0.005960",
1634
+ "semantic_entropy_norm": "0.968348",
1635
+ "strongest_group_by_lift": "cat:information",
1636
+ "strongest_group_lift": "0.000000",
1637
+ "suggested_probe_bundle": "species_taxonomy",
1638
+ "needs_glossary": "1",
1639
+ "combined_score": "0.035614",
1640
+ "actionable_score": "0.007483"
1641
+ },
1642
+ {
1643
+ "tag": "long_hair",
1644
+ "sample_occurrences": "62",
1645
+ "fluffyrock_count": "261330",
1646
+ "prevalence": "0.062000",
1647
+ "ig_sum_bits": "0.355282",
1648
+ "ig_mean_bits": "0.006703",
1649
+ "delta_top5_mass": "0.327226",
1650
+ "mean_abs_log2_lift": "0.500167",
1651
+ "semantic_top_group": "cat:hair",
1652
+ "semantic_margin": "0.091200",
1653
+ "semantic_entropy_norm": "0.962667",
1654
+ "strongest_group_by_lift": "cat:hair",
1655
+ "strongest_group_lift": "2.680965",
1656
+ "suggested_probe_bundle": "other",
1657
+ "needs_glossary": "0",
1658
+ "combined_score": "0.044438",
1659
+ "actionable_score": "0.007477"
1660
+ },
1661
+ {
1662
+ "tag": "white_fur",
1663
+ "sample_occurrences": "133",
1664
+ "fluffyrock_count": "426284",
1665
+ "prevalence": "0.133000",
1666
+ "ig_sum_bits": "0.279758",
1667
+ "ig_mean_bits": "0.005278",
1668
+ "delta_top5_mass": "0.190917",
1669
+ "mean_abs_log2_lift": "0.799621",
1670
+ "semantic_top_group": "cat:posture",
1671
+ "semantic_margin": "0.004182",
1672
+ "semantic_entropy_norm": "0.963464",
1673
+ "strongest_group_by_lift": "cat:information",
1674
+ "strongest_group_lift": "0.000000",
1675
+ "suggested_probe_bundle": "other",
1676
+ "needs_glossary": "0",
1677
+ "combined_score": "0.049235",
1678
+ "actionable_score": "0.007363"
1679
+ },
1680
+ {
1681
+ "tag": "jewelry",
1682
+ "sample_occurrences": "53",
1683
+ "fluffyrock_count": "185593",
1684
+ "prevalence": "0.053000",
1685
+ "ig_sum_bits": "0.405028",
1686
+ "ig_mean_bits": "0.007642",
1687
+ "delta_top5_mass": "0.329660",
1688
+ "mean_abs_log2_lift": "1.548813",
1689
+ "semantic_top_group": "cat:body_decor",
1690
+ "semantic_margin": "0.065941",
1691
+ "semantic_entropy_norm": "0.966169",
1692
+ "strongest_group_by_lift": "wiki:character_relations",
1693
+ "strongest_group_lift": "0.000000",
1694
+ "suggested_probe_bundle": "other",
1695
+ "needs_glossary": "0",
1696
+ "combined_score": "0.046905",
1697
+ "actionable_score": "0.007316"
1698
+ },
1699
+ {
1700
+ "tag": "cleavage",
1701
+ "sample_occurrences": "41",
1702
+ "fluffyrock_count": "116981",
1703
+ "prevalence": "0.041000",
1704
+ "ig_sum_bits": "0.368920",
1705
+ "ig_mean_bits": "0.006961",
1706
+ "delta_top5_mass": "0.424000",
1707
+ "mean_abs_log2_lift": "1.077146",
1708
+ "semantic_top_group": "cat:breasts",
1709
+ "semantic_margin": "0.048045",
1710
+ "semantic_entropy_norm": "0.952018",
1711
+ "strongest_group_by_lift": "wiki:themes",
1712
+ "strongest_group_lift": "0.000000",
1713
+ "suggested_probe_bundle": "other",
1714
+ "needs_glossary": "0",
1715
+ "combined_score": "0.038332",
1716
+ "actionable_score": "0.006721"
1717
+ },
1718
+ {
1719
+ "tag": "duo",
1720
+ "sample_occurrences": "246",
1721
+ "fluffyrock_count": "1327850",
1722
+ "prevalence": "0.246000",
1723
+ "ig_sum_bits": "0.196565",
1724
+ "ig_mean_bits": "0.003709",
1725
+ "delta_top5_mass": "0.131317",
1726
+ "mean_abs_log2_lift": "0.251101",
1727
+ "semantic_top_group": "cat:gaze",
1728
+ "semantic_margin": "0.011348",
1729
+ "semantic_entropy_norm": "0.964475",
1730
+ "strongest_group_by_lift": "wiki:character_relations",
1731
+ "strongest_group_lift": "2.405840",
1732
+ "suggested_probe_bundle": "count_cardinality",
1733
+ "needs_glossary": "1",
1734
+ "combined_score": "0.043832",
1735
+ "actionable_score": "0.006575"
1736
+ }
1737
+ ],
1738
+ "bundle_scores": [
1739
+ {
1740
+ "bundle": "clothing_state",
1741
+ "bundle_score_top5_actionable": 0.669466,
1742
+ "top_tags": [
1743
+ "clothing",
1744
+ "clothed",
1745
+ "topwear",
1746
+ "bottomwear",
1747
+ "shirt"
1748
+ ],
1749
+ "glossary_rate_top5": 0.4
1750
+ },
1751
+ {
1752
+ "bundle": "other",
1753
+ "bundle_score_top5_actionable": 0.425525,
1754
+ "top_tags": [
1755
+ "hair",
1756
+ "female",
1757
+ "male",
1758
+ "canine",
1759
+ "fingers"
1760
+ ],
1761
+ "glossary_rate_top5": 0.2
1762
+ },
1763
+ {
1764
+ "bundle": "scene_pose",
1765
+ "bundle_score_top5_actionable": 0.322049,
1766
+ "top_tags": [
1767
+ "simple_background",
1768
+ "standing",
1769
+ "white_background",
1770
+ "outside",
1771
+ "sitting"
1772
+ ],
1773
+ "glossary_rate_top5": 0.0
1774
+ },
1775
+ {
1776
+ "bundle": "gaze_expression",
1777
+ "bundle_score_top5_actionable": 0.320334,
1778
+ "top_tags": [
1779
+ "smile",
1780
+ "looking_at_viewer",
1781
+ "open_mouth",
1782
+ "blush",
1783
+ "eyes_closed"
1784
+ ],
1785
+ "glossary_rate_top5": 0.0
1786
+ },
1787
+ {
1788
+ "bundle": "body_type_presence",
1789
+ "bundle_score_top5_actionable": 0.158529,
1790
+ "top_tags": [
1791
+ "anthro",
1792
+ "biped",
1793
+ "feral",
1794
+ "humanoid",
1795
+ "quadruped"
1796
+ ],
1797
+ "glossary_rate_top5": 0.6
1798
+ },
1799
+ {
1800
+ "bundle": "text_symbols",
1801
+ "bundle_score_top5_actionable": 0.125053,
1802
+ "top_tags": [
1803
+ "text",
1804
+ "dialogue",
1805
+ "<3"
1806
+ ],
1807
+ "glossary_rate_top5": 0.333
1808
+ },
1809
+ {
1810
+ "bundle": "body_shape_breasts",
1811
+ "bundle_score_top5_actionable": 0.079118,
1812
+ "top_tags": [
1813
+ "breasts",
1814
+ "big_breasts",
1815
+ "wide_hips",
1816
+ "thick_thighs",
1817
+ "muscular_male"
1818
+ ],
1819
+ "glossary_rate_top5": 0.0
1820
+ },
1821
+ {
1822
+ "bundle": "species_taxonomy",
1823
+ "bundle_score_top5_actionable": 0.065936,
1824
+ "top_tags": [
1825
+ "canid",
1826
+ "mammal",
1827
+ "canis",
1828
+ "equid",
1829
+ "leporid"
1830
+ ],
1831
+ "glossary_rate_top5": 0.8
1832
+ },
1833
+ {
1834
+ "bundle": "count_cardinality",
1835
+ "bundle_score_top5_actionable": 0.023691,
1836
+ "top_tags": [
1837
+ "solo",
1838
+ "duo",
1839
+ "group",
1840
+ "trio",
1841
+ "husky"
1842
+ ],
1843
+ "glossary_rate_top5": 0.6
1844
+ }
1845
+ ],
1846
+ "diversified_probe_shortlist": [
1847
+ "clothing",
1848
+ "hair",
1849
+ "simple_background",
1850
+ "clothed",
1851
+ "text",
1852
+ "smile",
1853
+ "looking_at_viewer",
1854
+ "anthro",
1855
+ "breasts",
1856
+ "standing",
1857
+ "dialogue",
1858
+ "female",
1859
+ "solo",
1860
+ "open_mouth",
1861
+ "male"
1862
+ ],
1863
+ "outputs": {
1864
+ "csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_informativeness.csv",
1865
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_informativeness_summary.json"
1866
+ }
1867
+ }
data/analysis/probe_policy_simulation.csv ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mode,strategy,budget,top_m,hit_at_m,recall_at_m,precision_at_m,ndcg_at_m,true_mass,topm_true_mass
2
+ cold_start,baseline_no_probe,3,5,0.997000,0.294097,0.915200,0.920657,0.560906,0.243309
3
+ cold_start,baseline_no_probe,3,8,0.998000,0.434738,0.852500,0.880656,0.560906,0.342338
4
+ cold_start,fixed_top_actionable,3,5,0.999000,0.298718,0.930400,0.937397,0.627974,0.286392
5
+ cold_start,fixed_top_actionable,3,8,0.999000,0.455276,0.894625,0.914544,0.627974,0.403060
6
+ cold_start,fixed_mmr,3,5,0.999000,0.303410,0.941000,0.947166,0.640523,0.273048
7
+ cold_start,fixed_mmr,3,8,1.000000,0.470316,0.919500,0.934291,0.640523,0.408458
8
+ cold_start,adaptive_entropy,3,5,0.999000,0.300981,0.933000,0.942117,0.648580,0.299866
9
+ cold_start,adaptive_entropy,3,8,1.000000,0.467337,0.912250,0.928783,0.648580,0.424741
10
+ cold_start,baseline_no_probe,5,5,0.997000,0.294097,0.915200,0.920657,0.560906,0.243309
11
+ cold_start,baseline_no_probe,5,8,0.998000,0.434738,0.852500,0.880656,0.560906,0.342338
12
+ cold_start,fixed_top_actionable,5,5,0.999000,0.302319,0.940200,0.944141,0.672677,0.300715
13
+ cold_start,fixed_top_actionable,5,8,0.999000,0.469518,0.918500,0.931777,0.672677,0.437220
14
+ cold_start,fixed_mmr,5,5,0.999000,0.309303,0.955400,0.962393,0.677985,0.305600
15
+ cold_start,fixed_mmr,5,8,0.999000,0.472797,0.923625,0.942412,0.677985,0.438226
16
+ cold_start,adaptive_entropy,5,5,0.999000,0.308379,0.949800,0.957865,0.687326,0.339214
17
+ cold_start,adaptive_entropy,5,8,0.999000,0.472845,0.920625,0.939458,0.687326,0.464909
18
+ cold_start,baseline_no_probe,8,5,0.997000,0.294097,0.915200,0.920657,0.560906,0.243309
19
+ cold_start,baseline_no_probe,8,8,0.998000,0.434738,0.852500,0.880656,0.560906,0.342338
20
+ cold_start,fixed_top_actionable,8,5,0.999000,0.306435,0.950000,0.957627,0.720929,0.327657
21
+ cold_start,fixed_top_actionable,8,8,0.999000,0.474425,0.927375,0.943717,0.720929,0.472782
22
+ cold_start,fixed_mmr,8,5,0.999000,0.306435,0.950000,0.957627,0.720929,0.327657
23
+ cold_start,fixed_mmr,8,8,0.999000,0.474425,0.927375,0.943717,0.720929,0.472782
24
+ cold_start,adaptive_entropy,8,5,0.999000,0.307421,0.949600,0.957299,0.736404,0.378668
25
+ cold_start,adaptive_entropy,8,8,0.999000,0.476696,0.926000,0.942598,0.736404,0.512029
26
+ warm_start_easy2,baseline_no_probe,3,5,1.000000,0.319580,0.974200,0.981660,0.584248,0.247201
27
+ warm_start_easy2,baseline_no_probe,3,8,1.000000,0.488842,0.945375,0.964276,0.584248,0.360456
28
+ warm_start_easy2,fixed_top_actionable,3,5,1.000000,0.316842,0.968000,0.973513,0.667247,0.293420
29
+ warm_start_easy2,fixed_top_actionable,3,8,1.000000,0.482525,0.935500,0.954054,0.667247,0.422666
30
+ warm_start_easy2,fixed_mmr,3,5,1.000000,0.316553,0.967800,0.973695,0.668689,0.294775
31
+ warm_start_easy2,fixed_mmr,3,8,1.000000,0.483474,0.937000,0.955154,0.668689,0.423177
32
+ warm_start_easy2,adaptive_entropy,3,5,1.000000,0.316812,0.968000,0.975042,0.674402,0.318080
33
+ warm_start_easy2,adaptive_entropy,3,8,1.000000,0.485218,0.938625,0.957273,0.674402,0.441954
34
+ warm_start_easy2,baseline_no_probe,5,5,1.000000,0.319580,0.974200,0.981660,0.584248,0.247201
35
+ warm_start_easy2,baseline_no_probe,5,8,1.000000,0.488842,0.945375,0.964276,0.584248,0.360456
36
+ warm_start_easy2,fixed_top_actionable,5,5,1.000000,0.312521,0.958600,0.966792,0.707247,0.314510
37
+ warm_start_easy2,fixed_top_actionable,5,8,1.000000,0.479840,0.932875,0.951098,0.707247,0.454423
38
+ warm_start_easy2,fixed_mmr,5,5,1.000000,0.313391,0.960400,0.968231,0.710294,0.315666
39
+ warm_start_easy2,fixed_mmr,5,8,1.000000,0.481800,0.935375,0.953103,0.710294,0.456495
40
+ warm_start_easy2,adaptive_entropy,5,5,1.000000,0.313410,0.958400,0.966606,0.717603,0.356699
41
+ warm_start_easy2,adaptive_entropy,5,8,1.000000,0.482998,0.933500,0.951525,0.717603,0.486616
42
+ warm_start_easy2,baseline_no_probe,8,5,1.000000,0.319580,0.974200,0.981660,0.584248,0.247201
43
+ warm_start_easy2,baseline_no_probe,8,8,1.000000,0.488842,0.945375,0.964276,0.584248,0.360456
44
+ warm_start_easy2,fixed_top_actionable,8,5,1.000000,0.312792,0.960400,0.967993,0.733608,0.344500
45
+ warm_start_easy2,fixed_top_actionable,8,8,1.000000,0.478613,0.930625,0.949616,0.733608,0.486487
46
+ warm_start_easy2,fixed_mmr,8,5,1.000000,0.311775,0.957600,0.964796,0.740706,0.351629
47
+ warm_start_easy2,fixed_mmr,8,8,1.000000,0.479598,0.932250,0.949454,0.740706,0.494811
48
+ warm_start_easy2,adaptive_entropy,8,5,1.000000,0.311895,0.956400,0.965453,0.755812,0.406757
49
+ warm_start_easy2,adaptive_entropy,8,8,1.000000,0.479927,0.928625,0.948161,0.755812,0.536603
data/analysis/probe_policy_simulation_summary.json ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "min_count": 200,
4
+ "min_group_images": 20,
5
+ "min_probe_images": 5,
6
+ "probe_pool_size": 120,
7
+ "prevalence_min": 0.02,
8
+ "prevalence_max": 0.6,
9
+ "budgets": [
10
+ 3,
11
+ 5,
12
+ 8
13
+ ],
14
+ "top_m_values": [
15
+ 5,
16
+ 8
17
+ ],
18
+ "modes": [
19
+ "cold_start",
20
+ "warm_start_easy2"
21
+ ],
22
+ "laplace_alpha": 1.0,
23
+ "note": "Oracle probe answers from GT tags; optimistic upper bound."
24
+ },
25
+ "n_images": 1000,
26
+ "n_active_groups": 53,
27
+ "n_candidate_probes": 120,
28
+ "excluded_wiki_groups": [
29
+ "ambiguous_tags",
30
+ "disambiguations",
31
+ "e621",
32
+ "multiple_tags",
33
+ "sound_effects",
34
+ "unknown_tags"
35
+ ],
36
+ "probe_pool_head": [
37
+ "clothing",
38
+ "hair",
39
+ "clothed",
40
+ "simple_background",
41
+ "smile",
42
+ "text",
43
+ "looking_at_viewer",
44
+ "breasts",
45
+ "female",
46
+ "standing",
47
+ "male",
48
+ "dialogue",
49
+ "open_mouth",
50
+ "topwear",
51
+ "blush",
52
+ "canine",
53
+ "biped",
54
+ "canid",
55
+ "white_background",
56
+ "bottomwear",
57
+ "fingers",
58
+ "feral",
59
+ "fur",
60
+ "shirt",
61
+ "outside",
62
+ "solo",
63
+ "sitting",
64
+ "holding_object",
65
+ "eyes_closed",
66
+ "white_body"
67
+ ],
68
+ "mmr_head": [
69
+ "clothing",
70
+ "hair",
71
+ "simple_background",
72
+ "clothed",
73
+ "text",
74
+ "smile",
75
+ "looking_at_viewer",
76
+ "breasts",
77
+ "standing",
78
+ "dialogue",
79
+ "female",
80
+ "solo",
81
+ "open_mouth",
82
+ "male",
83
+ "topwear",
84
+ "blush",
85
+ "canine",
86
+ "biped",
87
+ "canid",
88
+ "white_background",
89
+ "bottomwear",
90
+ "fingers",
91
+ "feral",
92
+ "fur",
93
+ "shirt",
94
+ "outside",
95
+ "sitting",
96
+ "holding_object",
97
+ "eyes_closed",
98
+ "white_body"
99
+ ],
100
+ "likely_useful_snapshot_budget5_top5": [
101
+ {
102
+ "mode": "cold_start",
103
+ "baseline_ndcg_at_5": 0.920657,
104
+ "fixed_top_ndcg_at_5": 0.944141,
105
+ "adaptive_ndcg_at_5": 0.957865,
106
+ "adaptive_minus_fixed_top_ndcg_at_5": 0.013723999999999958,
107
+ "adaptive_minus_baseline_ndcg_at_5": 0.03720800000000002
108
+ },
109
+ {
110
+ "mode": "warm_start_easy2",
111
+ "baseline_ndcg_at_5": 0.98166,
112
+ "fixed_top_ndcg_at_5": 0.966792,
113
+ "adaptive_ndcg_at_5": 0.966606,
114
+ "adaptive_minus_fixed_top_ndcg_at_5": -0.00018600000000001948,
115
+ "adaptive_minus_baseline_ndcg_at_5": -0.015054000000000012
116
+ }
117
+ ],
118
+ "outputs": {
119
+ "csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_policy_simulation.csv",
120
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_policy_simulation_summary.json"
121
+ }
122
+ }
data/analysis/probe_reliability_n100.csv ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,bundle,needs_glossary,support_pos,support_neg,tp_explicit,fp_explicit,fn_explicit,precision_explicit,recall_explicit,f1_explicit,tp_strong,fp_strong,fn_strong,precision_strong,recall_strong,f1_strong
2
+ wide_hips,body_shape_breasts,0,1,99,1,0,0,1.000000,1.000000,1.000000,1,0,0,1.000000,1.000000,1.000000
3
+ anthro,body_type_presence,1,68,32,63,19,5,0.768293,0.926471,0.840000,67,25,1,0.728261,0.985294,0.837500
4
+ felid,species_taxonomy,1,18,82,11,0,7,1.000000,0.611111,0.758621,12,1,6,0.923077,0.666667,0.774194
5
+ group,count_cardinality,0,16,84,10,1,6,0.909091,0.625000,0.740741,10,1,6,0.909091,0.625000,0.740741
6
+ blush,gaze_expression,0,13,87,10,2,3,0.833333,0.769231,0.800000,10,4,3,0.714286,0.769231,0.740741
7
+ clothing,clothing_state,0,59,41,42,21,17,0.666667,0.711864,0.688525,42,21,17,0.666667,0.711864,0.688525
8
+ canid,species_taxonomy,1,37,63,21,7,16,0.750000,0.567568,0.646154,24,11,13,0.685714,0.648649,0.666667
9
+ <3,text_symbols,1,6,94,3,0,3,1.000000,0.500000,0.666667,3,0,3,1.000000,0.500000,0.666667
10
+ thick_thighs,body_shape_breasts,0,1,99,1,1,0,0.500000,1.000000,0.666667,1,1,0,0.500000,1.000000,0.666667
11
+ bird,species_taxonomy,0,6,94,4,3,2,0.571429,0.666667,0.615385,4,3,2,0.571429,0.666667,0.615385
12
+ bear,species_taxonomy,0,5,95,3,4,2,0.428571,0.600000,0.500000,4,4,1,0.500000,0.800000,0.615385
13
+ text,text_symbols,0,23,77,15,10,8,0.600000,0.652174,0.625000,15,11,8,0.576923,0.652174,0.612245
14
+ simple_background,scene_pose,0,27,73,15,8,12,0.652174,0.555556,0.600000,15,8,12,0.652174,0.555556,0.600000
15
+ eyes_closed,gaze_expression,0,4,96,3,3,1,0.500000,0.750000,0.600000,3,3,1,0.500000,0.750000,0.600000
16
+ duo,count_cardinality,1,20,80,11,9,9,0.550000,0.550000,0.550000,12,9,8,0.571429,0.600000,0.585366
17
+ solo,count_cardinality,1,57,43,24,3,33,0.888889,0.421053,0.571429,24,3,33,0.888889,0.421053,0.571429
18
+ dialogue,text_symbols,0,11,89,10,14,1,0.416667,0.909091,0.571429,10,14,1,0.416667,0.909091,0.571429
19
+ clothed,clothing_state,0,32,68,29,45,3,0.391892,0.906250,0.547170,29,45,3,0.391892,0.906250,0.547170
20
+ sitting,scene_pose,0,9,91,8,15,1,0.347826,0.888889,0.500000,8,15,1,0.347826,0.888889,0.500000
21
+ outside,scene_pose,0,10,90,6,13,4,0.315789,0.600000,0.413793,6,13,4,0.315789,0.600000,0.413793
22
+ looking_at_viewer,gaze_expression,0,20,80,4,0,16,1.000000,0.200000,0.333333,5,1,15,0.833333,0.250000,0.384615
23
+ leporid,species_taxonomy,1,8,92,2,1,6,0.666667,0.250000,0.363636,2,1,6,0.666667,0.250000,0.363636
24
+ open_mouth,gaze_expression,0,16,84,6,26,10,0.187500,0.375000,0.250000,8,29,8,0.216216,0.500000,0.301887
25
+ topwear,clothing_state,1,14,86,12,53,2,0.184615,0.857143,0.303797,12,54,2,0.181818,0.857143,0.300000
26
+ standing,scene_pose,0,10,90,7,45,3,0.134615,0.700000,0.225806,8,49,2,0.140351,0.800000,0.238806
27
+ feral,body_type_presence,1,23,77,2,1,21,0.666667,0.086957,0.153846,3,1,20,0.750000,0.130435,0.222222
28
+ canis,species_taxonomy,1,23,77,3,4,20,0.428571,0.130435,0.200000,3,4,20,0.428571,0.130435,0.200000
29
+ bottomwear,clothing_state,1,18,82,3,19,15,0.136364,0.166667,0.150000,3,19,15,0.136364,0.166667,0.150000
30
+ smile,gaze_expression,0,19,81,2,14,17,0.125000,0.105263,0.114286,2,16,17,0.111111,0.105263,0.108108
31
+ biped,body_type_presence,0,9,91,2,25,7,0.074074,0.222222,0.111111,2,29,7,0.064516,0.222222,0.100000
32
+ breasts,body_shape_breasts,0,13,87,0,0,13,0.000000,0.000000,0.000000,0,1,13,0.000000,0.000000,0.000000
33
+ topless,clothing_state,0,6,94,0,4,6,0.000000,0.000000,0.000000,0,4,6,0.000000,0.000000,0.000000
34
+ humanoid,body_type_presence,1,6,94,0,26,6,0.000000,0.000000,0.000000,0,31,6,0.000000,0.000000,0.000000
35
+ big_breasts,body_shape_breasts,0,3,97,0,11,3,0.000000,0.000000,0.000000,0,11,3,0.000000,0.000000,0.000000
36
+ nude,clothing_state,0,3,97,0,2,3,0.000000,0.000000,0.000000,0,2,3,0.000000,0.000000,0.000000
data/analysis/probe_reliability_n100.json ADDED
@@ -0,0 +1,422 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "settings": {
3
+ "n": 100,
4
+ "seed": 42,
5
+ "caption_field": "caption_cogvlm",
6
+ "probe_count": 35,
7
+ "retries": 2,
8
+ "temperature": 0.0,
9
+ "max_tokens": 900,
10
+ "model_env": "meta-llama/llama-3.1-8b-instruct"
11
+ },
12
+ "overall_metrics": {
13
+ "explicit": {
14
+ "tp": 333,
15
+ "fp": 409,
16
+ "fn": 281,
17
+ "precision": 0.448787,
18
+ "recall": 0.542345,
19
+ "f1": 0.49115
20
+ },
21
+ "strong": {
22
+ "tp": 348,
23
+ "fp": 444,
24
+ "fn": 266,
25
+ "precision": 0.439394,
26
+ "recall": 0.566775,
27
+ "f1": 0.495021
28
+ }
29
+ },
30
+ "diagnostics": {
31
+ "samples_with_attempt_failures": 0,
32
+ "samples_with_call_exhaustion": 0,
33
+ "avg_attempt_failure_rate": 0.0,
34
+ "avg_call_exhaustion_rate": 0.0
35
+ },
36
+ "top_tags_by_f1_strong": [
37
+ {
38
+ "tag": "wide_hips",
39
+ "bundle": "body_shape_breasts",
40
+ "needs_glossary": "0",
41
+ "support_pos": "1",
42
+ "support_neg": "99",
43
+ "tp_explicit": "1",
44
+ "fp_explicit": "0",
45
+ "fn_explicit": "0",
46
+ "precision_explicit": "1.000000",
47
+ "recall_explicit": "1.000000",
48
+ "f1_explicit": "1.000000",
49
+ "tp_strong": "1",
50
+ "fp_strong": "0",
51
+ "fn_strong": "0",
52
+ "precision_strong": "1.000000",
53
+ "recall_strong": "1.000000",
54
+ "f1_strong": "1.000000"
55
+ },
56
+ {
57
+ "tag": "anthro",
58
+ "bundle": "body_type_presence",
59
+ "needs_glossary": "1",
60
+ "support_pos": "68",
61
+ "support_neg": "32",
62
+ "tp_explicit": "63",
63
+ "fp_explicit": "19",
64
+ "fn_explicit": "5",
65
+ "precision_explicit": "0.768293",
66
+ "recall_explicit": "0.926471",
67
+ "f1_explicit": "0.840000",
68
+ "tp_strong": "67",
69
+ "fp_strong": "25",
70
+ "fn_strong": "1",
71
+ "precision_strong": "0.728261",
72
+ "recall_strong": "0.985294",
73
+ "f1_strong": "0.837500"
74
+ },
75
+ {
76
+ "tag": "felid",
77
+ "bundle": "species_taxonomy",
78
+ "needs_glossary": "1",
79
+ "support_pos": "18",
80
+ "support_neg": "82",
81
+ "tp_explicit": "11",
82
+ "fp_explicit": "0",
83
+ "fn_explicit": "7",
84
+ "precision_explicit": "1.000000",
85
+ "recall_explicit": "0.611111",
86
+ "f1_explicit": "0.758621",
87
+ "tp_strong": "12",
88
+ "fp_strong": "1",
89
+ "fn_strong": "6",
90
+ "precision_strong": "0.923077",
91
+ "recall_strong": "0.666667",
92
+ "f1_strong": "0.774194"
93
+ },
94
+ {
95
+ "tag": "group",
96
+ "bundle": "count_cardinality",
97
+ "needs_glossary": "0",
98
+ "support_pos": "16",
99
+ "support_neg": "84",
100
+ "tp_explicit": "10",
101
+ "fp_explicit": "1",
102
+ "fn_explicit": "6",
103
+ "precision_explicit": "0.909091",
104
+ "recall_explicit": "0.625000",
105
+ "f1_explicit": "0.740741",
106
+ "tp_strong": "10",
107
+ "fp_strong": "1",
108
+ "fn_strong": "6",
109
+ "precision_strong": "0.909091",
110
+ "recall_strong": "0.625000",
111
+ "f1_strong": "0.740741"
112
+ },
113
+ {
114
+ "tag": "blush",
115
+ "bundle": "gaze_expression",
116
+ "needs_glossary": "0",
117
+ "support_pos": "13",
118
+ "support_neg": "87",
119
+ "tp_explicit": "10",
120
+ "fp_explicit": "2",
121
+ "fn_explicit": "3",
122
+ "precision_explicit": "0.833333",
123
+ "recall_explicit": "0.769231",
124
+ "f1_explicit": "0.800000",
125
+ "tp_strong": "10",
126
+ "fp_strong": "4",
127
+ "fn_strong": "3",
128
+ "precision_strong": "0.714286",
129
+ "recall_strong": "0.769231",
130
+ "f1_strong": "0.740741"
131
+ },
132
+ {
133
+ "tag": "clothing",
134
+ "bundle": "clothing_state",
135
+ "needs_glossary": "0",
136
+ "support_pos": "59",
137
+ "support_neg": "41",
138
+ "tp_explicit": "42",
139
+ "fp_explicit": "21",
140
+ "fn_explicit": "17",
141
+ "precision_explicit": "0.666667",
142
+ "recall_explicit": "0.711864",
143
+ "f1_explicit": "0.688525",
144
+ "tp_strong": "42",
145
+ "fp_strong": "21",
146
+ "fn_strong": "17",
147
+ "precision_strong": "0.666667",
148
+ "recall_strong": "0.711864",
149
+ "f1_strong": "0.688525"
150
+ },
151
+ {
152
+ "tag": "canid",
153
+ "bundle": "species_taxonomy",
154
+ "needs_glossary": "1",
155
+ "support_pos": "37",
156
+ "support_neg": "63",
157
+ "tp_explicit": "21",
158
+ "fp_explicit": "7",
159
+ "fn_explicit": "16",
160
+ "precision_explicit": "0.750000",
161
+ "recall_explicit": "0.567568",
162
+ "f1_explicit": "0.646154",
163
+ "tp_strong": "24",
164
+ "fp_strong": "11",
165
+ "fn_strong": "13",
166
+ "precision_strong": "0.685714",
167
+ "recall_strong": "0.648649",
168
+ "f1_strong": "0.666667"
169
+ },
170
+ {
171
+ "tag": "<3",
172
+ "bundle": "text_symbols",
173
+ "needs_glossary": "1",
174
+ "support_pos": "6",
175
+ "support_neg": "94",
176
+ "tp_explicit": "3",
177
+ "fp_explicit": "0",
178
+ "fn_explicit": "3",
179
+ "precision_explicit": "1.000000",
180
+ "recall_explicit": "0.500000",
181
+ "f1_explicit": "0.666667",
182
+ "tp_strong": "3",
183
+ "fp_strong": "0",
184
+ "fn_strong": "3",
185
+ "precision_strong": "1.000000",
186
+ "recall_strong": "0.500000",
187
+ "f1_strong": "0.666667"
188
+ },
189
+ {
190
+ "tag": "thick_thighs",
191
+ "bundle": "body_shape_breasts",
192
+ "needs_glossary": "0",
193
+ "support_pos": "1",
194
+ "support_neg": "99",
195
+ "tp_explicit": "1",
196
+ "fp_explicit": "1",
197
+ "fn_explicit": "0",
198
+ "precision_explicit": "0.500000",
199
+ "recall_explicit": "1.000000",
200
+ "f1_explicit": "0.666667",
201
+ "tp_strong": "1",
202
+ "fp_strong": "1",
203
+ "fn_strong": "0",
204
+ "precision_strong": "0.500000",
205
+ "recall_strong": "1.000000",
206
+ "f1_strong": "0.666667"
207
+ },
208
+ {
209
+ "tag": "bird",
210
+ "bundle": "species_taxonomy",
211
+ "needs_glossary": "0",
212
+ "support_pos": "6",
213
+ "support_neg": "94",
214
+ "tp_explicit": "4",
215
+ "fp_explicit": "3",
216
+ "fn_explicit": "2",
217
+ "precision_explicit": "0.571429",
218
+ "recall_explicit": "0.666667",
219
+ "f1_explicit": "0.615385",
220
+ "tp_strong": "4",
221
+ "fp_strong": "3",
222
+ "fn_strong": "2",
223
+ "precision_strong": "0.571429",
224
+ "recall_strong": "0.666667",
225
+ "f1_strong": "0.615385"
226
+ },
227
+ {
228
+ "tag": "bear",
229
+ "bundle": "species_taxonomy",
230
+ "needs_glossary": "0",
231
+ "support_pos": "5",
232
+ "support_neg": "95",
233
+ "tp_explicit": "3",
234
+ "fp_explicit": "4",
235
+ "fn_explicit": "2",
236
+ "precision_explicit": "0.428571",
237
+ "recall_explicit": "0.600000",
238
+ "f1_explicit": "0.500000",
239
+ "tp_strong": "4",
240
+ "fp_strong": "4",
241
+ "fn_strong": "1",
242
+ "precision_strong": "0.500000",
243
+ "recall_strong": "0.800000",
244
+ "f1_strong": "0.615385"
245
+ },
246
+ {
247
+ "tag": "text",
248
+ "bundle": "text_symbols",
249
+ "needs_glossary": "0",
250
+ "support_pos": "23",
251
+ "support_neg": "77",
252
+ "tp_explicit": "15",
253
+ "fp_explicit": "10",
254
+ "fn_explicit": "8",
255
+ "precision_explicit": "0.600000",
256
+ "recall_explicit": "0.652174",
257
+ "f1_explicit": "0.625000",
258
+ "tp_strong": "15",
259
+ "fp_strong": "11",
260
+ "fn_strong": "8",
261
+ "precision_strong": "0.576923",
262
+ "recall_strong": "0.652174",
263
+ "f1_strong": "0.612245"
264
+ },
265
+ {
266
+ "tag": "simple_background",
267
+ "bundle": "scene_pose",
268
+ "needs_glossary": "0",
269
+ "support_pos": "27",
270
+ "support_neg": "73",
271
+ "tp_explicit": "15",
272
+ "fp_explicit": "8",
273
+ "fn_explicit": "12",
274
+ "precision_explicit": "0.652174",
275
+ "recall_explicit": "0.555556",
276
+ "f1_explicit": "0.600000",
277
+ "tp_strong": "15",
278
+ "fp_strong": "8",
279
+ "fn_strong": "12",
280
+ "precision_strong": "0.652174",
281
+ "recall_strong": "0.555556",
282
+ "f1_strong": "0.600000"
283
+ },
284
+ {
285
+ "tag": "eyes_closed",
286
+ "bundle": "gaze_expression",
287
+ "needs_glossary": "0",
288
+ "support_pos": "4",
289
+ "support_neg": "96",
290
+ "tp_explicit": "3",
291
+ "fp_explicit": "3",
292
+ "fn_explicit": "1",
293
+ "precision_explicit": "0.500000",
294
+ "recall_explicit": "0.750000",
295
+ "f1_explicit": "0.600000",
296
+ "tp_strong": "3",
297
+ "fp_strong": "3",
298
+ "fn_strong": "1",
299
+ "precision_strong": "0.500000",
300
+ "recall_strong": "0.750000",
301
+ "f1_strong": "0.600000"
302
+ },
303
+ {
304
+ "tag": "duo",
305
+ "bundle": "count_cardinality",
306
+ "needs_glossary": "1",
307
+ "support_pos": "20",
308
+ "support_neg": "80",
309
+ "tp_explicit": "11",
310
+ "fp_explicit": "9",
311
+ "fn_explicit": "9",
312
+ "precision_explicit": "0.550000",
313
+ "recall_explicit": "0.550000",
314
+ "f1_explicit": "0.550000",
315
+ "tp_strong": "12",
316
+ "fp_strong": "9",
317
+ "fn_strong": "8",
318
+ "precision_strong": "0.571429",
319
+ "recall_strong": "0.600000",
320
+ "f1_strong": "0.585366"
321
+ },
322
+ {
323
+ "tag": "solo",
324
+ "bundle": "count_cardinality",
325
+ "needs_glossary": "1",
326
+ "support_pos": "57",
327
+ "support_neg": "43",
328
+ "tp_explicit": "24",
329
+ "fp_explicit": "3",
330
+ "fn_explicit": "33",
331
+ "precision_explicit": "0.888889",
332
+ "recall_explicit": "0.421053",
333
+ "f1_explicit": "0.571429",
334
+ "tp_strong": "24",
335
+ "fp_strong": "3",
336
+ "fn_strong": "33",
337
+ "precision_strong": "0.888889",
338
+ "recall_strong": "0.421053",
339
+ "f1_strong": "0.571429"
340
+ },
341
+ {
342
+ "tag": "dialogue",
343
+ "bundle": "text_symbols",
344
+ "needs_glossary": "0",
345
+ "support_pos": "11",
346
+ "support_neg": "89",
347
+ "tp_explicit": "10",
348
+ "fp_explicit": "14",
349
+ "fn_explicit": "1",
350
+ "precision_explicit": "0.416667",
351
+ "recall_explicit": "0.909091",
352
+ "f1_explicit": "0.571429",
353
+ "tp_strong": "10",
354
+ "fp_strong": "14",
355
+ "fn_strong": "1",
356
+ "precision_strong": "0.416667",
357
+ "recall_strong": "0.909091",
358
+ "f1_strong": "0.571429"
359
+ },
360
+ {
361
+ "tag": "clothed",
362
+ "bundle": "clothing_state",
363
+ "needs_glossary": "0",
364
+ "support_pos": "32",
365
+ "support_neg": "68",
366
+ "tp_explicit": "29",
367
+ "fp_explicit": "45",
368
+ "fn_explicit": "3",
369
+ "precision_explicit": "0.391892",
370
+ "recall_explicit": "0.906250",
371
+ "f1_explicit": "0.547170",
372
+ "tp_strong": "29",
373
+ "fp_strong": "45",
374
+ "fn_strong": "3",
375
+ "precision_strong": "0.391892",
376
+ "recall_strong": "0.906250",
377
+ "f1_strong": "0.547170"
378
+ },
379
+ {
380
+ "tag": "sitting",
381
+ "bundle": "scene_pose",
382
+ "needs_glossary": "0",
383
+ "support_pos": "9",
384
+ "support_neg": "91",
385
+ "tp_explicit": "8",
386
+ "fp_explicit": "15",
387
+ "fn_explicit": "1",
388
+ "precision_explicit": "0.347826",
389
+ "recall_explicit": "0.888889",
390
+ "f1_explicit": "0.500000",
391
+ "tp_strong": "8",
392
+ "fp_strong": "15",
393
+ "fn_strong": "1",
394
+ "precision_strong": "0.347826",
395
+ "recall_strong": "0.888889",
396
+ "f1_strong": "0.500000"
397
+ },
398
+ {
399
+ "tag": "outside",
400
+ "bundle": "scene_pose",
401
+ "needs_glossary": "0",
402
+ "support_pos": "10",
403
+ "support_neg": "90",
404
+ "tp_explicit": "6",
405
+ "fp_explicit": "13",
406
+ "fn_explicit": "4",
407
+ "precision_explicit": "0.315789",
408
+ "recall_explicit": "0.600000",
409
+ "f1_explicit": "0.413793",
410
+ "tp_strong": "6",
411
+ "fp_strong": "13",
412
+ "fn_strong": "4",
413
+ "precision_strong": "0.315789",
414
+ "recall_strong": "0.600000",
415
+ "f1_strong": "0.413793"
416
+ }
417
+ ],
418
+ "outputs": {
419
+ "csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_reliability_n100.csv",
420
+ "json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_reliability_n100.json"
421
+ }
422
+ }
data/analysis/probe_reliability_sanity10.csv ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,bundle,needs_glossary,support_pos,support_neg,tp_explicit,fp_explicit,fn_explicit,precision_explicit,recall_explicit,f1_explicit,tp_strong,fp_strong,fn_strong,precision_strong,recall_strong,f1_strong
2
+ outside,scene_pose,0,2,8,2,0,0,1.000000,1.000000,1.000000,2,0,0,1.000000,1.000000,1.000000
3
+ eyes_closed,gaze_expression,0,1,9,1,0,0,1.000000,1.000000,1.000000,1,0,0,1.000000,1.000000,1.000000
4
+ group,count_cardinality,0,1,9,1,0,0,1.000000,1.000000,1.000000,1,0,0,1.000000,1.000000,1.000000
5
+ feral,body_type_presence,1,1,9,1,0,0,1.000000,1.000000,1.000000,1,0,0,1.000000,1.000000,1.000000
6
+ <3,text_symbols,1,1,9,1,0,0,1.000000,1.000000,1.000000,1,0,0,1.000000,1.000000,1.000000
7
+ clothing,clothing_state,0,8,2,8,1,0,0.888889,1.000000,0.941176,8,1,0,0.888889,1.000000,0.941176
8
+ anthro,body_type_presence,1,8,2,8,2,0,0.800000,1.000000,0.888889,8,2,0,0.800000,1.000000,0.888889
9
+ bear,species_taxonomy,0,2,8,2,1,0,0.666667,1.000000,0.800000,2,1,0,0.666667,1.000000,0.800000
10
+ duo,count_cardinality,1,2,8,2,1,0,0.666667,1.000000,0.800000,2,1,0,0.666667,1.000000,0.800000
11
+ solo,count_cardinality,1,7,3,4,0,3,1.000000,0.571429,0.727273,4,0,3,1.000000,0.571429,0.727273
12
+ clothed,clothing_state,0,5,5,5,4,0,0.555556,1.000000,0.714286,5,4,0,0.555556,1.000000,0.714286
13
+ bird,species_taxonomy,0,2,8,1,0,1,1.000000,0.500000,0.666667,1,0,1,1.000000,0.500000,0.666667
14
+ leporid,species_taxonomy,1,2,8,1,0,1,1.000000,0.500000,0.666667,1,0,1,1.000000,0.500000,0.666667
15
+ felid,species_taxonomy,1,2,8,1,0,1,1.000000,0.500000,0.666667,1,0,1,1.000000,0.500000,0.666667
16
+ canis,species_taxonomy,1,1,9,1,1,0,0.500000,1.000000,0.666667,1,1,0,0.500000,1.000000,0.666667
17
+ simple_background,scene_pose,0,3,7,3,4,0,0.428571,1.000000,0.600000,3,4,0,0.428571,1.000000,0.600000
18
+ canid,species_taxonomy,1,3,7,2,2,1,0.500000,0.666667,0.571429,2,2,1,0.500000,0.666667,0.571429
19
+ looking_at_viewer,gaze_expression,0,3,7,1,0,2,1.000000,0.333333,0.500000,1,0,2,1.000000,0.333333,0.500000
20
+ standing,scene_pose,0,2,8,2,4,0,0.333333,1.000000,0.500000,2,4,0,0.333333,1.000000,0.500000
21
+ biped,body_type_presence,0,3,7,2,4,1,0.333333,0.666667,0.444444,2,4,1,0.333333,0.666667,0.444444
22
+ smile,gaze_expression,0,3,7,0,2,3,0.000000,0.000000,0.000000,0,2,3,0.000000,0.000000,0.000000
23
+ topless,clothing_state,0,2,8,0,0,2,0.000000,0.000000,0.000000,0,0,2,0.000000,0.000000,0.000000
24
+ text,text_symbols,0,1,9,0,3,1,0.000000,0.000000,0.000000,0,3,1,0.000000,0.000000,0.000000
25
+ open_mouth,gaze_expression,0,1,9,0,3,1,0.000000,0.000000,0.000000,0,3,1,0.000000,0.000000,0.000000
26
+ blush,gaze_expression,0,1,9,0,0,1,0.000000,0.000000,0.000000,0,0,1,0.000000,0.000000,0.000000
27
+ bottomwear,clothing_state,1,1,9,0,5,1,0.000000,0.000000,0.000000,0,5,1,0.000000,0.000000,0.000000
28
+ breasts,body_shape_breasts,0,0,10,0,0,0,0.000000,0.000000,0.000000,0,0,0,0.000000,0.000000,0.000000
29
+ dialogue,text_symbols,0,0,10,0,1,0,0.000000,0.000000,0.000000,0,1,0,0.000000,0.000000,0.000000
30
+ sitting,scene_pose,0,0,10,0,3,0,0.000000,0.000000,0.000000,0,3,0,0.000000,0.000000,0.000000
31
+ big_breasts,body_shape_breasts,0,0,10,0,1,0,0.000000,0.000000,0.000000,0,1,0,0.000000,0.000000,0.000000
32
+ nude,clothing_state,0,0,10,0,0,0,0.000000,0.000000,0.000000,0,0,0,0.000000,0.000000,0.000000
33
+ wide_hips,body_shape_breasts,0,0,10,0,0,0,0.000000,0.000000,0.000000,0,0,0,0.000000,0.000000,0.000000
34
+ thick_thighs,body_shape_breasts,0,0,10,0,0,0,0.000000,0.000000,0.000000,0,0,0,0.000000,0.000000,0.000000
35
+ topwear,clothing_state,1,0,10,0,9,0,0.000000,0.000000,0.000000,0,9,0,0.000000,0.000000,0.000000
36
+ humanoid,body_type_presence,1,0,10,0,5,0,0.000000,0.000000,0.000000,0,5,0,0.000000,0.000000,0.000000
data/analysis/probe_reliability_sanity10.json ADDED
@@ -0,0 +1,422 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "settings": {
3
+ "n": 10,
4
+ "seed": 42,
5
+ "caption_field": "caption_cogvlm",
6
+ "probe_count": 35,
7
+ "retries": 2,
8
+ "temperature": 0.0,
9
+ "max_tokens": 900,
10
+ "model_env": "meta-llama/llama-3.1-8b-instruct"
11
+ },
12
+ "overall_metrics": {
13
+ "explicit": {
14
+ "tp": 49,
15
+ "fp": 56,
16
+ "fn": 19,
17
+ "precision": 0.466667,
18
+ "recall": 0.720588,
19
+ "f1": 0.566474
20
+ },
21
+ "strong": {
22
+ "tp": 49,
23
+ "fp": 56,
24
+ "fn": 19,
25
+ "precision": 0.466667,
26
+ "recall": 0.720588,
27
+ "f1": 0.566474
28
+ }
29
+ },
30
+ "diagnostics": {
31
+ "samples_with_attempt_failures": 0,
32
+ "samples_with_call_exhaustion": 0,
33
+ "avg_attempt_failure_rate": 0.0,
34
+ "avg_call_exhaustion_rate": 0.0
35
+ },
36
+ "top_tags_by_f1_strong": [
37
+ {
38
+ "tag": "outside",
39
+ "bundle": "scene_pose",
40
+ "needs_glossary": "0",
41
+ "support_pos": "2",
42
+ "support_neg": "8",
43
+ "tp_explicit": "2",
44
+ "fp_explicit": "0",
45
+ "fn_explicit": "0",
46
+ "precision_explicit": "1.000000",
47
+ "recall_explicit": "1.000000",
48
+ "f1_explicit": "1.000000",
49
+ "tp_strong": "2",
50
+ "fp_strong": "0",
51
+ "fn_strong": "0",
52
+ "precision_strong": "1.000000",
53
+ "recall_strong": "1.000000",
54
+ "f1_strong": "1.000000"
55
+ },
56
+ {
57
+ "tag": "eyes_closed",
58
+ "bundle": "gaze_expression",
59
+ "needs_glossary": "0",
60
+ "support_pos": "1",
61
+ "support_neg": "9",
62
+ "tp_explicit": "1",
63
+ "fp_explicit": "0",
64
+ "fn_explicit": "0",
65
+ "precision_explicit": "1.000000",
66
+ "recall_explicit": "1.000000",
67
+ "f1_explicit": "1.000000",
68
+ "tp_strong": "1",
69
+ "fp_strong": "0",
70
+ "fn_strong": "0",
71
+ "precision_strong": "1.000000",
72
+ "recall_strong": "1.000000",
73
+ "f1_strong": "1.000000"
74
+ },
75
+ {
76
+ "tag": "group",
77
+ "bundle": "count_cardinality",
78
+ "needs_glossary": "0",
79
+ "support_pos": "1",
80
+ "support_neg": "9",
81
+ "tp_explicit": "1",
82
+ "fp_explicit": "0",
83
+ "fn_explicit": "0",
84
+ "precision_explicit": "1.000000",
85
+ "recall_explicit": "1.000000",
86
+ "f1_explicit": "1.000000",
87
+ "tp_strong": "1",
88
+ "fp_strong": "0",
89
+ "fn_strong": "0",
90
+ "precision_strong": "1.000000",
91
+ "recall_strong": "1.000000",
92
+ "f1_strong": "1.000000"
93
+ },
94
+ {
95
+ "tag": "feral",
96
+ "bundle": "body_type_presence",
97
+ "needs_glossary": "1",
98
+ "support_pos": "1",
99
+ "support_neg": "9",
100
+ "tp_explicit": "1",
101
+ "fp_explicit": "0",
102
+ "fn_explicit": "0",
103
+ "precision_explicit": "1.000000",
104
+ "recall_explicit": "1.000000",
105
+ "f1_explicit": "1.000000",
106
+ "tp_strong": "1",
107
+ "fp_strong": "0",
108
+ "fn_strong": "0",
109
+ "precision_strong": "1.000000",
110
+ "recall_strong": "1.000000",
111
+ "f1_strong": "1.000000"
112
+ },
113
+ {
114
+ "tag": "<3",
115
+ "bundle": "text_symbols",
116
+ "needs_glossary": "1",
117
+ "support_pos": "1",
118
+ "support_neg": "9",
119
+ "tp_explicit": "1",
120
+ "fp_explicit": "0",
121
+ "fn_explicit": "0",
122
+ "precision_explicit": "1.000000",
123
+ "recall_explicit": "1.000000",
124
+ "f1_explicit": "1.000000",
125
+ "tp_strong": "1",
126
+ "fp_strong": "0",
127
+ "fn_strong": "0",
128
+ "precision_strong": "1.000000",
129
+ "recall_strong": "1.000000",
130
+ "f1_strong": "1.000000"
131
+ },
132
+ {
133
+ "tag": "clothing",
134
+ "bundle": "clothing_state",
135
+ "needs_glossary": "0",
136
+ "support_pos": "8",
137
+ "support_neg": "2",
138
+ "tp_explicit": "8",
139
+ "fp_explicit": "1",
140
+ "fn_explicit": "0",
141
+ "precision_explicit": "0.888889",
142
+ "recall_explicit": "1.000000",
143
+ "f1_explicit": "0.941176",
144
+ "tp_strong": "8",
145
+ "fp_strong": "1",
146
+ "fn_strong": "0",
147
+ "precision_strong": "0.888889",
148
+ "recall_strong": "1.000000",
149
+ "f1_strong": "0.941176"
150
+ },
151
+ {
152
+ "tag": "anthro",
153
+ "bundle": "body_type_presence",
154
+ "needs_glossary": "1",
155
+ "support_pos": "8",
156
+ "support_neg": "2",
157
+ "tp_explicit": "8",
158
+ "fp_explicit": "2",
159
+ "fn_explicit": "0",
160
+ "precision_explicit": "0.800000",
161
+ "recall_explicit": "1.000000",
162
+ "f1_explicit": "0.888889",
163
+ "tp_strong": "8",
164
+ "fp_strong": "2",
165
+ "fn_strong": "0",
166
+ "precision_strong": "0.800000",
167
+ "recall_strong": "1.000000",
168
+ "f1_strong": "0.888889"
169
+ },
170
+ {
171
+ "tag": "bear",
172
+ "bundle": "species_taxonomy",
173
+ "needs_glossary": "0",
174
+ "support_pos": "2",
175
+ "support_neg": "8",
176
+ "tp_explicit": "2",
177
+ "fp_explicit": "1",
178
+ "fn_explicit": "0",
179
+ "precision_explicit": "0.666667",
180
+ "recall_explicit": "1.000000",
181
+ "f1_explicit": "0.800000",
182
+ "tp_strong": "2",
183
+ "fp_strong": "1",
184
+ "fn_strong": "0",
185
+ "precision_strong": "0.666667",
186
+ "recall_strong": "1.000000",
187
+ "f1_strong": "0.800000"
188
+ },
189
+ {
190
+ "tag": "duo",
191
+ "bundle": "count_cardinality",
192
+ "needs_glossary": "1",
193
+ "support_pos": "2",
194
+ "support_neg": "8",
195
+ "tp_explicit": "2",
196
+ "fp_explicit": "1",
197
+ "fn_explicit": "0",
198
+ "precision_explicit": "0.666667",
199
+ "recall_explicit": "1.000000",
200
+ "f1_explicit": "0.800000",
201
+ "tp_strong": "2",
202
+ "fp_strong": "1",
203
+ "fn_strong": "0",
204
+ "precision_strong": "0.666667",
205
+ "recall_strong": "1.000000",
206
+ "f1_strong": "0.800000"
207
+ },
208
+ {
209
+ "tag": "solo",
210
+ "bundle": "count_cardinality",
211
+ "needs_glossary": "1",
212
+ "support_pos": "7",
213
+ "support_neg": "3",
214
+ "tp_explicit": "4",
215
+ "fp_explicit": "0",
216
+ "fn_explicit": "3",
217
+ "precision_explicit": "1.000000",
218
+ "recall_explicit": "0.571429",
219
+ "f1_explicit": "0.727273",
220
+ "tp_strong": "4",
221
+ "fp_strong": "0",
222
+ "fn_strong": "3",
223
+ "precision_strong": "1.000000",
224
+ "recall_strong": "0.571429",
225
+ "f1_strong": "0.727273"
226
+ },
227
+ {
228
+ "tag": "clothed",
229
+ "bundle": "clothing_state",
230
+ "needs_glossary": "0",
231
+ "support_pos": "5",
232
+ "support_neg": "5",
233
+ "tp_explicit": "5",
234
+ "fp_explicit": "4",
235
+ "fn_explicit": "0",
236
+ "precision_explicit": "0.555556",
237
+ "recall_explicit": "1.000000",
238
+ "f1_explicit": "0.714286",
239
+ "tp_strong": "5",
240
+ "fp_strong": "4",
241
+ "fn_strong": "0",
242
+ "precision_strong": "0.555556",
243
+ "recall_strong": "1.000000",
244
+ "f1_strong": "0.714286"
245
+ },
246
+ {
247
+ "tag": "bird",
248
+ "bundle": "species_taxonomy",
249
+ "needs_glossary": "0",
250
+ "support_pos": "2",
251
+ "support_neg": "8",
252
+ "tp_explicit": "1",
253
+ "fp_explicit": "0",
254
+ "fn_explicit": "1",
255
+ "precision_explicit": "1.000000",
256
+ "recall_explicit": "0.500000",
257
+ "f1_explicit": "0.666667",
258
+ "tp_strong": "1",
259
+ "fp_strong": "0",
260
+ "fn_strong": "1",
261
+ "precision_strong": "1.000000",
262
+ "recall_strong": "0.500000",
263
+ "f1_strong": "0.666667"
264
+ },
265
+ {
266
+ "tag": "leporid",
267
+ "bundle": "species_taxonomy",
268
+ "needs_glossary": "1",
269
+ "support_pos": "2",
270
+ "support_neg": "8",
271
+ "tp_explicit": "1",
272
+ "fp_explicit": "0",
273
+ "fn_explicit": "1",
274
+ "precision_explicit": "1.000000",
275
+ "recall_explicit": "0.500000",
276
+ "f1_explicit": "0.666667",
277
+ "tp_strong": "1",
278
+ "fp_strong": "0",
279
+ "fn_strong": "1",
280
+ "precision_strong": "1.000000",
281
+ "recall_strong": "0.500000",
282
+ "f1_strong": "0.666667"
283
+ },
284
+ {
285
+ "tag": "felid",
286
+ "bundle": "species_taxonomy",
287
+ "needs_glossary": "1",
288
+ "support_pos": "2",
289
+ "support_neg": "8",
290
+ "tp_explicit": "1",
291
+ "fp_explicit": "0",
292
+ "fn_explicit": "1",
293
+ "precision_explicit": "1.000000",
294
+ "recall_explicit": "0.500000",
295
+ "f1_explicit": "0.666667",
296
+ "tp_strong": "1",
297
+ "fp_strong": "0",
298
+ "fn_strong": "1",
299
+ "precision_strong": "1.000000",
300
+ "recall_strong": "0.500000",
301
+ "f1_strong": "0.666667"
302
+ },
303
+ {
304
+ "tag": "canis",
305
+ "bundle": "species_taxonomy",
306
+ "needs_glossary": "1",
307
+ "support_pos": "1",
308
+ "support_neg": "9",
309
+ "tp_explicit": "1",
310
+ "fp_explicit": "1",
311
+ "fn_explicit": "0",
312
+ "precision_explicit": "0.500000",
313
+ "recall_explicit": "1.000000",
314
+ "f1_explicit": "0.666667",
315
+ "tp_strong": "1",
316
+ "fp_strong": "1",
317
+ "fn_strong": "0",
318
+ "precision_strong": "0.500000",
319
+ "recall_strong": "1.000000",
320
+ "f1_strong": "0.666667"
321
+ },
322
+ {
323
+ "tag": "simple_background",
324
+ "bundle": "scene_pose",
325
+ "needs_glossary": "0",
326
+ "support_pos": "3",
327
+ "support_neg": "7",
328
+ "tp_explicit": "3",
329
+ "fp_explicit": "4",
330
+ "fn_explicit": "0",
331
+ "precision_explicit": "0.428571",
332
+ "recall_explicit": "1.000000",
333
+ "f1_explicit": "0.600000",
334
+ "tp_strong": "3",
335
+ "fp_strong": "4",
336
+ "fn_strong": "0",
337
+ "precision_strong": "0.428571",
338
+ "recall_strong": "1.000000",
339
+ "f1_strong": "0.600000"
340
+ },
341
+ {
342
+ "tag": "canid",
343
+ "bundle": "species_taxonomy",
344
+ "needs_glossary": "1",
345
+ "support_pos": "3",
346
+ "support_neg": "7",
347
+ "tp_explicit": "2",
348
+ "fp_explicit": "2",
349
+ "fn_explicit": "1",
350
+ "precision_explicit": "0.500000",
351
+ "recall_explicit": "0.666667",
352
+ "f1_explicit": "0.571429",
353
+ "tp_strong": "2",
354
+ "fp_strong": "2",
355
+ "fn_strong": "1",
356
+ "precision_strong": "0.500000",
357
+ "recall_strong": "0.666667",
358
+ "f1_strong": "0.571429"
359
+ },
360
+ {
361
+ "tag": "looking_at_viewer",
362
+ "bundle": "gaze_expression",
363
+ "needs_glossary": "0",
364
+ "support_pos": "3",
365
+ "support_neg": "7",
366
+ "tp_explicit": "1",
367
+ "fp_explicit": "0",
368
+ "fn_explicit": "2",
369
+ "precision_explicit": "1.000000",
370
+ "recall_explicit": "0.333333",
371
+ "f1_explicit": "0.500000",
372
+ "tp_strong": "1",
373
+ "fp_strong": "0",
374
+ "fn_strong": "2",
375
+ "precision_strong": "1.000000",
376
+ "recall_strong": "0.333333",
377
+ "f1_strong": "0.500000"
378
+ },
379
+ {
380
+ "tag": "standing",
381
+ "bundle": "scene_pose",
382
+ "needs_glossary": "0",
383
+ "support_pos": "2",
384
+ "support_neg": "8",
385
+ "tp_explicit": "2",
386
+ "fp_explicit": "4",
387
+ "fn_explicit": "0",
388
+ "precision_explicit": "0.333333",
389
+ "recall_explicit": "1.000000",
390
+ "f1_explicit": "0.500000",
391
+ "tp_strong": "2",
392
+ "fp_strong": "4",
393
+ "fn_strong": "0",
394
+ "precision_strong": "0.333333",
395
+ "recall_strong": "1.000000",
396
+ "f1_strong": "0.500000"
397
+ },
398
+ {
399
+ "tag": "biped",
400
+ "bundle": "body_type_presence",
401
+ "needs_glossary": "0",
402
+ "support_pos": "3",
403
+ "support_neg": "7",
404
+ "tp_explicit": "2",
405
+ "fp_explicit": "4",
406
+ "fn_explicit": "1",
407
+ "precision_explicit": "0.333333",
408
+ "recall_explicit": "0.666667",
409
+ "f1_explicit": "0.444444",
410
+ "tp_strong": "2",
411
+ "fp_strong": "4",
412
+ "fn_strong": "1",
413
+ "precision_strong": "0.333333",
414
+ "recall_strong": "0.666667",
415
+ "f1_strong": "0.444444"
416
+ }
417
+ ],
418
+ "outputs": {
419
+ "csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_reliability_sanity10.csv",
420
+ "json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_reliability_sanity10.json"
421
+ }
422
+ }
data/analysis/remaining_tag_centroid_simulation.json ADDED
@@ -0,0 +1,2572 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "min_count": 200,
3
+ "n_unique_tags_considered": 3068,
4
+ "n_uncovered_before_clustering": 1892,
5
+ "baseline_unique_coverage_pct": 38.33,
6
+ "baseline_top15_greedy_pct": 68.53,
7
+ "target_centroid_groups": [
8
+ "cat:anatomy_features",
9
+ "cat:body_build",
10
+ "cat:character_count",
11
+ "cat:color_markings",
12
+ "cat:relationship_pairing",
13
+ "cat:species_specific"
14
+ ],
15
+ "scenarios": [
16
+ {
17
+ "scenario": "very_strict",
18
+ "sim_min": 0.82,
19
+ "margin_min": 0.08,
20
+ "n_added_tags": 11,
21
+ "unique_coverage_pct": 40.19,
22
+ "top15_greedy_coverage_pct": 68.53,
23
+ "delta_unique_pct": 1.86,
24
+ "delta_top15_pct": 0.0,
25
+ "avg_sim": 0.8763,
26
+ "avg_margin": 0.1008,
27
+ "added_by_group": {
28
+ "cat:species_specific": 1,
29
+ "cat:body_build": 8,
30
+ "cat:relationship_pairing": 2
31
+ },
32
+ "sample_assignments_high_conf": [
33
+ {
34
+ "tag": "canis",
35
+ "fluffyrock_count": 587772,
36
+ "sample_occurrences": 159,
37
+ "best_group": "cat:species_specific",
38
+ "best_sim": 0.864142,
39
+ "second_group": "cat:character_count",
40
+ "second_sim": 0.779288,
41
+ "margin": 0.084854
42
+ },
43
+ {
44
+ "tag": "thick_thighs",
45
+ "fluffyrock_count": 339009,
46
+ "sample_occurrences": 25,
47
+ "best_group": "cat:body_build",
48
+ "best_sim": 0.915767,
49
+ "second_group": "cat:character_count",
50
+ "second_sim": 0.79776,
51
+ "margin": 0.118007
52
+ },
53
+ {
54
+ "tag": "interspecies",
55
+ "fluffyrock_count": 261649,
56
+ "sample_occurrences": 4,
57
+ "best_group": "cat:relationship_pairing",
58
+ "best_sim": 0.883739,
59
+ "second_group": "cat:character_count",
60
+ "second_sim": 0.799747,
61
+ "margin": 0.083992
62
+ },
63
+ {
64
+ "tag": "huge_breasts",
65
+ "fluffyrock_count": 261003,
66
+ "sample_occurrences": 13,
67
+ "best_group": "cat:body_build",
68
+ "best_sim": 0.833355,
69
+ "second_group": "cat:character_count",
70
+ "second_sim": 0.743262,
71
+ "margin": 0.090093
72
+ },
73
+ {
74
+ "tag": "wide_hips",
75
+ "fluffyrock_count": 235877,
76
+ "sample_occurrences": 30,
77
+ "best_group": "cat:body_build",
78
+ "best_sim": 0.876513,
79
+ "second_group": "cat:character_count",
80
+ "second_sim": 0.778367,
81
+ "margin": 0.098146
82
+ },
83
+ {
84
+ "tag": "slightly_chubby",
85
+ "fluffyrock_count": 118826,
86
+ "sample_occurrences": 21,
87
+ "best_group": "cat:body_build",
88
+ "best_sim": 0.959771,
89
+ "second_group": "cat:species_specific",
90
+ "second_sim": 0.837851,
91
+ "margin": 0.12192
92
+ },
93
+ {
94
+ "tag": "curvy_figure",
95
+ "fluffyrock_count": 106779,
96
+ "sample_occurrences": 7,
97
+ "best_group": "cat:body_build",
98
+ "best_sim": 0.833488,
99
+ "second_group": "cat:character_count",
100
+ "second_sim": 0.746921,
101
+ "margin": 0.086567
102
+ },
103
+ {
104
+ "tag": "huge_butt",
105
+ "fluffyrock_count": 87380,
106
+ "sample_occurrences": 2,
107
+ "best_group": "cat:body_build",
108
+ "best_sim": 0.85698,
109
+ "second_group": "cat:character_count",
110
+ "second_sim": 0.707513,
111
+ "margin": 0.149467
112
+ },
113
+ {
114
+ "tag": "larger_male",
115
+ "fluffyrock_count": 81280,
116
+ "sample_occurrences": 5,
117
+ "best_group": "cat:relationship_pairing",
118
+ "best_sim": 0.88921,
119
+ "second_group": "cat:character_count",
120
+ "second_sim": 0.806639,
121
+ "margin": 0.08257
122
+ },
123
+ {
124
+ "tag": "voluptuous",
125
+ "fluffyrock_count": 80599,
126
+ "sample_occurrences": 5,
127
+ "best_group": "cat:body_build",
128
+ "best_sim": 0.821224,
129
+ "second_group": "cat:character_count",
130
+ "second_sim": 0.712912,
131
+ "margin": 0.108312
132
+ },
133
+ {
134
+ "tag": "thong",
135
+ "fluffyrock_count": 30393,
136
+ "sample_occurrences": 1,
137
+ "best_group": "cat:body_build",
138
+ "best_sim": 0.905583,
139
+ "second_group": "cat:species_specific",
140
+ "second_sim": 0.820717,
141
+ "margin": 0.084866
142
+ }
143
+ ],
144
+ "sample_assignments_borderline": [
145
+ {
146
+ "tag": "bodily_fluids",
147
+ "fluffyrock_count": 1225999,
148
+ "sample_occurrences": 49,
149
+ "best_group": "cat:character_count",
150
+ "best_sim": 0.964808,
151
+ "second_group": "cat:relationship_pairing",
152
+ "second_sim": 0.953455,
153
+ "margin": 0.011353
154
+ },
155
+ {
156
+ "tag": "nipples",
157
+ "fluffyrock_count": 1079208,
158
+ "sample_occurrences": 17,
159
+ "best_group": "cat:character_count",
160
+ "best_sim": 0.93445,
161
+ "second_group": "cat:body_build",
162
+ "second_sim": 0.915792,
163
+ "margin": 0.018658
164
+ },
165
+ {
166
+ "tag": "feet",
167
+ "fluffyrock_count": 448730,
168
+ "sample_occurrences": 113,
169
+ "best_group": "cat:anatomy_features",
170
+ "best_sim": 0.907272,
171
+ "second_group": "cat:species_specific",
172
+ "second_sim": 0.90161,
173
+ "margin": 0.005662
174
+ },
175
+ {
176
+ "tag": "tuft",
177
+ "fluffyrock_count": 335534,
178
+ "sample_occurrences": 105,
179
+ "best_group": "cat:anatomy_features",
180
+ "best_sim": 0.938366,
181
+ "second_group": "cat:color_markings",
182
+ "second_sim": 0.929999,
183
+ "margin": 0.008367
184
+ },
185
+ {
186
+ "tag": "areola",
187
+ "fluffyrock_count": 320191,
188
+ "sample_occurrences": 1,
189
+ "best_group": "cat:character_count",
190
+ "best_sim": 0.86106,
191
+ "second_group": "cat:body_build",
192
+ "second_sim": 0.844252,
193
+ "margin": 0.016808
194
+ },
195
+ {
196
+ "tag": "dragon",
197
+ "fluffyrock_count": 309478,
198
+ "sample_occurrences": 84,
199
+ "best_group": "cat:anatomy_features",
200
+ "best_sim": 0.766118,
201
+ "second_group": "cat:character_count",
202
+ "second_sim": 0.750724,
203
+ "margin": 0.015395
204
+ },
205
+ {
206
+ "tag": "green_eyes",
207
+ "fluffyrock_count": 287579,
208
+ "sample_occurrences": 78,
209
+ "best_group": "cat:character_count",
210
+ "best_sim": 0.914274,
211
+ "second_group": "cat:species_specific",
212
+ "second_sim": 0.914113,
213
+ "margin": 0.000161
214
+ },
215
+ {
216
+ "tag": "black_nose",
217
+ "fluffyrock_count": 216679,
218
+ "sample_occurrences": 80,
219
+ "best_group": "cat:color_markings",
220
+ "best_sim": 0.89887,
221
+ "second_group": "cat:anatomy_features",
222
+ "second_sim": 0.887671,
223
+ "margin": 0.011199
224
+ },
225
+ {
226
+ "tag": "eyelashes",
227
+ "fluffyrock_count": 193540,
228
+ "sample_occurrences": 63,
229
+ "best_group": "cat:character_count",
230
+ "best_sim": 0.847013,
231
+ "second_group": "cat:anatomy_features",
232
+ "second_sim": 0.844852,
233
+ "margin": 0.002161
234
+ },
235
+ {
236
+ "tag": "saliva",
237
+ "fluffyrock_count": 187352,
238
+ "sample_occurrences": 10,
239
+ "best_group": "cat:relationship_pairing",
240
+ "best_sim": 0.930058,
241
+ "second_group": "cat:character_count",
242
+ "second_sim": 0.925649,
243
+ "margin": 0.004409
244
+ },
245
+ {
246
+ "tag": "black_hair",
247
+ "fluffyrock_count": 180936,
248
+ "sample_occurrences": 45,
249
+ "best_group": "cat:species_specific",
250
+ "best_sim": 0.892875,
251
+ "second_group": "cat:character_count",
252
+ "second_sim": 0.886585,
253
+ "margin": 0.006291
254
+ },
255
+ {
256
+ "tag": "eyebrows",
257
+ "fluffyrock_count": 170102,
258
+ "sample_occurrences": 50,
259
+ "best_group": "cat:anatomy_features",
260
+ "best_sim": 0.944516,
261
+ "second_group": "cat:color_markings",
262
+ "second_sim": 0.942686,
263
+ "margin": 0.00183
264
+ },
265
+ {
266
+ "tag": "vein",
267
+ "fluffyrock_count": 168934,
268
+ "sample_occurrences": 5,
269
+ "best_group": "cat:body_build",
270
+ "best_sim": 0.839476,
271
+ "second_group": "cat:character_count",
272
+ "second_sim": 0.835717,
273
+ "margin": 0.003759
274
+ },
275
+ {
276
+ "tag": "yellow_eyes",
277
+ "fluffyrock_count": 168780,
278
+ "sample_occurrences": 45,
279
+ "best_group": "cat:anatomy_features",
280
+ "best_sim": 0.896706,
281
+ "second_group": "cat:character_count",
282
+ "second_sim": 0.888449,
283
+ "margin": 0.008257
284
+ },
285
+ {
286
+ "tag": "panties",
287
+ "fluffyrock_count": 142486,
288
+ "sample_occurrences": 13,
289
+ "best_group": "cat:species_specific",
290
+ "best_sim": 0.718523,
291
+ "second_group": "cat:character_count",
292
+ "second_sim": 0.711868,
293
+ "margin": 0.006655
294
+ },
295
+ {
296
+ "tag": "white_hair",
297
+ "fluffyrock_count": 139416,
298
+ "sample_occurrences": 37,
299
+ "best_group": "cat:species_specific",
300
+ "best_sim": 0.849061,
301
+ "second_group": "cat:anatomy_features",
302
+ "second_sim": 0.844397,
303
+ "margin": 0.004665
304
+ },
305
+ {
306
+ "tag": "pose",
307
+ "fluffyrock_count": 130171,
308
+ "sample_occurrences": 25,
309
+ "best_group": "cat:species_specific",
310
+ "best_sim": 0.869967,
311
+ "second_group": "cat:anatomy_features",
312
+ "second_sim": 0.865188,
313
+ "margin": 0.004779
314
+ },
315
+ {
316
+ "tag": "cleavage",
317
+ "fluffyrock_count": 116981,
318
+ "sample_occurrences": 41,
319
+ "best_group": "cat:species_specific",
320
+ "best_sim": 0.750055,
321
+ "second_group": "cat:body_build",
322
+ "second_sim": 0.749197,
323
+ "margin": 0.000857
324
+ },
325
+ {
326
+ "tag": "red_hair",
327
+ "fluffyrock_count": 115770,
328
+ "sample_occurrences": 24,
329
+ "best_group": "cat:character_count",
330
+ "best_sim": 0.900872,
331
+ "second_group": "cat:species_specific",
332
+ "second_sim": 0.881749,
333
+ "margin": 0.019123
334
+ },
335
+ {
336
+ "tag": "accessory",
337
+ "fluffyrock_count": 114975,
338
+ "sample_occurrences": 29,
339
+ "best_group": "cat:character_count",
340
+ "best_sim": 0.862266,
341
+ "second_group": "cat:species_specific",
342
+ "second_sim": 0.853952,
343
+ "margin": 0.008314
344
+ },
345
+ {
346
+ "tag": "fluffy",
347
+ "fluffyrock_count": 113761,
348
+ "sample_occurrences": 35,
349
+ "best_group": "cat:anatomy_features",
350
+ "best_sim": 0.917693,
351
+ "second_group": "cat:color_markings",
352
+ "second_sim": 0.905509,
353
+ "margin": 0.012184
354
+ },
355
+ {
356
+ "tag": "thigh_highs",
357
+ "fluffyrock_count": 104261,
358
+ "sample_occurrences": 16,
359
+ "best_group": "cat:character_count",
360
+ "best_sim": 0.724052,
361
+ "second_group": "cat:color_markings",
362
+ "second_sim": 0.707402,
363
+ "margin": 0.01665
364
+ },
365
+ {
366
+ "tag": "submissive",
367
+ "fluffyrock_count": 102144,
368
+ "sample_occurrences": 1,
369
+ "best_group": "cat:relationship_pairing",
370
+ "best_sim": 0.73861,
371
+ "second_group": "cat:character_count",
372
+ "second_sim": 0.721952,
373
+ "margin": 0.016658
374
+ },
375
+ {
376
+ "tag": "stockings",
377
+ "fluffyrock_count": 96677,
378
+ "sample_occurrences": 6,
379
+ "best_group": "cat:character_count",
380
+ "best_sim": 0.799803,
381
+ "second_group": "cat:species_specific",
382
+ "second_sim": 0.784923,
383
+ "margin": 0.014881
384
+ },
385
+ {
386
+ "tag": "ring_piercing",
387
+ "fluffyrock_count": 96328,
388
+ "sample_occurrences": 21,
389
+ "best_group": "cat:body_build",
390
+ "best_sim": 0.823498,
391
+ "second_group": "cat:species_specific",
392
+ "second_sim": 0.808368,
393
+ "margin": 0.015131
394
+ }
395
+ ]
396
+ },
397
+ {
398
+ "scenario": "strict",
399
+ "sim_min": 0.78,
400
+ "margin_min": 0.06,
401
+ "n_added_tags": 38,
402
+ "unique_coverage_pct": 40.84,
403
+ "top15_greedy_coverage_pct": 68.74,
404
+ "delta_unique_pct": 2.51,
405
+ "delta_top15_pct": 0.21,
406
+ "avg_sim": 0.8472,
407
+ "avg_margin": 0.0803,
408
+ "added_by_group": {
409
+ "cat:species_specific": 16,
410
+ "cat:body_build": 10,
411
+ "cat:relationship_pairing": 6,
412
+ "cat:character_count": 1,
413
+ "cat:anatomy_features": 1,
414
+ "cat:color_markings": 4
415
+ },
416
+ "sample_assignments_high_conf": [
417
+ {
418
+ "tag": "canid",
419
+ "fluffyrock_count": 1141448,
420
+ "sample_occurrences": 304,
421
+ "best_group": "cat:species_specific",
422
+ "best_sim": 0.925909,
423
+ "second_group": "cat:character_count",
424
+ "second_sim": 0.853628,
425
+ "margin": 0.072281
426
+ },
427
+ {
428
+ "tag": "felid",
429
+ "fluffyrock_count": 598698,
430
+ "sample_occurrences": 158,
431
+ "best_group": "cat:species_specific",
432
+ "best_sim": 0.88795,
433
+ "second_group": "cat:character_count",
434
+ "second_sim": 0.819039,
435
+ "margin": 0.068911
436
+ },
437
+ {
438
+ "tag": "canis",
439
+ "fluffyrock_count": 587772,
440
+ "sample_occurrences": 159,
441
+ "best_group": "cat:species_specific",
442
+ "best_sim": 0.864142,
443
+ "second_group": "cat:character_count",
444
+ "second_sim": 0.779288,
445
+ "margin": 0.084854
446
+ },
447
+ {
448
+ "tag": "muscular",
449
+ "fluffyrock_count": 358573,
450
+ "sample_occurrences": 56,
451
+ "best_group": "cat:body_build",
452
+ "best_sim": 0.799827,
453
+ "second_group": "cat:species_specific",
454
+ "second_sim": 0.692788,
455
+ "margin": 0.10704
456
+ },
457
+ {
458
+ "tag": "thick_thighs",
459
+ "fluffyrock_count": 339009,
460
+ "sample_occurrences": 25,
461
+ "best_group": "cat:body_build",
462
+ "best_sim": 0.915767,
463
+ "second_group": "cat:character_count",
464
+ "second_sim": 0.79776,
465
+ "margin": 0.118007
466
+ },
467
+ {
468
+ "tag": "fox",
469
+ "fluffyrock_count": 337215,
470
+ "sample_occurrences": 105,
471
+ "best_group": "cat:species_specific",
472
+ "best_sim": 0.796958,
473
+ "second_group": "cat:character_count",
474
+ "second_sim": 0.693828,
475
+ "margin": 0.103131
476
+ },
477
+ {
478
+ "tag": "wolf",
479
+ "fluffyrock_count": 273053,
480
+ "sample_occurrences": 79,
481
+ "best_group": "cat:species_specific",
482
+ "best_sim": 0.848018,
483
+ "second_group": "cat:character_count",
484
+ "second_sim": 0.769764,
485
+ "margin": 0.078254
486
+ },
487
+ {
488
+ "tag": "interspecies",
489
+ "fluffyrock_count": 261649,
490
+ "sample_occurrences": 4,
491
+ "best_group": "cat:relationship_pairing",
492
+ "best_sim": 0.883739,
493
+ "second_group": "cat:character_count",
494
+ "second_sim": 0.799747,
495
+ "margin": 0.083992
496
+ },
497
+ {
498
+ "tag": "huge_breasts",
499
+ "fluffyrock_count": 261003,
500
+ "sample_occurrences": 13,
501
+ "best_group": "cat:body_build",
502
+ "best_sim": 0.833355,
503
+ "second_group": "cat:character_count",
504
+ "second_sim": 0.743262,
505
+ "margin": 0.090093
506
+ },
507
+ {
508
+ "tag": "felis",
509
+ "fluffyrock_count": 241656,
510
+ "sample_occurrences": 64,
511
+ "best_group": "cat:species_specific",
512
+ "best_sim": 0.822033,
513
+ "second_group": "cat:character_count",
514
+ "second_sim": 0.758581,
515
+ "margin": 0.063452
516
+ },
517
+ {
518
+ "tag": "domestic_cat",
519
+ "fluffyrock_count": 236967,
520
+ "sample_occurrences": 64,
521
+ "best_group": "cat:species_specific",
522
+ "best_sim": 0.81899,
523
+ "second_group": "cat:character_count",
524
+ "second_sim": 0.756082,
525
+ "margin": 0.062908
526
+ },
527
+ {
528
+ "tag": "wide_hips",
529
+ "fluffyrock_count": 235877,
530
+ "sample_occurrences": 30,
531
+ "best_group": "cat:body_build",
532
+ "best_sim": 0.876513,
533
+ "second_group": "cat:character_count",
534
+ "second_sim": 0.778367,
535
+ "margin": 0.098146
536
+ },
537
+ {
538
+ "tag": "slightly_chubby",
539
+ "fluffyrock_count": 118826,
540
+ "sample_occurrences": 21,
541
+ "best_group": "cat:body_build",
542
+ "best_sim": 0.959771,
543
+ "second_group": "cat:species_specific",
544
+ "second_sim": 0.837851,
545
+ "margin": 0.12192
546
+ },
547
+ {
548
+ "tag": "curvy_figure",
549
+ "fluffyrock_count": 106779,
550
+ "sample_occurrences": 7,
551
+ "best_group": "cat:body_build",
552
+ "best_sim": 0.833488,
553
+ "second_group": "cat:character_count",
554
+ "second_sim": 0.746921,
555
+ "margin": 0.086567
556
+ },
557
+ {
558
+ "tag": "huge_butt",
559
+ "fluffyrock_count": 87380,
560
+ "sample_occurrences": 2,
561
+ "best_group": "cat:body_build",
562
+ "best_sim": 0.85698,
563
+ "second_group": "cat:character_count",
564
+ "second_sim": 0.707513,
565
+ "margin": 0.149467
566
+ },
567
+ {
568
+ "tag": "larger_male",
569
+ "fluffyrock_count": 81280,
570
+ "sample_occurrences": 5,
571
+ "best_group": "cat:relationship_pairing",
572
+ "best_sim": 0.88921,
573
+ "second_group": "cat:character_count",
574
+ "second_sim": 0.806639,
575
+ "margin": 0.08257
576
+ },
577
+ {
578
+ "tag": "voluptuous",
579
+ "fluffyrock_count": 80599,
580
+ "sample_occurrences": 5,
581
+ "best_group": "cat:body_build",
582
+ "best_sim": 0.821224,
583
+ "second_group": "cat:character_count",
584
+ "second_sim": 0.712912,
585
+ "margin": 0.108312
586
+ },
587
+ {
588
+ "tag": "human_on_anthro",
589
+ "fluffyrock_count": 68297,
590
+ "sample_occurrences": 2,
591
+ "best_group": "cat:relationship_pairing",
592
+ "best_sim": 0.818607,
593
+ "second_group": "cat:character_count",
594
+ "second_sim": 0.749736,
595
+ "margin": 0.068871
596
+ },
597
+ {
598
+ "tag": "green_hair",
599
+ "fluffyrock_count": 53768,
600
+ "sample_occurrences": 9,
601
+ "best_group": "cat:character_count",
602
+ "best_sim": 0.860165,
603
+ "second_group": "cat:species_specific",
604
+ "second_sim": 0.783589,
605
+ "margin": 0.076577
606
+ },
607
+ {
608
+ "tag": "first_person_view",
609
+ "fluffyrock_count": 50405,
610
+ "sample_occurrences": 1,
611
+ "best_group": "cat:relationship_pairing",
612
+ "best_sim": 0.867916,
613
+ "second_group": "cat:character_count",
614
+ "second_sim": 0.807474,
615
+ "margin": 0.060442
616
+ },
617
+ {
618
+ "tag": "smaller_female",
619
+ "fluffyrock_count": 47686,
620
+ "sample_occurrences": 4,
621
+ "best_group": "cat:relationship_pairing",
622
+ "best_sim": 0.901325,
623
+ "second_group": "cat:character_count",
624
+ "second_sim": 0.829112,
625
+ "margin": 0.072213
626
+ },
627
+ {
628
+ "tag": "nipple_outline",
629
+ "fluffyrock_count": 46144,
630
+ "sample_occurrences": 2,
631
+ "best_group": "cat:body_build",
632
+ "best_sim": 0.814549,
633
+ "second_group": "cat:character_count",
634
+ "second_sim": 0.746128,
635
+ "margin": 0.068421
636
+ },
637
+ {
638
+ "tag": "procyonid",
639
+ "fluffyrock_count": 38706,
640
+ "sample_occurrences": 11,
641
+ "best_group": "cat:species_specific",
642
+ "best_sim": 0.808664,
643
+ "second_group": "cat:character_count",
644
+ "second_sim": 0.74563,
645
+ "margin": 0.063034
646
+ },
647
+ {
648
+ "tag": "raccoon",
649
+ "fluffyrock_count": 36911,
650
+ "sample_occurrences": 11,
651
+ "best_group": "cat:species_specific",
652
+ "best_sim": 0.804551,
653
+ "second_group": "cat:character_count",
654
+ "second_sim": 0.743717,
655
+ "margin": 0.060833
656
+ },
657
+ {
658
+ "tag": "thong",
659
+ "fluffyrock_count": 30393,
660
+ "sample_occurrences": 1,
661
+ "best_group": "cat:body_build",
662
+ "best_sim": 0.905583,
663
+ "second_group": "cat:species_specific",
664
+ "second_sim": 0.820717,
665
+ "margin": 0.084866
666
+ }
667
+ ],
668
+ "sample_assignments_borderline": [
669
+ {
670
+ "tag": "bodily_fluids",
671
+ "fluffyrock_count": 1225999,
672
+ "sample_occurrences": 49,
673
+ "best_group": "cat:character_count",
674
+ "best_sim": 0.964808,
675
+ "second_group": "cat:relationship_pairing",
676
+ "second_sim": 0.953455,
677
+ "margin": 0.011353
678
+ },
679
+ {
680
+ "tag": "nipples",
681
+ "fluffyrock_count": 1079208,
682
+ "sample_occurrences": 17,
683
+ "best_group": "cat:character_count",
684
+ "best_sim": 0.93445,
685
+ "second_group": "cat:body_build",
686
+ "second_sim": 0.915792,
687
+ "margin": 0.018658
688
+ },
689
+ {
690
+ "tag": "feet",
691
+ "fluffyrock_count": 448730,
692
+ "sample_occurrences": 113,
693
+ "best_group": "cat:anatomy_features",
694
+ "best_sim": 0.907272,
695
+ "second_group": "cat:species_specific",
696
+ "second_sim": 0.90161,
697
+ "margin": 0.005662
698
+ },
699
+ {
700
+ "tag": "tuft",
701
+ "fluffyrock_count": 335534,
702
+ "sample_occurrences": 105,
703
+ "best_group": "cat:anatomy_features",
704
+ "best_sim": 0.938366,
705
+ "second_group": "cat:color_markings",
706
+ "second_sim": 0.929999,
707
+ "margin": 0.008367
708
+ },
709
+ {
710
+ "tag": "areola",
711
+ "fluffyrock_count": 320191,
712
+ "sample_occurrences": 1,
713
+ "best_group": "cat:character_count",
714
+ "best_sim": 0.86106,
715
+ "second_group": "cat:body_build",
716
+ "second_sim": 0.844252,
717
+ "margin": 0.016808
718
+ },
719
+ {
720
+ "tag": "dragon",
721
+ "fluffyrock_count": 309478,
722
+ "sample_occurrences": 84,
723
+ "best_group": "cat:anatomy_features",
724
+ "best_sim": 0.766118,
725
+ "second_group": "cat:character_count",
726
+ "second_sim": 0.750724,
727
+ "margin": 0.015395
728
+ },
729
+ {
730
+ "tag": "green_eyes",
731
+ "fluffyrock_count": 287579,
732
+ "sample_occurrences": 78,
733
+ "best_group": "cat:character_count",
734
+ "best_sim": 0.914274,
735
+ "second_group": "cat:species_specific",
736
+ "second_sim": 0.914113,
737
+ "margin": 0.000161
738
+ },
739
+ {
740
+ "tag": "black_nose",
741
+ "fluffyrock_count": 216679,
742
+ "sample_occurrences": 80,
743
+ "best_group": "cat:color_markings",
744
+ "best_sim": 0.89887,
745
+ "second_group": "cat:anatomy_features",
746
+ "second_sim": 0.887671,
747
+ "margin": 0.011199
748
+ },
749
+ {
750
+ "tag": "eyelashes",
751
+ "fluffyrock_count": 193540,
752
+ "sample_occurrences": 63,
753
+ "best_group": "cat:character_count",
754
+ "best_sim": 0.847013,
755
+ "second_group": "cat:anatomy_features",
756
+ "second_sim": 0.844852,
757
+ "margin": 0.002161
758
+ },
759
+ {
760
+ "tag": "saliva",
761
+ "fluffyrock_count": 187352,
762
+ "sample_occurrences": 10,
763
+ "best_group": "cat:relationship_pairing",
764
+ "best_sim": 0.930058,
765
+ "second_group": "cat:character_count",
766
+ "second_sim": 0.925649,
767
+ "margin": 0.004409
768
+ },
769
+ {
770
+ "tag": "black_hair",
771
+ "fluffyrock_count": 180936,
772
+ "sample_occurrences": 45,
773
+ "best_group": "cat:species_specific",
774
+ "best_sim": 0.892875,
775
+ "second_group": "cat:character_count",
776
+ "second_sim": 0.886585,
777
+ "margin": 0.006291
778
+ },
779
+ {
780
+ "tag": "eyebrows",
781
+ "fluffyrock_count": 170102,
782
+ "sample_occurrences": 50,
783
+ "best_group": "cat:anatomy_features",
784
+ "best_sim": 0.944516,
785
+ "second_group": "cat:color_markings",
786
+ "second_sim": 0.942686,
787
+ "margin": 0.00183
788
+ },
789
+ {
790
+ "tag": "vein",
791
+ "fluffyrock_count": 168934,
792
+ "sample_occurrences": 5,
793
+ "best_group": "cat:body_build",
794
+ "best_sim": 0.839476,
795
+ "second_group": "cat:character_count",
796
+ "second_sim": 0.835717,
797
+ "margin": 0.003759
798
+ },
799
+ {
800
+ "tag": "yellow_eyes",
801
+ "fluffyrock_count": 168780,
802
+ "sample_occurrences": 45,
803
+ "best_group": "cat:anatomy_features",
804
+ "best_sim": 0.896706,
805
+ "second_group": "cat:character_count",
806
+ "second_sim": 0.888449,
807
+ "margin": 0.008257
808
+ },
809
+ {
810
+ "tag": "panties",
811
+ "fluffyrock_count": 142486,
812
+ "sample_occurrences": 13,
813
+ "best_group": "cat:species_specific",
814
+ "best_sim": 0.718523,
815
+ "second_group": "cat:character_count",
816
+ "second_sim": 0.711868,
817
+ "margin": 0.006655
818
+ },
819
+ {
820
+ "tag": "white_hair",
821
+ "fluffyrock_count": 139416,
822
+ "sample_occurrences": 37,
823
+ "best_group": "cat:species_specific",
824
+ "best_sim": 0.849061,
825
+ "second_group": "cat:anatomy_features",
826
+ "second_sim": 0.844397,
827
+ "margin": 0.004665
828
+ },
829
+ {
830
+ "tag": "pose",
831
+ "fluffyrock_count": 130171,
832
+ "sample_occurrences": 25,
833
+ "best_group": "cat:species_specific",
834
+ "best_sim": 0.869967,
835
+ "second_group": "cat:anatomy_features",
836
+ "second_sim": 0.865188,
837
+ "margin": 0.004779
838
+ },
839
+ {
840
+ "tag": "cleavage",
841
+ "fluffyrock_count": 116981,
842
+ "sample_occurrences": 41,
843
+ "best_group": "cat:species_specific",
844
+ "best_sim": 0.750055,
845
+ "second_group": "cat:body_build",
846
+ "second_sim": 0.749197,
847
+ "margin": 0.000857
848
+ },
849
+ {
850
+ "tag": "red_hair",
851
+ "fluffyrock_count": 115770,
852
+ "sample_occurrences": 24,
853
+ "best_group": "cat:character_count",
854
+ "best_sim": 0.900872,
855
+ "second_group": "cat:species_specific",
856
+ "second_sim": 0.881749,
857
+ "margin": 0.019123
858
+ },
859
+ {
860
+ "tag": "accessory",
861
+ "fluffyrock_count": 114975,
862
+ "sample_occurrences": 29,
863
+ "best_group": "cat:character_count",
864
+ "best_sim": 0.862266,
865
+ "second_group": "cat:species_specific",
866
+ "second_sim": 0.853952,
867
+ "margin": 0.008314
868
+ },
869
+ {
870
+ "tag": "fluffy",
871
+ "fluffyrock_count": 113761,
872
+ "sample_occurrences": 35,
873
+ "best_group": "cat:anatomy_features",
874
+ "best_sim": 0.917693,
875
+ "second_group": "cat:color_markings",
876
+ "second_sim": 0.905509,
877
+ "margin": 0.012184
878
+ },
879
+ {
880
+ "tag": "thigh_highs",
881
+ "fluffyrock_count": 104261,
882
+ "sample_occurrences": 16,
883
+ "best_group": "cat:character_count",
884
+ "best_sim": 0.724052,
885
+ "second_group": "cat:color_markings",
886
+ "second_sim": 0.707402,
887
+ "margin": 0.01665
888
+ },
889
+ {
890
+ "tag": "submissive",
891
+ "fluffyrock_count": 102144,
892
+ "sample_occurrences": 1,
893
+ "best_group": "cat:relationship_pairing",
894
+ "best_sim": 0.73861,
895
+ "second_group": "cat:character_count",
896
+ "second_sim": 0.721952,
897
+ "margin": 0.016658
898
+ },
899
+ {
900
+ "tag": "stockings",
901
+ "fluffyrock_count": 96677,
902
+ "sample_occurrences": 6,
903
+ "best_group": "cat:character_count",
904
+ "best_sim": 0.799803,
905
+ "second_group": "cat:species_specific",
906
+ "second_sim": 0.784923,
907
+ "margin": 0.014881
908
+ },
909
+ {
910
+ "tag": "ring_piercing",
911
+ "fluffyrock_count": 96328,
912
+ "sample_occurrences": 21,
913
+ "best_group": "cat:body_build",
914
+ "best_sim": 0.823498,
915
+ "second_group": "cat:species_specific",
916
+ "second_sim": 0.808368,
917
+ "margin": 0.015131
918
+ }
919
+ ]
920
+ },
921
+ {
922
+ "scenario": "moderate",
923
+ "sim_min": 0.74,
924
+ "margin_min": 0.04,
925
+ "n_added_tags": 131,
926
+ "unique_coverage_pct": 43.58,
927
+ "top15_greedy_coverage_pct": 70.19,
928
+ "delta_unique_pct": 5.25,
929
+ "delta_top15_pct": 1.66,
930
+ "avg_sim": 0.8151,
931
+ "avg_margin": 0.0686,
932
+ "added_by_group": {
933
+ "cat:species_specific": 52,
934
+ "cat:relationship_pairing": 16,
935
+ "cat:body_build": 24,
936
+ "cat:character_count": 7,
937
+ "cat:color_markings": 20,
938
+ "cat:anatomy_features": 12
939
+ },
940
+ "sample_assignments_high_conf": [
941
+ {
942
+ "tag": "canid",
943
+ "fluffyrock_count": 1141448,
944
+ "sample_occurrences": 304,
945
+ "best_group": "cat:species_specific",
946
+ "best_sim": 0.925909,
947
+ "second_group": "cat:character_count",
948
+ "second_sim": 0.853628,
949
+ "margin": 0.072281
950
+ },
951
+ {
952
+ "tag": "felid",
953
+ "fluffyrock_count": 598698,
954
+ "sample_occurrences": 158,
955
+ "best_group": "cat:species_specific",
956
+ "best_sim": 0.88795,
957
+ "second_group": "cat:character_count",
958
+ "second_sim": 0.819039,
959
+ "margin": 0.068911
960
+ },
961
+ {
962
+ "tag": "canis",
963
+ "fluffyrock_count": 587772,
964
+ "sample_occurrences": 159,
965
+ "best_group": "cat:species_specific",
966
+ "best_sim": 0.864142,
967
+ "second_group": "cat:character_count",
968
+ "second_sim": 0.779288,
969
+ "margin": 0.084854
970
+ },
971
+ {
972
+ "tag": "male/female",
973
+ "fluffyrock_count": 551565,
974
+ "sample_occurrences": 12,
975
+ "best_group": "cat:relationship_pairing",
976
+ "best_sim": 0.958212,
977
+ "second_group": "cat:character_count",
978
+ "second_sim": 0.908632,
979
+ "margin": 0.049581
980
+ },
981
+ {
982
+ "tag": "muscular",
983
+ "fluffyrock_count": 358573,
984
+ "sample_occurrences": 56,
985
+ "best_group": "cat:body_build",
986
+ "best_sim": 0.799827,
987
+ "second_group": "cat:species_specific",
988
+ "second_sim": 0.692788,
989
+ "margin": 0.10704
990
+ },
991
+ {
992
+ "tag": "thick_thighs",
993
+ "fluffyrock_count": 339009,
994
+ "sample_occurrences": 25,
995
+ "best_group": "cat:body_build",
996
+ "best_sim": 0.915767,
997
+ "second_group": "cat:character_count",
998
+ "second_sim": 0.79776,
999
+ "margin": 0.118007
1000
+ },
1001
+ {
1002
+ "tag": "fox",
1003
+ "fluffyrock_count": 337215,
1004
+ "sample_occurrences": 105,
1005
+ "best_group": "cat:species_specific",
1006
+ "best_sim": 0.796958,
1007
+ "second_group": "cat:character_count",
1008
+ "second_sim": 0.693828,
1009
+ "margin": 0.103131
1010
+ },
1011
+ {
1012
+ "tag": "domestic_dog",
1013
+ "fluffyrock_count": 300258,
1014
+ "sample_occurrences": 77,
1015
+ "best_group": "cat:species_specific",
1016
+ "best_sim": 0.751079,
1017
+ "second_group": "cat:character_count",
1018
+ "second_sim": 0.669701,
1019
+ "margin": 0.081378
1020
+ },
1021
+ {
1022
+ "tag": "wolf",
1023
+ "fluffyrock_count": 273053,
1024
+ "sample_occurrences": 79,
1025
+ "best_group": "cat:species_specific",
1026
+ "best_sim": 0.848018,
1027
+ "second_group": "cat:character_count",
1028
+ "second_sim": 0.769764,
1029
+ "margin": 0.078254
1030
+ },
1031
+ {
1032
+ "tag": "big_butt",
1033
+ "fluffyrock_count": 262825,
1034
+ "sample_occurrences": 11,
1035
+ "best_group": "cat:body_build",
1036
+ "best_sim": 0.894649,
1037
+ "second_group": "cat:character_count",
1038
+ "second_sim": 0.844478,
1039
+ "margin": 0.050171
1040
+ },
1041
+ {
1042
+ "tag": "interspecies",
1043
+ "fluffyrock_count": 261649,
1044
+ "sample_occurrences": 4,
1045
+ "best_group": "cat:relationship_pairing",
1046
+ "best_sim": 0.883739,
1047
+ "second_group": "cat:character_count",
1048
+ "second_sim": 0.799747,
1049
+ "margin": 0.083992
1050
+ },
1051
+ {
1052
+ "tag": "huge_breasts",
1053
+ "fluffyrock_count": 261003,
1054
+ "sample_occurrences": 13,
1055
+ "best_group": "cat:body_build",
1056
+ "best_sim": 0.833355,
1057
+ "second_group": "cat:character_count",
1058
+ "second_sim": 0.743262,
1059
+ "margin": 0.090093
1060
+ },
1061
+ {
1062
+ "tag": "felis",
1063
+ "fluffyrock_count": 241656,
1064
+ "sample_occurrences": 64,
1065
+ "best_group": "cat:species_specific",
1066
+ "best_sim": 0.822033,
1067
+ "second_group": "cat:character_count",
1068
+ "second_sim": 0.758581,
1069
+ "margin": 0.063452
1070
+ },
1071
+ {
1072
+ "tag": "domestic_cat",
1073
+ "fluffyrock_count": 236967,
1074
+ "sample_occurrences": 64,
1075
+ "best_group": "cat:species_specific",
1076
+ "best_sim": 0.81899,
1077
+ "second_group": "cat:character_count",
1078
+ "second_sim": 0.756082,
1079
+ "margin": 0.062908
1080
+ },
1081
+ {
1082
+ "tag": "wide_hips",
1083
+ "fluffyrock_count": 235877,
1084
+ "sample_occurrences": 30,
1085
+ "best_group": "cat:body_build",
1086
+ "best_sim": 0.876513,
1087
+ "second_group": "cat:character_count",
1088
+ "second_sim": 0.778367,
1089
+ "margin": 0.098146
1090
+ },
1091
+ {
1092
+ "tag": "red_eyes",
1093
+ "fluffyrock_count": 213545,
1094
+ "sample_occurrences": 53,
1095
+ "best_group": "cat:character_count",
1096
+ "best_sim": 0.889482,
1097
+ "second_group": "cat:anatomy_features",
1098
+ "second_sim": 0.845381,
1099
+ "margin": 0.044101
1100
+ },
1101
+ {
1102
+ "tag": "belly",
1103
+ "fluffyrock_count": 194310,
1104
+ "sample_occurrences": 31,
1105
+ "best_group": "cat:body_build",
1106
+ "best_sim": 0.765762,
1107
+ "second_group": "cat:species_specific",
1108
+ "second_sim": 0.56733,
1109
+ "margin": 0.198432
1110
+ },
1111
+ {
1112
+ "tag": "pantherine",
1113
+ "fluffyrock_count": 181019,
1114
+ "sample_occurrences": 45,
1115
+ "best_group": "cat:species_specific",
1116
+ "best_sim": 0.770992,
1117
+ "second_group": "cat:body_build",
1118
+ "second_sim": 0.699266,
1119
+ "margin": 0.071726
1120
+ },
1121
+ {
1122
+ "tag": "abs",
1123
+ "fluffyrock_count": 171761,
1124
+ "sample_occurrences": 23,
1125
+ "best_group": "cat:body_build",
1126
+ "best_sim": 0.770712,
1127
+ "second_group": "cat:species_specific",
1128
+ "second_sim": 0.683944,
1129
+ "margin": 0.086768
1130
+ },
1131
+ {
1132
+ "tag": "muscular_anthro",
1133
+ "fluffyrock_count": 164876,
1134
+ "sample_occurrences": 30,
1135
+ "best_group": "cat:body_build",
1136
+ "best_sim": 0.754172,
1137
+ "second_group": "cat:species_specific",
1138
+ "second_sim": 0.62991,
1139
+ "margin": 0.124263
1140
+ },
1141
+ {
1142
+ "tag": "hyper",
1143
+ "fluffyrock_count": 137712,
1144
+ "sample_occurrences": 2,
1145
+ "best_group": "cat:body_build",
1146
+ "best_sim": 0.769747,
1147
+ "second_group": "cat:character_count",
1148
+ "second_sim": 0.584672,
1149
+ "margin": 0.185075
1150
+ },
1151
+ {
1152
+ "tag": "stripes",
1153
+ "fluffyrock_count": 132789,
1154
+ "sample_occurrences": 32,
1155
+ "best_group": "cat:color_markings",
1156
+ "best_sim": 0.909492,
1157
+ "second_group": "cat:anatomy_features",
1158
+ "second_sim": 0.854684,
1159
+ "margin": 0.054808
1160
+ },
1161
+ {
1162
+ "tag": "slightly_chubby",
1163
+ "fluffyrock_count": 118826,
1164
+ "sample_occurrences": 21,
1165
+ "best_group": "cat:body_build",
1166
+ "best_sim": 0.959771,
1167
+ "second_group": "cat:species_specific",
1168
+ "second_sim": 0.837851,
1169
+ "margin": 0.12192
1170
+ },
1171
+ {
1172
+ "tag": "scales",
1173
+ "fluffyrock_count": 118407,
1174
+ "sample_occurrences": 36,
1175
+ "best_group": "cat:anatomy_features",
1176
+ "best_sim": 0.744621,
1177
+ "second_group": "cat:color_markings",
1178
+ "second_sim": 0.67907,
1179
+ "margin": 0.065551
1180
+ },
1181
+ {
1182
+ "tag": "membrane_(anatomy)",
1183
+ "fluffyrock_count": 107123,
1184
+ "sample_occurrences": 34,
1185
+ "best_group": "cat:anatomy_features",
1186
+ "best_sim": 0.777309,
1187
+ "second_group": "cat:character_count",
1188
+ "second_sim": 0.719666,
1189
+ "margin": 0.057643
1190
+ }
1191
+ ],
1192
+ "sample_assignments_borderline": [
1193
+ {
1194
+ "tag": "bodily_fluids",
1195
+ "fluffyrock_count": 1225999,
1196
+ "sample_occurrences": 49,
1197
+ "best_group": "cat:character_count",
1198
+ "best_sim": 0.964808,
1199
+ "second_group": "cat:relationship_pairing",
1200
+ "second_sim": 0.953455,
1201
+ "margin": 0.011353
1202
+ },
1203
+ {
1204
+ "tag": "nipples",
1205
+ "fluffyrock_count": 1079208,
1206
+ "sample_occurrences": 17,
1207
+ "best_group": "cat:character_count",
1208
+ "best_sim": 0.93445,
1209
+ "second_group": "cat:body_build",
1210
+ "second_sim": 0.915792,
1211
+ "margin": 0.018658
1212
+ },
1213
+ {
1214
+ "tag": "feet",
1215
+ "fluffyrock_count": 448730,
1216
+ "sample_occurrences": 113,
1217
+ "best_group": "cat:anatomy_features",
1218
+ "best_sim": 0.907272,
1219
+ "second_group": "cat:species_specific",
1220
+ "second_sim": 0.90161,
1221
+ "margin": 0.005662
1222
+ },
1223
+ {
1224
+ "tag": "tuft",
1225
+ "fluffyrock_count": 335534,
1226
+ "sample_occurrences": 105,
1227
+ "best_group": "cat:anatomy_features",
1228
+ "best_sim": 0.938366,
1229
+ "second_group": "cat:color_markings",
1230
+ "second_sim": 0.929999,
1231
+ "margin": 0.008367
1232
+ },
1233
+ {
1234
+ "tag": "areola",
1235
+ "fluffyrock_count": 320191,
1236
+ "sample_occurrences": 1,
1237
+ "best_group": "cat:character_count",
1238
+ "best_sim": 0.86106,
1239
+ "second_group": "cat:body_build",
1240
+ "second_sim": 0.844252,
1241
+ "margin": 0.016808
1242
+ },
1243
+ {
1244
+ "tag": "dragon",
1245
+ "fluffyrock_count": 309478,
1246
+ "sample_occurrences": 84,
1247
+ "best_group": "cat:anatomy_features",
1248
+ "best_sim": 0.766118,
1249
+ "second_group": "cat:character_count",
1250
+ "second_sim": 0.750724,
1251
+ "margin": 0.015395
1252
+ },
1253
+ {
1254
+ "tag": "green_eyes",
1255
+ "fluffyrock_count": 287579,
1256
+ "sample_occurrences": 78,
1257
+ "best_group": "cat:character_count",
1258
+ "best_sim": 0.914274,
1259
+ "second_group": "cat:species_specific",
1260
+ "second_sim": 0.914113,
1261
+ "margin": 0.000161
1262
+ },
1263
+ {
1264
+ "tag": "black_nose",
1265
+ "fluffyrock_count": 216679,
1266
+ "sample_occurrences": 80,
1267
+ "best_group": "cat:color_markings",
1268
+ "best_sim": 0.89887,
1269
+ "second_group": "cat:anatomy_features",
1270
+ "second_sim": 0.887671,
1271
+ "margin": 0.011199
1272
+ },
1273
+ {
1274
+ "tag": "eyelashes",
1275
+ "fluffyrock_count": 193540,
1276
+ "sample_occurrences": 63,
1277
+ "best_group": "cat:character_count",
1278
+ "best_sim": 0.847013,
1279
+ "second_group": "cat:anatomy_features",
1280
+ "second_sim": 0.844852,
1281
+ "margin": 0.002161
1282
+ },
1283
+ {
1284
+ "tag": "saliva",
1285
+ "fluffyrock_count": 187352,
1286
+ "sample_occurrences": 10,
1287
+ "best_group": "cat:relationship_pairing",
1288
+ "best_sim": 0.930058,
1289
+ "second_group": "cat:character_count",
1290
+ "second_sim": 0.925649,
1291
+ "margin": 0.004409
1292
+ },
1293
+ {
1294
+ "tag": "black_hair",
1295
+ "fluffyrock_count": 180936,
1296
+ "sample_occurrences": 45,
1297
+ "best_group": "cat:species_specific",
1298
+ "best_sim": 0.892875,
1299
+ "second_group": "cat:character_count",
1300
+ "second_sim": 0.886585,
1301
+ "margin": 0.006291
1302
+ },
1303
+ {
1304
+ "tag": "eyebrows",
1305
+ "fluffyrock_count": 170102,
1306
+ "sample_occurrences": 50,
1307
+ "best_group": "cat:anatomy_features",
1308
+ "best_sim": 0.944516,
1309
+ "second_group": "cat:color_markings",
1310
+ "second_sim": 0.942686,
1311
+ "margin": 0.00183
1312
+ },
1313
+ {
1314
+ "tag": "vein",
1315
+ "fluffyrock_count": 168934,
1316
+ "sample_occurrences": 5,
1317
+ "best_group": "cat:body_build",
1318
+ "best_sim": 0.839476,
1319
+ "second_group": "cat:character_count",
1320
+ "second_sim": 0.835717,
1321
+ "margin": 0.003759
1322
+ },
1323
+ {
1324
+ "tag": "yellow_eyes",
1325
+ "fluffyrock_count": 168780,
1326
+ "sample_occurrences": 45,
1327
+ "best_group": "cat:anatomy_features",
1328
+ "best_sim": 0.896706,
1329
+ "second_group": "cat:character_count",
1330
+ "second_sim": 0.888449,
1331
+ "margin": 0.008257
1332
+ },
1333
+ {
1334
+ "tag": "panties",
1335
+ "fluffyrock_count": 142486,
1336
+ "sample_occurrences": 13,
1337
+ "best_group": "cat:species_specific",
1338
+ "best_sim": 0.718523,
1339
+ "second_group": "cat:character_count",
1340
+ "second_sim": 0.711868,
1341
+ "margin": 0.006655
1342
+ },
1343
+ {
1344
+ "tag": "white_hair",
1345
+ "fluffyrock_count": 139416,
1346
+ "sample_occurrences": 37,
1347
+ "best_group": "cat:species_specific",
1348
+ "best_sim": 0.849061,
1349
+ "second_group": "cat:anatomy_features",
1350
+ "second_sim": 0.844397,
1351
+ "margin": 0.004665
1352
+ },
1353
+ {
1354
+ "tag": "pose",
1355
+ "fluffyrock_count": 130171,
1356
+ "sample_occurrences": 25,
1357
+ "best_group": "cat:species_specific",
1358
+ "best_sim": 0.869967,
1359
+ "second_group": "cat:anatomy_features",
1360
+ "second_sim": 0.865188,
1361
+ "margin": 0.004779
1362
+ },
1363
+ {
1364
+ "tag": "cleavage",
1365
+ "fluffyrock_count": 116981,
1366
+ "sample_occurrences": 41,
1367
+ "best_group": "cat:species_specific",
1368
+ "best_sim": 0.750055,
1369
+ "second_group": "cat:body_build",
1370
+ "second_sim": 0.749197,
1371
+ "margin": 0.000857
1372
+ },
1373
+ {
1374
+ "tag": "red_hair",
1375
+ "fluffyrock_count": 115770,
1376
+ "sample_occurrences": 24,
1377
+ "best_group": "cat:character_count",
1378
+ "best_sim": 0.900872,
1379
+ "second_group": "cat:species_specific",
1380
+ "second_sim": 0.881749,
1381
+ "margin": 0.019123
1382
+ },
1383
+ {
1384
+ "tag": "accessory",
1385
+ "fluffyrock_count": 114975,
1386
+ "sample_occurrences": 29,
1387
+ "best_group": "cat:character_count",
1388
+ "best_sim": 0.862266,
1389
+ "second_group": "cat:species_specific",
1390
+ "second_sim": 0.853952,
1391
+ "margin": 0.008314
1392
+ },
1393
+ {
1394
+ "tag": "fluffy",
1395
+ "fluffyrock_count": 113761,
1396
+ "sample_occurrences": 35,
1397
+ "best_group": "cat:anatomy_features",
1398
+ "best_sim": 0.917693,
1399
+ "second_group": "cat:color_markings",
1400
+ "second_sim": 0.905509,
1401
+ "margin": 0.012184
1402
+ },
1403
+ {
1404
+ "tag": "thigh_highs",
1405
+ "fluffyrock_count": 104261,
1406
+ "sample_occurrences": 16,
1407
+ "best_group": "cat:character_count",
1408
+ "best_sim": 0.724052,
1409
+ "second_group": "cat:color_markings",
1410
+ "second_sim": 0.707402,
1411
+ "margin": 0.01665
1412
+ },
1413
+ {
1414
+ "tag": "submissive",
1415
+ "fluffyrock_count": 102144,
1416
+ "sample_occurrences": 1,
1417
+ "best_group": "cat:relationship_pairing",
1418
+ "best_sim": 0.73861,
1419
+ "second_group": "cat:character_count",
1420
+ "second_sim": 0.721952,
1421
+ "margin": 0.016658
1422
+ },
1423
+ {
1424
+ "tag": "stockings",
1425
+ "fluffyrock_count": 96677,
1426
+ "sample_occurrences": 6,
1427
+ "best_group": "cat:character_count",
1428
+ "best_sim": 0.799803,
1429
+ "second_group": "cat:species_specific",
1430
+ "second_sim": 0.784923,
1431
+ "margin": 0.014881
1432
+ },
1433
+ {
1434
+ "tag": "ring_piercing",
1435
+ "fluffyrock_count": 96328,
1436
+ "sample_occurrences": 21,
1437
+ "best_group": "cat:body_build",
1438
+ "best_sim": 0.823498,
1439
+ "second_group": "cat:species_specific",
1440
+ "second_sim": 0.808368,
1441
+ "margin": 0.015131
1442
+ }
1443
+ ]
1444
+ },
1445
+ {
1446
+ "scenario": "loose",
1447
+ "sim_min": 0.7,
1448
+ "margin_min": 0.02,
1449
+ "n_added_tags": 363,
1450
+ "unique_coverage_pct": 50.85,
1451
+ "top15_greedy_coverage_pct": 75.92,
1452
+ "delta_unique_pct": 12.52,
1453
+ "delta_top15_pct": 7.39,
1454
+ "avg_sim": 0.8096,
1455
+ "avg_margin": 0.0486,
1456
+ "added_by_group": {
1457
+ "cat:species_specific": 117,
1458
+ "cat:character_count": 41,
1459
+ "cat:relationship_pairing": 26,
1460
+ "cat:body_build": 47,
1461
+ "cat:color_markings": 59,
1462
+ "cat:anatomy_features": 73
1463
+ },
1464
+ "sample_assignments_high_conf": [
1465
+ {
1466
+ "tag": "fur",
1467
+ "fluffyrock_count": 1417091,
1468
+ "sample_occurrences": 422,
1469
+ "best_group": "cat:species_specific",
1470
+ "best_sim": 0.966513,
1471
+ "second_group": "cat:character_count",
1472
+ "second_sim": 0.937298,
1473
+ "margin": 0.029215
1474
+ },
1475
+ {
1476
+ "tag": "canid",
1477
+ "fluffyrock_count": 1141448,
1478
+ "sample_occurrences": 304,
1479
+ "best_group": "cat:species_specific",
1480
+ "best_sim": 0.925909,
1481
+ "second_group": "cat:character_count",
1482
+ "second_sim": 0.853628,
1483
+ "margin": 0.072281
1484
+ },
1485
+ {
1486
+ "tag": "butt",
1487
+ "fluffyrock_count": 828096,
1488
+ "sample_occurrences": 25,
1489
+ "best_group": "cat:character_count",
1490
+ "best_sim": 0.961395,
1491
+ "second_group": "cat:relationship_pairing",
1492
+ "second_sim": 0.938435,
1493
+ "margin": 0.02296
1494
+ },
1495
+ {
1496
+ "tag": "felid",
1497
+ "fluffyrock_count": 598698,
1498
+ "sample_occurrences": 158,
1499
+ "best_group": "cat:species_specific",
1500
+ "best_sim": 0.88795,
1501
+ "second_group": "cat:character_count",
1502
+ "second_sim": 0.819039,
1503
+ "margin": 0.068911
1504
+ },
1505
+ {
1506
+ "tag": "canis",
1507
+ "fluffyrock_count": 587772,
1508
+ "sample_occurrences": 159,
1509
+ "best_group": "cat:species_specific",
1510
+ "best_sim": 0.864142,
1511
+ "second_group": "cat:character_count",
1512
+ "second_sim": 0.779288,
1513
+ "margin": 0.084854
1514
+ },
1515
+ {
1516
+ "tag": "male/female",
1517
+ "fluffyrock_count": 551565,
1518
+ "sample_occurrences": 12,
1519
+ "best_group": "cat:relationship_pairing",
1520
+ "best_sim": 0.958212,
1521
+ "second_group": "cat:character_count",
1522
+ "second_sim": 0.908632,
1523
+ "margin": 0.049581
1524
+ },
1525
+ {
1526
+ "tag": "blue_eyes",
1527
+ "fluffyrock_count": 451508,
1528
+ "sample_occurrences": 144,
1529
+ "best_group": "cat:character_count",
1530
+ "best_sim": 0.94142,
1531
+ "second_group": "cat:species_specific",
1532
+ "second_sim": 0.914454,
1533
+ "margin": 0.026967
1534
+ },
1535
+ {
1536
+ "tag": "male/male",
1537
+ "fluffyrock_count": 427119,
1538
+ "sample_occurrences": 26,
1539
+ "best_group": "cat:relationship_pairing",
1540
+ "best_sim": 0.852538,
1541
+ "second_group": "cat:character_count",
1542
+ "second_sim": 0.823404,
1543
+ "margin": 0.029134
1544
+ },
1545
+ {
1546
+ "tag": "navel",
1547
+ "fluffyrock_count": 373746,
1548
+ "sample_occurrences": 45,
1549
+ "best_group": "cat:body_build",
1550
+ "best_sim": 0.923345,
1551
+ "second_group": "cat:species_specific",
1552
+ "second_sim": 0.888708,
1553
+ "margin": 0.034636
1554
+ },
1555
+ {
1556
+ "tag": "muscular",
1557
+ "fluffyrock_count": 358573,
1558
+ "sample_occurrences": 56,
1559
+ "best_group": "cat:body_build",
1560
+ "best_sim": 0.799827,
1561
+ "second_group": "cat:species_specific",
1562
+ "second_sim": 0.692788,
1563
+ "margin": 0.10704
1564
+ },
1565
+ {
1566
+ "tag": "thick_thighs",
1567
+ "fluffyrock_count": 339009,
1568
+ "sample_occurrences": 25,
1569
+ "best_group": "cat:body_build",
1570
+ "best_sim": 0.915767,
1571
+ "second_group": "cat:character_count",
1572
+ "second_sim": 0.79776,
1573
+ "margin": 0.118007
1574
+ },
1575
+ {
1576
+ "tag": "fox",
1577
+ "fluffyrock_count": 337215,
1578
+ "sample_occurrences": 105,
1579
+ "best_group": "cat:species_specific",
1580
+ "best_sim": 0.796958,
1581
+ "second_group": "cat:character_count",
1582
+ "second_sim": 0.693828,
1583
+ "margin": 0.103131
1584
+ },
1585
+ {
1586
+ "tag": "domestic_dog",
1587
+ "fluffyrock_count": 300258,
1588
+ "sample_occurrences": 77,
1589
+ "best_group": "cat:species_specific",
1590
+ "best_sim": 0.751079,
1591
+ "second_group": "cat:character_count",
1592
+ "second_sim": 0.669701,
1593
+ "margin": 0.081378
1594
+ },
1595
+ {
1596
+ "tag": "wolf",
1597
+ "fluffyrock_count": 273053,
1598
+ "sample_occurrences": 79,
1599
+ "best_group": "cat:species_specific",
1600
+ "best_sim": 0.848018,
1601
+ "second_group": "cat:character_count",
1602
+ "second_sim": 0.769764,
1603
+ "margin": 0.078254
1604
+ },
1605
+ {
1606
+ "tag": "plant",
1607
+ "fluffyrock_count": 265700,
1608
+ "sample_occurrences": 97,
1609
+ "best_group": "cat:character_count",
1610
+ "best_sim": 0.887749,
1611
+ "second_group": "cat:species_specific",
1612
+ "second_sim": 0.858598,
1613
+ "margin": 0.029152
1614
+ },
1615
+ {
1616
+ "tag": "big_butt",
1617
+ "fluffyrock_count": 262825,
1618
+ "sample_occurrences": 11,
1619
+ "best_group": "cat:body_build",
1620
+ "best_sim": 0.894649,
1621
+ "second_group": "cat:character_count",
1622
+ "second_sim": 0.844478,
1623
+ "margin": 0.050171
1624
+ },
1625
+ {
1626
+ "tag": "interspecies",
1627
+ "fluffyrock_count": 261649,
1628
+ "sample_occurrences": 4,
1629
+ "best_group": "cat:relationship_pairing",
1630
+ "best_sim": 0.883739,
1631
+ "second_group": "cat:character_count",
1632
+ "second_sim": 0.799747,
1633
+ "margin": 0.083992
1634
+ },
1635
+ {
1636
+ "tag": "huge_breasts",
1637
+ "fluffyrock_count": 261003,
1638
+ "sample_occurrences": 13,
1639
+ "best_group": "cat:body_build",
1640
+ "best_sim": 0.833355,
1641
+ "second_group": "cat:character_count",
1642
+ "second_sim": 0.743262,
1643
+ "margin": 0.090093
1644
+ },
1645
+ {
1646
+ "tag": "eyewear",
1647
+ "fluffyrock_count": 251234,
1648
+ "sample_occurrences": 93,
1649
+ "best_group": "cat:species_specific",
1650
+ "best_sim": 0.919188,
1651
+ "second_group": "cat:character_count",
1652
+ "second_sim": 0.898806,
1653
+ "margin": 0.020382
1654
+ },
1655
+ {
1656
+ "tag": "sweat",
1657
+ "fluffyrock_count": 250842,
1658
+ "sample_occurrences": 20,
1659
+ "best_group": "cat:character_count",
1660
+ "best_sim": 0.932303,
1661
+ "second_group": "cat:body_build",
1662
+ "second_sim": 0.89766,
1663
+ "margin": 0.034643
1664
+ },
1665
+ {
1666
+ "tag": "felis",
1667
+ "fluffyrock_count": 241656,
1668
+ "sample_occurrences": 64,
1669
+ "best_group": "cat:species_specific",
1670
+ "best_sim": 0.822033,
1671
+ "second_group": "cat:character_count",
1672
+ "second_sim": 0.758581,
1673
+ "margin": 0.063452
1674
+ },
1675
+ {
1676
+ "tag": "domestic_cat",
1677
+ "fluffyrock_count": 236967,
1678
+ "sample_occurrences": 64,
1679
+ "best_group": "cat:species_specific",
1680
+ "best_sim": 0.81899,
1681
+ "second_group": "cat:character_count",
1682
+ "second_sim": 0.756082,
1683
+ "margin": 0.062908
1684
+ },
1685
+ {
1686
+ "tag": "wide_hips",
1687
+ "fluffyrock_count": 235877,
1688
+ "sample_occurrences": 30,
1689
+ "best_group": "cat:body_build",
1690
+ "best_sim": 0.876513,
1691
+ "second_group": "cat:character_count",
1692
+ "second_sim": 0.778367,
1693
+ "margin": 0.098146
1694
+ },
1695
+ {
1696
+ "tag": "red_eyes",
1697
+ "fluffyrock_count": 213545,
1698
+ "sample_occurrences": 53,
1699
+ "best_group": "cat:character_count",
1700
+ "best_sim": 0.889482,
1701
+ "second_group": "cat:anatomy_features",
1702
+ "second_sim": 0.845381,
1703
+ "margin": 0.044101
1704
+ },
1705
+ {
1706
+ "tag": "belly",
1707
+ "fluffyrock_count": 194310,
1708
+ "sample_occurrences": 31,
1709
+ "best_group": "cat:body_build",
1710
+ "best_sim": 0.765762,
1711
+ "second_group": "cat:species_specific",
1712
+ "second_sim": 0.56733,
1713
+ "margin": 0.198432
1714
+ }
1715
+ ],
1716
+ "sample_assignments_borderline": [
1717
+ {
1718
+ "tag": "bodily_fluids",
1719
+ "fluffyrock_count": 1225999,
1720
+ "sample_occurrences": 49,
1721
+ "best_group": "cat:character_count",
1722
+ "best_sim": 0.964808,
1723
+ "second_group": "cat:relationship_pairing",
1724
+ "second_sim": 0.953455,
1725
+ "margin": 0.011353
1726
+ },
1727
+ {
1728
+ "tag": "nipples",
1729
+ "fluffyrock_count": 1079208,
1730
+ "sample_occurrences": 17,
1731
+ "best_group": "cat:character_count",
1732
+ "best_sim": 0.93445,
1733
+ "second_group": "cat:body_build",
1734
+ "second_sim": 0.915792,
1735
+ "margin": 0.018658
1736
+ },
1737
+ {
1738
+ "tag": "feet",
1739
+ "fluffyrock_count": 448730,
1740
+ "sample_occurrences": 113,
1741
+ "best_group": "cat:anatomy_features",
1742
+ "best_sim": 0.907272,
1743
+ "second_group": "cat:species_specific",
1744
+ "second_sim": 0.90161,
1745
+ "margin": 0.005662
1746
+ },
1747
+ {
1748
+ "tag": "tuft",
1749
+ "fluffyrock_count": 335534,
1750
+ "sample_occurrences": 105,
1751
+ "best_group": "cat:anatomy_features",
1752
+ "best_sim": 0.938366,
1753
+ "second_group": "cat:color_markings",
1754
+ "second_sim": 0.929999,
1755
+ "margin": 0.008367
1756
+ },
1757
+ {
1758
+ "tag": "areola",
1759
+ "fluffyrock_count": 320191,
1760
+ "sample_occurrences": 1,
1761
+ "best_group": "cat:character_count",
1762
+ "best_sim": 0.86106,
1763
+ "second_group": "cat:body_build",
1764
+ "second_sim": 0.844252,
1765
+ "margin": 0.016808
1766
+ },
1767
+ {
1768
+ "tag": "dragon",
1769
+ "fluffyrock_count": 309478,
1770
+ "sample_occurrences": 84,
1771
+ "best_group": "cat:anatomy_features",
1772
+ "best_sim": 0.766118,
1773
+ "second_group": "cat:character_count",
1774
+ "second_sim": 0.750724,
1775
+ "margin": 0.015395
1776
+ },
1777
+ {
1778
+ "tag": "green_eyes",
1779
+ "fluffyrock_count": 287579,
1780
+ "sample_occurrences": 78,
1781
+ "best_group": "cat:character_count",
1782
+ "best_sim": 0.914274,
1783
+ "second_group": "cat:species_specific",
1784
+ "second_sim": 0.914113,
1785
+ "margin": 0.000161
1786
+ },
1787
+ {
1788
+ "tag": "black_nose",
1789
+ "fluffyrock_count": 216679,
1790
+ "sample_occurrences": 80,
1791
+ "best_group": "cat:color_markings",
1792
+ "best_sim": 0.89887,
1793
+ "second_group": "cat:anatomy_features",
1794
+ "second_sim": 0.887671,
1795
+ "margin": 0.011199
1796
+ },
1797
+ {
1798
+ "tag": "eyelashes",
1799
+ "fluffyrock_count": 193540,
1800
+ "sample_occurrences": 63,
1801
+ "best_group": "cat:character_count",
1802
+ "best_sim": 0.847013,
1803
+ "second_group": "cat:anatomy_features",
1804
+ "second_sim": 0.844852,
1805
+ "margin": 0.002161
1806
+ },
1807
+ {
1808
+ "tag": "saliva",
1809
+ "fluffyrock_count": 187352,
1810
+ "sample_occurrences": 10,
1811
+ "best_group": "cat:relationship_pairing",
1812
+ "best_sim": 0.930058,
1813
+ "second_group": "cat:character_count",
1814
+ "second_sim": 0.925649,
1815
+ "margin": 0.004409
1816
+ },
1817
+ {
1818
+ "tag": "black_hair",
1819
+ "fluffyrock_count": 180936,
1820
+ "sample_occurrences": 45,
1821
+ "best_group": "cat:species_specific",
1822
+ "best_sim": 0.892875,
1823
+ "second_group": "cat:character_count",
1824
+ "second_sim": 0.886585,
1825
+ "margin": 0.006291
1826
+ },
1827
+ {
1828
+ "tag": "eyebrows",
1829
+ "fluffyrock_count": 170102,
1830
+ "sample_occurrences": 50,
1831
+ "best_group": "cat:anatomy_features",
1832
+ "best_sim": 0.944516,
1833
+ "second_group": "cat:color_markings",
1834
+ "second_sim": 0.942686,
1835
+ "margin": 0.00183
1836
+ },
1837
+ {
1838
+ "tag": "vein",
1839
+ "fluffyrock_count": 168934,
1840
+ "sample_occurrences": 5,
1841
+ "best_group": "cat:body_build",
1842
+ "best_sim": 0.839476,
1843
+ "second_group": "cat:character_count",
1844
+ "second_sim": 0.835717,
1845
+ "margin": 0.003759
1846
+ },
1847
+ {
1848
+ "tag": "yellow_eyes",
1849
+ "fluffyrock_count": 168780,
1850
+ "sample_occurrences": 45,
1851
+ "best_group": "cat:anatomy_features",
1852
+ "best_sim": 0.896706,
1853
+ "second_group": "cat:character_count",
1854
+ "second_sim": 0.888449,
1855
+ "margin": 0.008257
1856
+ },
1857
+ {
1858
+ "tag": "panties",
1859
+ "fluffyrock_count": 142486,
1860
+ "sample_occurrences": 13,
1861
+ "best_group": "cat:species_specific",
1862
+ "best_sim": 0.718523,
1863
+ "second_group": "cat:character_count",
1864
+ "second_sim": 0.711868,
1865
+ "margin": 0.006655
1866
+ },
1867
+ {
1868
+ "tag": "white_hair",
1869
+ "fluffyrock_count": 139416,
1870
+ "sample_occurrences": 37,
1871
+ "best_group": "cat:species_specific",
1872
+ "best_sim": 0.849061,
1873
+ "second_group": "cat:anatomy_features",
1874
+ "second_sim": 0.844397,
1875
+ "margin": 0.004665
1876
+ },
1877
+ {
1878
+ "tag": "pose",
1879
+ "fluffyrock_count": 130171,
1880
+ "sample_occurrences": 25,
1881
+ "best_group": "cat:species_specific",
1882
+ "best_sim": 0.869967,
1883
+ "second_group": "cat:anatomy_features",
1884
+ "second_sim": 0.865188,
1885
+ "margin": 0.004779
1886
+ },
1887
+ {
1888
+ "tag": "cleavage",
1889
+ "fluffyrock_count": 116981,
1890
+ "sample_occurrences": 41,
1891
+ "best_group": "cat:species_specific",
1892
+ "best_sim": 0.750055,
1893
+ "second_group": "cat:body_build",
1894
+ "second_sim": 0.749197,
1895
+ "margin": 0.000857
1896
+ },
1897
+ {
1898
+ "tag": "red_hair",
1899
+ "fluffyrock_count": 115770,
1900
+ "sample_occurrences": 24,
1901
+ "best_group": "cat:character_count",
1902
+ "best_sim": 0.900872,
1903
+ "second_group": "cat:species_specific",
1904
+ "second_sim": 0.881749,
1905
+ "margin": 0.019123
1906
+ },
1907
+ {
1908
+ "tag": "accessory",
1909
+ "fluffyrock_count": 114975,
1910
+ "sample_occurrences": 29,
1911
+ "best_group": "cat:character_count",
1912
+ "best_sim": 0.862266,
1913
+ "second_group": "cat:species_specific",
1914
+ "second_sim": 0.853952,
1915
+ "margin": 0.008314
1916
+ },
1917
+ {
1918
+ "tag": "fluffy",
1919
+ "fluffyrock_count": 113761,
1920
+ "sample_occurrences": 35,
1921
+ "best_group": "cat:anatomy_features",
1922
+ "best_sim": 0.917693,
1923
+ "second_group": "cat:color_markings",
1924
+ "second_sim": 0.905509,
1925
+ "margin": 0.012184
1926
+ },
1927
+ {
1928
+ "tag": "thigh_highs",
1929
+ "fluffyrock_count": 104261,
1930
+ "sample_occurrences": 16,
1931
+ "best_group": "cat:character_count",
1932
+ "best_sim": 0.724052,
1933
+ "second_group": "cat:color_markings",
1934
+ "second_sim": 0.707402,
1935
+ "margin": 0.01665
1936
+ },
1937
+ {
1938
+ "tag": "submissive",
1939
+ "fluffyrock_count": 102144,
1940
+ "sample_occurrences": 1,
1941
+ "best_group": "cat:relationship_pairing",
1942
+ "best_sim": 0.73861,
1943
+ "second_group": "cat:character_count",
1944
+ "second_sim": 0.721952,
1945
+ "margin": 0.016658
1946
+ },
1947
+ {
1948
+ "tag": "stockings",
1949
+ "fluffyrock_count": 96677,
1950
+ "sample_occurrences": 6,
1951
+ "best_group": "cat:character_count",
1952
+ "best_sim": 0.799803,
1953
+ "second_group": "cat:species_specific",
1954
+ "second_sim": 0.784923,
1955
+ "margin": 0.014881
1956
+ },
1957
+ {
1958
+ "tag": "ring_piercing",
1959
+ "fluffyrock_count": 96328,
1960
+ "sample_occurrences": 21,
1961
+ "best_group": "cat:body_build",
1962
+ "best_sim": 0.823498,
1963
+ "second_group": "cat:species_specific",
1964
+ "second_sim": 0.808368,
1965
+ "margin": 0.015131
1966
+ }
1967
+ ]
1968
+ }
1969
+ ],
1970
+ "top_uncovered_before_clustering": [
1971
+ {
1972
+ "tag": "fur",
1973
+ "fluffyrock_count": 1417091,
1974
+ "sample_occurrences": 422
1975
+ },
1976
+ {
1977
+ "tag": "bodily_fluids",
1978
+ "fluffyrock_count": 1225999,
1979
+ "sample_occurrences": 49
1980
+ },
1981
+ {
1982
+ "tag": "canid",
1983
+ "fluffyrock_count": 1141448,
1984
+ "sample_occurrences": 304
1985
+ },
1986
+ {
1987
+ "tag": "nipples",
1988
+ "fluffyrock_count": 1079208,
1989
+ "sample_occurrences": 17
1990
+ },
1991
+ {
1992
+ "tag": "butt",
1993
+ "fluffyrock_count": 828096,
1994
+ "sample_occurrences": 25
1995
+ },
1996
+ {
1997
+ "tag": "felid",
1998
+ "fluffyrock_count": 598698,
1999
+ "sample_occurrences": 158
2000
+ },
2001
+ {
2002
+ "tag": "canis",
2003
+ "fluffyrock_count": 587772,
2004
+ "sample_occurrences": 159
2005
+ },
2006
+ {
2007
+ "tag": "male/female",
2008
+ "fluffyrock_count": 551565,
2009
+ "sample_occurrences": 12
2010
+ },
2011
+ {
2012
+ "tag": "blue_eyes",
2013
+ "fluffyrock_count": 451508,
2014
+ "sample_occurrences": 144
2015
+ },
2016
+ {
2017
+ "tag": "feet",
2018
+ "fluffyrock_count": 448730,
2019
+ "sample_occurrences": 113
2020
+ },
2021
+ {
2022
+ "tag": "male/male",
2023
+ "fluffyrock_count": 427119,
2024
+ "sample_occurrences": 26
2025
+ },
2026
+ {
2027
+ "tag": "equid",
2028
+ "fluffyrock_count": 399764,
2029
+ "sample_occurrences": 97
2030
+ },
2031
+ {
2032
+ "tag": "navel",
2033
+ "fluffyrock_count": 373746,
2034
+ "sample_occurrences": 45
2035
+ },
2036
+ {
2037
+ "tag": "muscular",
2038
+ "fluffyrock_count": 358573,
2039
+ "sample_occurrences": 56
2040
+ },
2041
+ {
2042
+ "tag": "thick_thighs",
2043
+ "fluffyrock_count": 339009,
2044
+ "sample_occurrences": 25
2045
+ },
2046
+ {
2047
+ "tag": "fox",
2048
+ "fluffyrock_count": 337215,
2049
+ "sample_occurrences": 105
2050
+ },
2051
+ {
2052
+ "tag": "tuft",
2053
+ "fluffyrock_count": 335534,
2054
+ "sample_occurrences": 105
2055
+ },
2056
+ {
2057
+ "tag": "areola",
2058
+ "fluffyrock_count": 320191,
2059
+ "sample_occurrences": 1
2060
+ },
2061
+ {
2062
+ "tag": "dragon",
2063
+ "fluffyrock_count": 309478,
2064
+ "sample_occurrences": 84
2065
+ },
2066
+ {
2067
+ "tag": "domestic_dog",
2068
+ "fluffyrock_count": 300258,
2069
+ "sample_occurrences": 77
2070
+ },
2071
+ {
2072
+ "tag": "<3",
2073
+ "fluffyrock_count": 293372,
2074
+ "sample_occurrences": 50
2075
+ },
2076
+ {
2077
+ "tag": "green_eyes",
2078
+ "fluffyrock_count": 287579,
2079
+ "sample_occurrences": 78
2080
+ },
2081
+ {
2082
+ "tag": "wolf",
2083
+ "fluffyrock_count": 273053,
2084
+ "sample_occurrences": 79
2085
+ },
2086
+ {
2087
+ "tag": "plant",
2088
+ "fluffyrock_count": 265700,
2089
+ "sample_occurrences": 97
2090
+ },
2091
+ {
2092
+ "tag": "big_butt",
2093
+ "fluffyrock_count": 262825,
2094
+ "sample_occurrences": 11
2095
+ },
2096
+ {
2097
+ "tag": "interspecies",
2098
+ "fluffyrock_count": 261649,
2099
+ "sample_occurrences": 4
2100
+ },
2101
+ {
2102
+ "tag": "huge_breasts",
2103
+ "fluffyrock_count": 261003,
2104
+ "sample_occurrences": 13
2105
+ },
2106
+ {
2107
+ "tag": "eyewear",
2108
+ "fluffyrock_count": 251234,
2109
+ "sample_occurrences": 93
2110
+ },
2111
+ {
2112
+ "tag": "sweat",
2113
+ "fluffyrock_count": 250842,
2114
+ "sample_occurrences": 20
2115
+ },
2116
+ {
2117
+ "tag": "felis",
2118
+ "fluffyrock_count": 241656,
2119
+ "sample_occurrences": 64
2120
+ },
2121
+ {
2122
+ "tag": "young",
2123
+ "fluffyrock_count": 238777,
2124
+ "sample_occurrences": 40
2125
+ },
2126
+ {
2127
+ "tag": "domestic_cat",
2128
+ "fluffyrock_count": 236967,
2129
+ "sample_occurrences": 64
2130
+ },
2131
+ {
2132
+ "tag": "wide_hips",
2133
+ "fluffyrock_count": 235877,
2134
+ "sample_occurrences": 30
2135
+ },
2136
+ {
2137
+ "tag": "black_nose",
2138
+ "fluffyrock_count": 216679,
2139
+ "sample_occurrences": 80
2140
+ },
2141
+ {
2142
+ "tag": "red_eyes",
2143
+ "fluffyrock_count": 213545,
2144
+ "sample_occurrences": 53
2145
+ },
2146
+ {
2147
+ "tag": "reptile",
2148
+ "fluffyrock_count": 206968,
2149
+ "sample_occurrences": 47
2150
+ },
2151
+ {
2152
+ "tag": "leporid",
2153
+ "fluffyrock_count": 204404,
2154
+ "sample_occurrences": 73
2155
+ },
2156
+ {
2157
+ "tag": "rabbit",
2158
+ "fluffyrock_count": 199906,
2159
+ "sample_occurrences": 73
2160
+ },
2161
+ {
2162
+ "tag": "belly",
2163
+ "fluffyrock_count": 194310,
2164
+ "sample_occurrences": 31
2165
+ },
2166
+ {
2167
+ "tag": "eyelashes",
2168
+ "fluffyrock_count": 193540,
2169
+ "sample_occurrences": 63
2170
+ },
2171
+ {
2172
+ "tag": "horse",
2173
+ "fluffyrock_count": 193042,
2174
+ "sample_occurrences": 40
2175
+ },
2176
+ {
2177
+ "tag": "feathers",
2178
+ "fluffyrock_count": 192797,
2179
+ "sample_occurrences": 76
2180
+ },
2181
+ {
2182
+ "tag": "saliva",
2183
+ "fluffyrock_count": 187352,
2184
+ "sample_occurrences": 10
2185
+ },
2186
+ {
2187
+ "tag": "brown_hair",
2188
+ "fluffyrock_count": 183486,
2189
+ "sample_occurrences": 48
2190
+ },
2191
+ {
2192
+ "tag": "blonde_hair",
2193
+ "fluffyrock_count": 183468,
2194
+ "sample_occurrences": 47
2195
+ },
2196
+ {
2197
+ "tag": "pantherine",
2198
+ "fluffyrock_count": 181019,
2199
+ "sample_occurrences": 45
2200
+ },
2201
+ {
2202
+ "tag": "black_hair",
2203
+ "fluffyrock_count": 180936,
2204
+ "sample_occurrences": 45
2205
+ },
2206
+ {
2207
+ "tag": "overweight",
2208
+ "fluffyrock_count": 176776,
2209
+ "sample_occurrences": 36
2210
+ },
2211
+ {
2212
+ "tag": "abs",
2213
+ "fluffyrock_count": 171761,
2214
+ "sample_occurrences": 23
2215
+ },
2216
+ {
2217
+ "tag": "eyebrows",
2218
+ "fluffyrock_count": 170102,
2219
+ "sample_occurrences": 50
2220
+ },
2221
+ {
2222
+ "tag": "vein",
2223
+ "fluffyrock_count": 168934,
2224
+ "sample_occurrences": 5
2225
+ },
2226
+ {
2227
+ "tag": "yellow_eyes",
2228
+ "fluffyrock_count": 168780,
2229
+ "sample_occurrences": 45
2230
+ },
2231
+ {
2232
+ "tag": "muscular_anthro",
2233
+ "fluffyrock_count": 164876,
2234
+ "sample_occurrences": 30
2235
+ },
2236
+ {
2237
+ "tag": "half-closed_eyes",
2238
+ "fluffyrock_count": 158388,
2239
+ "sample_occurrences": 19
2240
+ },
2241
+ {
2242
+ "tag": "pecs",
2243
+ "fluffyrock_count": 158385,
2244
+ "sample_occurrences": 23
2245
+ },
2246
+ {
2247
+ "tag": "bovid",
2248
+ "fluffyrock_count": 156674,
2249
+ "sample_occurrences": 35
2250
+ },
2251
+ {
2252
+ "tag": "barefoot",
2253
+ "fluffyrock_count": 148056,
2254
+ "sample_occurrences": 46
2255
+ },
2256
+ {
2257
+ "tag": "cub",
2258
+ "fluffyrock_count": 147547,
2259
+ "sample_occurrences": 21
2260
+ },
2261
+ {
2262
+ "tag": "panties",
2263
+ "fluffyrock_count": 142486,
2264
+ "sample_occurrences": 13
2265
+ },
2266
+ {
2267
+ "tag": "white_hair",
2268
+ "fluffyrock_count": 139416,
2269
+ "sample_occurrences": 37
2270
+ },
2271
+ {
2272
+ "tag": "animal_humanoid",
2273
+ "fluffyrock_count": 139252,
2274
+ "sample_occurrences": 31
2275
+ },
2276
+ {
2277
+ "tag": "blue_hair",
2278
+ "fluffyrock_count": 138674,
2279
+ "sample_occurrences": 48
2280
+ },
2281
+ {
2282
+ "tag": "hyper",
2283
+ "fluffyrock_count": 137712,
2284
+ "sample_occurrences": 2
2285
+ },
2286
+ {
2287
+ "tag": "hybrid",
2288
+ "fluffyrock_count": 134372,
2289
+ "sample_occurrences": 30
2290
+ },
2291
+ {
2292
+ "tag": "stripes",
2293
+ "fluffyrock_count": 132789,
2294
+ "sample_occurrences": 32
2295
+ },
2296
+ {
2297
+ "tag": "purple_eyes",
2298
+ "fluffyrock_count": 131683,
2299
+ "sample_occurrences": 34
2300
+ },
2301
+ {
2302
+ "tag": "pose",
2303
+ "fluffyrock_count": 130171,
2304
+ "sample_occurrences": 25
2305
+ },
2306
+ {
2307
+ "tag": "pony",
2308
+ "fluffyrock_count": 122813,
2309
+ "sample_occurrences": 35
2310
+ },
2311
+ {
2312
+ "tag": "slightly_chubby",
2313
+ "fluffyrock_count": 118826,
2314
+ "sample_occurrences": 21
2315
+ },
2316
+ {
2317
+ "tag": "scales",
2318
+ "fluffyrock_count": 118407,
2319
+ "sample_occurrences": 36
2320
+ },
2321
+ {
2322
+ "tag": "cleavage",
2323
+ "fluffyrock_count": 116981,
2324
+ "sample_occurrences": 41
2325
+ },
2326
+ {
2327
+ "tag": "red_hair",
2328
+ "fluffyrock_count": 115770,
2329
+ "sample_occurrences": 24
2330
+ },
2331
+ {
2332
+ "tag": "bird",
2333
+ "fluffyrock_count": 115422,
2334
+ "sample_occurrences": 42
2335
+ },
2336
+ {
2337
+ "tag": "accessory",
2338
+ "fluffyrock_count": 114975,
2339
+ "sample_occurrences": 29
2340
+ },
2341
+ {
2342
+ "tag": "fluffy",
2343
+ "fluffyrock_count": 113761,
2344
+ "sample_occurrences": 35
2345
+ },
2346
+ {
2347
+ "tag": "unicorn",
2348
+ "fluffyrock_count": 112328,
2349
+ "sample_occurrences": 39
2350
+ },
2351
+ {
2352
+ "tag": "bear",
2353
+ "fluffyrock_count": 111640,
2354
+ "sample_occurrences": 38
2355
+ },
2356
+ {
2357
+ "tag": "membrane_(anatomy)",
2358
+ "fluffyrock_count": 107123,
2359
+ "sample_occurrences": 34
2360
+ },
2361
+ {
2362
+ "tag": "curvy_figure",
2363
+ "fluffyrock_count": 106779,
2364
+ "sample_occurrences": 7
2365
+ },
2366
+ {
2367
+ "tag": "biceps",
2368
+ "fluffyrock_count": 106240,
2369
+ "sample_occurrences": 22
2370
+ },
2371
+ {
2372
+ "tag": "thigh_highs",
2373
+ "fluffyrock_count": 104261,
2374
+ "sample_occurrences": 16
2375
+ },
2376
+ {
2377
+ "tag": "cutie_mark",
2378
+ "fluffyrock_count": 104225,
2379
+ "sample_occurrences": 29
2380
+ },
2381
+ {
2382
+ "tag": "submissive",
2383
+ "fluffyrock_count": 102144,
2384
+ "sample_occurrences": 1
2385
+ },
2386
+ {
2387
+ "tag": "purple_hair",
2388
+ "fluffyrock_count": 101470,
2389
+ "sample_occurrences": 27
2390
+ },
2391
+ {
2392
+ "tag": "pink_hair",
2393
+ "fluffyrock_count": 99350,
2394
+ "sample_occurrences": 25
2395
+ },
2396
+ {
2397
+ "tag": "stockings",
2398
+ "fluffyrock_count": 96677,
2399
+ "sample_occurrences": 6
2400
+ },
2401
+ {
2402
+ "tag": "ring_piercing",
2403
+ "fluffyrock_count": 96328,
2404
+ "sample_occurrences": 21
2405
+ },
2406
+ {
2407
+ "tag": "dripping",
2408
+ "fluffyrock_count": 94534,
2409
+ "sample_occurrences": 1
2410
+ },
2411
+ {
2412
+ "tag": "makeup",
2413
+ "fluffyrock_count": 93493,
2414
+ "sample_occurrences": 14
2415
+ },
2416
+ {
2417
+ "tag": "mammal_humanoid",
2418
+ "fluffyrock_count": 91171,
2419
+ "sample_occurrences": 19
2420
+ },
2421
+ {
2422
+ "tag": "machine",
2423
+ "fluffyrock_count": 90867,
2424
+ "sample_occurrences": 34
2425
+ },
2426
+ {
2427
+ "tag": "female/female",
2428
+ "fluffyrock_count": 89943,
2429
+ "sample_occurrences": 6
2430
+ },
2431
+ {
2432
+ "tag": "digitigrade",
2433
+ "fluffyrock_count": 87950,
2434
+ "sample_occurrences": 23
2435
+ },
2436
+ {
2437
+ "tag": "humanoid_hands",
2438
+ "fluffyrock_count": 87880,
2439
+ "sample_occurrences": 28
2440
+ },
2441
+ {
2442
+ "tag": "huge_butt",
2443
+ "fluffyrock_count": 87380,
2444
+ "sample_occurrences": 2
2445
+ },
2446
+ {
2447
+ "tag": "drooling",
2448
+ "fluffyrock_count": 86754,
2449
+ "sample_occurrences": 5
2450
+ },
2451
+ {
2452
+ "tag": "demon",
2453
+ "fluffyrock_count": 84604,
2454
+ "sample_occurrences": 16
2455
+ },
2456
+ {
2457
+ "tag": "faceless_character",
2458
+ "fluffyrock_count": 83520,
2459
+ "sample_occurrences": 2
2460
+ },
2461
+ {
2462
+ "tag": "brown_eyes",
2463
+ "fluffyrock_count": 82369,
2464
+ "sample_occurrences": 24
2465
+ },
2466
+ {
2467
+ "tag": "larger_male",
2468
+ "fluffyrock_count": 81280,
2469
+ "sample_occurrences": 5
2470
+ },
2471
+ {
2472
+ "tag": "smaller_male",
2473
+ "fluffyrock_count": 81264,
2474
+ "sample_occurrences": 4
2475
+ },
2476
+ {
2477
+ "tag": "voluptuous",
2478
+ "fluffyrock_count": 80599,
2479
+ "sample_occurrences": 5
2480
+ },
2481
+ {
2482
+ "tag": "tiger",
2483
+ "fluffyrock_count": 79829,
2484
+ "sample_occurrences": 22
2485
+ },
2486
+ {
2487
+ "tag": "plantigrade",
2488
+ "fluffyrock_count": 78753,
2489
+ "sample_occurrences": 16
2490
+ },
2491
+ {
2492
+ "tag": "big_belly",
2493
+ "fluffyrock_count": 78444,
2494
+ "sample_occurrences": 8
2495
+ },
2496
+ {
2497
+ "tag": "glistening",
2498
+ "fluffyrock_count": 77473,
2499
+ "sample_occurrences": 22
2500
+ },
2501
+ {
2502
+ "tag": "glowing",
2503
+ "fluffyrock_count": 77098,
2504
+ "sample_occurrences": 21
2505
+ },
2506
+ {
2507
+ "tag": "overweight_anthro",
2508
+ "fluffyrock_count": 75733,
2509
+ "sample_occurrences": 18
2510
+ },
2511
+ {
2512
+ "tag": "spots",
2513
+ "fluffyrock_count": 75219,
2514
+ "sample_occurrences": 16
2515
+ },
2516
+ {
2517
+ "tag": "pegasus",
2518
+ "fluffyrock_count": 72559,
2519
+ "sample_occurrences": 19
2520
+ },
2521
+ {
2522
+ "tag": "pink_nipples",
2523
+ "fluffyrock_count": 71412,
2524
+ "sample_occurrences": 2
2525
+ },
2526
+ {
2527
+ "tag": "pupils",
2528
+ "fluffyrock_count": 69130,
2529
+ "sample_occurrences": 27
2530
+ },
2531
+ {
2532
+ "tag": "hair_accessory",
2533
+ "fluffyrock_count": 69018,
2534
+ "sample_occurrences": 25
2535
+ },
2536
+ {
2537
+ "tag": "fish",
2538
+ "fluffyrock_count": 69011,
2539
+ "sample_occurrences": 17
2540
+ },
2541
+ {
2542
+ "tag": "bikini",
2543
+ "fluffyrock_count": 68724,
2544
+ "sample_occurrences": 12
2545
+ },
2546
+ {
2547
+ "tag": "soles",
2548
+ "fluffyrock_count": 68647,
2549
+ "sample_occurrences": 8
2550
+ },
2551
+ {
2552
+ "tag": "facial_hair",
2553
+ "fluffyrock_count": 68586,
2554
+ "sample_occurrences": 18
2555
+ },
2556
+ {
2557
+ "tag": "human_on_anthro",
2558
+ "fluffyrock_count": 68297,
2559
+ "sample_occurrences": 2
2560
+ },
2561
+ {
2562
+ "tag": "earth_pony",
2563
+ "fluffyrock_count": 67495,
2564
+ "sample_occurrences": 22
2565
+ },
2566
+ {
2567
+ "tag": "deer",
2568
+ "fluffyrock_count": 67106,
2569
+ "sample_occurrences": 13
2570
+ }
2571
+ ]
2572
+ }
data/analysis/simplified_probe_tags.csv ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,bundle,needs_glossary,prevalence,actionable_score,selected_initial,support_pos,precision_strong,recall_strong,f1_strong,reliability_weight,final_score,selected_final,reliability_note
2
+ clothing,clothing_state,0,0.574000,0.347207,1,59,0.666667,0.711864,0.688525,0.688525,0.266097,1,"support=59, f1=0.689, prec=0.667, rec=0.712"
3
+ clothed,clothing_state,0,0.337000,0.257132,1,32,0.391892,0.906250,0.547170,0.547170,0.169804,0,"support=32, f1=0.547, prec=0.392, rec=0.906"
4
+ simple_background,scene_pose,0,0.345000,0.216830,1,27,0.652174,0.555556,0.600000,0.600000,0.151781,1,"support=27, f1=0.600, prec=0.652, rec=0.556"
5
+ anthro,body_type_presence,1,0.623000,0.111983,1,68,0.728261,0.985294,0.837500,0.837500,0.098335,1,"support=68, f1=0.838, prec=0.728, rec=0.985"
6
+ text,text_symbols,0,0.275000,0.092079,1,23,0.576923,0.652174,0.612245,0.612245,0.065301,1,"support=23, f1=0.612, prec=0.577, rec=0.652"
7
+ smile,gaze_expression,0,0.230000,0.174446,1,19,0.111111,0.105263,0.108108,0.108108,0.057756,0,"support=19, f1=0.108, prec=0.111, rec=0.105"
8
+ looking_at_viewer,gaze_expression,0,0.155000,0.072879,1,20,0.833333,0.250000,0.384615,0.384615,0.039243,0,"support=20, f1=0.385, prec=0.833, rec=0.250"
9
+ standing,scene_pose,0,0.130000,0.056088,1,10,0.140351,0.800000,0.238806,0.238806,0.024068,0,"support=10, f1=0.239, prec=0.140, rec=0.800"
10
+ blush,gaze_expression,0,0.110000,0.028668,1,13,0.714286,0.769231,0.740741,0.740741,0.023094,1,"support=13, f1=0.741, prec=0.714, rec=0.769"
11
+ dialogue,text_symbols,0,0.112000,0.032610,1,11,0.416667,0.909091,0.571429,0.571429,0.022128,0,"support=11, f1=0.571, prec=0.417, rec=0.909"
12
+ canid,species_taxonomy,1,0.304000,0.025518,1,37,0.685714,0.648649,0.666667,0.666667,0.019139,1,"support=37, f1=0.667, prec=0.686, rec=0.649"
13
+ breasts,body_shape_breasts,0,0.155000,0.069761,1,13,0.000000,0.000000,0.000000,0.000000,0.017440,0,"support=13, f1=0.000, prec=0.000, rec=0.000"
14
+ open_mouth,gaze_expression,0,0.179000,0.032152,1,16,0.216216,0.500000,0.301887,0.301887,0.015318,0,"support=16, f1=0.302, prec=0.216, rec=0.500"
15
+ topwear,clothing_state,1,0.183000,0.032059,1,14,0.181818,0.857143,0.300000,0.300000,0.015228,0,"support=14, f1=0.300, prec=0.182, rec=0.857"
16
+ solo,count_cardinality,1,0.572000,0.012913,1,57,0.888889,0.421053,0.571429,0.571429,0.008762,1,"support=57, f1=0.571, prec=0.889, rec=0.421"
17
+ eyes_closed,gaze_expression,0,0.068000,0.012189,1,4,0.500000,0.750000,0.600000,0.600000,0.008532,0,"support=4, f1=0.600, prec=0.500, rec=0.750"
18
+ biped,body_type_presence,0,0.095000,0.026207,1,9,0.064516,0.222222,0.100000,0.100000,0.008517,0,"support=9, f1=0.100, prec=0.065, rec=0.222"
19
+ sitting,scene_pose,0,0.074000,0.012630,1,9,0.347826,0.888889,0.500000,0.500000,0.007894,0,"support=9, f1=0.500, prec=0.348, rec=0.889"
20
+ outside,scene_pose,0,0.093000,0.013410,1,10,0.315789,0.600000,0.413793,0.413793,0.007514,0,"support=10, f1=0.414, prec=0.316, rec=0.600"
21
+ bottomwear,clothing_state,1,0.135000,0.019568,1,18,0.136364,0.166667,0.150000,0.150000,0.007093,0,"support=18, f1=0.150, prec=0.136, rec=0.167"
22
+ feral,body_type_presence,1,0.219000,0.016378,1,23,0.750000,0.130435,0.222222,0.222222,0.006824,0,"support=23, f1=0.222, prec=0.750, rec=0.130"
23
+ duo,count_cardinality,1,0.246000,0.006575,1,20,0.571429,0.600000,0.585366,0.585366,0.004530,1,"support=20, f1=0.585, prec=0.571, rec=0.600"
24
+ canis,species_taxonomy,1,0.159000,0.007483,1,23,0.428571,0.130435,0.200000,0.200000,0.002993,0,"support=23, f1=0.200, prec=0.429, rec=0.130"
25
+ group,count_cardinality,0,0.120000,0.003704,1,16,0.909091,0.625000,0.740741,0.740741,0.002984,1,"support=16, f1=0.741, prec=0.909, rec=0.625"
26
+ felid,species_taxonomy,1,0.158000,0.002667,1,18,0.923077,0.666667,0.774194,0.774194,0.002215,1,"support=18, f1=0.774, prec=0.923, rec=0.667"
27
+ topless,clothing_state,0,0.045000,0.007936,1,6,0.000000,0.000000,0.000000,0.000000,0.001984,0,"support=6, f1=0.000, prec=0.000, rec=0.000"
28
+ wide_hips,body_shape_breasts,0,0.030000,0.001969,1,1,1.000000,1.000000,1.000000,1.000000,0.001969,0,"support=1, f1=1.000, prec=1.000, rec=1.000"
29
+ leporid,species_taxonomy,1,0.073000,0.003274,1,8,0.666667,0.250000,0.363636,0.363636,0.001711,0,"support=8, f1=0.364, prec=0.667, rec=0.250"
30
+ big_breasts,body_shape_breasts,0,0.041000,0.004877,1,3,0.000000,0.000000,0.000000,0.000000,0.001219,0,"support=3, f1=0.000, prec=0.000, rec=0.000"
31
+ thick_thighs,body_shape_breasts,0,0.025000,0.001367,1,1,0.500000,1.000000,0.666667,0.666667,0.001025,0,"support=1, f1=0.667, prec=0.500, rec=1.000"
32
+ nude,clothing_state,0,0.057000,0.004049,1,3,0.000000,0.000000,0.000000,0.000000,0.001012,0,"support=3, f1=0.000, prec=0.000, rec=0.000"
33
+ humanoid,body_type_presence,1,0.076000,0.003484,1,6,0.000000,0.000000,0.000000,0.000000,0.000871,0,"support=6, f1=0.000, prec=0.000, rec=0.000"
34
+ bird,species_taxonomy,0,0.042000,0.001184,1,6,0.571429,0.666667,0.615385,0.615385,0.000842,1,"support=6, f1=0.615, prec=0.571, rec=0.667"
35
+ bear,species_taxonomy,0,0.038000,0.001141,1,5,0.500000,0.800000,0.615385,0.615385,0.000812,1,"support=5, f1=0.615, prec=0.500, rec=0.800"
36
+ <3,text_symbols,1,0.050000,0.000364,1,6,1.000000,0.500000,0.666667,0.666667,0.000273,1,"support=6, f1=0.667, prec=1.000, rec=0.500"
data/analysis/simplified_probe_tags_summary.json ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "probe_info_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\probe_informativeness.csv",
3
+ "reliability_csv": "data\\analysis\\probe_reliability_n100.csv",
4
+ "n_selected_initial": 35,
5
+ "n_selected_final": 13,
6
+ "selected_final_tags": [
7
+ "clothing",
8
+ "simple_background",
9
+ "anthro",
10
+ "text",
11
+ "blush",
12
+ "canid",
13
+ "solo",
14
+ "duo",
15
+ "group",
16
+ "felid",
17
+ "bird",
18
+ "bear",
19
+ "<3"
20
+ ],
21
+ "bundle_specs": {
22
+ "clothing_state": {
23
+ "cap": 6,
24
+ "force": [
25
+ "clothing",
26
+ "clothed",
27
+ "topwear",
28
+ "bottomwear",
29
+ "topless",
30
+ "nude"
31
+ ],
32
+ "deny": [
33
+ "5_fingers"
34
+ ]
35
+ },
36
+ "scene_pose": {
37
+ "cap": 4,
38
+ "force": [
39
+ "simple_background",
40
+ "standing",
41
+ "sitting",
42
+ "outside"
43
+ ],
44
+ "deny": []
45
+ },
46
+ "gaze_expression": {
47
+ "cap": 5,
48
+ "force": [
49
+ "smile",
50
+ "looking_at_viewer",
51
+ "open_mouth",
52
+ "blush",
53
+ "eyes_closed"
54
+ ],
55
+ "deny": []
56
+ },
57
+ "text_symbols": {
58
+ "cap": 3,
59
+ "force": [
60
+ "text",
61
+ "dialogue",
62
+ "<3"
63
+ ],
64
+ "deny": []
65
+ },
66
+ "body_type_presence": {
67
+ "cap": 4,
68
+ "force": [
69
+ "anthro",
70
+ "feral",
71
+ "biped",
72
+ "humanoid"
73
+ ],
74
+ "deny": []
75
+ },
76
+ "count_cardinality": {
77
+ "cap": 5,
78
+ "force": [
79
+ "zero_pictured",
80
+ "solo",
81
+ "duo",
82
+ "trio",
83
+ "group"
84
+ ],
85
+ "deny": [
86
+ "black_bars",
87
+ "husky",
88
+ "marsupial"
89
+ ]
90
+ },
91
+ "body_shape_breasts": {
92
+ "cap": 4,
93
+ "force": [
94
+ "breasts",
95
+ "big_breasts",
96
+ "wide_hips",
97
+ "thick_thighs"
98
+ ],
99
+ "deny": []
100
+ },
101
+ "species_taxonomy": {
102
+ "cap": 6,
103
+ "force": [
104
+ "canid",
105
+ "canis",
106
+ "felid",
107
+ "leporid",
108
+ "bird",
109
+ "bear",
110
+ "unicorn",
111
+ "equid"
112
+ ],
113
+ "deny": [
114
+ "mammal"
115
+ ]
116
+ }
117
+ },
118
+ "thresholds": {
119
+ "min_prevalence": 0.01,
120
+ "max_prevalence": 0.7,
121
+ "min_support_pos": 5,
122
+ "min_f1_strong": 0.45,
123
+ "min_precision_strong": 0.5
124
+ },
125
+ "outputs": {
126
+ "csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\simplified_probe_tags.csv",
127
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\simplified_probe_tags_summary.json"
128
+ }
129
+ }
data/analysis/tag_group_coverage_ranked.csv ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ group,group_size,covered_occurrences,covered_occ_pct,covered_unique_tags,covered_unique_pct,covered_images,covered_images_pct
2
+ species,100,1741,6.26,41,1.34,878,87.8
3
+ genders,336,1209,4.35,23,0.75,927,92.7
4
+ body_types,22,1035,3.72,9,0.29,909,90.9
5
+ facial_expressions,86,927,3.33,43,1.4,515,51.5
6
+ outfits,55,644,2.32,14,0.46,576,57.6
7
+ pose,189,622,2.24,66,2.15,417,41.7
8
+ romance,59,555,2.0,28,0.91,357,35.7
9
+ narrative_elements,113,536,1.93,26,0.85,317,31.7
10
+ art,111,536,1.93,21,0.68,477,47.7
11
+ nudity,146,501,1.8,21,0.68,396,39.6
12
+ background,19,443,1.59,10,0.33,418,41.8
13
+ language,13,438,1.58,7,0.23,278,27.8
14
+ hair,22,373,1.34,1,0.03,373,37.3
15
+ food,360,203,0.73,48,1.56,104,10.4
16
+ settings,186,196,0.7,39,1.27,125,12.5
17
+ piercing,99,178,0.64,16,0.52,97,9.7
18
+ unknown_tags,20,172,0.62,2,0.07,170,17.0
19
+ ambiguous_tags,18,169,0.61,3,0.1,167,16.7
20
+ bondage,89,124,0.45,16,0.52,98,9.8
21
+ sound_effects,7,120,0.43,1,0.03,120,12.0
22
+ feral_anatomy,4,114,0.41,2,0.07,110,11.0
23
+ bdsm,72,83,0.3,11,0.36,70,7.0
24
+ misc,13,81,0.29,5,0.16,80,8.0
25
+ clothes,14,80,0.29,3,0.1,75,7.5
26
+ professions,118,73,0.26,24,0.78,53,5.3
27
+ art_style,17,71,0.26,9,0.29,70,7.0
28
+ character_relations,48,51,0.18,6,0.2,49,4.9
29
+ themes,9,40,0.14,3,0.1,39,3.9
30
+ multiple_tags,45,21,0.08,7,0.23,18,1.8
31
+ optics,9,12,0.04,1,0.03,12,1.2
32
+ sex_positions,29,10,0.04,2,0.07,10,1.0
33
+ sports,4,4,0.01,1,0.03,4,0.4
34
+ nazi_germany,20,1,0.0,1,0.03,1,0.1
35
+ magic,5,0,0.0,0,0.0,0,0.0
36
+ colors,88,0,0.0,0,0.0,0,0.0
37
+ weapons,4,0,0.0,0,0.0,0,0.0
38
+ markings,5,0,0.0,0,0.0,0,0.0
39
+ fictional_species,19,0,0.0,0,0.0,0,0.0
40
+ anatomy,1,0,0.0,0,0.0,0,0.0
41
+ e621,9,0,0.0,0,0.0,0,0.0
42
+ video_games,7,0,0.0,0,0.0,0,0.0
43
+ countries,203,0,0.0,0,0.0,0,0.0
44
+ penetration_tags,8,0,0.0,0,0.0,0,0.0
45
+ touching,2,0,0.0,0,0.0,0,0.0
46
+ file_attributes,9,0,0.0,0,0.0,0,0.0
47
+ events/holidays,3,0,0.0,0,0.0,0,0.0
data/analysis/tag_group_coverage_ranked_baseline200.csv ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ group,group_size,covered_occurrences,covered_occ_pct,covered_unique_tags,covered_unique_pct,covered_images,covered_images_pct
2
+ species,100,1741,6.26,41,1.34,878,87.8
3
+ genders,336,1209,4.35,23,0.75,927,92.7
4
+ body_types,22,1035,3.72,9,0.29,909,90.9
5
+ facial_expressions,86,927,3.33,43,1.4,515,51.5
6
+ outfits,55,644,2.32,14,0.46,576,57.6
7
+ pose,189,622,2.24,66,2.15,417,41.7
8
+ romance,59,555,2.0,28,0.91,357,35.7
9
+ narrative_elements,113,536,1.93,26,0.85,317,31.7
10
+ art,111,536,1.93,21,0.68,477,47.7
11
+ nudity,146,501,1.8,21,0.68,396,39.6
12
+ background,19,443,1.59,10,0.33,418,41.8
13
+ language,13,438,1.58,7,0.23,278,27.8
14
+ hair,22,373,1.34,1,0.03,373,37.3
15
+ food,360,203,0.73,48,1.56,104,10.4
16
+ settings,186,196,0.7,39,1.27,125,12.5
17
+ piercing,99,178,0.64,16,0.52,97,9.7
18
+ unknown_tags,20,172,0.62,2,0.07,170,17.0
19
+ ambiguous_tags,18,169,0.61,3,0.1,167,16.7
20
+ bondage,89,124,0.45,16,0.52,98,9.8
21
+ sound_effects,7,120,0.43,1,0.03,120,12.0
22
+ feral_anatomy,4,114,0.41,2,0.07,110,11.0
23
+ bdsm,72,83,0.3,11,0.36,70,7.0
24
+ misc,13,81,0.29,5,0.16,80,8.0
25
+ clothes,14,80,0.29,3,0.1,75,7.5
26
+ professions,118,73,0.26,24,0.78,53,5.3
27
+ art_style,17,71,0.26,9,0.29,70,7.0
28
+ character_relations,48,51,0.18,6,0.2,49,4.9
29
+ themes,9,40,0.14,3,0.1,39,3.9
30
+ multiple_tags,45,21,0.08,7,0.23,18,1.8
31
+ optics,9,12,0.04,1,0.03,12,1.2
32
+ sex_positions,29,10,0.04,2,0.07,10,1.0
33
+ sports,4,4,0.01,1,0.03,4,0.4
34
+ nazi_germany,20,1,0.0,1,0.03,1,0.1
35
+ magic,5,0,0.0,0,0.0,0,0.0
36
+ colors,88,0,0.0,0,0.0,0,0.0
37
+ weapons,4,0,0.0,0,0.0,0,0.0
38
+ markings,5,0,0.0,0,0.0,0,0.0
39
+ fictional_species,19,0,0.0,0,0.0,0,0.0
40
+ anatomy,1,0,0.0,0,0.0,0,0.0
41
+ e621,9,0,0.0,0,0.0,0,0.0
42
+ video_games,7,0,0.0,0,0.0,0,0.0
43
+ countries,203,0,0.0,0,0.0,0,0.0
44
+ penetration_tags,8,0,0.0,0,0.0,0,0.0
45
+ touching,2,0,0.0,0,0.0,0,0.0
46
+ file_attributes,9,0,0.0,0,0.0,0,0.0
47
+ events/holidays,3,0,0.0,0,0.0,0,0.0
data/analysis/tag_group_coverage_ranked_combined200.csv ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ group,group_size,covered_occurrences,covered_occ_pct,covered_unique_tags,covered_unique_pct,covered_images,covered_images_pct
2
+ cat:clothing_detail,423,2800,10.07,218,7.11,595,59.5
3
+ cat:anatomy_features,410,2410,8.67,206,6.71,660,66.0
4
+ species,100,1741,6.26,41,1.34,878,87.8
5
+ cat:color_markings,244,1722,6.19,112,3.65,520,52.0
6
+ genders,336,1209,4.35,23,0.75,927,92.7
7
+ cat:background_composition,85,1089,3.92,50,1.63,544,54.4
8
+ cat:gender,3,1089,3.92,3,0.1,927,92.7
9
+ body_types,22,1035,3.72,9,0.29,909,90.9
10
+ cat:count,5,946,3.4,4,0.13,933,93.3
11
+ cat:body_type,7,932,3.35,6,0.2,880,88.0
12
+ facial_expressions,86,927,3.33,43,1.4,515,51.5
13
+ cat:species,13,879,3.16,13,0.42,706,70.6
14
+ cat:objects_props,266,870,3.13,141,4.6,312,31.2
15
+ cat:expression_detail,34,700,2.52,20,0.65,438,43.8
16
+ outfits,55,644,2.32,14,0.46,576,57.6
17
+ pose,189,622,2.24,66,2.15,417,41.7
18
+ romance,59,555,2.0,28,0.91,357,35.7
19
+ narrative_elements,113,536,1.93,26,0.85,317,31.7
20
+ art,111,536,1.93,21,0.68,477,47.7
21
+ nudity,146,501,1.8,21,0.68,396,39.6
22
+ cat:pose_action_detail,94,500,1.8,51,1.66,336,33.6
23
+ cat:hair,3,472,1.7,3,0.1,373,37.3
24
+ cat:expression,8,447,1.61,8,0.26,349,34.9
25
+ background,19,443,1.59,10,0.33,418,41.8
26
+ language,13,438,1.58,7,0.23,278,27.8
27
+ hair,22,373,1.34,1,0.03,373,37.3
28
+ cat:gaze_detail,22,303,1.09,13,0.42,267,26.7
29
+ cat:body_decor,9,259,0.93,9,0.29,213,21.3
30
+ cat:franchise_series,86,252,0.91,19,0.62,106,10.6
31
+ cat:gaze,4,244,0.88,4,0.13,234,23.4
32
+ cat:posture,6,235,0.85,6,0.2,226,22.6
33
+ food,360,203,0.73,48,1.56,104,10.4
34
+ cat:breasts,4,197,0.71,3,0.1,155,15.5
35
+ settings,186,196,0.7,39,1.27,125,12.5
36
+ piercing,99,178,0.64,16,0.52,97,9.7
37
+ cat:text,4,156,0.56,3,0.1,117,11.7
38
+ cat:location,5,151,0.54,5,0.16,132,13.2
39
+ cat:clothing,6,143,0.51,6,0.2,123,12.3
40
+ bondage,89,124,0.45,16,0.52,98,9.8
41
+ feral_anatomy,4,114,0.41,2,0.07,110,11.0
42
+ bdsm,72,83,0.3,11,0.36,70,7.0
43
+ misc,13,81,0.29,5,0.16,80,8.0
44
+ clothes,14,80,0.29,3,0.1,75,7.5
45
+ professions,118,73,0.26,24,0.78,53,5.3
46
+ art_style,17,71,0.26,9,0.29,70,7.0
47
+ cat:general_activity_if_any,6,62,0.22,6,0.2,60,6.0
48
+ cat:perspective,6,58,0.21,6,0.2,55,5.5
49
+ character_relations,48,51,0.18,6,0.2,49,4.9
50
+ cat:fur_style,2,48,0.17,2,0.07,47,4.7
51
+ themes,9,40,0.14,3,0.1,39,3.9
52
+ cat:information,3,30,0.11,1,0.03,30,3.0
53
+ cat:organization,8,21,0.08,3,0.1,21,2.1
54
+ cat:limbs,5,20,0.07,5,0.16,20,2.0
55
+ cat:style,3,15,0.05,2,0.07,15,1.5
56
+ optics,9,12,0.04,1,0.03,12,1.2
57
+ sex_positions,29,10,0.04,2,0.07,10,1.0
58
+ sports,4,4,0.01,1,0.03,4,0.4
59
+ nazi_germany,20,1,0.0,1,0.03,1,0.1
60
+ magic,5,0,0.0,0,0.0,0,0.0
61
+ colors,88,0,0.0,0,0.0,0,0.0
62
+ weapons,4,0,0.0,0,0.0,0,0.0
63
+ markings,5,0,0.0,0,0.0,0,0.0
64
+ fictional_species,19,0,0.0,0,0.0,0,0.0
65
+ anatomy,1,0,0.0,0,0.0,0,0.0
66
+ video_games,7,0,0.0,0,0.0,0,0.0
67
+ countries,203,0,0.0,0,0.0,0,0.0
68
+ penetration_tags,8,0,0.0,0,0.0,0,0.0
69
+ touching,2,0,0.0,0,0.0,0,0.0
70
+ file_attributes,9,0,0.0,0,0.0,0,0.0
71
+ events/holidays,3,0,0.0,0,0.0,0,0.0
72
+ cat:resolution,3,0,0.0,0,0.0,0,0.0
73
+ cat:quality,6,0,0.0,0,0.0,0,0.0
74
+ cat:requests,2,0,0.0,0,0.0,0,0.0
data/analysis/tag_group_coverage_summary.json ADDED
@@ -0,0 +1,1042 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "sample_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\eval_samples\\e621_sfw_sample_1000_seed123_buffer10000.jsonl",
3
+ "min_count": 200,
4
+ "n_images": 1000,
5
+ "n_groups": 46,
6
+ "total_tag_occurrences": 27806,
7
+ "n_unique_tags": 3068,
8
+ "n_unique_tags_covered_by_any_group": 470,
9
+ "unique_tag_coverage_by_any_group_pct": 15.32,
10
+ "top_groups_by_occurrence_coverage": [
11
+ {
12
+ "group": "species",
13
+ "group_size": 100,
14
+ "covered_occurrences": 1741,
15
+ "covered_occ_pct": 6.26,
16
+ "covered_unique_tags": 41,
17
+ "covered_unique_pct": 1.34,
18
+ "covered_images": 878,
19
+ "covered_images_pct": 87.8
20
+ },
21
+ {
22
+ "group": "genders",
23
+ "group_size": 336,
24
+ "covered_occurrences": 1209,
25
+ "covered_occ_pct": 4.35,
26
+ "covered_unique_tags": 23,
27
+ "covered_unique_pct": 0.75,
28
+ "covered_images": 927,
29
+ "covered_images_pct": 92.7
30
+ },
31
+ {
32
+ "group": "body_types",
33
+ "group_size": 22,
34
+ "covered_occurrences": 1035,
35
+ "covered_occ_pct": 3.72,
36
+ "covered_unique_tags": 9,
37
+ "covered_unique_pct": 0.29,
38
+ "covered_images": 909,
39
+ "covered_images_pct": 90.9
40
+ },
41
+ {
42
+ "group": "facial_expressions",
43
+ "group_size": 86,
44
+ "covered_occurrences": 927,
45
+ "covered_occ_pct": 3.33,
46
+ "covered_unique_tags": 43,
47
+ "covered_unique_pct": 1.4,
48
+ "covered_images": 515,
49
+ "covered_images_pct": 51.5
50
+ },
51
+ {
52
+ "group": "outfits",
53
+ "group_size": 55,
54
+ "covered_occurrences": 644,
55
+ "covered_occ_pct": 2.32,
56
+ "covered_unique_tags": 14,
57
+ "covered_unique_pct": 0.46,
58
+ "covered_images": 576,
59
+ "covered_images_pct": 57.6
60
+ },
61
+ {
62
+ "group": "pose",
63
+ "group_size": 189,
64
+ "covered_occurrences": 622,
65
+ "covered_occ_pct": 2.24,
66
+ "covered_unique_tags": 66,
67
+ "covered_unique_pct": 2.15,
68
+ "covered_images": 417,
69
+ "covered_images_pct": 41.7
70
+ },
71
+ {
72
+ "group": "romance",
73
+ "group_size": 59,
74
+ "covered_occurrences": 555,
75
+ "covered_occ_pct": 2.0,
76
+ "covered_unique_tags": 28,
77
+ "covered_unique_pct": 0.91,
78
+ "covered_images": 357,
79
+ "covered_images_pct": 35.7
80
+ },
81
+ {
82
+ "group": "narrative_elements",
83
+ "group_size": 113,
84
+ "covered_occurrences": 536,
85
+ "covered_occ_pct": 1.93,
86
+ "covered_unique_tags": 26,
87
+ "covered_unique_pct": 0.85,
88
+ "covered_images": 317,
89
+ "covered_images_pct": 31.7
90
+ },
91
+ {
92
+ "group": "art",
93
+ "group_size": 111,
94
+ "covered_occurrences": 536,
95
+ "covered_occ_pct": 1.93,
96
+ "covered_unique_tags": 21,
97
+ "covered_unique_pct": 0.68,
98
+ "covered_images": 477,
99
+ "covered_images_pct": 47.7
100
+ },
101
+ {
102
+ "group": "nudity",
103
+ "group_size": 146,
104
+ "covered_occurrences": 501,
105
+ "covered_occ_pct": 1.8,
106
+ "covered_unique_tags": 21,
107
+ "covered_unique_pct": 0.68,
108
+ "covered_images": 396,
109
+ "covered_images_pct": 39.6
110
+ },
111
+ {
112
+ "group": "background",
113
+ "group_size": 19,
114
+ "covered_occurrences": 443,
115
+ "covered_occ_pct": 1.59,
116
+ "covered_unique_tags": 10,
117
+ "covered_unique_pct": 0.33,
118
+ "covered_images": 418,
119
+ "covered_images_pct": 41.8
120
+ },
121
+ {
122
+ "group": "language",
123
+ "group_size": 13,
124
+ "covered_occurrences": 438,
125
+ "covered_occ_pct": 1.58,
126
+ "covered_unique_tags": 7,
127
+ "covered_unique_pct": 0.23,
128
+ "covered_images": 278,
129
+ "covered_images_pct": 27.8
130
+ },
131
+ {
132
+ "group": "hair",
133
+ "group_size": 22,
134
+ "covered_occurrences": 373,
135
+ "covered_occ_pct": 1.34,
136
+ "covered_unique_tags": 1,
137
+ "covered_unique_pct": 0.03,
138
+ "covered_images": 373,
139
+ "covered_images_pct": 37.3
140
+ },
141
+ {
142
+ "group": "food",
143
+ "group_size": 360,
144
+ "covered_occurrences": 203,
145
+ "covered_occ_pct": 0.73,
146
+ "covered_unique_tags": 48,
147
+ "covered_unique_pct": 1.56,
148
+ "covered_images": 104,
149
+ "covered_images_pct": 10.4
150
+ },
151
+ {
152
+ "group": "settings",
153
+ "group_size": 186,
154
+ "covered_occurrences": 196,
155
+ "covered_occ_pct": 0.7,
156
+ "covered_unique_tags": 39,
157
+ "covered_unique_pct": 1.27,
158
+ "covered_images": 125,
159
+ "covered_images_pct": 12.5
160
+ },
161
+ {
162
+ "group": "piercing",
163
+ "group_size": 99,
164
+ "covered_occurrences": 178,
165
+ "covered_occ_pct": 0.64,
166
+ "covered_unique_tags": 16,
167
+ "covered_unique_pct": 0.52,
168
+ "covered_images": 97,
169
+ "covered_images_pct": 9.7
170
+ },
171
+ {
172
+ "group": "unknown_tags",
173
+ "group_size": 20,
174
+ "covered_occurrences": 172,
175
+ "covered_occ_pct": 0.62,
176
+ "covered_unique_tags": 2,
177
+ "covered_unique_pct": 0.07,
178
+ "covered_images": 170,
179
+ "covered_images_pct": 17.0
180
+ },
181
+ {
182
+ "group": "ambiguous_tags",
183
+ "group_size": 18,
184
+ "covered_occurrences": 169,
185
+ "covered_occ_pct": 0.61,
186
+ "covered_unique_tags": 3,
187
+ "covered_unique_pct": 0.1,
188
+ "covered_images": 167,
189
+ "covered_images_pct": 16.7
190
+ },
191
+ {
192
+ "group": "bondage",
193
+ "group_size": 89,
194
+ "covered_occurrences": 124,
195
+ "covered_occ_pct": 0.45,
196
+ "covered_unique_tags": 16,
197
+ "covered_unique_pct": 0.52,
198
+ "covered_images": 98,
199
+ "covered_images_pct": 9.8
200
+ },
201
+ {
202
+ "group": "sound_effects",
203
+ "group_size": 7,
204
+ "covered_occurrences": 120,
205
+ "covered_occ_pct": 0.43,
206
+ "covered_unique_tags": 1,
207
+ "covered_unique_pct": 0.03,
208
+ "covered_images": 120,
209
+ "covered_images_pct": 12.0
210
+ }
211
+ ],
212
+ "greedy_cover_steps": [
213
+ {
214
+ "step": 1,
215
+ "group": "species",
216
+ "gain_occurrences": 1741,
217
+ "cumulative_covered_occurrences": 1741,
218
+ "cumulative_covered_pct": 6.26,
219
+ "new_tags_covered": 41
220
+ },
221
+ {
222
+ "step": 2,
223
+ "group": "genders",
224
+ "gain_occurrences": 1209,
225
+ "cumulative_covered_occurrences": 2950,
226
+ "cumulative_covered_pct": 10.61,
227
+ "new_tags_covered": 23
228
+ },
229
+ {
230
+ "step": 3,
231
+ "group": "body_types",
232
+ "gain_occurrences": 1035,
233
+ "cumulative_covered_occurrences": 3985,
234
+ "cumulative_covered_pct": 14.33,
235
+ "new_tags_covered": 9
236
+ },
237
+ {
238
+ "step": 4,
239
+ "group": "facial_expressions",
240
+ "gain_occurrences": 927,
241
+ "cumulative_covered_occurrences": 4912,
242
+ "cumulative_covered_pct": 17.67,
243
+ "new_tags_covered": 43
244
+ },
245
+ {
246
+ "step": 5,
247
+ "group": "outfits",
248
+ "gain_occurrences": 644,
249
+ "cumulative_covered_occurrences": 5556,
250
+ "cumulative_covered_pct": 19.98,
251
+ "new_tags_covered": 14
252
+ },
253
+ {
254
+ "step": 6,
255
+ "group": "pose",
256
+ "gain_occurrences": 622,
257
+ "cumulative_covered_occurrences": 6178,
258
+ "cumulative_covered_pct": 22.22,
259
+ "new_tags_covered": 66
260
+ },
261
+ {
262
+ "step": 7,
263
+ "group": "art",
264
+ "gain_occurrences": 536,
265
+ "cumulative_covered_occurrences": 6714,
266
+ "cumulative_covered_pct": 24.15,
267
+ "new_tags_covered": 21
268
+ },
269
+ {
270
+ "step": 8,
271
+ "group": "narrative_elements",
272
+ "gain_occurrences": 536,
273
+ "cumulative_covered_occurrences": 7250,
274
+ "cumulative_covered_pct": 26.07,
275
+ "new_tags_covered": 26
276
+ },
277
+ {
278
+ "step": 9,
279
+ "group": "nudity",
280
+ "gain_occurrences": 501,
281
+ "cumulative_covered_occurrences": 7751,
282
+ "cumulative_covered_pct": 27.88,
283
+ "new_tags_covered": 21
284
+ },
285
+ {
286
+ "step": 10,
287
+ "group": "hair",
288
+ "gain_occurrences": 373,
289
+ "cumulative_covered_occurrences": 8124,
290
+ "cumulative_covered_pct": 29.22,
291
+ "new_tags_covered": 1
292
+ },
293
+ {
294
+ "step": 11,
295
+ "group": "food",
296
+ "gain_occurrences": 203,
297
+ "cumulative_covered_occurrences": 8327,
298
+ "cumulative_covered_pct": 29.95,
299
+ "new_tags_covered": 48
300
+ },
301
+ {
302
+ "step": 12,
303
+ "group": "settings",
304
+ "gain_occurrences": 196,
305
+ "cumulative_covered_occurrences": 8523,
306
+ "cumulative_covered_pct": 30.65,
307
+ "new_tags_covered": 39
308
+ },
309
+ {
310
+ "step": 13,
311
+ "group": "piercing",
312
+ "gain_occurrences": 178,
313
+ "cumulative_covered_occurrences": 8701,
314
+ "cumulative_covered_pct": 31.29,
315
+ "new_tags_covered": 16
316
+ },
317
+ {
318
+ "step": 14,
319
+ "group": "romance",
320
+ "gain_occurrences": 140,
321
+ "cumulative_covered_occurrences": 8841,
322
+ "cumulative_covered_pct": 31.8,
323
+ "new_tags_covered": 21
324
+ },
325
+ {
326
+ "step": 15,
327
+ "group": "bondage",
328
+ "gain_occurrences": 124,
329
+ "cumulative_covered_occurrences": 8965,
330
+ "cumulative_covered_pct": 32.24,
331
+ "new_tags_covered": 16
332
+ },
333
+ {
334
+ "step": 16,
335
+ "group": "sound_effects",
336
+ "gain_occurrences": 120,
337
+ "cumulative_covered_occurrences": 9085,
338
+ "cumulative_covered_pct": 32.67,
339
+ "new_tags_covered": 1
340
+ },
341
+ {
342
+ "step": 17,
343
+ "group": "feral_anatomy",
344
+ "gain_occurrences": 114,
345
+ "cumulative_covered_occurrences": 9199,
346
+ "cumulative_covered_pct": 33.08,
347
+ "new_tags_covered": 2
348
+ },
349
+ {
350
+ "step": 18,
351
+ "group": "professions",
352
+ "gain_occurrences": 61,
353
+ "cumulative_covered_occurrences": 9260,
354
+ "cumulative_covered_pct": 33.3,
355
+ "new_tags_covered": 21
356
+ },
357
+ {
358
+ "step": 19,
359
+ "group": "clothes",
360
+ "gain_occurrences": 58,
361
+ "cumulative_covered_occurrences": 9318,
362
+ "cumulative_covered_pct": 33.51,
363
+ "new_tags_covered": 2
364
+ },
365
+ {
366
+ "step": 20,
367
+ "group": "character_relations",
368
+ "gain_occurrences": 51,
369
+ "cumulative_covered_occurrences": 9369,
370
+ "cumulative_covered_pct": 33.69,
371
+ "new_tags_covered": 6
372
+ },
373
+ {
374
+ "step": 21,
375
+ "group": "themes",
376
+ "gain_occurrences": 40,
377
+ "cumulative_covered_occurrences": 9409,
378
+ "cumulative_covered_pct": 33.84,
379
+ "new_tags_covered": 3
380
+ },
381
+ {
382
+ "step": 22,
383
+ "group": "background",
384
+ "gain_occurrences": 31,
385
+ "cumulative_covered_occurrences": 9440,
386
+ "cumulative_covered_pct": 33.95,
387
+ "new_tags_covered": 8
388
+ },
389
+ {
390
+ "step": 23,
391
+ "group": "multiple_tags",
392
+ "gain_occurrences": 21,
393
+ "cumulative_covered_occurrences": 9461,
394
+ "cumulative_covered_pct": 34.03,
395
+ "new_tags_covered": 7
396
+ },
397
+ {
398
+ "step": 24,
399
+ "group": "misc",
400
+ "gain_occurrences": 13,
401
+ "cumulative_covered_occurrences": 9474,
402
+ "cumulative_covered_pct": 34.07,
403
+ "new_tags_covered": 2
404
+ },
405
+ {
406
+ "step": 25,
407
+ "group": "optics",
408
+ "gain_occurrences": 12,
409
+ "cumulative_covered_occurrences": 9486,
410
+ "cumulative_covered_pct": 34.11,
411
+ "new_tags_covered": 1
412
+ }
413
+ ],
414
+ "uncovered_after_top_n_groups": {
415
+ "top_n_groups": 15,
416
+ "selected_groups": [
417
+ "species",
418
+ "genders",
419
+ "body_types",
420
+ "facial_expressions",
421
+ "outfits",
422
+ "pose",
423
+ "art",
424
+ "narrative_elements",
425
+ "nudity",
426
+ "hair",
427
+ "food",
428
+ "settings",
429
+ "piercing",
430
+ "romance",
431
+ "bondage"
432
+ ],
433
+ "n_uncovered_unique_tags": 2663,
434
+ "top_uncovered_tags": [
435
+ {
436
+ "tag": "solo",
437
+ "fluffyrock_count": 2021510,
438
+ "sample_occurrences": 572
439
+ },
440
+ {
441
+ "tag": "breasts",
442
+ "fluffyrock_count": 1503887,
443
+ "sample_occurrences": 155
444
+ },
445
+ {
446
+ "tag": "fur",
447
+ "fluffyrock_count": 1417091,
448
+ "sample_occurrences": 422
449
+ },
450
+ {
451
+ "tag": "duo",
452
+ "fluffyrock_count": 1327850,
453
+ "sample_occurrences": 246
454
+ },
455
+ {
456
+ "tag": "bodily_fluids",
457
+ "fluffyrock_count": 1225999,
458
+ "sample_occurrences": 49
459
+ },
460
+ {
461
+ "tag": "canid",
462
+ "fluffyrock_count": 1141448,
463
+ "sample_occurrences": 304
464
+ },
465
+ {
466
+ "tag": "nipples",
467
+ "fluffyrock_count": 1079208,
468
+ "sample_occurrences": 17
469
+ },
470
+ {
471
+ "tag": "butt",
472
+ "fluffyrock_count": 828096,
473
+ "sample_occurrences": 25
474
+ },
475
+ {
476
+ "tag": "tongue",
477
+ "fluffyrock_count": 725619,
478
+ "sample_occurrences": 109
479
+ },
480
+ {
481
+ "tag": "tail",
482
+ "fluffyrock_count": 686059,
483
+ "sample_occurrences": 110
484
+ },
485
+ {
486
+ "tag": "big_breasts",
487
+ "fluffyrock_count": 682494,
488
+ "sample_occurrences": 41
489
+ },
490
+ {
491
+ "tag": "felid",
492
+ "fluffyrock_count": 598698,
493
+ "sample_occurrences": 158
494
+ },
495
+ {
496
+ "tag": "canis",
497
+ "fluffyrock_count": 587772,
498
+ "sample_occurrences": 159
499
+ },
500
+ {
501
+ "tag": "horn",
502
+ "fluffyrock_count": 568235,
503
+ "sample_occurrences": 154
504
+ },
505
+ {
506
+ "tag": "male/female",
507
+ "fluffyrock_count": 551565,
508
+ "sample_occurrences": 12
509
+ },
510
+ {
511
+ "tag": "teeth",
512
+ "fluffyrock_count": 546600,
513
+ "sample_occurrences": 132
514
+ },
515
+ {
516
+ "tag": "white_body",
517
+ "fluffyrock_count": 541452,
518
+ "sample_occurrences": 171
519
+ },
520
+ {
521
+ "tag": "blue_eyes",
522
+ "fluffyrock_count": 451508,
523
+ "sample_occurrences": 144
524
+ },
525
+ {
526
+ "tag": "feet",
527
+ "fluffyrock_count": 448730,
528
+ "sample_occurrences": 113
529
+ },
530
+ {
531
+ "tag": "pokemon_(species)",
532
+ "fluffyrock_count": 446455,
533
+ "sample_occurrences": 98
534
+ },
535
+ {
536
+ "tag": "claws",
537
+ "fluffyrock_count": 434361,
538
+ "sample_occurrences": 111
539
+ },
540
+ {
541
+ "tag": "male/male",
542
+ "fluffyrock_count": 427119,
543
+ "sample_occurrences": 26
544
+ },
545
+ {
546
+ "tag": "white_fur",
547
+ "fluffyrock_count": 426284,
548
+ "sample_occurrences": 133
549
+ },
550
+ {
551
+ "tag": "topwear",
552
+ "fluffyrock_count": 412942,
553
+ "sample_occurrences": 183
554
+ },
555
+ {
556
+ "tag": "equid",
557
+ "fluffyrock_count": 399764,
558
+ "sample_occurrences": 97
559
+ },
560
+ {
561
+ "tag": "group",
562
+ "fluffyrock_count": 391223,
563
+ "sample_occurrences": 120
564
+ },
565
+ {
566
+ "tag": "fingers",
567
+ "fluffyrock_count": 383191,
568
+ "sample_occurrences": 146
569
+ },
570
+ {
571
+ "tag": "navel",
572
+ "fluffyrock_count": 373746,
573
+ "sample_occurrences": 45
574
+ },
575
+ {
576
+ "tag": "biped",
577
+ "fluffyrock_count": 372051,
578
+ "sample_occurrences": 95
579
+ },
580
+ {
581
+ "tag": "wings",
582
+ "fluffyrock_count": 364542,
583
+ "sample_occurrences": 114
584
+ },
585
+ {
586
+ "tag": "muscular",
587
+ "fluffyrock_count": 358573,
588
+ "sample_occurrences": 56
589
+ },
590
+ {
591
+ "tag": "thick_thighs",
592
+ "fluffyrock_count": 339009,
593
+ "sample_occurrences": 25
594
+ },
595
+ {
596
+ "tag": "fox",
597
+ "fluffyrock_count": 337215,
598
+ "sample_occurrences": 105
599
+ },
600
+ {
601
+ "tag": "tuft",
602
+ "fluffyrock_count": 335534,
603
+ "sample_occurrences": 105
604
+ },
605
+ {
606
+ "tag": "toes",
607
+ "fluffyrock_count": 326285,
608
+ "sample_occurrences": 82
609
+ },
610
+ {
611
+ "tag": "piercing",
612
+ "fluffyrock_count": 321077,
613
+ "sample_occurrences": 56
614
+ },
615
+ {
616
+ "tag": "areola",
617
+ "fluffyrock_count": 320191,
618
+ "sample_occurrences": 1
619
+ },
620
+ {
621
+ "tag": "brown_body",
622
+ "fluffyrock_count": 316754,
623
+ "sample_occurrences": 87
624
+ },
625
+ {
626
+ "tag": "bottomwear",
627
+ "fluffyrock_count": 316556,
628
+ "sample_occurrences": 135
629
+ },
630
+ {
631
+ "tag": "underwear",
632
+ "fluffyrock_count": 316540,
633
+ "sample_occurrences": 35
634
+ },
635
+ {
636
+ "tag": "white_background",
637
+ "fluffyrock_count": 310041,
638
+ "sample_occurrences": 115
639
+ },
640
+ {
641
+ "tag": "dragon",
642
+ "fluffyrock_count": 309478,
643
+ "sample_occurrences": 84
644
+ },
645
+ {
646
+ "tag": "domestic_dog",
647
+ "fluffyrock_count": 300258,
648
+ "sample_occurrences": 77
649
+ },
650
+ {
651
+ "tag": "<3",
652
+ "fluffyrock_count": 293372,
653
+ "sample_occurrences": 50
654
+ },
655
+ {
656
+ "tag": "green_eyes",
657
+ "fluffyrock_count": 287579,
658
+ "sample_occurrences": 78
659
+ },
660
+ {
661
+ "tag": "blue_body",
662
+ "fluffyrock_count": 277335,
663
+ "sample_occurrences": 76
664
+ },
665
+ {
666
+ "tag": "black_body",
667
+ "fluffyrock_count": 273271,
668
+ "sample_occurrences": 78
669
+ },
670
+ {
671
+ "tag": "wolf",
672
+ "fluffyrock_count": 273053,
673
+ "sample_occurrences": 79
674
+ },
675
+ {
676
+ "tag": "plant",
677
+ "fluffyrock_count": 265700,
678
+ "sample_occurrences": 97
679
+ },
680
+ {
681
+ "tag": "size_difference",
682
+ "fluffyrock_count": 265067,
683
+ "sample_occurrences": 34
684
+ },
685
+ {
686
+ "tag": "brown_fur",
687
+ "fluffyrock_count": 264096,
688
+ "sample_occurrences": 72
689
+ },
690
+ {
691
+ "tag": "furniture",
692
+ "fluffyrock_count": 263294,
693
+ "sample_occurrences": 48
694
+ },
695
+ {
696
+ "tag": "big_butt",
697
+ "fluffyrock_count": 262825,
698
+ "sample_occurrences": 11
699
+ },
700
+ {
701
+ "tag": "interspecies",
702
+ "fluffyrock_count": 261649,
703
+ "sample_occurrences": 4
704
+ },
705
+ {
706
+ "tag": "long_hair",
707
+ "fluffyrock_count": 261330,
708
+ "sample_occurrences": 62
709
+ },
710
+ {
711
+ "tag": "huge_breasts",
712
+ "fluffyrock_count": 261003,
713
+ "sample_occurrences": 13
714
+ },
715
+ {
716
+ "tag": "legwear",
717
+ "fluffyrock_count": 254808,
718
+ "sample_occurrences": 37
719
+ },
720
+ {
721
+ "tag": "markings",
722
+ "fluffyrock_count": 253976,
723
+ "sample_occurrences": 61
724
+ },
725
+ {
726
+ "tag": "shirt",
727
+ "fluffyrock_count": 253553,
728
+ "sample_occurrences": 103
729
+ },
730
+ {
731
+ "tag": "outside",
732
+ "fluffyrock_count": 253364,
733
+ "sample_occurrences": 93
734
+ },
735
+ {
736
+ "tag": "eyewear",
737
+ "fluffyrock_count": 251234,
738
+ "sample_occurrences": 93
739
+ },
740
+ {
741
+ "tag": "sweat",
742
+ "fluffyrock_count": 250842,
743
+ "sample_occurrences": 20
744
+ },
745
+ {
746
+ "tag": "felis",
747
+ "fluffyrock_count": 241656,
748
+ "sample_occurrences": 64
749
+ },
750
+ {
751
+ "tag": "young",
752
+ "fluffyrock_count": 238777,
753
+ "sample_occurrences": 40
754
+ },
755
+ {
756
+ "tag": "domestic_cat",
757
+ "fluffyrock_count": 236967,
758
+ "sample_occurrences": 64
759
+ },
760
+ {
761
+ "tag": "headgear",
762
+ "fluffyrock_count": 236145,
763
+ "sample_occurrences": 92
764
+ },
765
+ {
766
+ "tag": "wide_hips",
767
+ "fluffyrock_count": 235877,
768
+ "sample_occurrences": 30
769
+ },
770
+ {
771
+ "tag": "grey_body",
772
+ "fluffyrock_count": 233510,
773
+ "sample_occurrences": 74
774
+ },
775
+ {
776
+ "tag": "paws",
777
+ "fluffyrock_count": 220932,
778
+ "sample_occurrences": 36
779
+ },
780
+ {
781
+ "tag": "5_fingers",
782
+ "fluffyrock_count": 220474,
783
+ "sample_occurrences": 88
784
+ },
785
+ {
786
+ "tag": "black_nose",
787
+ "fluffyrock_count": 216679,
788
+ "sample_occurrences": 80
789
+ },
790
+ {
791
+ "tag": "red_eyes",
792
+ "fluffyrock_count": 213545,
793
+ "sample_occurrences": 53
794
+ },
795
+ {
796
+ "tag": "footwear",
797
+ "fluffyrock_count": 211229,
798
+ "sample_occurrences": 73
799
+ },
800
+ {
801
+ "tag": "reptile",
802
+ "fluffyrock_count": 206968,
803
+ "sample_occurrences": 47
804
+ },
805
+ {
806
+ "tag": "leporid",
807
+ "fluffyrock_count": 204404,
808
+ "sample_occurrences": 73
809
+ },
810
+ {
811
+ "tag": "rabbit",
812
+ "fluffyrock_count": 199906,
813
+ "sample_occurrences": 73
814
+ },
815
+ {
816
+ "tag": "black_fur",
817
+ "fluffyrock_count": 197688,
818
+ "sample_occurrences": 60
819
+ },
820
+ {
821
+ "tag": "belly",
822
+ "fluffyrock_count": 194310,
823
+ "sample_occurrences": 31
824
+ },
825
+ {
826
+ "tag": "eyelashes",
827
+ "fluffyrock_count": 193540,
828
+ "sample_occurrences": 63
829
+ },
830
+ {
831
+ "tag": "horse",
832
+ "fluffyrock_count": 193042,
833
+ "sample_occurrences": 40
834
+ },
835
+ {
836
+ "tag": "feathers",
837
+ "fluffyrock_count": 192797,
838
+ "sample_occurrences": 76
839
+ },
840
+ {
841
+ "tag": "pawpads",
842
+ "fluffyrock_count": 191398,
843
+ "sample_occurrences": 24
844
+ },
845
+ {
846
+ "tag": "solo_focus",
847
+ "fluffyrock_count": 191284,
848
+ "sample_occurrences": 18
849
+ },
850
+ {
851
+ "tag": "fangs",
852
+ "fluffyrock_count": 190633,
853
+ "sample_occurrences": 49
854
+ },
855
+ {
856
+ "tag": "saliva",
857
+ "fluffyrock_count": 187352,
858
+ "sample_occurrences": 10
859
+ },
860
+ {
861
+ "tag": "brown_hair",
862
+ "fluffyrock_count": 183486,
863
+ "sample_occurrences": 48
864
+ },
865
+ {
866
+ "tag": "blonde_hair",
867
+ "fluffyrock_count": 183468,
868
+ "sample_occurrences": 47
869
+ },
870
+ {
871
+ "tag": "pantherine",
872
+ "fluffyrock_count": 181019,
873
+ "sample_occurrences": 45
874
+ },
875
+ {
876
+ "tag": "black_hair",
877
+ "fluffyrock_count": 180936,
878
+ "sample_occurrences": 45
879
+ },
880
+ {
881
+ "tag": "yellow_body",
882
+ "fluffyrock_count": 180032,
883
+ "sample_occurrences": 46
884
+ },
885
+ {
886
+ "tag": "glasses",
887
+ "fluffyrock_count": 177552,
888
+ "sample_occurrences": 49
889
+ },
890
+ {
891
+ "tag": "headwear",
892
+ "fluffyrock_count": 177119,
893
+ "sample_occurrences": 69
894
+ },
895
+ {
896
+ "tag": "overweight",
897
+ "fluffyrock_count": 176776,
898
+ "sample_occurrences": 36
899
+ },
900
+ {
901
+ "tag": "grey_fur",
902
+ "fluffyrock_count": 175923,
903
+ "sample_occurrences": 61
904
+ },
905
+ {
906
+ "tag": "abs",
907
+ "fluffyrock_count": 171761,
908
+ "sample_occurrences": 23
909
+ },
910
+ {
911
+ "tag": "eyebrows",
912
+ "fluffyrock_count": 170102,
913
+ "sample_occurrences": 50
914
+ },
915
+ {
916
+ "tag": "vein",
917
+ "fluffyrock_count": 168934,
918
+ "sample_occurrences": 5
919
+ },
920
+ {
921
+ "tag": "yellow_eyes",
922
+ "fluffyrock_count": 168780,
923
+ "sample_occurrences": 45
924
+ },
925
+ {
926
+ "tag": "hat",
927
+ "fluffyrock_count": 166176,
928
+ "sample_occurrences": 67
929
+ },
930
+ {
931
+ "tag": "muscular_anthro",
932
+ "fluffyrock_count": 164876,
933
+ "sample_occurrences": 30
934
+ },
935
+ {
936
+ "tag": "handwear",
937
+ "fluffyrock_count": 162603,
938
+ "sample_occurrences": 56
939
+ },
940
+ {
941
+ "tag": "not_furry",
942
+ "fluffyrock_count": 160518,
943
+ "sample_occurrences": 30
944
+ },
945
+ {
946
+ "tag": "bed",
947
+ "fluffyrock_count": 159493,
948
+ "sample_occurrences": 25
949
+ },
950
+ {
951
+ "tag": "half-closed_eyes",
952
+ "fluffyrock_count": 158388,
953
+ "sample_occurrences": 19
954
+ },
955
+ {
956
+ "tag": "pecs",
957
+ "fluffyrock_count": 158385,
958
+ "sample_occurrences": 23
959
+ },
960
+ {
961
+ "tag": "gloves",
962
+ "fluffyrock_count": 156718,
963
+ "sample_occurrences": 52
964
+ },
965
+ {
966
+ "tag": "bovid",
967
+ "fluffyrock_count": 156674,
968
+ "sample_occurrences": 35
969
+ },
970
+ {
971
+ "tag": "orange_body",
972
+ "fluffyrock_count": 153299,
973
+ "sample_occurrences": 47
974
+ },
975
+ {
976
+ "tag": "blue_fur",
977
+ "fluffyrock_count": 153027,
978
+ "sample_occurrences": 48
979
+ },
980
+ {
981
+ "tag": "barefoot",
982
+ "fluffyrock_count": 148056,
983
+ "sample_occurrences": 46
984
+ },
985
+ {
986
+ "tag": "cub",
987
+ "fluffyrock_count": 147547,
988
+ "sample_occurrences": 21
989
+ },
990
+ {
991
+ "tag": "inside",
992
+ "fluffyrock_count": 145110,
993
+ "sample_occurrences": 30
994
+ },
995
+ {
996
+ "tag": "panties",
997
+ "fluffyrock_count": 142486,
998
+ "sample_occurrences": 13
999
+ },
1000
+ {
1001
+ "tag": "two_tone_body",
1002
+ "fluffyrock_count": 140258,
1003
+ "sample_occurrences": 28
1004
+ },
1005
+ {
1006
+ "tag": "anthrofied",
1007
+ "fluffyrock_count": 139874,
1008
+ "sample_occurrences": 11
1009
+ },
1010
+ {
1011
+ "tag": "white_hair",
1012
+ "fluffyrock_count": 139416,
1013
+ "sample_occurrences": 37
1014
+ },
1015
+ {
1016
+ "tag": "animal_humanoid",
1017
+ "fluffyrock_count": 139252,
1018
+ "sample_occurrences": 31
1019
+ },
1020
+ {
1021
+ "tag": "blue_hair",
1022
+ "fluffyrock_count": 138674,
1023
+ "sample_occurrences": 48
1024
+ },
1025
+ {
1026
+ "tag": "holding_object",
1027
+ "fluffyrock_count": 137895,
1028
+ "sample_occurrences": 84
1029
+ },
1030
+ {
1031
+ "tag": "hyper",
1032
+ "fluffyrock_count": 137712,
1033
+ "sample_occurrences": 2
1034
+ }
1035
+ ]
1036
+ },
1037
+ "outputs": {
1038
+ "ranked_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_ranked.csv",
1039
+ "uncovered_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_uncovered_after_topn.csv",
1040
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_summary.json"
1041
+ }
1042
+ }
data/analysis/tag_group_coverage_summary_baseline200.json ADDED
@@ -0,0 +1,1045 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "sample_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\eval_samples\\e621_sfw_sample_1000_seed123_buffer10000.jsonl",
3
+ "min_count": 200,
4
+ "n_images": 1000,
5
+ "n_groups": 46,
6
+ "n_wiki_groups": 46,
7
+ "n_category_groups_added": 0,
8
+ "include_category_groups": false,
9
+ "total_tag_occurrences": 27806,
10
+ "n_unique_tags": 3068,
11
+ "n_unique_tags_covered_by_any_group": 470,
12
+ "unique_tag_coverage_by_any_group_pct": 15.32,
13
+ "top_groups_by_occurrence_coverage": [
14
+ {
15
+ "group": "species",
16
+ "group_size": 100,
17
+ "covered_occurrences": 1741,
18
+ "covered_occ_pct": 6.26,
19
+ "covered_unique_tags": 41,
20
+ "covered_unique_pct": 1.34,
21
+ "covered_images": 878,
22
+ "covered_images_pct": 87.8
23
+ },
24
+ {
25
+ "group": "genders",
26
+ "group_size": 336,
27
+ "covered_occurrences": 1209,
28
+ "covered_occ_pct": 4.35,
29
+ "covered_unique_tags": 23,
30
+ "covered_unique_pct": 0.75,
31
+ "covered_images": 927,
32
+ "covered_images_pct": 92.7
33
+ },
34
+ {
35
+ "group": "body_types",
36
+ "group_size": 22,
37
+ "covered_occurrences": 1035,
38
+ "covered_occ_pct": 3.72,
39
+ "covered_unique_tags": 9,
40
+ "covered_unique_pct": 0.29,
41
+ "covered_images": 909,
42
+ "covered_images_pct": 90.9
43
+ },
44
+ {
45
+ "group": "facial_expressions",
46
+ "group_size": 86,
47
+ "covered_occurrences": 927,
48
+ "covered_occ_pct": 3.33,
49
+ "covered_unique_tags": 43,
50
+ "covered_unique_pct": 1.4,
51
+ "covered_images": 515,
52
+ "covered_images_pct": 51.5
53
+ },
54
+ {
55
+ "group": "outfits",
56
+ "group_size": 55,
57
+ "covered_occurrences": 644,
58
+ "covered_occ_pct": 2.32,
59
+ "covered_unique_tags": 14,
60
+ "covered_unique_pct": 0.46,
61
+ "covered_images": 576,
62
+ "covered_images_pct": 57.6
63
+ },
64
+ {
65
+ "group": "pose",
66
+ "group_size": 189,
67
+ "covered_occurrences": 622,
68
+ "covered_occ_pct": 2.24,
69
+ "covered_unique_tags": 66,
70
+ "covered_unique_pct": 2.15,
71
+ "covered_images": 417,
72
+ "covered_images_pct": 41.7
73
+ },
74
+ {
75
+ "group": "romance",
76
+ "group_size": 59,
77
+ "covered_occurrences": 555,
78
+ "covered_occ_pct": 2.0,
79
+ "covered_unique_tags": 28,
80
+ "covered_unique_pct": 0.91,
81
+ "covered_images": 357,
82
+ "covered_images_pct": 35.7
83
+ },
84
+ {
85
+ "group": "narrative_elements",
86
+ "group_size": 113,
87
+ "covered_occurrences": 536,
88
+ "covered_occ_pct": 1.93,
89
+ "covered_unique_tags": 26,
90
+ "covered_unique_pct": 0.85,
91
+ "covered_images": 317,
92
+ "covered_images_pct": 31.7
93
+ },
94
+ {
95
+ "group": "art",
96
+ "group_size": 111,
97
+ "covered_occurrences": 536,
98
+ "covered_occ_pct": 1.93,
99
+ "covered_unique_tags": 21,
100
+ "covered_unique_pct": 0.68,
101
+ "covered_images": 477,
102
+ "covered_images_pct": 47.7
103
+ },
104
+ {
105
+ "group": "nudity",
106
+ "group_size": 146,
107
+ "covered_occurrences": 501,
108
+ "covered_occ_pct": 1.8,
109
+ "covered_unique_tags": 21,
110
+ "covered_unique_pct": 0.68,
111
+ "covered_images": 396,
112
+ "covered_images_pct": 39.6
113
+ },
114
+ {
115
+ "group": "background",
116
+ "group_size": 19,
117
+ "covered_occurrences": 443,
118
+ "covered_occ_pct": 1.59,
119
+ "covered_unique_tags": 10,
120
+ "covered_unique_pct": 0.33,
121
+ "covered_images": 418,
122
+ "covered_images_pct": 41.8
123
+ },
124
+ {
125
+ "group": "language",
126
+ "group_size": 13,
127
+ "covered_occurrences": 438,
128
+ "covered_occ_pct": 1.58,
129
+ "covered_unique_tags": 7,
130
+ "covered_unique_pct": 0.23,
131
+ "covered_images": 278,
132
+ "covered_images_pct": 27.8
133
+ },
134
+ {
135
+ "group": "hair",
136
+ "group_size": 22,
137
+ "covered_occurrences": 373,
138
+ "covered_occ_pct": 1.34,
139
+ "covered_unique_tags": 1,
140
+ "covered_unique_pct": 0.03,
141
+ "covered_images": 373,
142
+ "covered_images_pct": 37.3
143
+ },
144
+ {
145
+ "group": "food",
146
+ "group_size": 360,
147
+ "covered_occurrences": 203,
148
+ "covered_occ_pct": 0.73,
149
+ "covered_unique_tags": 48,
150
+ "covered_unique_pct": 1.56,
151
+ "covered_images": 104,
152
+ "covered_images_pct": 10.4
153
+ },
154
+ {
155
+ "group": "settings",
156
+ "group_size": 186,
157
+ "covered_occurrences": 196,
158
+ "covered_occ_pct": 0.7,
159
+ "covered_unique_tags": 39,
160
+ "covered_unique_pct": 1.27,
161
+ "covered_images": 125,
162
+ "covered_images_pct": 12.5
163
+ },
164
+ {
165
+ "group": "piercing",
166
+ "group_size": 99,
167
+ "covered_occurrences": 178,
168
+ "covered_occ_pct": 0.64,
169
+ "covered_unique_tags": 16,
170
+ "covered_unique_pct": 0.52,
171
+ "covered_images": 97,
172
+ "covered_images_pct": 9.7
173
+ },
174
+ {
175
+ "group": "unknown_tags",
176
+ "group_size": 20,
177
+ "covered_occurrences": 172,
178
+ "covered_occ_pct": 0.62,
179
+ "covered_unique_tags": 2,
180
+ "covered_unique_pct": 0.07,
181
+ "covered_images": 170,
182
+ "covered_images_pct": 17.0
183
+ },
184
+ {
185
+ "group": "ambiguous_tags",
186
+ "group_size": 18,
187
+ "covered_occurrences": 169,
188
+ "covered_occ_pct": 0.61,
189
+ "covered_unique_tags": 3,
190
+ "covered_unique_pct": 0.1,
191
+ "covered_images": 167,
192
+ "covered_images_pct": 16.7
193
+ },
194
+ {
195
+ "group": "bondage",
196
+ "group_size": 89,
197
+ "covered_occurrences": 124,
198
+ "covered_occ_pct": 0.45,
199
+ "covered_unique_tags": 16,
200
+ "covered_unique_pct": 0.52,
201
+ "covered_images": 98,
202
+ "covered_images_pct": 9.8
203
+ },
204
+ {
205
+ "group": "sound_effects",
206
+ "group_size": 7,
207
+ "covered_occurrences": 120,
208
+ "covered_occ_pct": 0.43,
209
+ "covered_unique_tags": 1,
210
+ "covered_unique_pct": 0.03,
211
+ "covered_images": 120,
212
+ "covered_images_pct": 12.0
213
+ }
214
+ ],
215
+ "greedy_cover_steps": [
216
+ {
217
+ "step": 1,
218
+ "group": "species",
219
+ "gain_occurrences": 1741,
220
+ "cumulative_covered_occurrences": 1741,
221
+ "cumulative_covered_pct": 6.26,
222
+ "new_tags_covered": 41
223
+ },
224
+ {
225
+ "step": 2,
226
+ "group": "genders",
227
+ "gain_occurrences": 1209,
228
+ "cumulative_covered_occurrences": 2950,
229
+ "cumulative_covered_pct": 10.61,
230
+ "new_tags_covered": 23
231
+ },
232
+ {
233
+ "step": 3,
234
+ "group": "body_types",
235
+ "gain_occurrences": 1035,
236
+ "cumulative_covered_occurrences": 3985,
237
+ "cumulative_covered_pct": 14.33,
238
+ "new_tags_covered": 9
239
+ },
240
+ {
241
+ "step": 4,
242
+ "group": "facial_expressions",
243
+ "gain_occurrences": 927,
244
+ "cumulative_covered_occurrences": 4912,
245
+ "cumulative_covered_pct": 17.67,
246
+ "new_tags_covered": 43
247
+ },
248
+ {
249
+ "step": 5,
250
+ "group": "outfits",
251
+ "gain_occurrences": 644,
252
+ "cumulative_covered_occurrences": 5556,
253
+ "cumulative_covered_pct": 19.98,
254
+ "new_tags_covered": 14
255
+ },
256
+ {
257
+ "step": 6,
258
+ "group": "pose",
259
+ "gain_occurrences": 622,
260
+ "cumulative_covered_occurrences": 6178,
261
+ "cumulative_covered_pct": 22.22,
262
+ "new_tags_covered": 66
263
+ },
264
+ {
265
+ "step": 7,
266
+ "group": "art",
267
+ "gain_occurrences": 536,
268
+ "cumulative_covered_occurrences": 6714,
269
+ "cumulative_covered_pct": 24.15,
270
+ "new_tags_covered": 21
271
+ },
272
+ {
273
+ "step": 8,
274
+ "group": "narrative_elements",
275
+ "gain_occurrences": 536,
276
+ "cumulative_covered_occurrences": 7250,
277
+ "cumulative_covered_pct": 26.07,
278
+ "new_tags_covered": 26
279
+ },
280
+ {
281
+ "step": 9,
282
+ "group": "nudity",
283
+ "gain_occurrences": 501,
284
+ "cumulative_covered_occurrences": 7751,
285
+ "cumulative_covered_pct": 27.88,
286
+ "new_tags_covered": 21
287
+ },
288
+ {
289
+ "step": 10,
290
+ "group": "hair",
291
+ "gain_occurrences": 373,
292
+ "cumulative_covered_occurrences": 8124,
293
+ "cumulative_covered_pct": 29.22,
294
+ "new_tags_covered": 1
295
+ },
296
+ {
297
+ "step": 11,
298
+ "group": "food",
299
+ "gain_occurrences": 203,
300
+ "cumulative_covered_occurrences": 8327,
301
+ "cumulative_covered_pct": 29.95,
302
+ "new_tags_covered": 48
303
+ },
304
+ {
305
+ "step": 12,
306
+ "group": "settings",
307
+ "gain_occurrences": 196,
308
+ "cumulative_covered_occurrences": 8523,
309
+ "cumulative_covered_pct": 30.65,
310
+ "new_tags_covered": 39
311
+ },
312
+ {
313
+ "step": 13,
314
+ "group": "piercing",
315
+ "gain_occurrences": 178,
316
+ "cumulative_covered_occurrences": 8701,
317
+ "cumulative_covered_pct": 31.29,
318
+ "new_tags_covered": 16
319
+ },
320
+ {
321
+ "step": 14,
322
+ "group": "romance",
323
+ "gain_occurrences": 140,
324
+ "cumulative_covered_occurrences": 8841,
325
+ "cumulative_covered_pct": 31.8,
326
+ "new_tags_covered": 21
327
+ },
328
+ {
329
+ "step": 15,
330
+ "group": "bondage",
331
+ "gain_occurrences": 124,
332
+ "cumulative_covered_occurrences": 8965,
333
+ "cumulative_covered_pct": 32.24,
334
+ "new_tags_covered": 16
335
+ },
336
+ {
337
+ "step": 16,
338
+ "group": "sound_effects",
339
+ "gain_occurrences": 120,
340
+ "cumulative_covered_occurrences": 9085,
341
+ "cumulative_covered_pct": 32.67,
342
+ "new_tags_covered": 1
343
+ },
344
+ {
345
+ "step": 17,
346
+ "group": "feral_anatomy",
347
+ "gain_occurrences": 114,
348
+ "cumulative_covered_occurrences": 9199,
349
+ "cumulative_covered_pct": 33.08,
350
+ "new_tags_covered": 2
351
+ },
352
+ {
353
+ "step": 18,
354
+ "group": "professions",
355
+ "gain_occurrences": 61,
356
+ "cumulative_covered_occurrences": 9260,
357
+ "cumulative_covered_pct": 33.3,
358
+ "new_tags_covered": 21
359
+ },
360
+ {
361
+ "step": 19,
362
+ "group": "clothes",
363
+ "gain_occurrences": 58,
364
+ "cumulative_covered_occurrences": 9318,
365
+ "cumulative_covered_pct": 33.51,
366
+ "new_tags_covered": 2
367
+ },
368
+ {
369
+ "step": 20,
370
+ "group": "character_relations",
371
+ "gain_occurrences": 51,
372
+ "cumulative_covered_occurrences": 9369,
373
+ "cumulative_covered_pct": 33.69,
374
+ "new_tags_covered": 6
375
+ },
376
+ {
377
+ "step": 21,
378
+ "group": "themes",
379
+ "gain_occurrences": 40,
380
+ "cumulative_covered_occurrences": 9409,
381
+ "cumulative_covered_pct": 33.84,
382
+ "new_tags_covered": 3
383
+ },
384
+ {
385
+ "step": 22,
386
+ "group": "background",
387
+ "gain_occurrences": 31,
388
+ "cumulative_covered_occurrences": 9440,
389
+ "cumulative_covered_pct": 33.95,
390
+ "new_tags_covered": 8
391
+ },
392
+ {
393
+ "step": 23,
394
+ "group": "multiple_tags",
395
+ "gain_occurrences": 21,
396
+ "cumulative_covered_occurrences": 9461,
397
+ "cumulative_covered_pct": 34.03,
398
+ "new_tags_covered": 7
399
+ },
400
+ {
401
+ "step": 24,
402
+ "group": "misc",
403
+ "gain_occurrences": 13,
404
+ "cumulative_covered_occurrences": 9474,
405
+ "cumulative_covered_pct": 34.07,
406
+ "new_tags_covered": 2
407
+ },
408
+ {
409
+ "step": 25,
410
+ "group": "optics",
411
+ "gain_occurrences": 12,
412
+ "cumulative_covered_occurrences": 9486,
413
+ "cumulative_covered_pct": 34.11,
414
+ "new_tags_covered": 1
415
+ }
416
+ ],
417
+ "uncovered_after_top_n_groups": {
418
+ "top_n_groups": 15,
419
+ "selected_groups": [
420
+ "species",
421
+ "genders",
422
+ "body_types",
423
+ "facial_expressions",
424
+ "outfits",
425
+ "pose",
426
+ "art",
427
+ "narrative_elements",
428
+ "nudity",
429
+ "hair",
430
+ "food",
431
+ "settings",
432
+ "piercing",
433
+ "romance",
434
+ "bondage"
435
+ ],
436
+ "n_uncovered_unique_tags": 2663,
437
+ "top_uncovered_tags": [
438
+ {
439
+ "tag": "solo",
440
+ "fluffyrock_count": 2021510,
441
+ "sample_occurrences": 572
442
+ },
443
+ {
444
+ "tag": "breasts",
445
+ "fluffyrock_count": 1503887,
446
+ "sample_occurrences": 155
447
+ },
448
+ {
449
+ "tag": "fur",
450
+ "fluffyrock_count": 1417091,
451
+ "sample_occurrences": 422
452
+ },
453
+ {
454
+ "tag": "duo",
455
+ "fluffyrock_count": 1327850,
456
+ "sample_occurrences": 246
457
+ },
458
+ {
459
+ "tag": "bodily_fluids",
460
+ "fluffyrock_count": 1225999,
461
+ "sample_occurrences": 49
462
+ },
463
+ {
464
+ "tag": "canid",
465
+ "fluffyrock_count": 1141448,
466
+ "sample_occurrences": 304
467
+ },
468
+ {
469
+ "tag": "nipples",
470
+ "fluffyrock_count": 1079208,
471
+ "sample_occurrences": 17
472
+ },
473
+ {
474
+ "tag": "butt",
475
+ "fluffyrock_count": 828096,
476
+ "sample_occurrences": 25
477
+ },
478
+ {
479
+ "tag": "tongue",
480
+ "fluffyrock_count": 725619,
481
+ "sample_occurrences": 109
482
+ },
483
+ {
484
+ "tag": "tail",
485
+ "fluffyrock_count": 686059,
486
+ "sample_occurrences": 110
487
+ },
488
+ {
489
+ "tag": "big_breasts",
490
+ "fluffyrock_count": 682494,
491
+ "sample_occurrences": 41
492
+ },
493
+ {
494
+ "tag": "felid",
495
+ "fluffyrock_count": 598698,
496
+ "sample_occurrences": 158
497
+ },
498
+ {
499
+ "tag": "canis",
500
+ "fluffyrock_count": 587772,
501
+ "sample_occurrences": 159
502
+ },
503
+ {
504
+ "tag": "horn",
505
+ "fluffyrock_count": 568235,
506
+ "sample_occurrences": 154
507
+ },
508
+ {
509
+ "tag": "male/female",
510
+ "fluffyrock_count": 551565,
511
+ "sample_occurrences": 12
512
+ },
513
+ {
514
+ "tag": "teeth",
515
+ "fluffyrock_count": 546600,
516
+ "sample_occurrences": 132
517
+ },
518
+ {
519
+ "tag": "white_body",
520
+ "fluffyrock_count": 541452,
521
+ "sample_occurrences": 171
522
+ },
523
+ {
524
+ "tag": "blue_eyes",
525
+ "fluffyrock_count": 451508,
526
+ "sample_occurrences": 144
527
+ },
528
+ {
529
+ "tag": "feet",
530
+ "fluffyrock_count": 448730,
531
+ "sample_occurrences": 113
532
+ },
533
+ {
534
+ "tag": "pokemon_(species)",
535
+ "fluffyrock_count": 446455,
536
+ "sample_occurrences": 98
537
+ },
538
+ {
539
+ "tag": "claws",
540
+ "fluffyrock_count": 434361,
541
+ "sample_occurrences": 111
542
+ },
543
+ {
544
+ "tag": "male/male",
545
+ "fluffyrock_count": 427119,
546
+ "sample_occurrences": 26
547
+ },
548
+ {
549
+ "tag": "white_fur",
550
+ "fluffyrock_count": 426284,
551
+ "sample_occurrences": 133
552
+ },
553
+ {
554
+ "tag": "topwear",
555
+ "fluffyrock_count": 412942,
556
+ "sample_occurrences": 183
557
+ },
558
+ {
559
+ "tag": "equid",
560
+ "fluffyrock_count": 399764,
561
+ "sample_occurrences": 97
562
+ },
563
+ {
564
+ "tag": "group",
565
+ "fluffyrock_count": 391223,
566
+ "sample_occurrences": 120
567
+ },
568
+ {
569
+ "tag": "fingers",
570
+ "fluffyrock_count": 383191,
571
+ "sample_occurrences": 146
572
+ },
573
+ {
574
+ "tag": "navel",
575
+ "fluffyrock_count": 373746,
576
+ "sample_occurrences": 45
577
+ },
578
+ {
579
+ "tag": "biped",
580
+ "fluffyrock_count": 372051,
581
+ "sample_occurrences": 95
582
+ },
583
+ {
584
+ "tag": "wings",
585
+ "fluffyrock_count": 364542,
586
+ "sample_occurrences": 114
587
+ },
588
+ {
589
+ "tag": "muscular",
590
+ "fluffyrock_count": 358573,
591
+ "sample_occurrences": 56
592
+ },
593
+ {
594
+ "tag": "thick_thighs",
595
+ "fluffyrock_count": 339009,
596
+ "sample_occurrences": 25
597
+ },
598
+ {
599
+ "tag": "fox",
600
+ "fluffyrock_count": 337215,
601
+ "sample_occurrences": 105
602
+ },
603
+ {
604
+ "tag": "tuft",
605
+ "fluffyrock_count": 335534,
606
+ "sample_occurrences": 105
607
+ },
608
+ {
609
+ "tag": "toes",
610
+ "fluffyrock_count": 326285,
611
+ "sample_occurrences": 82
612
+ },
613
+ {
614
+ "tag": "piercing",
615
+ "fluffyrock_count": 321077,
616
+ "sample_occurrences": 56
617
+ },
618
+ {
619
+ "tag": "areola",
620
+ "fluffyrock_count": 320191,
621
+ "sample_occurrences": 1
622
+ },
623
+ {
624
+ "tag": "brown_body",
625
+ "fluffyrock_count": 316754,
626
+ "sample_occurrences": 87
627
+ },
628
+ {
629
+ "tag": "bottomwear",
630
+ "fluffyrock_count": 316556,
631
+ "sample_occurrences": 135
632
+ },
633
+ {
634
+ "tag": "underwear",
635
+ "fluffyrock_count": 316540,
636
+ "sample_occurrences": 35
637
+ },
638
+ {
639
+ "tag": "white_background",
640
+ "fluffyrock_count": 310041,
641
+ "sample_occurrences": 115
642
+ },
643
+ {
644
+ "tag": "dragon",
645
+ "fluffyrock_count": 309478,
646
+ "sample_occurrences": 84
647
+ },
648
+ {
649
+ "tag": "domestic_dog",
650
+ "fluffyrock_count": 300258,
651
+ "sample_occurrences": 77
652
+ },
653
+ {
654
+ "tag": "<3",
655
+ "fluffyrock_count": 293372,
656
+ "sample_occurrences": 50
657
+ },
658
+ {
659
+ "tag": "green_eyes",
660
+ "fluffyrock_count": 287579,
661
+ "sample_occurrences": 78
662
+ },
663
+ {
664
+ "tag": "blue_body",
665
+ "fluffyrock_count": 277335,
666
+ "sample_occurrences": 76
667
+ },
668
+ {
669
+ "tag": "black_body",
670
+ "fluffyrock_count": 273271,
671
+ "sample_occurrences": 78
672
+ },
673
+ {
674
+ "tag": "wolf",
675
+ "fluffyrock_count": 273053,
676
+ "sample_occurrences": 79
677
+ },
678
+ {
679
+ "tag": "plant",
680
+ "fluffyrock_count": 265700,
681
+ "sample_occurrences": 97
682
+ },
683
+ {
684
+ "tag": "size_difference",
685
+ "fluffyrock_count": 265067,
686
+ "sample_occurrences": 34
687
+ },
688
+ {
689
+ "tag": "brown_fur",
690
+ "fluffyrock_count": 264096,
691
+ "sample_occurrences": 72
692
+ },
693
+ {
694
+ "tag": "furniture",
695
+ "fluffyrock_count": 263294,
696
+ "sample_occurrences": 48
697
+ },
698
+ {
699
+ "tag": "big_butt",
700
+ "fluffyrock_count": 262825,
701
+ "sample_occurrences": 11
702
+ },
703
+ {
704
+ "tag": "interspecies",
705
+ "fluffyrock_count": 261649,
706
+ "sample_occurrences": 4
707
+ },
708
+ {
709
+ "tag": "long_hair",
710
+ "fluffyrock_count": 261330,
711
+ "sample_occurrences": 62
712
+ },
713
+ {
714
+ "tag": "huge_breasts",
715
+ "fluffyrock_count": 261003,
716
+ "sample_occurrences": 13
717
+ },
718
+ {
719
+ "tag": "legwear",
720
+ "fluffyrock_count": 254808,
721
+ "sample_occurrences": 37
722
+ },
723
+ {
724
+ "tag": "markings",
725
+ "fluffyrock_count": 253976,
726
+ "sample_occurrences": 61
727
+ },
728
+ {
729
+ "tag": "shirt",
730
+ "fluffyrock_count": 253553,
731
+ "sample_occurrences": 103
732
+ },
733
+ {
734
+ "tag": "outside",
735
+ "fluffyrock_count": 253364,
736
+ "sample_occurrences": 93
737
+ },
738
+ {
739
+ "tag": "eyewear",
740
+ "fluffyrock_count": 251234,
741
+ "sample_occurrences": 93
742
+ },
743
+ {
744
+ "tag": "sweat",
745
+ "fluffyrock_count": 250842,
746
+ "sample_occurrences": 20
747
+ },
748
+ {
749
+ "tag": "felis",
750
+ "fluffyrock_count": 241656,
751
+ "sample_occurrences": 64
752
+ },
753
+ {
754
+ "tag": "young",
755
+ "fluffyrock_count": 238777,
756
+ "sample_occurrences": 40
757
+ },
758
+ {
759
+ "tag": "domestic_cat",
760
+ "fluffyrock_count": 236967,
761
+ "sample_occurrences": 64
762
+ },
763
+ {
764
+ "tag": "headgear",
765
+ "fluffyrock_count": 236145,
766
+ "sample_occurrences": 92
767
+ },
768
+ {
769
+ "tag": "wide_hips",
770
+ "fluffyrock_count": 235877,
771
+ "sample_occurrences": 30
772
+ },
773
+ {
774
+ "tag": "grey_body",
775
+ "fluffyrock_count": 233510,
776
+ "sample_occurrences": 74
777
+ },
778
+ {
779
+ "tag": "paws",
780
+ "fluffyrock_count": 220932,
781
+ "sample_occurrences": 36
782
+ },
783
+ {
784
+ "tag": "5_fingers",
785
+ "fluffyrock_count": 220474,
786
+ "sample_occurrences": 88
787
+ },
788
+ {
789
+ "tag": "black_nose",
790
+ "fluffyrock_count": 216679,
791
+ "sample_occurrences": 80
792
+ },
793
+ {
794
+ "tag": "red_eyes",
795
+ "fluffyrock_count": 213545,
796
+ "sample_occurrences": 53
797
+ },
798
+ {
799
+ "tag": "footwear",
800
+ "fluffyrock_count": 211229,
801
+ "sample_occurrences": 73
802
+ },
803
+ {
804
+ "tag": "reptile",
805
+ "fluffyrock_count": 206968,
806
+ "sample_occurrences": 47
807
+ },
808
+ {
809
+ "tag": "leporid",
810
+ "fluffyrock_count": 204404,
811
+ "sample_occurrences": 73
812
+ },
813
+ {
814
+ "tag": "rabbit",
815
+ "fluffyrock_count": 199906,
816
+ "sample_occurrences": 73
817
+ },
818
+ {
819
+ "tag": "black_fur",
820
+ "fluffyrock_count": 197688,
821
+ "sample_occurrences": 60
822
+ },
823
+ {
824
+ "tag": "belly",
825
+ "fluffyrock_count": 194310,
826
+ "sample_occurrences": 31
827
+ },
828
+ {
829
+ "tag": "eyelashes",
830
+ "fluffyrock_count": 193540,
831
+ "sample_occurrences": 63
832
+ },
833
+ {
834
+ "tag": "horse",
835
+ "fluffyrock_count": 193042,
836
+ "sample_occurrences": 40
837
+ },
838
+ {
839
+ "tag": "feathers",
840
+ "fluffyrock_count": 192797,
841
+ "sample_occurrences": 76
842
+ },
843
+ {
844
+ "tag": "pawpads",
845
+ "fluffyrock_count": 191398,
846
+ "sample_occurrences": 24
847
+ },
848
+ {
849
+ "tag": "solo_focus",
850
+ "fluffyrock_count": 191284,
851
+ "sample_occurrences": 18
852
+ },
853
+ {
854
+ "tag": "fangs",
855
+ "fluffyrock_count": 190633,
856
+ "sample_occurrences": 49
857
+ },
858
+ {
859
+ "tag": "saliva",
860
+ "fluffyrock_count": 187352,
861
+ "sample_occurrences": 10
862
+ },
863
+ {
864
+ "tag": "brown_hair",
865
+ "fluffyrock_count": 183486,
866
+ "sample_occurrences": 48
867
+ },
868
+ {
869
+ "tag": "blonde_hair",
870
+ "fluffyrock_count": 183468,
871
+ "sample_occurrences": 47
872
+ },
873
+ {
874
+ "tag": "pantherine",
875
+ "fluffyrock_count": 181019,
876
+ "sample_occurrences": 45
877
+ },
878
+ {
879
+ "tag": "black_hair",
880
+ "fluffyrock_count": 180936,
881
+ "sample_occurrences": 45
882
+ },
883
+ {
884
+ "tag": "yellow_body",
885
+ "fluffyrock_count": 180032,
886
+ "sample_occurrences": 46
887
+ },
888
+ {
889
+ "tag": "glasses",
890
+ "fluffyrock_count": 177552,
891
+ "sample_occurrences": 49
892
+ },
893
+ {
894
+ "tag": "headwear",
895
+ "fluffyrock_count": 177119,
896
+ "sample_occurrences": 69
897
+ },
898
+ {
899
+ "tag": "overweight",
900
+ "fluffyrock_count": 176776,
901
+ "sample_occurrences": 36
902
+ },
903
+ {
904
+ "tag": "grey_fur",
905
+ "fluffyrock_count": 175923,
906
+ "sample_occurrences": 61
907
+ },
908
+ {
909
+ "tag": "abs",
910
+ "fluffyrock_count": 171761,
911
+ "sample_occurrences": 23
912
+ },
913
+ {
914
+ "tag": "eyebrows",
915
+ "fluffyrock_count": 170102,
916
+ "sample_occurrences": 50
917
+ },
918
+ {
919
+ "tag": "vein",
920
+ "fluffyrock_count": 168934,
921
+ "sample_occurrences": 5
922
+ },
923
+ {
924
+ "tag": "yellow_eyes",
925
+ "fluffyrock_count": 168780,
926
+ "sample_occurrences": 45
927
+ },
928
+ {
929
+ "tag": "hat",
930
+ "fluffyrock_count": 166176,
931
+ "sample_occurrences": 67
932
+ },
933
+ {
934
+ "tag": "muscular_anthro",
935
+ "fluffyrock_count": 164876,
936
+ "sample_occurrences": 30
937
+ },
938
+ {
939
+ "tag": "handwear",
940
+ "fluffyrock_count": 162603,
941
+ "sample_occurrences": 56
942
+ },
943
+ {
944
+ "tag": "not_furry",
945
+ "fluffyrock_count": 160518,
946
+ "sample_occurrences": 30
947
+ },
948
+ {
949
+ "tag": "bed",
950
+ "fluffyrock_count": 159493,
951
+ "sample_occurrences": 25
952
+ },
953
+ {
954
+ "tag": "half-closed_eyes",
955
+ "fluffyrock_count": 158388,
956
+ "sample_occurrences": 19
957
+ },
958
+ {
959
+ "tag": "pecs",
960
+ "fluffyrock_count": 158385,
961
+ "sample_occurrences": 23
962
+ },
963
+ {
964
+ "tag": "gloves",
965
+ "fluffyrock_count": 156718,
966
+ "sample_occurrences": 52
967
+ },
968
+ {
969
+ "tag": "bovid",
970
+ "fluffyrock_count": 156674,
971
+ "sample_occurrences": 35
972
+ },
973
+ {
974
+ "tag": "orange_body",
975
+ "fluffyrock_count": 153299,
976
+ "sample_occurrences": 47
977
+ },
978
+ {
979
+ "tag": "blue_fur",
980
+ "fluffyrock_count": 153027,
981
+ "sample_occurrences": 48
982
+ },
983
+ {
984
+ "tag": "barefoot",
985
+ "fluffyrock_count": 148056,
986
+ "sample_occurrences": 46
987
+ },
988
+ {
989
+ "tag": "cub",
990
+ "fluffyrock_count": 147547,
991
+ "sample_occurrences": 21
992
+ },
993
+ {
994
+ "tag": "inside",
995
+ "fluffyrock_count": 145110,
996
+ "sample_occurrences": 30
997
+ },
998
+ {
999
+ "tag": "panties",
1000
+ "fluffyrock_count": 142486,
1001
+ "sample_occurrences": 13
1002
+ },
1003
+ {
1004
+ "tag": "two_tone_body",
1005
+ "fluffyrock_count": 140258,
1006
+ "sample_occurrences": 28
1007
+ },
1008
+ {
1009
+ "tag": "anthrofied",
1010
+ "fluffyrock_count": 139874,
1011
+ "sample_occurrences": 11
1012
+ },
1013
+ {
1014
+ "tag": "white_hair",
1015
+ "fluffyrock_count": 139416,
1016
+ "sample_occurrences": 37
1017
+ },
1018
+ {
1019
+ "tag": "animal_humanoid",
1020
+ "fluffyrock_count": 139252,
1021
+ "sample_occurrences": 31
1022
+ },
1023
+ {
1024
+ "tag": "blue_hair",
1025
+ "fluffyrock_count": 138674,
1026
+ "sample_occurrences": 48
1027
+ },
1028
+ {
1029
+ "tag": "holding_object",
1030
+ "fluffyrock_count": 137895,
1031
+ "sample_occurrences": 84
1032
+ },
1033
+ {
1034
+ "tag": "hyper",
1035
+ "fluffyrock_count": 137712,
1036
+ "sample_occurrences": 2
1037
+ }
1038
+ ]
1039
+ },
1040
+ "outputs": {
1041
+ "ranked_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_ranked_baseline200.csv",
1042
+ "uncovered_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_uncovered_after_topn_baseline200.csv",
1043
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_summary_baseline200.json"
1044
+ }
1045
+ }
data/analysis/tag_group_coverage_summary_combined200.json ADDED
@@ -0,0 +1,853 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "sample_file": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\eval_samples\\e621_sfw_sample_1000_seed123_buffer10000.jsonl",
3
+ "min_count": 200,
4
+ "n_images": 1000,
5
+ "n_groups": 73,
6
+ "n_wiki_groups": 41,
7
+ "excluded_wiki_groups": [
8
+ "ambiguous_tags",
9
+ "disambiguations",
10
+ "e621",
11
+ "multiple_tags",
12
+ "sound_effects",
13
+ "unknown_tags"
14
+ ],
15
+ "n_category_groups_added": 32,
16
+ "include_category_groups": true,
17
+ "total_tag_occurrences": 27806,
18
+ "n_unique_tags": 3068,
19
+ "n_unique_tags_covered_by_any_group": 1170,
20
+ "unique_tag_coverage_by_any_group_pct": 38.14,
21
+ "top_groups_by_occurrence_coverage": [
22
+ {
23
+ "group": "cat:clothing_detail",
24
+ "group_size": 423,
25
+ "covered_occurrences": 2800,
26
+ "covered_occ_pct": 10.07,
27
+ "covered_unique_tags": 218,
28
+ "covered_unique_pct": 7.11,
29
+ "covered_images": 595,
30
+ "covered_images_pct": 59.5
31
+ },
32
+ {
33
+ "group": "cat:anatomy_features",
34
+ "group_size": 410,
35
+ "covered_occurrences": 2410,
36
+ "covered_occ_pct": 8.67,
37
+ "covered_unique_tags": 206,
38
+ "covered_unique_pct": 6.71,
39
+ "covered_images": 660,
40
+ "covered_images_pct": 66.0
41
+ },
42
+ {
43
+ "group": "species",
44
+ "group_size": 100,
45
+ "covered_occurrences": 1741,
46
+ "covered_occ_pct": 6.26,
47
+ "covered_unique_tags": 41,
48
+ "covered_unique_pct": 1.34,
49
+ "covered_images": 878,
50
+ "covered_images_pct": 87.8
51
+ },
52
+ {
53
+ "group": "cat:color_markings",
54
+ "group_size": 244,
55
+ "covered_occurrences": 1722,
56
+ "covered_occ_pct": 6.19,
57
+ "covered_unique_tags": 112,
58
+ "covered_unique_pct": 3.65,
59
+ "covered_images": 520,
60
+ "covered_images_pct": 52.0
61
+ },
62
+ {
63
+ "group": "genders",
64
+ "group_size": 336,
65
+ "covered_occurrences": 1209,
66
+ "covered_occ_pct": 4.35,
67
+ "covered_unique_tags": 23,
68
+ "covered_unique_pct": 0.75,
69
+ "covered_images": 927,
70
+ "covered_images_pct": 92.7
71
+ },
72
+ {
73
+ "group": "cat:background_composition",
74
+ "group_size": 85,
75
+ "covered_occurrences": 1089,
76
+ "covered_occ_pct": 3.92,
77
+ "covered_unique_tags": 50,
78
+ "covered_unique_pct": 1.63,
79
+ "covered_images": 544,
80
+ "covered_images_pct": 54.4
81
+ },
82
+ {
83
+ "group": "cat:gender",
84
+ "group_size": 3,
85
+ "covered_occurrences": 1089,
86
+ "covered_occ_pct": 3.92,
87
+ "covered_unique_tags": 3,
88
+ "covered_unique_pct": 0.1,
89
+ "covered_images": 927,
90
+ "covered_images_pct": 92.7
91
+ },
92
+ {
93
+ "group": "body_types",
94
+ "group_size": 22,
95
+ "covered_occurrences": 1035,
96
+ "covered_occ_pct": 3.72,
97
+ "covered_unique_tags": 9,
98
+ "covered_unique_pct": 0.29,
99
+ "covered_images": 909,
100
+ "covered_images_pct": 90.9
101
+ },
102
+ {
103
+ "group": "cat:count",
104
+ "group_size": 5,
105
+ "covered_occurrences": 946,
106
+ "covered_occ_pct": 3.4,
107
+ "covered_unique_tags": 4,
108
+ "covered_unique_pct": 0.13,
109
+ "covered_images": 933,
110
+ "covered_images_pct": 93.3
111
+ },
112
+ {
113
+ "group": "cat:body_type",
114
+ "group_size": 7,
115
+ "covered_occurrences": 932,
116
+ "covered_occ_pct": 3.35,
117
+ "covered_unique_tags": 6,
118
+ "covered_unique_pct": 0.2,
119
+ "covered_images": 880,
120
+ "covered_images_pct": 88.0
121
+ },
122
+ {
123
+ "group": "facial_expressions",
124
+ "group_size": 86,
125
+ "covered_occurrences": 927,
126
+ "covered_occ_pct": 3.33,
127
+ "covered_unique_tags": 43,
128
+ "covered_unique_pct": 1.4,
129
+ "covered_images": 515,
130
+ "covered_images_pct": 51.5
131
+ },
132
+ {
133
+ "group": "cat:species",
134
+ "group_size": 13,
135
+ "covered_occurrences": 879,
136
+ "covered_occ_pct": 3.16,
137
+ "covered_unique_tags": 13,
138
+ "covered_unique_pct": 0.42,
139
+ "covered_images": 706,
140
+ "covered_images_pct": 70.6
141
+ },
142
+ {
143
+ "group": "cat:objects_props",
144
+ "group_size": 266,
145
+ "covered_occurrences": 870,
146
+ "covered_occ_pct": 3.13,
147
+ "covered_unique_tags": 141,
148
+ "covered_unique_pct": 4.6,
149
+ "covered_images": 312,
150
+ "covered_images_pct": 31.2
151
+ },
152
+ {
153
+ "group": "cat:expression_detail",
154
+ "group_size": 34,
155
+ "covered_occurrences": 700,
156
+ "covered_occ_pct": 2.52,
157
+ "covered_unique_tags": 20,
158
+ "covered_unique_pct": 0.65,
159
+ "covered_images": 438,
160
+ "covered_images_pct": 43.8
161
+ },
162
+ {
163
+ "group": "outfits",
164
+ "group_size": 55,
165
+ "covered_occurrences": 644,
166
+ "covered_occ_pct": 2.32,
167
+ "covered_unique_tags": 14,
168
+ "covered_unique_pct": 0.46,
169
+ "covered_images": 576,
170
+ "covered_images_pct": 57.6
171
+ },
172
+ {
173
+ "group": "pose",
174
+ "group_size": 189,
175
+ "covered_occurrences": 622,
176
+ "covered_occ_pct": 2.24,
177
+ "covered_unique_tags": 66,
178
+ "covered_unique_pct": 2.15,
179
+ "covered_images": 417,
180
+ "covered_images_pct": 41.7
181
+ },
182
+ {
183
+ "group": "romance",
184
+ "group_size": 59,
185
+ "covered_occurrences": 555,
186
+ "covered_occ_pct": 2.0,
187
+ "covered_unique_tags": 28,
188
+ "covered_unique_pct": 0.91,
189
+ "covered_images": 357,
190
+ "covered_images_pct": 35.7
191
+ },
192
+ {
193
+ "group": "narrative_elements",
194
+ "group_size": 113,
195
+ "covered_occurrences": 536,
196
+ "covered_occ_pct": 1.93,
197
+ "covered_unique_tags": 26,
198
+ "covered_unique_pct": 0.85,
199
+ "covered_images": 317,
200
+ "covered_images_pct": 31.7
201
+ },
202
+ {
203
+ "group": "art",
204
+ "group_size": 111,
205
+ "covered_occurrences": 536,
206
+ "covered_occ_pct": 1.93,
207
+ "covered_unique_tags": 21,
208
+ "covered_unique_pct": 0.68,
209
+ "covered_images": 477,
210
+ "covered_images_pct": 47.7
211
+ },
212
+ {
213
+ "group": "nudity",
214
+ "group_size": 146,
215
+ "covered_occurrences": 501,
216
+ "covered_occ_pct": 1.8,
217
+ "covered_unique_tags": 21,
218
+ "covered_unique_pct": 0.68,
219
+ "covered_images": 396,
220
+ "covered_images_pct": 39.6
221
+ }
222
+ ],
223
+ "greedy_cover_steps": [
224
+ {
225
+ "step": 1,
226
+ "group": "cat:clothing_detail",
227
+ "gain_occurrences": 2800,
228
+ "cumulative_covered_occurrences": 2800,
229
+ "cumulative_covered_pct": 10.07,
230
+ "new_tags_covered": 218
231
+ },
232
+ {
233
+ "step": 2,
234
+ "group": "cat:anatomy_features",
235
+ "gain_occurrences": 2406,
236
+ "cumulative_covered_occurrences": 5206,
237
+ "cumulative_covered_pct": 18.72,
238
+ "new_tags_covered": 204
239
+ },
240
+ {
241
+ "step": 3,
242
+ "group": "species",
243
+ "gain_occurrences": 1741,
244
+ "cumulative_covered_occurrences": 6947,
245
+ "cumulative_covered_pct": 24.98,
246
+ "new_tags_covered": 41
247
+ },
248
+ {
249
+ "step": 4,
250
+ "group": "cat:color_markings",
251
+ "gain_occurrences": 1634,
252
+ "cumulative_covered_occurrences": 8581,
253
+ "cumulative_covered_pct": 30.86,
254
+ "new_tags_covered": 93
255
+ },
256
+ {
257
+ "step": 5,
258
+ "group": "genders",
259
+ "gain_occurrences": 1209,
260
+ "cumulative_covered_occurrences": 9790,
261
+ "cumulative_covered_pct": 35.21,
262
+ "new_tags_covered": 23
263
+ },
264
+ {
265
+ "step": 6,
266
+ "group": "cat:background_composition",
267
+ "gain_occurrences": 1077,
268
+ "cumulative_covered_occurrences": 10867,
269
+ "cumulative_covered_pct": 39.08,
270
+ "new_tags_covered": 46
271
+ },
272
+ {
273
+ "step": 7,
274
+ "group": "body_types",
275
+ "gain_occurrences": 1035,
276
+ "cumulative_covered_occurrences": 11902,
277
+ "cumulative_covered_pct": 42.8,
278
+ "new_tags_covered": 9
279
+ },
280
+ {
281
+ "step": 8,
282
+ "group": "cat:count",
283
+ "gain_occurrences": 946,
284
+ "cumulative_covered_occurrences": 12848,
285
+ "cumulative_covered_pct": 46.21,
286
+ "new_tags_covered": 4
287
+ },
288
+ {
289
+ "step": 9,
290
+ "group": "facial_expressions",
291
+ "gain_occurrences": 881,
292
+ "cumulative_covered_occurrences": 13729,
293
+ "cumulative_covered_pct": 49.37,
294
+ "new_tags_covered": 41
295
+ },
296
+ {
297
+ "step": 10,
298
+ "group": "cat:objects_props",
299
+ "gain_occurrences": 863,
300
+ "cumulative_covered_occurrences": 14592,
301
+ "cumulative_covered_pct": 52.48,
302
+ "new_tags_covered": 135
303
+ },
304
+ {
305
+ "step": 11,
306
+ "group": "pose",
307
+ "gain_occurrences": 622,
308
+ "cumulative_covered_occurrences": 15214,
309
+ "cumulative_covered_pct": 54.71,
310
+ "new_tags_covered": 66
311
+ },
312
+ {
313
+ "step": 12,
314
+ "group": "narrative_elements",
315
+ "gain_occurrences": 535,
316
+ "cumulative_covered_occurrences": 15749,
317
+ "cumulative_covered_pct": 56.64,
318
+ "new_tags_covered": 25
319
+ },
320
+ {
321
+ "step": 13,
322
+ "group": "cat:hair",
323
+ "gain_occurrences": 472,
324
+ "cumulative_covered_occurrences": 16221,
325
+ "cumulative_covered_pct": 58.34,
326
+ "new_tags_covered": 3
327
+ },
328
+ {
329
+ "step": 14,
330
+ "group": "cat:franchise_series",
331
+ "gain_occurrences": 252,
332
+ "cumulative_covered_occurrences": 16473,
333
+ "cumulative_covered_pct": 59.24,
334
+ "new_tags_covered": 19
335
+ },
336
+ {
337
+ "step": 15,
338
+ "group": "cat:body_decor",
339
+ "gain_occurrences": 192,
340
+ "cumulative_covered_occurrences": 16665,
341
+ "cumulative_covered_pct": 59.93,
342
+ "new_tags_covered": 8
343
+ },
344
+ {
345
+ "step": 16,
346
+ "group": "nudity",
347
+ "gain_occurrences": 145,
348
+ "cumulative_covered_occurrences": 16810,
349
+ "cumulative_covered_pct": 60.45,
350
+ "new_tags_covered": 15
351
+ },
352
+ {
353
+ "step": 17,
354
+ "group": "romance",
355
+ "gain_occurrences": 139,
356
+ "cumulative_covered_occurrences": 16949,
357
+ "cumulative_covered_pct": 60.95,
358
+ "new_tags_covered": 20
359
+ },
360
+ {
361
+ "step": 18,
362
+ "group": "art",
363
+ "gain_occurrences": 118,
364
+ "cumulative_covered_occurrences": 17067,
365
+ "cumulative_covered_pct": 61.38,
366
+ "new_tags_covered": 16
367
+ },
368
+ {
369
+ "step": 19,
370
+ "group": "feral_anatomy",
371
+ "gain_occurrences": 114,
372
+ "cumulative_covered_occurrences": 17181,
373
+ "cumulative_covered_pct": 61.79,
374
+ "new_tags_covered": 2
375
+ },
376
+ {
377
+ "step": 20,
378
+ "group": "piercing",
379
+ "gain_occurrences": 111,
380
+ "cumulative_covered_occurrences": 17292,
381
+ "cumulative_covered_pct": 62.19,
382
+ "new_tags_covered": 12
383
+ },
384
+ {
385
+ "step": 21,
386
+ "group": "food",
387
+ "gain_occurrences": 103,
388
+ "cumulative_covered_occurrences": 17395,
389
+ "cumulative_covered_pct": 62.56,
390
+ "new_tags_covered": 34
391
+ },
392
+ {
393
+ "step": 22,
394
+ "group": "settings",
395
+ "gain_occurrences": 97,
396
+ "cumulative_covered_occurrences": 17492,
397
+ "cumulative_covered_pct": 62.91,
398
+ "new_tags_covered": 33
399
+ },
400
+ {
401
+ "step": 23,
402
+ "group": "bondage",
403
+ "gain_occurrences": 95,
404
+ "cumulative_covered_occurrences": 17587,
405
+ "cumulative_covered_pct": 63.25,
406
+ "new_tags_covered": 15
407
+ },
408
+ {
409
+ "step": 24,
410
+ "group": "outfits",
411
+ "gain_occurrences": 70,
412
+ "cumulative_covered_occurrences": 17657,
413
+ "cumulative_covered_pct": 63.5,
414
+ "new_tags_covered": 13
415
+ },
416
+ {
417
+ "step": 25,
418
+ "group": "cat:perspective",
419
+ "gain_occurrences": 58,
420
+ "cumulative_covered_occurrences": 17715,
421
+ "cumulative_covered_pct": 63.71,
422
+ "new_tags_covered": 6
423
+ }
424
+ ],
425
+ "uncovered_after_top_n_groups": {
426
+ "top_n_groups": 15,
427
+ "selected_groups": [
428
+ "cat:clothing_detail",
429
+ "cat:anatomy_features",
430
+ "species",
431
+ "cat:color_markings",
432
+ "genders",
433
+ "cat:background_composition",
434
+ "body_types",
435
+ "cat:count",
436
+ "facial_expressions",
437
+ "cat:objects_props",
438
+ "pose",
439
+ "narrative_elements",
440
+ "cat:hair",
441
+ "cat:franchise_series",
442
+ "cat:body_decor"
443
+ ],
444
+ "n_uncovered_unique_tags": 2133,
445
+ "top_uncovered_tags": [
446
+ {
447
+ "tag": "fur",
448
+ "fluffyrock_count": 1417091,
449
+ "sample_occurrences": 422
450
+ },
451
+ {
452
+ "tag": "bodily_fluids",
453
+ "fluffyrock_count": 1225999,
454
+ "sample_occurrences": 49
455
+ },
456
+ {
457
+ "tag": "canid",
458
+ "fluffyrock_count": 1141448,
459
+ "sample_occurrences": 304
460
+ },
461
+ {
462
+ "tag": "nude",
463
+ "fluffyrock_count": 1103510,
464
+ "sample_occurrences": 57
465
+ },
466
+ {
467
+ "tag": "nipples",
468
+ "fluffyrock_count": 1079208,
469
+ "sample_occurrences": 17
470
+ },
471
+ {
472
+ "tag": "butt",
473
+ "fluffyrock_count": 828096,
474
+ "sample_occurrences": 25
475
+ },
476
+ {
477
+ "tag": "felid",
478
+ "fluffyrock_count": 598698,
479
+ "sample_occurrences": 158
480
+ },
481
+ {
482
+ "tag": "canis",
483
+ "fluffyrock_count": 587772,
484
+ "sample_occurrences": 159
485
+ },
486
+ {
487
+ "tag": "male/female",
488
+ "fluffyrock_count": 551565,
489
+ "sample_occurrences": 12
490
+ },
491
+ {
492
+ "tag": "blue_eyes",
493
+ "fluffyrock_count": 451508,
494
+ "sample_occurrences": 144
495
+ },
496
+ {
497
+ "tag": "feet",
498
+ "fluffyrock_count": 448730,
499
+ "sample_occurrences": 113
500
+ },
501
+ {
502
+ "tag": "male/male",
503
+ "fluffyrock_count": 427119,
504
+ "sample_occurrences": 26
505
+ },
506
+ {
507
+ "tag": "equid",
508
+ "fluffyrock_count": 399764,
509
+ "sample_occurrences": 97
510
+ },
511
+ {
512
+ "tag": "navel",
513
+ "fluffyrock_count": 373746,
514
+ "sample_occurrences": 45
515
+ },
516
+ {
517
+ "tag": "biped",
518
+ "fluffyrock_count": 372051,
519
+ "sample_occurrences": 95
520
+ },
521
+ {
522
+ "tag": "muscular",
523
+ "fluffyrock_count": 358573,
524
+ "sample_occurrences": 56
525
+ },
526
+ {
527
+ "tag": "thick_thighs",
528
+ "fluffyrock_count": 339009,
529
+ "sample_occurrences": 25
530
+ },
531
+ {
532
+ "tag": "fox",
533
+ "fluffyrock_count": 337215,
534
+ "sample_occurrences": 105
535
+ },
536
+ {
537
+ "tag": "tuft",
538
+ "fluffyrock_count": 335534,
539
+ "sample_occurrences": 105
540
+ },
541
+ {
542
+ "tag": "areola",
543
+ "fluffyrock_count": 320191,
544
+ "sample_occurrences": 1
545
+ },
546
+ {
547
+ "tag": "dragon",
548
+ "fluffyrock_count": 309478,
549
+ "sample_occurrences": 84
550
+ },
551
+ {
552
+ "tag": "domestic_dog",
553
+ "fluffyrock_count": 300258,
554
+ "sample_occurrences": 77
555
+ },
556
+ {
557
+ "tag": "<3",
558
+ "fluffyrock_count": 293372,
559
+ "sample_occurrences": 50
560
+ },
561
+ {
562
+ "tag": "green_eyes",
563
+ "fluffyrock_count": 287579,
564
+ "sample_occurrences": 78
565
+ },
566
+ {
567
+ "tag": "wolf",
568
+ "fluffyrock_count": 273053,
569
+ "sample_occurrences": 79
570
+ },
571
+ {
572
+ "tag": "plant",
573
+ "fluffyrock_count": 265700,
574
+ "sample_occurrences": 97
575
+ },
576
+ {
577
+ "tag": "size_difference",
578
+ "fluffyrock_count": 265067,
579
+ "sample_occurrences": 34
580
+ },
581
+ {
582
+ "tag": "big_butt",
583
+ "fluffyrock_count": 262825,
584
+ "sample_occurrences": 11
585
+ },
586
+ {
587
+ "tag": "interspecies",
588
+ "fluffyrock_count": 261649,
589
+ "sample_occurrences": 4
590
+ },
591
+ {
592
+ "tag": "huge_breasts",
593
+ "fluffyrock_count": 261003,
594
+ "sample_occurrences": 13
595
+ },
596
+ {
597
+ "tag": "eyewear",
598
+ "fluffyrock_count": 251234,
599
+ "sample_occurrences": 93
600
+ },
601
+ {
602
+ "tag": "sweat",
603
+ "fluffyrock_count": 250842,
604
+ "sample_occurrences": 20
605
+ },
606
+ {
607
+ "tag": "felis",
608
+ "fluffyrock_count": 241656,
609
+ "sample_occurrences": 64
610
+ },
611
+ {
612
+ "tag": "young",
613
+ "fluffyrock_count": 238777,
614
+ "sample_occurrences": 40
615
+ },
616
+ {
617
+ "tag": "domestic_cat",
618
+ "fluffyrock_count": 236967,
619
+ "sample_occurrences": 64
620
+ },
621
+ {
622
+ "tag": "wide_hips",
623
+ "fluffyrock_count": 235877,
624
+ "sample_occurrences": 30
625
+ },
626
+ {
627
+ "tag": "black_nose",
628
+ "fluffyrock_count": 216679,
629
+ "sample_occurrences": 80
630
+ },
631
+ {
632
+ "tag": "red_eyes",
633
+ "fluffyrock_count": 213545,
634
+ "sample_occurrences": 53
635
+ },
636
+ {
637
+ "tag": "reptile",
638
+ "fluffyrock_count": 206968,
639
+ "sample_occurrences": 47
640
+ },
641
+ {
642
+ "tag": "leporid",
643
+ "fluffyrock_count": 204404,
644
+ "sample_occurrences": 73
645
+ },
646
+ {
647
+ "tag": "rabbit",
648
+ "fluffyrock_count": 199906,
649
+ "sample_occurrences": 73
650
+ },
651
+ {
652
+ "tag": "belly",
653
+ "fluffyrock_count": 194310,
654
+ "sample_occurrences": 31
655
+ },
656
+ {
657
+ "tag": "eyelashes",
658
+ "fluffyrock_count": 193540,
659
+ "sample_occurrences": 63
660
+ },
661
+ {
662
+ "tag": "horse",
663
+ "fluffyrock_count": 193042,
664
+ "sample_occurrences": 40
665
+ },
666
+ {
667
+ "tag": "feathers",
668
+ "fluffyrock_count": 192797,
669
+ "sample_occurrences": 76
670
+ },
671
+ {
672
+ "tag": "solo_focus",
673
+ "fluffyrock_count": 191284,
674
+ "sample_occurrences": 18
675
+ },
676
+ {
677
+ "tag": "saliva",
678
+ "fluffyrock_count": 187352,
679
+ "sample_occurrences": 10
680
+ },
681
+ {
682
+ "tag": "jewelry",
683
+ "fluffyrock_count": 185593,
684
+ "sample_occurrences": 53
685
+ },
686
+ {
687
+ "tag": "brown_hair",
688
+ "fluffyrock_count": 183486,
689
+ "sample_occurrences": 48
690
+ },
691
+ {
692
+ "tag": "blonde_hair",
693
+ "fluffyrock_count": 183468,
694
+ "sample_occurrences": 47
695
+ },
696
+ {
697
+ "tag": "pantherine",
698
+ "fluffyrock_count": 181019,
699
+ "sample_occurrences": 45
700
+ },
701
+ {
702
+ "tag": "black_hair",
703
+ "fluffyrock_count": 180936,
704
+ "sample_occurrences": 45
705
+ },
706
+ {
707
+ "tag": "overweight",
708
+ "fluffyrock_count": 176776,
709
+ "sample_occurrences": 36
710
+ },
711
+ {
712
+ "tag": "abs",
713
+ "fluffyrock_count": 171761,
714
+ "sample_occurrences": 23
715
+ },
716
+ {
717
+ "tag": "eyebrows",
718
+ "fluffyrock_count": 170102,
719
+ "sample_occurrences": 50
720
+ },
721
+ {
722
+ "tag": "vein",
723
+ "fluffyrock_count": 168934,
724
+ "sample_occurrences": 5
725
+ },
726
+ {
727
+ "tag": "yellow_eyes",
728
+ "fluffyrock_count": 168780,
729
+ "sample_occurrences": 45
730
+ },
731
+ {
732
+ "tag": "muscular_anthro",
733
+ "fluffyrock_count": 164876,
734
+ "sample_occurrences": 30
735
+ },
736
+ {
737
+ "tag": "half-closed_eyes",
738
+ "fluffyrock_count": 158388,
739
+ "sample_occurrences": 19
740
+ },
741
+ {
742
+ "tag": "pecs",
743
+ "fluffyrock_count": 158385,
744
+ "sample_occurrences": 23
745
+ },
746
+ {
747
+ "tag": "bovid",
748
+ "fluffyrock_count": 156674,
749
+ "sample_occurrences": 35
750
+ },
751
+ {
752
+ "tag": "barefoot",
753
+ "fluffyrock_count": 148056,
754
+ "sample_occurrences": 46
755
+ },
756
+ {
757
+ "tag": "cub",
758
+ "fluffyrock_count": 147547,
759
+ "sample_occurrences": 21
760
+ },
761
+ {
762
+ "tag": "bound",
763
+ "fluffyrock_count": 143669,
764
+ "sample_occurrences": 3
765
+ },
766
+ {
767
+ "tag": "panties",
768
+ "fluffyrock_count": 142486,
769
+ "sample_occurrences": 13
770
+ },
771
+ {
772
+ "tag": "anthrofied",
773
+ "fluffyrock_count": 139874,
774
+ "sample_occurrences": 11
775
+ },
776
+ {
777
+ "tag": "white_hair",
778
+ "fluffyrock_count": 139416,
779
+ "sample_occurrences": 37
780
+ },
781
+ {
782
+ "tag": "animal_humanoid",
783
+ "fluffyrock_count": 139252,
784
+ "sample_occurrences": 31
785
+ },
786
+ {
787
+ "tag": "blue_hair",
788
+ "fluffyrock_count": 138674,
789
+ "sample_occurrences": 48
790
+ },
791
+ {
792
+ "tag": "hyper",
793
+ "fluffyrock_count": 137712,
794
+ "sample_occurrences": 2
795
+ },
796
+ {
797
+ "tag": "hybrid",
798
+ "fluffyrock_count": 134372,
799
+ "sample_occurrences": 30
800
+ },
801
+ {
802
+ "tag": "stripes",
803
+ "fluffyrock_count": 132789,
804
+ "sample_occurrences": 32
805
+ },
806
+ {
807
+ "tag": "bottomless",
808
+ "fluffyrock_count": 132767,
809
+ "sample_occurrences": 7
810
+ },
811
+ {
812
+ "tag": "purple_eyes",
813
+ "fluffyrock_count": 131683,
814
+ "sample_occurrences": 34
815
+ },
816
+ {
817
+ "tag": "pose",
818
+ "fluffyrock_count": 130171,
819
+ "sample_occurrences": 25
820
+ },
821
+ {
822
+ "tag": "rear_view",
823
+ "fluffyrock_count": 128927,
824
+ "sample_occurrences": 7
825
+ },
826
+ {
827
+ "tag": "licking",
828
+ "fluffyrock_count": 126818,
829
+ "sample_occurrences": 9
830
+ },
831
+ {
832
+ "tag": "pony",
833
+ "fluffyrock_count": 122813,
834
+ "sample_occurrences": 35
835
+ },
836
+ {
837
+ "tag": "slightly_chubby",
838
+ "fluffyrock_count": 118826,
839
+ "sample_occurrences": 21
840
+ },
841
+ {
842
+ "tag": "scales",
843
+ "fluffyrock_count": 118407,
844
+ "sample_occurrences": 36
845
+ }
846
+ ]
847
+ },
848
+ "outputs": {
849
+ "ranked_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_ranked_combined200.csv",
850
+ "uncovered_csv": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_uncovered_after_topn_combined200.csv",
851
+ "summary_json": "E:\\image\\backup\\Prompt_Squirrel_RAG\\data\\analysis\\tag_group_coverage_summary_combined200.json"
852
+ }
853
+ }
data/analysis/tag_group_uncovered_after_topn.csv ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,fluffyrock_count,sample_occurrences
2
+ solo,2021510,572
3
+ breasts,1503887,155
4
+ fur,1417091,422
5
+ duo,1327850,246
6
+ bodily_fluids,1225999,49
7
+ canid,1141448,304
8
+ nipples,1079208,17
9
+ butt,828096,25
10
+ tongue,725619,109
11
+ tail,686059,110
12
+ big_breasts,682494,41
13
+ felid,598698,158
14
+ canis,587772,159
15
+ horn,568235,154
16
+ male/female,551565,12
17
+ teeth,546600,132
18
+ white_body,541452,171
19
+ blue_eyes,451508,144
20
+ feet,448730,113
21
+ pokemon_(species),446455,98
22
+ claws,434361,111
23
+ male/male,427119,26
24
+ white_fur,426284,133
25
+ topwear,412942,183
26
+ equid,399764,97
27
+ group,391223,120
28
+ fingers,383191,146
29
+ navel,373746,45
30
+ biped,372051,95
31
+ wings,364542,114
32
+ muscular,358573,56
33
+ thick_thighs,339009,25
34
+ fox,337215,105
35
+ tuft,335534,105
36
+ toes,326285,82
37
+ piercing,321077,56
38
+ areola,320191,1
39
+ brown_body,316754,87
40
+ bottomwear,316556,135
41
+ underwear,316540,35
42
+ white_background,310041,115
43
+ dragon,309478,84
44
+ domestic_dog,300258,77
45
+ <3,293372,50
46
+ green_eyes,287579,78
47
+ blue_body,277335,76
48
+ black_body,273271,78
49
+ wolf,273053,79
50
+ plant,265700,97
51
+ size_difference,265067,34
52
+ brown_fur,264096,72
53
+ furniture,263294,48
54
+ big_butt,262825,11
55
+ interspecies,261649,4
56
+ long_hair,261330,62
57
+ huge_breasts,261003,13
58
+ legwear,254808,37
59
+ markings,253976,61
60
+ shirt,253553,103
61
+ outside,253364,93
62
+ eyewear,251234,93
63
+ sweat,250842,20
64
+ felis,241656,64
65
+ young,238777,40
66
+ domestic_cat,236967,64
67
+ headgear,236145,92
68
+ wide_hips,235877,30
69
+ grey_body,233510,74
70
+ paws,220932,36
71
+ 5_fingers,220474,88
72
+ black_nose,216679,80
73
+ red_eyes,213545,53
74
+ footwear,211229,73
75
+ reptile,206968,47
76
+ leporid,204404,73
77
+ rabbit,199906,73
78
+ black_fur,197688,60
79
+ belly,194310,31
80
+ eyelashes,193540,63
81
+ horse,193042,40
82
+ feathers,192797,76
83
+ pawpads,191398,24
84
+ solo_focus,191284,18
85
+ fangs,190633,49
86
+ saliva,187352,10
87
+ brown_hair,183486,48
88
+ blonde_hair,183468,47
89
+ pantherine,181019,45
90
+ black_hair,180936,45
91
+ yellow_body,180032,46
92
+ glasses,177552,49
93
+ headwear,177119,69
94
+ overweight,176776,36
95
+ grey_fur,175923,61
96
+ abs,171761,23
97
+ eyebrows,170102,50
98
+ vein,168934,5
99
+ yellow_eyes,168780,45
100
+ hat,166176,67
101
+ muscular_anthro,164876,30
102
+ handwear,162603,56
103
+ not_furry,160518,30
104
+ bed,159493,25
105
+ half-closed_eyes,158388,19
106
+ pecs,158385,23
107
+ gloves,156718,52
108
+ bovid,156674,35
109
+ orange_body,153299,47
110
+ blue_fur,153027,48
111
+ barefoot,148056,46
112
+ cub,147547,21
113
+ inside,145110,30
114
+ panties,142486,13
115
+ two_tone_body,140258,28
116
+ anthrofied,139874,11
117
+ white_hair,139416,37
118
+ animal_humanoid,139252,31
119
+ blue_hair,138674,48
120
+ holding_object,137895,84
121
+ hyper,137712,2
data/analysis/tag_group_uncovered_after_topn_baseline200.csv ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,fluffyrock_count,sample_occurrences
2
+ solo,2021510,572
3
+ breasts,1503887,155
4
+ fur,1417091,422
5
+ duo,1327850,246
6
+ bodily_fluids,1225999,49
7
+ canid,1141448,304
8
+ nipples,1079208,17
9
+ butt,828096,25
10
+ tongue,725619,109
11
+ tail,686059,110
12
+ big_breasts,682494,41
13
+ felid,598698,158
14
+ canis,587772,159
15
+ horn,568235,154
16
+ male/female,551565,12
17
+ teeth,546600,132
18
+ white_body,541452,171
19
+ blue_eyes,451508,144
20
+ feet,448730,113
21
+ pokemon_(species),446455,98
22
+ claws,434361,111
23
+ male/male,427119,26
24
+ white_fur,426284,133
25
+ topwear,412942,183
26
+ equid,399764,97
27
+ group,391223,120
28
+ fingers,383191,146
29
+ navel,373746,45
30
+ biped,372051,95
31
+ wings,364542,114
32
+ muscular,358573,56
33
+ thick_thighs,339009,25
34
+ fox,337215,105
35
+ tuft,335534,105
36
+ toes,326285,82
37
+ piercing,321077,56
38
+ areola,320191,1
39
+ brown_body,316754,87
40
+ bottomwear,316556,135
41
+ underwear,316540,35
42
+ white_background,310041,115
43
+ dragon,309478,84
44
+ domestic_dog,300258,77
45
+ <3,293372,50
46
+ green_eyes,287579,78
47
+ blue_body,277335,76
48
+ black_body,273271,78
49
+ wolf,273053,79
50
+ plant,265700,97
51
+ size_difference,265067,34
52
+ brown_fur,264096,72
53
+ furniture,263294,48
54
+ big_butt,262825,11
55
+ interspecies,261649,4
56
+ long_hair,261330,62
57
+ huge_breasts,261003,13
58
+ legwear,254808,37
59
+ markings,253976,61
60
+ shirt,253553,103
61
+ outside,253364,93
62
+ eyewear,251234,93
63
+ sweat,250842,20
64
+ felis,241656,64
65
+ young,238777,40
66
+ domestic_cat,236967,64
67
+ headgear,236145,92
68
+ wide_hips,235877,30
69
+ grey_body,233510,74
70
+ paws,220932,36
71
+ 5_fingers,220474,88
72
+ black_nose,216679,80
73
+ red_eyes,213545,53
74
+ footwear,211229,73
75
+ reptile,206968,47
76
+ leporid,204404,73
77
+ rabbit,199906,73
78
+ black_fur,197688,60
79
+ belly,194310,31
80
+ eyelashes,193540,63
81
+ horse,193042,40
82
+ feathers,192797,76
83
+ pawpads,191398,24
84
+ solo_focus,191284,18
85
+ fangs,190633,49
86
+ saliva,187352,10
87
+ brown_hair,183486,48
88
+ blonde_hair,183468,47
89
+ pantherine,181019,45
90
+ black_hair,180936,45
91
+ yellow_body,180032,46
92
+ glasses,177552,49
93
+ headwear,177119,69
94
+ overweight,176776,36
95
+ grey_fur,175923,61
96
+ abs,171761,23
97
+ eyebrows,170102,50
98
+ vein,168934,5
99
+ yellow_eyes,168780,45
100
+ hat,166176,67
101
+ muscular_anthro,164876,30
102
+ handwear,162603,56
103
+ not_furry,160518,30
104
+ bed,159493,25
105
+ half-closed_eyes,158388,19
106
+ pecs,158385,23
107
+ gloves,156718,52
108
+ bovid,156674,35
109
+ orange_body,153299,47
110
+ blue_fur,153027,48
111
+ barefoot,148056,46
112
+ cub,147547,21
113
+ inside,145110,30
114
+ panties,142486,13
115
+ two_tone_body,140258,28
116
+ anthrofied,139874,11
117
+ white_hair,139416,37
118
+ animal_humanoid,139252,31
119
+ blue_hair,138674,48
120
+ holding_object,137895,84
121
+ hyper,137712,2
data/analysis/tag_group_uncovered_after_topn_combined200.csv ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ tag,fluffyrock_count,sample_occurrences
2
+ fur,1417091,422
3
+ bodily_fluids,1225999,49
4
+ canid,1141448,304
5
+ nude,1103510,57
6
+ nipples,1079208,17
7
+ butt,828096,25
8
+ felid,598698,158
9
+ canis,587772,159
10
+ male/female,551565,12
11
+ blue_eyes,451508,144
12
+ feet,448730,113
13
+ male/male,427119,26
14
+ equid,399764,97
15
+ navel,373746,45
16
+ biped,372051,95
17
+ muscular,358573,56
18
+ thick_thighs,339009,25
19
+ fox,337215,105
20
+ tuft,335534,105
21
+ areola,320191,1
22
+ dragon,309478,84
23
+ domestic_dog,300258,77
24
+ <3,293372,50
25
+ green_eyes,287579,78
26
+ wolf,273053,79
27
+ plant,265700,97
28
+ size_difference,265067,34
29
+ big_butt,262825,11
30
+ interspecies,261649,4
31
+ huge_breasts,261003,13
32
+ eyewear,251234,93
33
+ sweat,250842,20
34
+ felis,241656,64
35
+ young,238777,40
36
+ domestic_cat,236967,64
37
+ wide_hips,235877,30
38
+ black_nose,216679,80
39
+ red_eyes,213545,53
40
+ reptile,206968,47
41
+ leporid,204404,73
42
+ rabbit,199906,73
43
+ belly,194310,31
44
+ eyelashes,193540,63
45
+ horse,193042,40
46
+ feathers,192797,76
47
+ solo_focus,191284,18
48
+ saliva,187352,10
49
+ jewelry,185593,53
50
+ brown_hair,183486,48
51
+ blonde_hair,183468,47
52
+ pantherine,181019,45
53
+ black_hair,180936,45
54
+ overweight,176776,36
55
+ abs,171761,23
56
+ eyebrows,170102,50
57
+ vein,168934,5
58
+ yellow_eyes,168780,45
59
+ muscular_anthro,164876,30
60
+ half-closed_eyes,158388,19
61
+ pecs,158385,23
62
+ bovid,156674,35
63
+ barefoot,148056,46
64
+ cub,147547,21
65
+ bound,143669,3
66
+ panties,142486,13
67
+ anthrofied,139874,11
68
+ white_hair,139416,37
69
+ animal_humanoid,139252,31
70
+ blue_hair,138674,48
71
+ hyper,137712,2
72
+ hybrid,134372,30
73
+ stripes,132789,32
74
+ bottomless,132767,7
75
+ purple_eyes,131683,34
76
+ pose,130171,25
77
+ rear_view,128927,7
78
+ licking,126818,9
79
+ pony,122813,35
80
+ slightly_chubby,118826,21
81
+ scales,118407,36
data/analysis/tfidf_category_insights.json ADDED
@@ -0,0 +1,795 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "registry_file": "data\\analysis\\category_registry.csv",
3
+ "n_existing_categories_with_centroids": 23,
4
+ "existing_category_centroid_sizes": {
5
+ "body_decor": 9,
6
+ "body_type": 7,
7
+ "breasts": 4,
8
+ "clothing": 6,
9
+ "count": 5,
10
+ "expression": 8,
11
+ "fur_style": 2,
12
+ "gaze": 4,
13
+ "gender": 3,
14
+ "general_activity_if_any": 6,
15
+ "hair": 3,
16
+ "information": 3,
17
+ "limbs": 5,
18
+ "location": 5,
19
+ "organization": 8,
20
+ "perspective": 6,
21
+ "posture": 6,
22
+ "quality": 6,
23
+ "requests": 2,
24
+ "resolution": 3,
25
+ "species": 13,
26
+ "style": 3,
27
+ "text": 4
28
+ },
29
+ "n_uncategorized_tags_with_vectors": 5089,
30
+ "n_soft_assignments": 5089,
31
+ "soft_assignment_csv": "data\\analysis\\tfidf_uncategorized_soft_assignments.csv",
32
+ "strong_suggestions_by_existing_category": {
33
+ "body_type": [
34
+ {
35
+ "tag": "equid",
36
+ "count": 399764,
37
+ "sim": 0.6387,
38
+ "margin": 0.0706
39
+ },
40
+ {
41
+ "tag": "friendship_is_magic",
42
+ "count": 254760,
43
+ "sim": 0.5903,
44
+ "margin": 0.0648
45
+ },
46
+ {
47
+ "tag": "horse",
48
+ "count": 193042,
49
+ "sim": 0.6347,
50
+ "margin": 0.0589
51
+ },
52
+ {
53
+ "tag": "feathers",
54
+ "count": 192797,
55
+ "sim": 0.6778,
56
+ "margin": 0.0342
57
+ },
58
+ {
59
+ "tag": "pony",
60
+ "count": 122813,
61
+ "sim": 0.579,
62
+ "margin": 0.0623
63
+ },
64
+ {
65
+ "tag": "unicorn",
66
+ "count": 112328,
67
+ "sim": 0.5911,
68
+ "margin": 0.0641
69
+ },
70
+ {
71
+ "tag": "cutie_mark",
72
+ "count": 104225,
73
+ "sim": 0.5776,
74
+ "margin": 0.0699
75
+ },
76
+ {
77
+ "tag": "pegasus",
78
+ "count": 72559,
79
+ "sim": 0.5737,
80
+ "margin": 0.0658
81
+ },
82
+ {
83
+ "tag": "mythology",
84
+ "count": 70578,
85
+ "sim": 0.7804,
86
+ "margin": 0.0444
87
+ },
88
+ {
89
+ "tag": "earth_pony",
90
+ "count": 67495,
91
+ "sim": 0.5658,
92
+ "margin": 0.0629
93
+ },
94
+ {
95
+ "tag": "winged_unicorn",
96
+ "count": 47307,
97
+ "sim": 0.5658,
98
+ "margin": 0.0681
99
+ },
100
+ {
101
+ "tag": "twilight_sparkle_(mlp)",
102
+ "count": 37425,
103
+ "sim": 0.5623,
104
+ "margin": 0.0662
105
+ },
106
+ {
107
+ "tag": "magic",
108
+ "count": 34045,
109
+ "sim": 0.7569,
110
+ "margin": 0.0321
111
+ },
112
+ {
113
+ "tag": "blue_feathers",
114
+ "count": 32326,
115
+ "sim": 0.6117,
116
+ "margin": 0.0467
117
+ }
118
+ ],
119
+ "breasts": [
120
+ {
121
+ "tag": "thick_thighs",
122
+ "count": 339009,
123
+ "sim": 0.9322,
124
+ "margin": 0.1074
125
+ },
126
+ {
127
+ "tag": "wide_hips",
128
+ "count": 235877,
129
+ "sim": 0.9492,
130
+ "margin": 0.1399
131
+ },
132
+ {
133
+ "tag": "cleavage",
134
+ "count": 116981,
135
+ "sim": 0.9351,
136
+ "margin": 0.048
137
+ },
138
+ {
139
+ "tag": "curvy_figure",
140
+ "count": 106779,
141
+ "sim": 0.9519,
142
+ "margin": 0.1504
143
+ },
144
+ {
145
+ "tag": "makeup",
146
+ "count": 93493,
147
+ "sim": 0.8957,
148
+ "margin": 0.0365
149
+ },
150
+ {
151
+ "tag": "voluptuous",
152
+ "count": 80599,
153
+ "sim": 0.9286,
154
+ "margin": 0.1621
155
+ },
156
+ {
157
+ "tag": "bikini",
158
+ "count": 68724,
159
+ "sim": 0.7648,
160
+ "margin": 0.1016
161
+ },
162
+ {
163
+ "tag": "mature_female",
164
+ "count": 56464,
165
+ "sim": 0.7331,
166
+ "margin": 0.073
167
+ },
168
+ {
169
+ "tag": "lips",
170
+ "count": 54711,
171
+ "sim": 0.9207,
172
+ "margin": 0.0435
173
+ },
174
+ {
175
+ "tag": "bat",
176
+ "count": 47413,
177
+ "sim": 0.57,
178
+ "margin": 0.0363
179
+ },
180
+ {
181
+ "tag": "high_heels",
182
+ "count": 46265,
183
+ "sim": 0.8766,
184
+ "margin": 0.069
185
+ },
186
+ {
187
+ "tag": "overweight_female",
188
+ "count": 41843,
189
+ "sim": 0.5167,
190
+ "margin": 0.0511
191
+ },
192
+ {
193
+ "tag": "lipstick",
194
+ "count": 33619,
195
+ "sim": 0.865,
196
+ "margin": 0.0906
197
+ },
198
+ {
199
+ "tag": "short_stack",
200
+ "count": 32606,
201
+ "sim": 0.7433,
202
+ "margin": 0.1192
203
+ }
204
+ ],
205
+ "hair": [
206
+ {
207
+ "tag": "blonde_hair",
208
+ "count": 183468,
209
+ "sim": 0.9439,
210
+ "margin": 0.0625
211
+ },
212
+ {
213
+ "tag": "animal_humanoid",
214
+ "count": 139252,
215
+ "sim": 0.5666,
216
+ "margin": 0.0467
217
+ },
218
+ {
219
+ "tag": "mammal_humanoid",
220
+ "count": 91171,
221
+ "sim": 0.5309,
222
+ "margin": 0.0651
223
+ },
224
+ {
225
+ "tag": "hair_accessory",
226
+ "count": 69018,
227
+ "sim": 0.881,
228
+ "margin": 0.0518
229
+ },
230
+ {
231
+ "tag": "translucent",
232
+ "count": 56824,
233
+ "sim": 0.9159,
234
+ "margin": 0.0338
235
+ },
236
+ {
237
+ "tag": "green_hair",
238
+ "count": 53768,
239
+ "sim": 0.9074,
240
+ "margin": 0.0322
241
+ },
242
+ {
243
+ "tag": "ponytail",
244
+ "count": 50191,
245
+ "sim": 0.9368,
246
+ "margin": 0.0343
247
+ },
248
+ {
249
+ "tag": "orange_hair",
250
+ "count": 42530,
251
+ "sim": 0.8961,
252
+ "margin": 0.0566
253
+ },
254
+ {
255
+ "tag": "grey_hair",
256
+ "count": 38953,
257
+ "sim": 0.8883,
258
+ "margin": 0.0369
259
+ }
260
+ ],
261
+ "clothing": [
262
+ {
263
+ "tag": "overweight",
264
+ "count": 176776,
265
+ "sim": 0.5174,
266
+ "margin": 0.0504
267
+ },
268
+ {
269
+ "tag": "bear",
270
+ "count": 111640,
271
+ "sim": 0.5774,
272
+ "margin": 0.0399
273
+ },
274
+ {
275
+ "tag": "overweight_male",
276
+ "count": 94070,
277
+ "sim": 0.4665,
278
+ "margin": 0.0552
279
+ },
280
+ {
281
+ "tag": "humanoid_hands",
282
+ "count": 87880,
283
+ "sim": 0.7098,
284
+ "margin": 0.066
285
+ },
286
+ {
287
+ "tag": "tiger",
288
+ "count": 79829,
289
+ "sim": 0.6916,
290
+ "margin": 0.0464
291
+ },
292
+ {
293
+ "tag": "overweight_anthro",
294
+ "count": 75733,
295
+ "sim": 0.4877,
296
+ "margin": 0.0376
297
+ },
298
+ {
299
+ "tag": "tank_top",
300
+ "count": 39957,
301
+ "sim": 0.894,
302
+ "margin": 0.0662
303
+ }
304
+ ],
305
+ "body_decor": [
306
+ {
307
+ "tag": "jewelry",
308
+ "count": 185593,
309
+ "sim": 0.982,
310
+ "margin": 0.0659
311
+ },
312
+ {
313
+ "tag": "ring_piercing",
314
+ "count": 96328,
315
+ "sim": 0.9299,
316
+ "margin": 0.0669
317
+ },
318
+ {
319
+ "tag": "spikes",
320
+ "count": 69719,
321
+ "sim": 0.7772,
322
+ "margin": 0.0432
323
+ },
324
+ {
325
+ "tag": "facial_piercing",
326
+ "count": 62968,
327
+ "sim": 0.8272,
328
+ "margin": 0.074
329
+ },
330
+ {
331
+ "tag": "nose_piercing",
332
+ "count": 42004,
333
+ "sim": 0.8219,
334
+ "margin": 0.0683
335
+ },
336
+ {
337
+ "tag": "furgonomics",
338
+ "count": 41726,
339
+ "sim": 0.9142,
340
+ "margin": 0.0364
341
+ },
342
+ {
343
+ "tag": "chain",
344
+ "count": 30193,
345
+ "sim": 0.7603,
346
+ "margin": 0.0372
347
+ }
348
+ ],
349
+ "perspective": [
350
+ {
351
+ "tag": "glistening",
352
+ "count": 77473,
353
+ "sim": 0.9013,
354
+ "margin": 0.0345
355
+ },
356
+ {
357
+ "tag": "pupils",
358
+ "count": 69130,
359
+ "sim": 0.91,
360
+ "margin": 0.0305
361
+ },
362
+ {
363
+ "tag": "facial_tuft",
364
+ "count": 66479,
365
+ "sim": 0.9038,
366
+ "margin": 0.0343
367
+ },
368
+ {
369
+ "tag": "cheek_tuft",
370
+ "count": 63157,
371
+ "sim": 0.902,
372
+ "margin": 0.0306
373
+ },
374
+ {
375
+ "tag": "full-length_portrait",
376
+ "count": 56605,
377
+ "sim": 0.8895,
378
+ "margin": 0.0365
379
+ },
380
+ {
381
+ "tag": "neck_tuft",
382
+ "count": 44495,
383
+ "sim": 0.9241,
384
+ "margin": 0.034
385
+ },
386
+ {
387
+ "tag": "colored",
388
+ "count": 42012,
389
+ "sim": 0.8533,
390
+ "margin": 0.0426
391
+ },
392
+ {
393
+ "tag": "light",
394
+ "count": 31841,
395
+ "sim": 0.9378,
396
+ "margin": 0.0324
397
+ }
398
+ ],
399
+ "species": [
400
+ {
401
+ "tag": "bird",
402
+ "count": 115422,
403
+ "sim": 0.5153,
404
+ "margin": 0.033
405
+ },
406
+ {
407
+ "tag": "fish",
408
+ "count": 69011,
409
+ "sim": 0.7118,
410
+ "margin": 0.0719
411
+ }
412
+ ],
413
+ "gender": [
414
+ {
415
+ "tag": "pokemorph",
416
+ "count": 61455,
417
+ "sim": 0.6524,
418
+ "margin": 0.0551
419
+ },
420
+ {
421
+ "tag": "eeveelution",
422
+ "count": 58150,
423
+ "sim": 0.6492,
424
+ "margin": 0.0838
425
+ },
426
+ {
427
+ "tag": "fire",
428
+ "count": 45251,
429
+ "sim": 0.8511,
430
+ "margin": 0.0378
431
+ }
432
+ ],
433
+ "fur_style": [
434
+ {
435
+ "tag": "lion",
436
+ "count": 60001,
437
+ "sim": 0.6967,
438
+ "margin": 0.0622
439
+ }
440
+ ],
441
+ "general_activity_if_any": [
442
+ {
443
+ "tag": "humor",
444
+ "count": 38344,
445
+ "sim": 0.8989,
446
+ "margin": 0.0663
447
+ }
448
+ ],
449
+ "organization": [
450
+ {
451
+ "tag": "surprise",
452
+ "count": 36982,
453
+ "sim": 0.7936,
454
+ "margin": 0.0792
455
+ }
456
+ ],
457
+ "style": [
458
+ {
459
+ "tag": "western_dragon",
460
+ "count": 32262,
461
+ "sim": 0.5275,
462
+ "margin": 0.0314
463
+ }
464
+ ],
465
+ "location": [
466
+ {
467
+ "tag": "window",
468
+ "count": 31314,
469
+ "sim": 0.9597,
470
+ "margin": 0.031
471
+ }
472
+ ]
473
+ },
474
+ "cluster_summary_uncategorized": [
475
+ {
476
+ "cluster_id": 0,
477
+ "size": 8,
478
+ "nearest_existing_category": "hair",
479
+ "nearest_existing_sim": 0.5456,
480
+ "top_tags": [
481
+ "animal_humanoid",
482
+ "mammal_humanoid",
483
+ "felid_humanoid",
484
+ "feline_humanoid",
485
+ "cat_humanoid",
486
+ "canid_humanoid",
487
+ "canine_humanoid",
488
+ "dark_skin"
489
+ ],
490
+ "top_tokens": [
491
+ "humanoid",
492
+ "animal",
493
+ "mammal",
494
+ "felid",
495
+ "feline",
496
+ "cat",
497
+ "canid",
498
+ "canine",
499
+ "dark",
500
+ "skin"
501
+ ]
502
+ },
503
+ {
504
+ "cluster_id": 1,
505
+ "size": 103,
506
+ "nearest_existing_category": "limbs",
507
+ "nearest_existing_sim": 0.9806,
508
+ "top_tags": [
509
+ "e621",
510
+ "fur",
511
+ "felid",
512
+ "feet",
513
+ "biped",
514
+ "tuft",
515
+ "plant",
516
+ "eyewear",
517
+ "felis",
518
+ "domestic_cat",
519
+ "black_nose",
520
+ "signature",
521
+ "brown_hair",
522
+ "black_hair",
523
+ "eyebrows",
524
+ "yellow_eyes",
525
+ "barefoot",
526
+ "stripes",
527
+ "pose",
528
+ "scales"
529
+ ],
530
+ "top_tokens": [
531
+ "tuft",
532
+ "black",
533
+ "eyes",
534
+ "hair",
535
+ "nose",
536
+ "brown",
537
+ "yellow",
538
+ "(artwork)",
539
+ "border",
540
+ "pupils",
541
+ "white",
542
+ "portrait"
543
+ ]
544
+ },
545
+ {
546
+ "cluster_id": 2,
547
+ "size": 67,
548
+ "nearest_existing_category": "species",
549
+ "nearest_existing_sim": 0.8735,
550
+ "top_tags": [
551
+ "leporid",
552
+ "rabbit",
553
+ "bovid",
554
+ "bird",
555
+ "sega",
556
+ "widescreen",
557
+ "3d_(artwork)",
558
+ "machine",
559
+ "caprine",
560
+ "alien",
561
+ "arthropod",
562
+ "tentacles",
563
+ "transformation",
564
+ "bandai_namco",
565
+ "robot",
566
+ "zootopia",
567
+ "undertale_(series)",
568
+ "bat",
569
+ "glowing_eyes",
570
+ "undertale"
571
+ ],
572
+ "top_tokens": [
573
+ "undertale",
574
+ "bat",
575
+ "fox",
576
+ "asian",
577
+ "mythology",
578
+ "leporid",
579
+ "rabbit",
580
+ "bovid",
581
+ "bird",
582
+ "sega",
583
+ "widescreen",
584
+ "(artwork)"
585
+ ]
586
+ },
587
+ {
588
+ "cluster_id": 3,
589
+ "size": 87,
590
+ "nearest_existing_category": "hair",
591
+ "nearest_existing_sim": 0.9635,
592
+ "top_tags": [
593
+ "mammal",
594
+ "text",
595
+ "blue_eyes",
596
+ "navel",
597
+ "thick_thighs",
598
+ "green_eyes",
599
+ "sweat",
600
+ "wide_hips",
601
+ "narrowed_eyes",
602
+ "red_eyes",
603
+ "eyelashes",
604
+ "jewelry",
605
+ "blonde_hair",
606
+ "half-closed_eyes",
607
+ "panties",
608
+ "white_hair",
609
+ "blue_hair",
610
+ "slightly_chubby",
611
+ "cleavage",
612
+ "red_hair"
613
+ ],
614
+ "top_tokens": [
615
+ "hair",
616
+ "eyes",
617
+ "blue",
618
+ "green",
619
+ "eye",
620
+ "skin",
621
+ "red",
622
+ "closed",
623
+ "white",
624
+ "accessory",
625
+ "one",
626
+ "mature"
627
+ ]
628
+ },
629
+ {
630
+ "cluster_id": 4,
631
+ "size": 76,
632
+ "nearest_existing_category": "posture",
633
+ "nearest_existing_sim": 0.9598,
634
+ "top_tags": [
635
+ "canid",
636
+ "canis",
637
+ "fox",
638
+ "dragon",
639
+ "domestic_dog",
640
+ "wolf",
641
+ "size_difference",
642
+ "young",
643
+ "reptile",
644
+ "hybrid",
645
+ "girly",
646
+ "membrane_(anatomy)",
647
+ "tears",
648
+ "digitigrade",
649
+ "drooling",
650
+ "larger_male",
651
+ "smaller_male",
652
+ "greyscale",
653
+ "soles",
654
+ "deer"
655
+ ],
656
+ "top_tokens": [
657
+ "dog",
658
+ "dragon",
659
+ "difference",
660
+ "larger",
661
+ "male",
662
+ "smaller",
663
+ "female",
664
+ "hand",
665
+ "focus",
666
+ "canid",
667
+ "canis",
668
+ "fox"
669
+ ]
670
+ },
671
+ {
672
+ "cluster_id": 5,
673
+ "size": 21,
674
+ "nearest_existing_category": "body_decor",
675
+ "nearest_existing_sim": 0.7824,
676
+ "top_tags": [
677
+ "muscular",
678
+ "muscular_male",
679
+ "pantherine",
680
+ "abs",
681
+ "muscular_anthro",
682
+ "pecs",
683
+ "biceps",
684
+ "tiger",
685
+ "facial_hair",
686
+ "facial_piercing",
687
+ "scar",
688
+ "body_hair",
689
+ "big_muscles",
690
+ "athletic",
691
+ "beard",
692
+ "nose_piercing",
693
+ "cattle",
694
+ "musclegut",
695
+ "nose_ring",
696
+ "loincloth"
697
+ ],
698
+ "top_tokens": [
699
+ "muscular",
700
+ "hair",
701
+ "facial",
702
+ "piercing",
703
+ "nose",
704
+ "male",
705
+ "pantherine",
706
+ "abs",
707
+ "anthro",
708
+ "pecs",
709
+ "biceps",
710
+ "tiger"
711
+ ]
712
+ },
713
+ {
714
+ "cluster_id": 6,
715
+ "size": 17,
716
+ "nearest_existing_category": "clothing",
717
+ "nearest_existing_sim": 0.5671,
718
+ "top_tags": [
719
+ "belly",
720
+ "overweight",
721
+ "bear",
722
+ "kemono",
723
+ "overweight_male",
724
+ "humanoid_hands",
725
+ "big_belly",
726
+ "overweight_anthro",
727
+ "overweight_female",
728
+ "moobs",
729
+ "obese",
730
+ "lifewonders",
731
+ "tokyo_afterschool_summoners",
732
+ "suina",
733
+ "ursine",
734
+ "suid",
735
+ "sus_(pig)"
736
+ ],
737
+ "top_tokens": [
738
+ "overweight",
739
+ "belly",
740
+ "bear",
741
+ "kemono",
742
+ "male",
743
+ "humanoid",
744
+ "hands",
745
+ "big",
746
+ "anthro",
747
+ "female",
748
+ "moobs",
749
+ "obese"
750
+ ]
751
+ },
752
+ {
753
+ "cluster_id": 7,
754
+ "size": 32,
755
+ "nearest_existing_category": "body_type",
756
+ "nearest_existing_sim": 0.6897,
757
+ "top_tags": [
758
+ "equid",
759
+ "friendship_is_magic",
760
+ "horse",
761
+ "feathers",
762
+ "purple_eyes",
763
+ "pony",
764
+ "fan_character",
765
+ "unicorn",
766
+ "cutie_mark",
767
+ "purple_hair",
768
+ "pink_hair",
769
+ "pegasus",
770
+ "mythology",
771
+ "earth_pony",
772
+ "winged_unicorn",
773
+ "freckles",
774
+ "quadruped",
775
+ "twilight_sparkle_(mlp)",
776
+ "crown",
777
+ "magic"
778
+ ],
779
+ "top_tokens": [
780
+ "(mlp)",
781
+ "feathers",
782
+ "hair",
783
+ "magic",
784
+ "purple",
785
+ "pony",
786
+ "unicorn",
787
+ "rainbow",
788
+ "princess",
789
+ "equid",
790
+ "friendship",
791
+ "horse"
792
+ ]
793
+ }
794
+ ]
795
+ }
data/analysis/tfidf_uncategorized_soft_assignments.csv ADDED
The diff for this file is too large to render. See raw diff
 
data/eval_results/eval_caption_cogvlm_n10_bootstrap_baseline_explicit.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:10:09.106479", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 15}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 151, "n_selected": 32, "n_implied": 0, "n_structural": 0, "ret_R": 0.25, "P": 0.0312, "R": 0.0833, "F1": 0.0455, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 26, "n_leaf_gt": 9, "ret_P": 0.0199, "sel_given_ret": 0.3333, "over_sel": 2.67, "why": {"explicit": 32}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 66, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "34": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0312, "gen_R": 0.0833, "gen_F1": 0.0455, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["black_fur", "black_hands", "black_necktie", "black_shirt", "blue_topwear", "business_attire", "business_suit", "businesswear", "domestic_cat", "felis", "green_vest", "grey_fur", "grey_shirt", "hair_bun", "holding_cup", "lagomorph", "neck_tuft", "polo_shirt", "shirt", "siamese", "suit_jacket", "sweater_vest", "teal_shirt", "undershirt", "vest", "white_dress_shirt", "white_jacket", "white_necktie", "white_shirt", "white_t-shirt", "yellow_shirt"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["black_fur", "black_hands", "black_necktie", "black_shirt", "blue_topwear", "business_attire", "business_suit", "businesswear", "domestic_cat", "feline", "felis", "green_vest", "grey_fur", "grey_shirt", "hair_bun", "holding_cup", "lagomorph", "neck_tuft", "polo_shirt", "shirt", "siamese", "suit_jacket", "sweater_vest", "teal_shirt", "undershirt", "vest", "white_dress_shirt", "white_jacket", "white_necktie", "white_shirt", "white_t-shirt", "yellow_shirt"], "stage3_selected": ["black_fur", "black_hands", "black_necktie", "black_shirt", "blue_topwear", "business_attire", "business_suit", "businesswear", "domestic_cat", "feline", "felis", "green_vest", "grey_fur", "grey_shirt", "hair_bun", "holding_cup", "lagomorph", "neck_tuft", "polo_shirt", "shirt", "siamese", "suit_jacket", "sweater_vest", "teal_shirt", "undershirt", "vest", "white_dress_shirt", "white_jacket", "white_necktie", "white_shirt", "white_t-shirt", "yellow_shirt"], "stage3_selected_scores": {"feline": 0.7023, "shirt": 0.7976, "felis": 0.633, "domestic_cat": 0.6294, "lagomorph": 0.3932, "black_fur": 0.7157, "grey_fur": 0.5738, "neck_tuft": 0.5874, "vest": 0.8387, "blue_topwear": 0.7266, "white_shirt": 0.737, "black_shirt": 0.7341, "holding_cup": 0.7652, "hair_bun": 0.6912, "siamese": 0.6206, "grey_shirt": 0.7575, "business_suit": 0.5728, "yellow_shirt": 0.7542, "black_hands": 0.4561, "undershirt": 0.7585, "polo_shirt": 0.7082, "sweater_vest": 0.7526, "white_t-shirt": 0.7323, "suit_jacket": 0.6849, "white_jacket": 0.6773, "black_necktie": 0.7125, "businesswear": 0.5699, "white_dress_shirt": 0.6876, "green_vest": 0.7228, "business_attire": 0.5642, "teal_shirt": 0.7466, "white_necktie": 0.6419}, "stage3_selected_ranks": {"feline": 45, "shirt": 5, "felis": 70, "domestic_cat": 73, "lagomorph": 153, "black_fur": 37, "grey_fur": 96, "neck_tuft": 88, "vest": 3, "blue_topwear": 30, "white_shirt": 22, "black_shirt": 25, "holding_cup": 10, "hair_bun": 52, "siamese": 76, "grey_shirt": 13, "business_suit": 97, "yellow_shirt": 15, "black_hands": 139, "undershirt": 11, "polo_shirt": 42, "sweater_vest": 16, "white_t-shirt": 26, "suit_jacket": 54, "white_jacket": 57, "black_necktie": 39, "businesswear": 99, "white_dress_shirt": 53, "green_vest": 33, "business_attire": 102, "teal_shirt": 19, "white_necktie": 65}, "stage3_selected_phrase_ranks": {"feline": 1, "shirt": 1, "felis": 3, "domestic_cat": 4, "lagomorph": 8, "black_fur": 1, "grey_fur": 8, "neck_tuft": 6, "vest": 1, "blue_topwear": 10, "white_shirt": 4, "black_shirt": 8, "holding_cup": 4, "hair_bun": 1, "siamese": 5, "grey_shirt": 1, "business_suit": 8, "yellow_shirt": 2, "black_hands": 6, "undershirt": 3, "polo_shirt": 6, "sweater_vest": 3, "white_t-shirt": 3, "suit_jacket": 4, "white_jacket": 7, "black_necktie": 2, "businesswear": 9, "white_dress_shirt": 5, "green_vest": 6, "business_attire": 1, "teal_shirt": 4, "white_necktie": 10}, "extra_evidence": {"black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "black_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4561}, "black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7125}, "black_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7341}, "blue_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7266}, "business_attire": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5642}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5728}, "businesswear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5699}, "domestic_cat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6294}, "felis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "green_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7228}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5738}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7575}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6912}, "holding_cup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7652}, "lagomorph": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5874}, "polo_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "siamese": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6206}, "suit_jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6849}, "sweater_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7526}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7585}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6876}, "white_jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6773}, "white_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6419}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.737}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7323}, "yellow_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7542}}, "structural": [], "t1": 2.17, "t2": 11.78, "t3": 13.93, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=154 entity=1 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 160, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.25, "R": 0.2857, "F1": 0.2667, "leaf_P": 0.0769, "leaf_R": 0.1, "leaf_F1": 0.087, "n_leaf_sel": 13, "n_leaf_gt": 10, "ret_P": 0.05, "sel_given_ret": 0.5, "over_sel": 1.14, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 52, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "40": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.25, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.25, "gen_R": 0.2857, "gen_F1": 0.2667, "missed": ["clothed", "clothing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal"], "extra": ["belly_dancer_outfit", "braided_hair", "dancewear", "gorilla", "grinning_at_viewer", "loincloth", "monkey", "raised_arm", "raised_arms", "relaxed_expression", "spread_arms", "ursine"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["ape", "bear", "belly_dancer_outfit", "braided_hair", "dancewear", "dancing", "gorilla", "grinning_at_viewer", "loincloth", "monkey", "primate", "raised_arm", "raised_arms", "relaxed_expression", "spread_arms", "ursine"], "stage3_selected": ["ape", "bear", "belly_dancer_outfit", "braided_hair", "dancewear", "dancing", "gorilla", "grinning_at_viewer", "loincloth", "monkey", "primate", "raised_arm", "raised_arms", "relaxed_expression", "spread_arms", "ursine"], "stage3_selected_scores": {"bear": 0.5551, "raised_arm": 0.4867, "braided_hair": 0.3274, "primate": 0.8894, "ursine": 0.4202, "loincloth": 0.5634, "dancing": 0.5523, "monkey": 0.7547, "ape": 0.9763, "raised_arms": 0.5407, "gorilla": 0.8287, "spread_arms": 0.3987, "dancewear": 0.3454, "grinning_at_viewer": 0.4379, "belly_dancer_outfit": 0.351, "relaxed_expression": 0.4112}, "stage3_selected_ranks": {"bear": 14, "raised_arm": 28, "braided_hair": 150, "primate": 2, "ursine": 56, "loincloth": 12, "dancing": 15, "monkey": 6, "ape": 1, "raised_arms": 19, "gorilla": 5, "spread_arms": 77, "dancewear": 123, "grinning_at_viewer": 40, "belly_dancer_outfit": 119, "relaxed_expression": 69}, "stage3_selected_phrase_ranks": {"bear": 1, "raised_arm": 1, "braided_hair": 8, "primate": 1, "ursine": 9, "loincloth": 1, "dancing": 1, "monkey": 6, "ape": 1, "raised_arms": 1, "gorilla": 3, "spread_arms": 5, "dancewear": 8, "grinning_at_viewer": 3, "belly_dancer_outfit": 7, "relaxed_expression": 7}, "extra_evidence": {"belly_dancer_outfit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.351}, "braided_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3274}, "dancewear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3454}, "gorilla": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8287}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4379}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4867}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4112}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "ursine": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}}, "structural": [], "t1": 2.24, "t2": 12.17, "t3": 30.41, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=160 entity=5 copyright_filtered=3 generic_char_to_general=1 unknown_type=3", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"style_or_meta\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"style_or_meta\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 133, "n_selected": 20, "n_implied": 0, "n_structural": 0, "ret_R": 0.7143, "P": 0.3, "R": 0.4286, "F1": 0.3529, "leaf_P": 0.2353, "leaf_R": 0.4444, "leaf_F1": 0.3077, "n_leaf_sel": 17, "n_leaf_gt": 9, "ret_P": 0.0752, "sel_given_ret": 0.6, "over_sel": 1.43, "why": {"explicit": 20}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 60, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3, "gen_R": 0.4286, "gen_F1": 0.3529, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "lagomorph", "leporid", "mammal"], "extra": ["cheek_spots", "coat", "confident", "eyes", "fur_coat", "holding_pillow", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "relationship", "romantic_ambiance", "round_ears", "winter_coat"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_spots", "coat", "confident", "duo", "eyes", "fur_coat", "holding_pillow", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "plushie", "rabbit", "relationship", "romantic", "romantic_ambiance", "romantic_couple", "round_ears", "teal_eyes", "winter_coat"], "stage3_selected": ["cheek_spots", "coat", "confident", "duo", "eyes", "fur_coat", "holding_pillow", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "plushie", "rabbit", "relationship", "romantic", "romantic_ambiance", "romantic_couple", "round_ears", "teal_eyes", "winter_coat"], "stage3_selected_scores": {"duo": 0.3624, "rabbit": 0.5936, "romantic": 0.5599, "romantic_couple": 0.5617, "coat": 0.638, "plushie": 0.7452, "romantic_ambiance": 0.5277, "teal_eyes": 0.6281, "lab_coat": 0.5158, "round_ears": 0.434, "inanimate_object": 0.4648, "cheek_spots": 0.4296, "confident": 0.5158, "holding_plushie": 0.7791, "fur_coat": 0.4905, "winter_coat": 0.4757, "holding_pillow": 0.4824, "holding_toy": 0.5853, "relationship": 0.6203, "eyes": 0.8767}, "stage3_selected_ranks": {"duo": 131, "rabbit": 13, "romantic": 19, "romantic_couple": 18, "coat": 7, "plushie": 3, "romantic_ambiance": 30, "teal_eyes": 8, "lab_coat": 40, "round_ears": 102, "inanimate_object": 73, "cheek_spots": 108, "confident": 39, "holding_plushie": 2, "fur_coat": 53, "winter_coat": 66, "holding_pillow": 61, "holding_toy": 14, "relationship": 9, "eyes": 1}, "stage3_selected_phrase_ranks": {"duo": 3, "rabbit": 1, "romantic": 2, "romantic_couple": 1, "coat": 1, "plushie": 1, "romantic_ambiance": 3, "teal_eyes": 1, "lab_coat": 5, "round_ears": 10, "inanimate_object": 9, "cheek_spots": 10, "confident": 7, "holding_plushie": 1, "fur_coat": 9, "winter_coat": 10, "holding_pillow": 8, "holding_toy": 4, "relationship": 1, "eyes": 1}, "extra_evidence": {"cheek_spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4296}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "fur_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4905}, "holding_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4824}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "holding_toy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5853}, "inanimate_object": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "lab_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "relationship": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6203}, "romantic_ambiance": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5277}, "round_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.434}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4757}}, "structural": [], "t1": 3.35, "t2": 10.68, "t3": 9.13, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=131 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 113, "n_selected": 48, "n_implied": 0, "n_structural": 0, "ret_R": 0.75, "P": 0.0625, "R": 0.75, "F1": 0.1154, "leaf_P": 0.0222, "leaf_R": 0.25, "leaf_F1": 0.0408, "n_leaf_sel": 45, "n_leaf_gt": 4, "ret_P": 0.0265, "sel_given_ret": 1.0, "over_sel": 12.0, "why": {"explicit": 49}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 55, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0625, "gen_R": 0.75, "gen_F1": 0.1154, "missed": ["solo"], "extra": ["8_eyes", "biceps", "big_ears", "blowup_background", "cartoon_physics", "cyan_background", "floating", "floating_hearts", "floating_island", "floating_limbs", "geometric_background", "glistening_eyes", "grey_nose", "half-closed_eyes", "light_nose", "looking_away", "looking_down_at_viewer", "midair", "mouth_closed", "muscular_male", "no_irises", "no_pupils", "nose", "pale_body", "pink_background", "red_spots", "round_nose", "smiling_at_viewer", "soviet_cartoon", "sparkling_background", "spots", "spotted_shoulders", "tan_belly", "tan_chest", "tan_countershading", "tan_face", "tan_fingers", "tan_hands", "tan_nose", "teal_background", "toony", "toony_eyes", "toothy_smile", "white_spots", "yellow_background"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["8_eyes", "biceps", "big_ears", "blowup_background", "cartoon_physics", "cyan_background", "floating", "floating_hearts", "floating_island", "floating_limbs", "geometric_background", "glistening_eyes", "grey_nose", "half-closed_eyes", "light_nose", "looking_away", "looking_down_at_viewer", "midair", "mouth_closed", "muscular_male", "no_irises", "no_pupils", "nose", "pale_body", "pink_background", "red_nose", "red_spots", "round_nose", "smile", "smiling_at_viewer", "soviet_cartoon", "sparkling_background", "spots", "spotted_shoulders", "tan_belly", "tan_body", "tan_chest", "tan_countershading", "tan_face", "tan_fingers", "tan_hands", "tan_nose", "teal_background", "toony", "toony_eyes", "toothy_smile", "white_spots", "yellow_background"], "stage3_selected": ["8_eyes", "biceps", "big_ears", "blowup_background", "cartoon_physics", "cyan_background", "floating", "floating_hearts", "floating_island", "floating_limbs", "geometric_background", "glistening_eyes", "grey_nose", "half-closed_eyes", "invalid_tag", "light_nose", "looking_away", "looking_down_at_viewer", "midair", "mouth_closed", "muscular_male", "no_irises", "no_pupils", "nose", "pale_body", "pink_background", "red_nose", "red_spots", "round_nose", "smile", "smiling_at_viewer", "soviet_cartoon", "sparkling_background", "spots", "spotted_shoulders", "tan_belly", "tan_body", "tan_chest", "tan_countershading", "tan_face", "tan_fingers", "tan_hands", "tan_nose", "teal_background", "toony", "toony_eyes", "toothy_smile", "white_spots", "yellow_background"], "stage3_selected_scores": {"smile": 0.5953, "muscular_male": 0.3003, "half-closed_eyes": 0.3988, "tan_body": 0.658, "biceps": 0.2959, "spots": 0.6222, "toony": 0.5972, "big_ears": 0.4491, "looking_away": 0.4923, "glistening_eyes": 0.5148, "pink_background": 0.507, "smiling_at_viewer": 0.5321, "mouth_closed": 0.6286, "red_nose": 0.7473, "yellow_background": 0.5326, "floating": 0.6452, "no_pupils": 0.4532, "tan_countershading": 0.6488, "grey_nose": 0.5181, "tan_belly": 0.6333, "white_spots": 0.6165, "looking_down_at_viewer": 0.4663, "midair": 0.4065, "tan_face": 0.6956, "invalid_tag": 0.5284, "geometric_background": 0.5063, "tan_nose": 0.6411, "teal_background": 0.5041, "tan_chest": 0.6867, "no_irises": 0.556, "blowup_background": 0.5293, "red_spots": 0.6014, "tan_hands": 0.6782, "light_nose": 0.6631, "cyan_background": 0.5072, "soviet_cartoon": 0.3873, "tan_fingers": 0.6852, "pale_body": 0.4676, "spotted_shoulders": 0.6126, "floating_island": 0.4737, "sparkling_background": 0.4255, "round_nose": 0.5282, "toony_eyes": 0.3677, "8_eyes": 0.3916, "toothy_smile": 0.43, "floating_limbs": 0.4312, "floating_hearts": 0.3953, "cartoon_physics": 0.4076, "nose": 0.8611}, "stage3_selected_ranks": {"smile": 37, "muscular_male": 116, "half-closed_eyes": 107, "tan_body": 15, "biceps": 118, "spots": 27, "toony": 36, "big_ears": 85, "looking_away": 72, "glistening_eyes": 60, "pink_background": 66, "smiling_at_viewer": 53, "mouth_closed": 26, "red_nose": 3, "yellow_background": 52, "floating": 20, "no_pupils": 84, "tan_countershading": 17, "grey_nose": 58, "tan_belly": 25, "white_spots": 29, "looking_down_at_viewer": 82, "midair": 105, "tan_face": 6, "invalid_tag": 56, "geometric_background": 67, "tan_nose": 22, "teal_background": 68, "tan_chest": 8, "no_irises": 48, "blowup_background": 55, "red_spots": 34, "tan_hands": 10, "light_nose": 14, "cyan_background": 65, "soviet_cartoon": 110, "tan_fingers": 9, "pale_body": 81, "spotted_shoulders": 31, "floating_island": 78, "sparkling_background": 100, "round_nose": 57, "toony_eyes": 112, "8_eyes": 109, "toothy_smile": 96, "floating_limbs": 94, "floating_hearts": 108, "cartoon_physics": 104, "nose": 2}, "stage3_selected_phrase_ranks": {"smile": 2, "muscular_male": 8, "half-closed_eyes": 9, "tan_body": 6, "biceps": 10, "spots": 7, "toony": 1, "big_ears": 7, "looking_away": 4, "glistening_eyes": 2, "pink_background": 10, "smiling_at_viewer": 3, "mouth_closed": 2, "red_nose": 1, "yellow_background": 2, "floating": 1, "no_pupils": 6, "tan_countershading": 7, "grey_nose": 10, "tan_belly": 10, "white_spots": 8, "looking_down_at_viewer": 7, "midair": 8, "tan_face": 1, "invalid_tag": 7, "geometric_background": 9, "tan_nose": 6, "teal_background": 10, "tan_chest": 2, "no_irises": 3, "blowup_background": 3, "red_spots": 10, "tan_hands": 4, "light_nose": 2, "cyan_background": 9, "soviet_cartoon": 7, "tan_fingers": 3, "pale_body": 3, "spotted_shoulders": 9, "floating_island": 2, "sparkling_background": 7, "round_nose": 9, "toony_eyes": 9, "8_eyes": 10, "toothy_smile": 10, "floating_limbs": 5, "floating_hearts": 10, "cartoon_physics": 6, "nose": 1}, "extra_evidence": {"8_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3916}, "biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2959}, "big_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5293}, "cartoon_physics": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4076}, "cyan_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5072}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "floating_hearts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3953}, "floating_island": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4737}, "floating_limbs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4312}, "geometric_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5063}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5148}, "grey_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5181}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3988}, "light_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6631}, "looking_away": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4923}, "looking_down_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4663}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4065}, "mouth_closed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6286}, "muscular_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3003}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.556}, "no_pupils": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4532}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8611}, "pale_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4676}, "pink_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.507}, "red_spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6014}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5282}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5321}, "soviet_cartoon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3873}, "sparkling_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4255}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6222}, "spotted_shoulders": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6126}, "tan_belly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6333}, "tan_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6867}, "tan_countershading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6488}, "tan_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6956}, "tan_fingers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6852}, "tan_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6782}, "tan_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6411}, "teal_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5041}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5972}, "toony_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3677}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "white_spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6165}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5326}}, "structural": [], "t1": 2.03, "t2": 11.34, "t3": 17.98, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=114 entity=0 copyright_filtered=4 generic_char_to_general=0 unknown_type=5"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 184, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.4091, "P": 0.2222, "R": 0.2727, "F1": 0.2449, "leaf_P": 0.1304, "leaf_R": 0.25, "leaf_F1": 0.1714, "n_leaf_sel": 23, "n_leaf_gt": 12, "ret_P": 0.0489, "sel_given_ret": 0.6667, "over_sel": 1.23, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 83, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2222, "gen_R": 0.2727, "gen_F1": 0.2449, "missed": ["anthro", "bottomwear", "chest_tuft", "clothed", "countershading", "felid", "fur", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "tiger", "topless"], "extra": ["actual_fur", "back_muscles", "bengal_tiger", "big_biceps", "cheek_tuft", "countershade_body", "facial_tuft", "flexing_bicep", "flexing_both_biceps", "flexing_muscles", "full-length_portrait", "gesture", "hand_on_chest", "hand_on_ear", "hand_on_own_head", "heterochromia", "muscular_arms", "neck_tuft", "striped_fur", "suggestive_pose", "white_chest"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["actual_fur", "back_muscles", "bengal_tiger", "big_biceps", "blue_eyes", "cheek_tuft", "clothing", "countershade_body", "facial_tuft", "flexing_bicep", "flexing_both_biceps", "flexing_muscles", "full-length_portrait", "gesture", "hand_on_chest", "hand_on_ear", "hand_on_head", "hand_on_own_head", "heterochromia", "muscular_arms", "neck_tuft", "shorts", "striped_fur", "stripes", "suggestive_pose", "tuft", "white_chest"], "stage3_selected": ["actual_fur", "back_muscles", "bengal_tiger", "big_biceps", "blue_eyes", "cheek_tuft", "clothing", "countershade_body", "facial_tuft", "flexing_bicep", "flexing_both_biceps", "flexing_muscles", "full-length_portrait", "gesture", "hand_on_chest", "hand_on_ear", "hand_on_head", "hand_on_own_head", "heterochromia", "muscular_arms", "neck_tuft", "shorts", "striped_fur", "stripes", "suggestive_pose", "tuft", "white_chest"], "stage3_selected_scores": {"clothing": 0.3357, "blue_eyes": 0.572, "tuft": 0.4899, "stripes": 0.4682, "shorts": 0.5791, "facial_tuft": 0.4826, "cheek_tuft": 0.4701, "gesture": 0.586, "full-length_portrait": 0.4406, "neck_tuft": 0.4664, "striped_fur": 0.6385, "heterochromia": 0.3978, "hand_on_head": 0.5934, "hand_on_chest": 0.4934, "back_muscles": 0.5862, "muscular_arms": 0.7949, "big_biceps": 0.6931, "flexing_bicep": 0.6626, "bengal_tiger": 0.4382, "white_chest": 0.92, "suggestive_pose": 0.6244, "countershade_body": 0.8756, "hand_on_ear": 0.4986, "flexing_both_biceps": 0.562, "hand_on_own_head": 0.5255, "flexing_muscles": 0.6013, "actual_fur": 0.4537}, "stage3_selected_ranks": {"clothing": 179, "blue_eyes": 63, "tuft": 94, "stripes": 119, "shorts": 62, "facial_tuft": 105, "cheek_tuft": 116, "gesture": 59, "full-length_portrait": 140, "neck_tuft": 120, "striped_fur": 36, "heterochromia": 162, "hand_on_head": 53, "hand_on_chest": 93, "back_muscles": 58, "muscular_arms": 14, "big_biceps": 24, "flexing_bicep": 30, "bengal_tiger": 143, "white_chest": 2, "suggestive_pose": 42, "countershade_body": 3, "hand_on_ear": 88, "flexing_both_biceps": 65, "hand_on_own_head": 73, "flexing_muscles": 49, "actual_fur": 128}, "stage3_selected_phrase_ranks": {"clothing": 9, "blue_eyes": 2, "tuft": 4, "stripes": 3, "shorts": 1, "facial_tuft": 6, "cheek_tuft": 8, "gesture": 1, "full-length_portrait": 9, "neck_tuft": 9, "striped_fur": 2, "heterochromia": 8, "hand_on_head": 2, "hand_on_chest": 9, "back_muscles": 7, "muscular_arms": 1, "big_biceps": 7, "flexing_bicep": 5, "bengal_tiger": 5, "white_chest": 1, "suggestive_pose": 1, "countershade_body": 1, "hand_on_ear": 7, "flexing_both_biceps": 8, "hand_on_own_head": 4, "flexing_muscles": 6, "actual_fur": 5}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4537}, "back_muscles": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5862}, "bengal_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4382}, "big_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6931}, "cheek_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4701}, "countershade_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8756}, "facial_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4826}, "flexing_bicep": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6626}, "flexing_both_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.562}, "flexing_muscles": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6013}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4406}, "gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.586}, "hand_on_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4934}, "hand_on_ear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4986}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5255}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3978}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7949}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4664}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6385}, "suggestive_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6244}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.92}}, "structural": [], "t1": 1.74, "t2": 1.88, "t3": 11.96, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=180 entity=3 copyright_filtered=2 generic_char_to_general=0 unknown_type=2"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 202, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.6923, "P": 0.25, "R": 0.3077, "F1": 0.2759, "leaf_P": 0.2, "leaf_R": 0.5, "leaf_F1": 0.2857, "n_leaf_sel": 15, "n_leaf_gt": 6, "ret_P": 0.0446, "sel_given_ret": 0.4444, "over_sel": 1.23, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 3, "calls_exhausted_retries": 1, "attempts_total": 6, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 76, "attempts_by_n_local": {"60": {"attempts": 5, "parse_ok": 2, "parse_fail": 0, "errors": 3}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.5, "call_exhaustion_rate": 0.25}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.25, "gen_R": 0.3077, "gen_F1": 0.2759, "missed": ["caprine", "fur", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["black_speech_bubble", "bubble", "dark", "dark_theme", "darkness", "english_text", "laying_on_ground", "leaning_on_wall", "light", "medieval_fantasy", "on_ground", "speech_bubble"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["black_speech_bubble", "bovid", "bubble", "dark", "dark_theme", "darkness", "dialogue", "english_text", "goat", "human", "laying_on_ground", "leaning_on_wall", "light", "medieval_fantasy", "on_ground", "speech_bubble"], "stage3_selected": ["black_speech_bubble", "bovid", "bubble", "dark", "dark_theme", "darkness", "dialogue", "english_text", "goat", "human", "laying_on_ground", "leaning_on_wall", "light", "medieval_fantasy", "on_ground", "speech_bubble"], "stage3_selected_scores": {"dialogue": 0.7402, "human": 0.6639, "speech_bubble": 0.7486, "bovid": 0.5984, "goat": 0.7748, "light": 0.7781, "on_ground": 0.6688, "bubble": 0.7514, "dark": 0.5867, "leaning_on_wall": 0.5031, "darkness": 0.8346, "dark_theme": 0.5937, "laying_on_ground": 0.7028, "black_speech_bubble": 0.6312, "medieval_fantasy": 0.4783, "english_text": 0.6161}, "stage3_selected_ranks": {"dialogue": 10, "human": 19, "speech_bubble": 9, "bovid": 46, "goat": 5, "light": 4, "on_ground": 18, "bubble": 8, "dark": 52, "leaning_on_wall": 145, "darkness": 2, "dark_theme": 48, "laying_on_ground": 12, "black_speech_bubble": 32, "medieval_fantasy": 170, "english_text": 34}, "stage3_selected_phrase_ranks": {"dialogue": 1, "human": 1, "speech_bubble": 1, "bovid": 3, "goat": 1, "light": 1, "on_ground": 3, "bubble": 1, "dark": 3, "leaning_on_wall": 10, "darkness": 1, "dark_theme": 2, "laying_on_ground": 2, "black_speech_bubble": 4, "medieval_fantasy": 5, "english_text": 4}, "extra_evidence": {"black_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6312}, "bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7514}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5867}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5937}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6161}, "laying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7028}, "leaning_on_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5031}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "medieval_fantasy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4783}, "on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6688}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7486}}, "structural": [], "t1": 1.79, "t2": 1.86, "t3": 37.92, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=191 entity=5 copyright_filtered=6 generic_char_to_general=2 unknown_type=2", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"style_or_meta\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 58, \"why\": \"weak_implied\"}, {\"i\": 59}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 59}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"style_or_meta\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 58, \"why\": \"weak_implied\"}, {\"i\": 59, \"why\": \"style_or_meta\"}, {\"i\": 60}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 60}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 43}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 207, "n_selected": 32, "n_implied": 0, "n_structural": 0, "ret_R": 0.8571, "P": 0.2188, "R": 0.5, "F1": 0.3043, "leaf_P": 0.2, "leaf_R": 0.5, "leaf_F1": 0.2857, "n_leaf_sel": 30, "n_leaf_gt": 12, "ret_P": 0.058, "sel_given_ret": 0.5833, "over_sel": 2.29, "why": {"explicit": 32}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 65, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "30": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2188, "gen_R": 0.5, "gen_F1": 0.3043, "missed": ["angry", "bed", "duo", "eyes_closed", "hair", "lying", "text"], "extra": ["annoyed_expression", "applying_makeup", "auburn_hair", "aurora_(phenomenon)", "bedding", "bedroom", "comic", "digital_media_(artwork)", "english_text", "expressions", "eyeliner", "eyes", "half-length_portrait", "head_on_pillow", "lavender_hair", "lipstick", "long_hair", "lying_on_bed", "pajamas", "pink_lipstick", "purple_background", "purple_eyes", "purple_hands", "resting", "sitting_on_bed"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["annoyed_expression", "applying_makeup", "auburn_hair", "aurora_(phenomenon)", "bedding", "bedroom", "blonde_hair", "blue_eyes", "comic", "digital_media_(artwork)", "english_text", "expressions", "eyeliner", "eyes", "eyeshadow", "green_eyes", "half-length_portrait", "head_on_pillow", "lavender_hair", "lipstick", "long_hair", "lying_on_bed", "makeup", "pajamas", "pink_lipstick", "purple_background", "purple_eyes", "purple_hair", "purple_hands", "resting", "sitting_on_bed", "sleeping"], "stage3_selected": ["annoyed_expression", "applying_makeup", "auburn_hair", "aurora_(phenomenon)", "bedding", "bedroom", "blonde_hair", "blue_eyes", "comic", "digital_media_(artwork)", "english_text", "expressions", "eyeliner", "eyes", "eyeshadow", "green_eyes", "half-length_portrait", "head_on_pillow", "lavender_hair", "lipstick", "long_hair", "lying_on_bed", "makeup", "pajamas", "pink_lipstick", "purple_background", "purple_eyes", "purple_hair", "purple_hands", "resting", "sitting_on_bed", "sleeping"], "stage3_selected_scores": {"digital_media_(artwork)": 0.3347, "blue_eyes": 0.595, "green_eyes": 0.5933, "comic": 0.3823, "long_hair": 0.4471, "blonde_hair": 0.5872, "purple_eyes": 0.4293, "purple_hair": 0.5591, "makeup": 0.5894, "eyeshadow": 0.4712, "lipstick": 0.4782, "bedroom": 0.4873, "sleeping": 0.5987, "bedding": 0.3861, "purple_background": 0.4897, "half-length_portrait": 0.4132, "eyeliner": 0.4399, "lying_on_bed": 0.4059, "pajamas": 0.4043, "sitting_on_bed": 0.3765, "pink_lipstick": 0.4482, "resting": 0.5015, "annoyed_expression": 0.7219, "aurora_(phenomenon)": 0.3131, "expressions": 0.5393, "head_on_pillow": 0.3867, "auburn_hair": 0.399, "lavender_hair": 0.4066, "purple_hands": 0.5404, "applying_makeup": 0.4698, "eyes": 0.8955, "english_text": 0.4128}, "stage3_selected_ranks": {"digital_media_(artwork)": 192, "blue_eyes": 6, "green_eyes": 9, "comic": 151, "long_hair": 61, "blonde_hair": 11, "purple_eyes": 76, "purple_hair": 14, "makeup": 10, "eyeshadow": 50, "lipstick": 45, "bedroom": 40, "sleeping": 5, "bedding": 147, "purple_background": 39, "half-length_portrait": 98, "eyeliner": 68, "lying_on_bed": 106, "pajamas": 108, "sitting_on_bed": 156, "pink_lipstick": 60, "resting": 31, "annoyed_expression": 2, "aurora_(phenomenon)": 198, "expressions": 19, "head_on_pillow": 146, "auburn_hair": 120, "lavender_hair": 105, "purple_hands": 17, "applying_makeup": 51, "eyes": 1, "english_text": 99}, "stage3_selected_phrase_ranks": {"digital_media_(artwork)": 10, "blue_eyes": 1, "green_eyes": 1, "comic": 10, "long_hair": 3, "blonde_hair": 1, "purple_eyes": 2, "purple_hair": 1, "makeup": 1, "eyeshadow": 3, "lipstick": 2, "bedroom": 1, "sleeping": 1, "bedding": 8, "purple_background": 4, "half-length_portrait": 7, "eyeliner": 9, "lying_on_bed": 4, "pajamas": 4, "sitting_on_bed": 10, "pink_lipstick": 7, "resting": 1, "annoyed_expression": 1, "aurora_(phenomenon)": 10, "expressions": 3, "head_on_pillow": 7, "auburn_hair": 7, "lavender_hair": 7, "purple_hands": 2, "applying_makeup": 4, "eyes": 1, "english_text": 4}, "extra_evidence": {"annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4698}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.399}, "aurora_(phenomenon)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3131}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3823}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5393}, "eyeliner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4399}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4132}, "head_on_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3867}, "lavender_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4066}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4782}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4471}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "pajamas": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4043}, "pink_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4482}, "purple_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4897}, "purple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4293}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5404}, "resting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sitting_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}}, "structural": [], "t1": 1.49, "t2": 1.97, "t3": 12.13, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=210 entity=0 copyright_filtered=0 generic_char_to_general=0 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 179, "n_selected": 20, "n_implied": 0, "n_structural": 0, "ret_R": 0.5, "P": 0.15, "R": 0.1364, "F1": 0.1429, "leaf_P": 0.1176, "leaf_R": 0.1538, "leaf_F1": 0.1333, "n_leaf_sel": 17, "n_leaf_gt": 13, "ret_P": 0.0615, "sel_given_ret": 0.2727, "over_sel": 0.91, "why": {"explicit": 20}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 73, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "59": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.15, "gen_R": 0.1364, "gen_F1": 0.1429, "missed": ["anthro", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "solo", "string_instrument", "tail", "torn_clothing"], "extra": ["4_claws", "blonde_hair", "curled_hair", "ear_ring", "electric_guitar", "holding_guitar", "long_tail", "maned_wolf", "notched_ear", "nub_tail", "playing_guitar", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_shirt", "torn_topwear", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_claws", "bass_guitar", "blonde_hair", "curled_hair", "ear_ring", "electric_guitar", "holding_guitar", "long_tail", "maned_wolf", "notched_ear", "nub_tail", "playing_guitar", "plucked_string_instrument", "spade_tail", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_shirt", "torn_topwear", "wolf"], "stage3_selected": ["4_claws", "bass_guitar", "blonde_hair", "curled_hair", "ear_ring", "electric_guitar", "holding_guitar", "long_tail", "maned_wolf", "notched_ear", "nub_tail", "playing_guitar", "plucked_string_instrument", "spade_tail", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_shirt", "torn_topwear", "wolf"], "stage3_selected_scores": {"wolf": 0.5777, "blonde_hair": 0.3817, "ear_ring": 0.3919, "long_tail": 0.4218, "spade_tail": 0.6177, "notched_ear": 0.4314, "torn_bottomwear": 0.436, "plucked_string_instrument": 0.8658, "curled_hair": 0.3871, "maned_wolf": 0.4595, "torn_topwear": 0.3942, "torn_shirt": 0.4046, "playing_guitar": 0.9316, "torn_jeans": 0.4822, "electric_guitar": 0.8663, "bass_guitar": 0.9118, "nub_tail": 0.4298, "4_claws": 0.4515, "holding_guitar": 0.8442, "torn_leggings": 0.424}, "stage3_selected_ranks": {"wolf": 16, "blonde_hair": 136, "ear_ring": 125, "long_tail": 101, "spade_tail": 13, "notched_ear": 87, "torn_bottomwear": 83, "plucked_string_instrument": 6, "curled_hair": 131, "maned_wolf": 60, "torn_topwear": 121, "torn_shirt": 112, "playing_guitar": 2, "torn_jeans": 43, "electric_guitar": 5, "bass_guitar": 3, "nub_tail": 93, "4_claws": 69, "holding_guitar": 11, "torn_leggings": 99}, "stage3_selected_phrase_ranks": {"wolf": 1, "blonde_hair": 6, "ear_ring": 4, "long_tail": 5, "spade_tail": 1, "notched_ear": 4, "torn_bottomwear": 3, "plucked_string_instrument": 5, "curled_hair": 5, "maned_wolf": 3, "torn_topwear": 10, "torn_shirt": 7, "playing_guitar": 1, "torn_jeans": 1, "electric_guitar": 4, "bass_guitar": 2, "nub_tail": 3, "4_claws": 4, "holding_guitar": 10, "torn_leggings": 5}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4515}, "blonde_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3817}, "curled_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3871}, "ear_ring": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3919}, "electric_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8663}, "holding_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8442}, "long_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4218}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4595}, "notched_ear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4314}, "nub_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4298}, "playing_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9316}, "torn_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.436}, "torn_jeans": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4822}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.424}, "torn_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4046}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3942}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5777}}, "structural": [], "t1": 0.93, "t2": 1.69, "t3": 12.37, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=179 entity=2 copyright_filtered=4 generic_char_to_general=0 unknown_type=2"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 161, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.6087, "P": 0.4444, "R": 0.3478, "F1": 0.3902, "leaf_P": 0.3846, "leaf_R": 0.3333, "leaf_F1": 0.3571, "n_leaf_sel": 13, "n_leaf_gt": 15, "ret_P": 0.087, "sel_given_ret": 0.5714, "over_sel": 0.78, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 94, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "41": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4444, "gen_R": 0.3478, "gen_F1": 0.3902, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "duo", "fox", "grey_background", "lagomorph", "looking_at_another", "mammal", "overalls", "pants", "standing"], "extra": ["arctic_fox", "black_bottomwear", "black_pants", "blue_clothing", "grey_shirt", "kit_fox", "lop_ears", "open_mouth", "white_shirt", "white_t-shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["arctic_fox", "black_bottomwear", "black_pants", "blue_clothing", "claws", "crossed_arms", "facial_markings", "fur", "grey_shirt", "kit_fox", "leporid", "lop_ears", "open_mouth", "rabbit", "shirt", "topwear", "white_shirt", "white_t-shirt"], "stage3_selected": ["arctic_fox", "black_bottomwear", "black_pants", "blue_clothing", "claws", "crossed_arms", "facial_markings", "fur", "grey_shirt", "kit_fox", "leporid", "lop_ears", "open_mouth", "rabbit", "shirt", "simple_background", "topwear", "white_shirt", "white_t-shirt"], "stage3_selected_scores": {"fur": 0.6531, "simple_background": 0.416, "open_mouth": 0.633, "claws": 0.6303, "topwear": 0.6439, "shirt": 0.7483, "leporid": 0.5826, "rabbit": 0.6511, "facial_markings": 0.6945, "blue_clothing": 0.6538, "crossed_arms": 0.7285, "black_bottomwear": 0.7384, "white_shirt": 0.8197, "arctic_fox": 0.4522, "black_pants": 0.833, "grey_shirt": 0.6923, "lop_ears": 0.4692, "white_t-shirt": 0.7504, "kit_fox": 0.4485}, "stage3_selected_ranks": {"fur": 41, "simple_background": 163, "open_mouth": 49, "claws": 51, "topwear": 44, "shirt": 18, "leporid": 77, "rabbit": 42, "facial_markings": 27, "blue_clothing": 40, "crossed_arms": 21, "black_bottomwear": 19, "white_shirt": 4, "arctic_fox": 155, "black_pants": 3, "grey_shirt": 28, "lop_ears": 150, "white_t-shirt": 15, "kit_fox": 156}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 8, "open_mouth": 1, "claws": 1, "topwear": 7, "shirt": 1, "leporid": 4, "rabbit": 1, "facial_markings": 1, "blue_clothing": 8, "crossed_arms": 1, "black_bottomwear": 5, "white_shirt": 1, "arctic_fox": 5, "black_pants": 1, "grey_shirt": 4, "lop_ears": 8, "white_t-shirt": 10, "kit_fox": 6}, "extra_evidence": {"arctic_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4522}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7384}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.833}, "blue_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6538}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6923}, "kit_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4485}, "lop_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4692}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7504}}, "structural": [], "t1": 1.35, "t2": 1.61, "t3": 21.66, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=161 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 189, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.6364, "P": 0.2222, "R": 0.5455, "F1": 0.3158, "leaf_P": 0.2174, "leaf_R": 0.7143, "leaf_F1": 0.3333, "n_leaf_sel": 23, "n_leaf_gt": 7, "ret_P": 0.037, "sel_given_ret": 0.8571, "over_sel": 2.45, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 110, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2222, "gen_R": 0.5455, "gen_F1": 0.3158, "missed": ["canid", "canine", "mammal", "solo", "white_body"], "extra": ["action_pose", "actual_fur", "blue_ears", "blue_mouth", "blue_stripes", "body_hair", "canine_humanoid", "curved_tail", "female_humanoid", "frog_humanoid", "glistening_eyes", "gradient_tail", "jumper", "jumping", "male_humanoid", "neck_tuft", "pink_eyes", "pose", "stripes", "tail", "tailed_humanoid"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["action_pose", "actual_fur", "blue_ears", "blue_eyes", "blue_mouth", "blue_nose", "blue_stripes", "body_hair", "canine_humanoid", "curved_tail", "female_humanoid", "frog_humanoid", "fur", "glistening_eyes", "gradient_tail", "jumper", "jumping", "male_humanoid", "neck_tuft", "open_mouth", "pink_eyes", "pose", "purple_body", "stripes", "tail", "tailed_humanoid", "white_fur"], "stage3_selected": ["action_pose", "actual_fur", "blue_ears", "blue_eyes", "blue_mouth", "blue_nose", "blue_stripes", "body_hair", "canine_humanoid", "curved_tail", "female_humanoid", "frog_humanoid", "fur", "glistening_eyes", "gradient_tail", "jumper", "jumping", "male_humanoid", "neck_tuft", "open_mouth", "pink_eyes", "pose", "purple_body", "stripes", "tail", "tailed_humanoid", "white_fur"], "stage3_selected_scores": {"fur": 0.5663, "open_mouth": 0.5847, "tail": 0.5894, "blue_eyes": 0.5818, "white_fur": 0.577, "stripes": 0.5778, "pose": 0.5982, "purple_body": 0.5474, "body_hair": 0.2969, "pink_eyes": 0.391, "neck_tuft": 0.3946, "canine_humanoid": 0.9127, "glistening_eyes": 0.4376, "blue_nose": 0.592, "blue_ears": 0.4792, "action_pose": 0.5938, "jumping": 0.5802, "blue_stripes": 0.5366, "blue_mouth": 0.425, "tailed_humanoid": 0.55, "frog_humanoid": 0.5238, "female_humanoid": 0.5577, "curved_tail": 0.5957, "male_humanoid": 0.5448, "gradient_tail": 0.5513, "jumper": 0.4004, "actual_fur": 0.4438}, "stage3_selected_ranks": {"fur": 28, "open_mouth": 21, "tail": 20, "blue_eyes": 22, "white_fur": 27, "stripes": 26, "pose": 15, "purple_body": 39, "body_hair": 189, "pink_eyes": 156, "neck_tuft": 152, "canine_humanoid": 1, "glistening_eyes": 128, "blue_nose": 19, "blue_ears": 89, "action_pose": 17, "jumping": 23, "blue_stripes": 48, "blue_mouth": 139, "tailed_humanoid": 38, "frog_humanoid": 54, "female_humanoid": 33, "curved_tail": 16, "male_humanoid": 44, "gradient_tail": 35, "jumper": 149, "actual_fur": 124}, "stage3_selected_phrase_ranks": {"fur": 1, "open_mouth": 1, "tail": 1, "blue_eyes": 1, "white_fur": 1, "stripes": 1, "pose": 1, "purple_body": 1, "body_hair": 10, "pink_eyes": 10, "neck_tuft": 10, "canine_humanoid": 1, "glistening_eyes": 3, "blue_nose": 1, "blue_ears": 7, "action_pose": 1, "jumping": 1, "blue_stripes": 3, "blue_mouth": 10, "tailed_humanoid": 8, "frog_humanoid": 10, "female_humanoid": 6, "curved_tail": 1, "male_humanoid": 9, "gradient_tail": 3, "jumper": 2, "actual_fur": 3}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5938}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4438}, "blue_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4792}, "blue_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.425}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5366}, "body_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2969}, "canine_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9127}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5957}, "female_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5577}, "frog_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5238}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "gradient_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5513}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5802}, "male_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5448}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3946}, "pink_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.391}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5982}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5778}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5894}, "tailed_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.55}}, "structural": [], "t1": 1.23, "t2": 1.8, "t3": 36.84, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=190 entity=4 copyright_filtered=2 generic_char_to_general=0 unknown_type=5"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_baseline_explicit_skiprewrite.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:13:19.547226", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 16}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 10, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.83, "why": {"explicit": 10}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 27, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["business_suit", "face_focus", "focused", "holding_topwear", "necktie", "shirt", "t-shirt", "translucent", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["business_suit", "face_focus", "focused", "holding_topwear", "necktie", "shirt", "t-shirt", "translucent", "white_t-shirt", "white_topwear"], "stage3_selected": ["business_suit", "face_focus", "focused", "holding_topwear", "necktie", "shirt", "t-shirt", "translucent", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "translucent": 0.7484, "necktie": 0.8082, "t-shirt": 0.8329, "white_topwear": 0.6638, "business_suit": 0.6178, "white_t-shirt": 0.6584, "focused": 0.6608, "face_focus": 0.4971, "holding_topwear": 0.595}, "stage3_selected_ranks": {"shirt": 1, "translucent": 10, "necktie": 3, "t-shirt": 2, "white_topwear": 13, "business_suit": 26, "white_t-shirt": 15, "focused": 14, "face_focus": 56, "holding_topwear": 36}, "stage3_selected_phrase_ranks": {"shirt": 1, "translucent": 1, "necktie": 1, "t-shirt": 2, "white_topwear": 1, "business_suit": 4, "white_t-shirt": 2, "focused": 1, "face_focus": 6, "holding_topwear": 3}, "extra_evidence": {"business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "holding_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "translucent": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7484}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.71, "t3": 18.27, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 20, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.15, "R": 0.2143, "F1": 0.1765, "leaf_P": 0.1053, "leaf_R": 0.2, "leaf_F1": 0.1379, "n_leaf_sel": 19, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 1.43, "why": {"explicit": 21}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 68, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.25, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.15, "gen_R": 0.2143, "gen_F1": 0.1765, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "breaking_the_fourth_wall", "expressions", "extreme_size_difference", "game_media", "grin", "hand_above_head", "hand_over_head", "larger_cub", "larger_male", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "smirk", "smug_grin", "straight_arms"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "breaking_the_fourth_wall", "expressions", "extreme_size_difference", "game_media", "grin", "group", "hand_above_head", "hand_over_head", "larger_cub", "larger_male", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "smirk", "smug_grin", "straight_arms"], "stage3_selected": ["animated_comic", "breaking_the_fourth_wall", "expressions", "extreme_size_difference", "game_media", "grin", "group", "hand_above_head", "hand_over_head", "larger_cub", "larger_male", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "simple_background", "smirk", "smug_grin", "straight_arms"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "looking_at_viewer": 0.5612, "group": 0.4099, "larger_male": 0.4954, "grin": 0.7174, "smirk": 0.5408, "raised_arm": 0.7205, "extreme_size_difference": 0.367, "raised_arms": 0.8636, "raised_hand": 0.6751, "smug_grin": 0.4997, "breaking_the_fourth_wall": 0.4209, "partially_clothed_anthro": 0.4576, "expressions": 0.4746, "hand_above_head": 0.6835, "straight_arms": 0.6554, "game_media": 0.353, "hand_over_head": 0.5861, "larger_cub": 0.4221, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "looking_at_viewer": 22, "group": 132, "larger_male": 68, "grin": 3, "smirk": 34, "raised_arm": 2, "extreme_size_difference": 149, "raised_arms": 1, "raised_hand": 5, "smug_grin": 64, "breaking_the_fourth_wall": 119, "partially_clothed_anthro": 100, "expressions": 86, "hand_above_head": 4, "straight_arms": 10, "game_media": 156, "hand_over_head": 18, "larger_cub": 117, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "looking_at_viewer": 2, "group": 6, "larger_male": 4, "grin": 1, "smirk": 4, "raised_arm": 2, "extreme_size_difference": 9, "raised_arms": 1, "raised_hand": 4, "smug_grin": 5, "breaking_the_fourth_wall": 5, "partially_clothed_anthro": 10, "expressions": 5, "hand_above_head": 3, "straight_arms": 2, "game_media": 5, "hand_over_head": 1, "larger_cub": 10, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4209}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4746}, "extreme_size_difference": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}, "game_media": {"source": "stage3", "why": "explicit", "retrieval_score": 0.353}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "hand_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6835}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5861}, "larger_cub": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4221}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6751}, "smirk": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5408}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "straight_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6554}}, "structural": [], "t1": 0.0, "t2": 10.06, "t3": 22.05, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.36.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.36.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.43, "why": {"explicit": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 16, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["bored_expression", "facial_expressions", "petting", "setting", "surprised_face", "toony_expression"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["bored_expression", "facial_expressions", "petting", "setting", "surprised_face", "toony_expression"], "stage3_selected": ["bored_expression", "facial_expressions", "petting", "setting", "surprised_face", "toony_expression"], "stage3_selected_scores": {"petting": 0.7411, "surprised_face": 0.5885, "bored_expression": 0.5856, "toony_expression": 0.5935, "facial_expressions": 0.5874, "setting": 1.0}, "stage3_selected_ranks": {"petting": 12, "surprised_face": 31, "bored_expression": 36, "toony_expression": 27, "facial_expressions": 33, "setting": 2}, "stage3_selected_phrase_ranks": {"petting": 2, "surprised_face": 9, "bored_expression": 9, "toony_expression": 4, "facial_expressions": 8, "setting": 1}, "extra_evidence": {"bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5856}, "facial_expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5874}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "surprised_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5885}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5935}}, "structural": [], "t1": 0.0, "t2": 8.79, "t3": 11.69, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 2, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 2, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 0.5, "why": {"explicit": 3}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 1, "calls_with_selection": 1, "calls_exhausted_retries": 0, "attempts_total": 1, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 1, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 10, "attempts_by_n_local": {"58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["round_eyes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["round_eyes", "toony"], "stage3_selected": ["round_eyes", "simple_background", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "toony": 0.763, "round_eyes": 0.6227}, "stage3_selected_ranks": {"simple_background": 10, "toony": 6, "round_eyes": 30}, "stage3_selected_phrase_ranks": {"simple_background": 5, "toony": 3, "round_eyes": 1}, "extra_evidence": {"round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.71, "t3": 9.09, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 17, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 15, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 0.0, "over_sel": 0.77, "why": {"explicit": 17}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 31, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}}, "attempt_failure_rate": 0.3333333333333333, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["action_pose", "blue_shorts", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "holding_face", "neck_tuft", "open_shorts", "playful", "pose", "raised_arm", "striped_background", "striped_markings", "striped_tail", "two_tone_fur"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "blue_shorts", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "holding_face", "neck_tuft", "open_shorts", "playful", "pose", "raised_arm", "striped_background", "striped_markings", "striped_tail", "two_tone_fur"], "stage3_selected": ["action_pose", "blue_shorts", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "holding_face", "neck_tuft", "open_shorts", "playful", "pose", "raised_arm", "striped_background", "striped_markings", "striped_tail", "two_tone_fur"], "stage3_selected_scores": {"pose": 0.847, "two_tone_fur": 0.5967, "happy": 0.457, "neck_tuft": 0.5847, "raised_arm": 0.6005, "striped_markings": 0.5966, "striped_tail": 0.59, "hand_on_face": 0.5657, "action_pose": 0.6359, "playful": 0.6803, "blue_shorts": 0.7288, "striped_background": 0.6234, "hand_gesture": 0.6065, "holding_face": 0.5005, "open_shorts": 0.7362, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"pose": 3, "two_tone_fur": 45, "happy": 87, "neck_tuft": 51, "raised_arm": 42, "striped_markings": 46, "striped_tail": 50, "hand_on_face": 55, "action_pose": 31, "playful": 18, "blue_shorts": 14, "striped_background": 34, "hand_gesture": 39, "holding_face": 77, "open_shorts": 11, "hand_over_head": 56, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"pose": 1, "two_tone_fur": 7, "happy": 5, "neck_tuft": 9, "raised_arm": 2, "striped_markings": 8, "striped_tail": 10, "hand_on_face": 4, "action_pose": 8, "playful": 1, "blue_shorts": 1, "striped_background": 3, "hand_gesture": 1, "holding_face": 9, "open_shorts": 8, "hand_over_head": 5, "hand_on_own_head": 6}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5657}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "holding_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5005}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "open_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7362}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "striped_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "striped_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.59}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5967}}, "structural": [], "t1": 0.0, "t2": 0.78, "t3": 16.74, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"strong_implied\"}, {\"i\": 2, \"why\": \"style_or_meta\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"style_or_meta\"}, {\"i\": 11, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"iconography\"}, {\"i\": 17, \"why\": \"strong_implied\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"other\"}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.12.why\n Input should be 'explicit', 'strong_implied', 'weak_implied', 'style_or_meta' or 'other' [type=literal_error, input_value='iconography', input_type=str]\n For further information visit https://errors.pydantic.dev/2.12/v/literal_error\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 33, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.0606, "R": 0.1538, "F1": 0.087, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 31, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.25, "over_sel": 2.54, "why": {"explicit": 33}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 1, "calls_exhausted_retries": 1, "attempts_total": 4, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 1, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 33, "attempts_by_n_local": {"60": {"attempts": 4, "parse_ok": 1, "parse_fail": 0, "errors": 3}}, "attempt_failure_rate": 0.75, "call_exhaustion_rate": 0.5}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0625, "gen_R": 0.1538, "gen_F1": 0.0889, "missed": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "4_panel_comic", "border", "chart", "confusion", "cracked_ground", "creepy_face", "dark", "dark_theme", "dim_lighting", "directed_motion_outline", "elemental_creature", "expressionless", "frown", "goat_demon", "holding_character", "iguanid", "larger_human", "light_bulb", "lighter", "lighting", "lights", "lizard_(divinity)", "lizard_tail", "one_page_comic", "sad", "scary", "shocked_face", "smaller_human", "sweatdrop_(iconography)", "unknown_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "4_panel_comic", "border", "chart", "confusion", "cracked_ground", "creepy_face", "dark", "dark_theme", "dim_lighting", "directed_motion_outline", "elemental_creature", "expressionless", "frown", "goat_demon", "holding_character", "iguanid", "larger_human", "light_bulb", "lighter", "lighting", "lights", "lizard_(divinity)", "lizard_tail", "one_page_comic", "reptile", "sad", "scalie", "scary", "shocked_face", "smaller_human", "sweatdrop_(iconography)", "unknown_character"], "stage3_selected": ["3_panel_comic", "4_panel_comic", "border", "chart", "confusion", "cracked_ground", "creepy_face", "dark", "dark_theme", "dim_lighting", "directed_motion_outline", "elemental_creature", "expressionless", "frown", "goat_demon", "holding_character", "iguanid", "larger_human", "light_bulb", "lighter", "lighting", "lights", "lizard_(divinity)", "lizard_tail", "one_page_comic", "reptile", "sad", "scalie", "scary", "shocked_face", "smaller_human", "sweatdrop_(iconography)", "unknown_character"], "stage3_selected_scores": {"scalie": 0.4916, "reptile": 0.5077, "border": 0.4053, "elemental_creature": 0.4767, "frown": 0.4011, "smaller_human": 0.4322, "lighting": 0.5533, "sad": 0.4777, "confusion": 0.4487, "dark": 0.5246, "larger_human": 0.4063, "unknown_character": 0.4404, "holding_character": 0.3838, "lights": 0.5706, "goat_demon": 0.5311, "chart": 0.3778, "iguanid": 0.5333, "dark_theme": 0.5093, "scary": 0.4653, "lighter": 0.567, "light_bulb": 0.5412, "directed_motion_outline": 0.4318, "lizard_tail": 0.5169, "shocked_face": 0.4417, "dim_lighting": 0.5529, "3_panel_comic": 0.4885, "4_panel_comic": 0.4744, "sweatdrop_(iconography)": 0.4447, "one_page_comic": 0.4707, "lizard_(divinity)": 0.5263, "creepy_face": 0.4758, "cracked_ground": 0.4027, "expressionless": 0.4291}, "stage3_selected_ranks": {"scalie": 53, "reptile": 46, "border": 115, "elemental_creature": 66, "frown": 119, "smaller_human": 102, "lighting": 21, "sad": 64, "confusion": 90, "dark": 37, "larger_human": 113, "unknown_character": 101, "holding_character": 124, "lights": 16, "goat_demon": 33, "chart": 127, "iguanid": 29, "dark_theme": 43, "scary": 79, "lighter": 18, "light_bulb": 25, "directed_motion_outline": 103, "lizard_tail": 41, "shocked_face": 98, "dim_lighting": 22, "3_panel_comic": 56, "4_panel_comic": 70, "sweatdrop_(iconography)": 96, "one_page_comic": 75, "lizard_(divinity)": 36, "creepy_face": 68, "cracked_ground": 116, "expressionless": 104}, "stage3_selected_phrase_ranks": {"scalie": 10, "reptile": 9, "border": 6, "elemental_creature": 7, "frown": 8, "smaller_human": 6, "lighting": 7, "sad": 6, "confusion": 4, "dark": 3, "larger_human": 9, "unknown_character": 6, "holding_character": 9, "lights": 5, "goat_demon": 7, "chart": 9, "iguanid": 5, "dark_theme": 4, "scary": 10, "lighter": 6, "light_bulb": 9, "directed_motion_outline": 9, "lizard_tail": 8, "shocked_face": 5, "dim_lighting": 8, "3_panel_comic": 2, "4_panel_comic": 4, "sweatdrop_(iconography)": 8, "one_page_comic": 5, "lizard_(divinity)": 5, "creepy_face": 6, "cracked_ground": 7, "expressionless": 3}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "4_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4744}, "border": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4053}, "chart": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3778}, "confusion": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4487}, "cracked_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4027}, "creepy_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4758}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5246}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "dim_lighting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5529}, "directed_motion_outline": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4318}, "elemental_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4767}, "expressionless": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4291}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5311}, "holding_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3838}, "iguanid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5333}, "larger_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4063}, "light_bulb": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5412}, "lighter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.567}, "lighting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5533}, "lights": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5706}, "lizard_(divinity)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5263}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5169}, "one_page_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4707}, "sad": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4777}, "scary": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4653}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4417}, "smaller_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4322}, "sweatdrop_(iconography)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4447}, "unknown_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4404}}, "structural": [], "t1": 0.0, "t2": 1.47, "t3": 26.62, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 37}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"style_or_meta\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 40}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 37}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 36, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.1667, "R": 0.4286, "F1": 0.24, "leaf_P": 0.125, "leaf_R": 0.3333, "leaf_F1": 0.1818, "n_leaf_sel": 32, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.75, "over_sel": 2.57, "why": {"explicit": 36}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 66, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 0.4286, "gen_F1": 0.24, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "hair", "lying", "purple_hair", "text"], "extra": ["8_eyes", "annoyed_expression", "auburn_hair", "bed_covers", "bed_sheet", "bedding", "bedroom", "blue_background", "blue_hair", "blue_lipstick", "eyes_mostly_closed", "hair_sticks", "lidded_eyes", "lipstick", "lipstick_on_face", "long_hair", "lying_on_bed", "no_irises", "pillow", "purple_background", "purple_eyelids", "purple_hands", "purple_legs", "purple_pupils", "red_hair", "red_lipstick", "restricted_palette", "sleeping_together", "sleepover", "sleepwear"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["8_eyes", "annoyed_expression", "auburn_hair", "bed", "bed_covers", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_background", "blue_eyes", "blue_hair", "blue_lipstick", "eyes_mostly_closed", "green_eyes", "hair_sticks", "lidded_eyes", "lipstick", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "no_irises", "pillow", "purple_background", "purple_eyelids", "purple_hands", "purple_legs", "purple_pupils", "red_hair", "red_lipstick", "restricted_palette", "sleeping", "sleeping_together", "sleepover", "sleepwear"], "stage3_selected": ["8_eyes", "annoyed_expression", "auburn_hair", "bed", "bed_covers", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_background", "blue_eyes", "blue_hair", "blue_lipstick", "eyes_mostly_closed", "green_eyes", "hair_sticks", "lidded_eyes", "lipstick", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "no_irises", "pillow", "purple_background", "purple_eyelids", "purple_hands", "purple_legs", "purple_pupils", "red_hair", "red_lipstick", "restricted_palette", "sleeping", "sleeping_together", "sleepover", "sleepwear"], "stage3_selected_scores": {"blue_eyes": 0.5401, "green_eyes": 0.538, "long_hair": 0.3962, "blonde_hair": 0.346, "bed": 0.5296, "blue_hair": 0.3597, "red_hair": 0.359, "makeup": 0.544, "pillow": 0.441, "lipstick": 0.4386, "bedroom": 0.4116, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "purple_background": 0.3682, "bed_sheet": 0.4395, "restricted_palette": 0.2856, "lidded_eyes": 0.3759, "lying_on_bed": 0.412, "red_lipstick": 0.4186, "blue_lipstick": 0.3955, "annoyed_expression": 0.3458, "bed_covers": 0.4011, "eyes_mostly_closed": 0.338, "no_irises": 0.3837, "purple_pupils": 0.3814, "auburn_hair": 0.3646, "lipstick_on_face": 0.3766, "sleeping_together": 0.408, "hair_sticks": 0.5043, "sleepover": 0.5015, "purple_eyelids": 0.405, "purple_hands": 0.6346, "sleepwear": 0.4346, "purple_legs": 0.6369, "8_eyes": 0.3787}, "stage3_selected_ranks": {"blue_eyes": 12, "green_eyes": 14, "long_hair": 48, "blonde_hair": 96, "bed": 15, "blue_hair": 77, "red_hair": 78, "makeup": 11, "pillow": 27, "lipstick": 29, "bedroom": 39, "blue_background": 105, "sleeping": 20, "bedding": 33, "purple_background": 67, "bed_sheet": 28, "restricted_palette": 138, "lidded_eyes": 63, "lying_on_bed": 38, "red_lipstick": 34, "blue_lipstick": 50, "annoyed_expression": 97, "bed_covers": 44, "eyes_mostly_closed": 112, "no_irises": 55, "purple_pupils": 57, "auburn_hair": 73, "lipstick_on_face": 62, "sleeping_together": 41, "hair_sticks": 18, "sleepover": 19, "purple_eyelids": 43, "purple_hands": 5, "sleepwear": 30, "purple_legs": 3, "8_eyes": 60}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "green_eyes": 1, "long_hair": 3, "blonde_hair": 8, "bed": 1, "blue_hair": 5, "red_hair": 5, "makeup": 1, "pillow": 3, "lipstick": 2, "bedroom": 7, "blue_background": 10, "sleeping": 2, "bedding": 5, "purple_background": 5, "bed_sheet": 4, "restricted_palette": 9, "lidded_eyes": 6, "lying_on_bed": 6, "red_lipstick": 4, "blue_lipstick": 10, "annoyed_expression": 9, "bed_covers": 9, "eyes_mostly_closed": 9, "no_irises": 2, "purple_pupils": 6, "auburn_hair": 4, "lipstick_on_face": 4, "sleeping_together": 1, "hair_sticks": 2, "sleepover": 1, "purple_eyelids": 3, "purple_hands": 1, "sleepwear": 5, "purple_legs": 1, "8_eyes": 4}, "extra_evidence": {"8_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3787}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3646}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "blue_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3597}, "blue_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3955}, "eyes_mostly_closed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.338}, "hair_sticks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5043}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3759}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "lipstick_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3766}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3962}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "purple_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3682}, "purple_eyelids": {"source": "stage3", "why": "explicit", "retrieval_score": 0.405}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6346}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "purple_pupils": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3814}, "red_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.359}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "restricted_palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2856}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "sleepover": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4346}}, "structural": [], "t1": 0.0, "t2": 1.39, "t3": 16.88, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.25, "R": 0.1818, "F1": 0.2105, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 10, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 0.73, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 40, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.25, "gen_R": 0.1818, "gen_F1": 0.2105, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["acoustic_guitar", "action_pose", "big_claws", "full-length_portrait", "maned_wolf", "orange_background", "personification", "portrait", "pose", "torn_leggings", "torn_shorts", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["acoustic_guitar", "action_pose", "big_claws", "canine", "claws", "full-length_portrait", "maned_wolf", "musical_instrument", "orange_background", "personification", "portrait", "pose", "torn_clothing", "torn_leggings", "torn_shorts", "wolf"], "stage3_selected": ["acoustic_guitar", "action_pose", "big_claws", "canine", "claws", "full-length_portrait", "maned_wolf", "musical_instrument", "orange_background", "personification", "portrait", "pose", "torn_clothing", "torn_leggings", "torn_shorts", "wolf"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "portrait": 0.6102, "pose": 0.8955, "full-length_portrait": 0.6018, "torn_clothing": 0.5179, "musical_instrument": 0.3362, "orange_background": 0.6518, "action_pose": 0.6726, "maned_wolf": 0.664, "big_claws": 0.7461, "torn_shorts": 0.4967, "personification": 0.4678, "torn_leggings": 0.4969, "acoustic_guitar": 0.3382}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "wolf": 23, "portrait": 47, "pose": 2, "full-length_portrait": 54, "torn_clothing": 73, "musical_instrument": 117, "orange_background": 24, "action_pose": 19, "maned_wolf": 21, "big_claws": 7, "torn_shorts": 80, "personification": 95, "torn_leggings": 79, "acoustic_guitar": 115}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "wolf": 2, "portrait": 7, "pose": 1, "full-length_portrait": 10, "torn_clothing": 4, "musical_instrument": 9, "orange_background": 2, "action_pose": 2, "maned_wolf": 1, "big_claws": 1, "torn_shorts": 7, "personification": 6, "torn_leggings": 6, "acoustic_guitar": 7}, "extra_evidence": {"acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3382}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6018}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "personification": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4678}, "portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6102}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}}, "structural": [], "t1": 0.0, "t2": 1.03, "t3": 14.46, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 30, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.1667, "R": 0.2174, "F1": 0.1887, "leaf_P": 0.0435, "leaf_R": 0.0667, "leaf_F1": 0.0526, "n_leaf_sel": 23, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 1.3, "why": {"explicit": 31}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 47, "attempts_by_n_local": {"60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 0.2174, "gen_F1": 0.1887, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "standing", "topwear"], "extra": ["4_claws", "arm_markings", "big_claws", "black_bottomwear", "claws_out", "clothed_anthro", "dress_shirt", "ear_markings", "finger_claws", "gloves_(marking)", "grey_shirt", "grey_text", "grey_theme", "grey_topwear", "half-closed_eyes", "hand_markings", "oversized_shirt", "partially_clothed_anthro", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_fur", "white_shirt", "white_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "arm_markings", "big_claws", "black_bottomwear", "claws", "claws_out", "clothed_anthro", "dress_shirt", "ear_markings", "facial_markings", "finger_claws", "fur", "gloves_(marking)", "grey_background", "grey_shirt", "grey_text", "grey_theme", "grey_topwear", "half-closed_eyes", "hand_markings", "oversized_shirt", "partially_clothed_anthro", "shirt", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_fur", "white_shirt", "white_topwear"], "stage3_selected": ["4_claws", "arm_markings", "big_claws", "black_bottomwear", "claws", "claws_out", "clothed_anthro", "dress_shirt", "ear_markings", "facial_markings", "finger_claws", "fur", "gloves_(marking)", "grey_background", "grey_shirt", "grey_text", "grey_theme", "grey_topwear", "half-closed_eyes", "hand_markings", "oversized_shirt", "partially_clothed_anthro", "shirt", "simple_background", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_fur", "white_shirt", "white_topwear"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "white_fur": 0.5776, "shirt": 0.7037, "half-closed_eyes": 0.4928, "tan_fur": 0.5593, "grey_background": 0.5635, "gloves_(marking)": 0.7072, "facial_markings": 0.9015, "finger_claws": 0.6026, "t-shirt": 0.6707, "white_topwear": 0.5694, "black_bottomwear": 0.5638, "white_shirt": 0.5609, "dress_shirt": 0.6075, "arm_markings": 0.7489, "grey_topwear": 0.5774, "ear_markings": 0.7178, "clothed_anthro": 0.5317, "snout_markings": 0.7146, "grey_shirt": 0.6272, "grey_text": 0.5626, "claws_out": 0.5754, "big_claws": 0.569, "undershirt": 0.6369, "partially_clothed_anthro": 0.5361, "grey_theme": 0.5547, "oversized_shirt": 0.6452, "4_claws": 0.6123, "hand_markings": 0.7033}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "white_fur": 44, "shirt": 13, "half-closed_eyes": 86, "tan_fur": 66, "grey_background": 58, "gloves_(marking)": 12, "facial_markings": 1, "finger_claws": 33, "t-shirt": 18, "white_topwear": 51, "black_bottomwear": 57, "white_shirt": 63, "dress_shirt": 31, "arm_markings": 4, "grey_topwear": 45, "ear_markings": 8, "clothed_anthro": 78, "snout_markings": 9, "grey_shirt": 25, "grey_text": 60, "claws_out": 46, "big_claws": 52, "undershirt": 22, "partially_clothed_anthro": 76, "grey_theme": 69, "oversized_shirt": 20, "4_claws": 28, "hand_markings": 14}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "white_fur": 3, "shirt": 1, "half-closed_eyes": 8, "tan_fur": 5, "grey_background": 3, "gloves_(marking)": 9, "facial_markings": 1, "finger_claws": 5, "t-shirt": 2, "white_topwear": 3, "black_bottomwear": 6, "white_shirt": 4, "dress_shirt": 9, "arm_markings": 3, "grey_topwear": 1, "ear_markings": 6, "clothed_anthro": 9, "snout_markings": 7, "grey_shirt": 7, "grey_text": 6, "claws_out": 7, "big_claws": 4, "undershirt": 4, "partially_clothed_anthro": 7, "grey_theme": 8, "oversized_shirt": 1, "4_claws": 3, "hand_markings": 10}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6123}, "arm_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7489}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.569}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5638}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5754}, "clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5317}, "dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6075}, "ear_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7178}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6026}, "gloves_(marking)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7072}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5626}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "hand_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7033}, "oversized_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5361}, "snout_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7146}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6707}, "tan_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5593}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5776}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5609}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5694}}, "structural": [], "t1": 0.0, "t2": 0.91, "t3": 17.59, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.037, "R": 0.0909, "F1": 0.0526, "leaf_P": 0.0417, "leaf_R": 0.1429, "leaf_F1": 0.0645, "n_leaf_sel": 24, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 2.45, "why": {"explicit": 28}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 74, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.037, "gen_R": 0.0909, "gen_F1": 0.0526, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "big_eyes", "blue_fur", "blue_markings", "blue_pawpads", "eyes", "full-length_portrait", "happy", "jumper", "jumpluff", "multiple_poses", "pawpads", "pink_stripes", "playful", "pose", "posed", "pride_colors", "progress_pride_colors", "raised_paw", "round_nose", "smiling_at_viewer", "striped_back", "striped_legs", "stripes", "suggestive_pose"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "big_eyes", "blue_fur", "blue_markings", "blue_nose", "blue_pawpads", "eyes", "full-length_portrait", "happy", "jumper", "jumpluff", "multiple_poses", "pawpads", "pink_stripes", "playful", "pose", "posed", "pride_colors", "progress_pride_colors", "raised_paw", "round_nose", "smiling_at_viewer", "striped_back", "striped_legs", "stripes", "suggestive_pose"], "stage3_selected": [">:)", ">:d", "big_eyes", "blue_fur", "blue_markings", "blue_nose", "blue_pawpads", "eyes", "full-length_portrait", "happy", "jumper", "jumpluff", "multiple_poses", "pawpads", "pink_stripes", "playful", "pose", "posed", "pride_colors", "progress_pride_colors", "raised_paw", "round_nose", "simple_background", "smiling_at_viewer", "striped_back", "striped_legs", "stripes", "suggestive_pose"], "stage3_selected_scores": {"simple_background": 0.401, "pawpads": 0.327, "blue_fur": 0.4171, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "full-length_portrait": 0.4491, "smiling_at_viewer": 0.451, "blue_nose": 0.5017, "big_eyes": 0.4619, "multiple_poses": 0.4969, "blue_pawpads": 0.4264, "blue_markings": 0.4512, "pride_colors": 0.2723, "playful": 0.6118, "striped_legs": 0.6654, "raised_paw": 0.3182, "suggestive_pose": 0.5892, "pink_stripes": 0.5235, "striped_back": 0.7273, ">:d": 0.3455, ">:)": 0.3449, "round_nose": 0.4996, "jumpluff": 0.3542, "posed": 0.4337, "jumper": 0.4127, "progress_pride_colors": 0.2713, "eyes": 0.97}, "stage3_selected_ranks": {"simple_background": 85, "pawpads": 125, "blue_fur": 77, "stripes": 18, "pose": 19, "happy": 101, "full-length_portrait": 63, "smiling_at_viewer": 62, "blue_nose": 45, "big_eyes": 57, "multiple_poses": 48, "blue_pawpads": 73, "blue_markings": 61, "pride_colors": 148, "playful": 17, "striped_legs": 12, "raised_paw": 134, "suggestive_pose": 21, "pink_stripes": 37, "striped_back": 4, ">:d": 111, ">:)": 113, "round_nose": 46, "jumpluff": 106, "posed": 69, "jumper": 81, "progress_pride_colors": 149, "eyes": 1}, "stage3_selected_phrase_ranks": {"simple_background": 4, "pawpads": 8, "blue_fur": 9, "stripes": 10, "pose": 1, "happy": 3, "full-length_portrait": 9, "smiling_at_viewer": 8, "blue_nose": 3, "big_eyes": 4, "multiple_poses": 5, "blue_pawpads": 7, "blue_markings": 6, "pride_colors": 6, "playful": 1, "striped_legs": 8, "raised_paw": 9, "suggestive_pose": 2, "pink_stripes": 1, "striped_back": 2, ">:d": 8, ">:)": 9, "round_nose": 9, "jumpluff": 6, "posed": 10, "jumper": 4, "progress_pride_colors": 7, "eyes": 1}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4619}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4512}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumpluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3542}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.327}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2723}, "progress_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2713}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4996}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7273}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6654}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "suggestive_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}}, "structural": [], "t1": 0.0, "t2": 1.25, "t3": 22.33, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_enabled_explicit.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:11:32.634111", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 14}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 133, "n_selected": 20, "n_implied": 0, "n_structural": 0, "ret_R": 0.4167, "P": 0.15, "R": 0.25, "F1": 0.1875, "leaf_P": 0.0625, "leaf_R": 0.1111, "leaf_F1": 0.08, "n_leaf_sel": 16, "n_leaf_gt": 9, "ret_P": 0.0376, "sel_given_ret": 0.6, "over_sel": 1.67, "why": {"explicit": 22}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 85, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.15, "gen_R": 0.25, "gen_F1": 0.1875, "missed": ["alpha_channel", "clothed", "clothing", "felid", "fingers", "hair", "male", "mammal", "solo"], "extra": ["black_fur", "bobcat", "business_attire", "hair_bun", "holding_container", "holding_mug", "jacket", "scowl", "shirt", "siamese", "spilled_drink", "teal_shirt", "topwear", "undershirt", "vest", "white_fur", "white_t-shirt"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "black_fur", "bobcat", "business_attire", "feline", "fur", "hair_bun", "holding_container", "holding_mug", "jacket", "scowl", "shirt", "siamese", "spilled_drink", "teal_shirt", "topwear", "undershirt", "vest", "white_fur", "white_t-shirt"], "stage3_selected": ["anthro", "black_fur", "bobcat", "business_attire", "feline", "fur", "hair_bun", "holding_container", "holding_mug", "invalid_background", "jacket", "scowl", "shirt", "siamese", "simple_background", "spilled_drink", "teal_shirt", "topwear", "undershirt", "vest", "white_fur", "white_t-shirt"], "stage3_selected_scores": {"anthro": 0.4903, "fur": 0.712, "simple_background": 0.6952, "white_fur": 0.5953, "topwear": 0.6992, "feline": 0.7023, "shirt": 0.7976, "black_fur": 0.7157, "jacket": 0.6655, "vest": 0.8387, "holding_container": 0.7583, "hair_bun": 0.4794, "siamese": 0.6206, "undershirt": 0.7585, "bobcat": 0.5752, "scowl": 0.5546, "holding_mug": 0.915, "white_t-shirt": 0.7323, "spilled_drink": 0.6346, "business_attire": 0.5642, "teal_shirt": 0.7466, "invalid_background": 0.6479}, "stage3_selected_ranks": {"anthro": 129, "fur": 40, "simple_background": 50, "white_fur": 85, "topwear": 49, "feline": 45, "shirt": 5, "black_fur": 37, "jacket": 58, "vest": 3, "holding_container": 12, "hair_bun": 132, "siamese": 75, "undershirt": 11, "bobcat": 92, "scowl": 103, "holding_mug": 1, "white_t-shirt": 26, "spilled_drink": 68, "business_attire": 100, "teal_shirt": 19, "invalid_background": 63}, "stage3_selected_phrase_ranks": {"anthro": 9, "fur": 1, "simple_background": 1, "white_fur": 3, "topwear": 10, "feline": 1, "shirt": 1, "black_fur": 1, "jacket": 8, "vest": 1, "holding_container": 5, "hair_bun": 8, "siamese": 5, "undershirt": 3, "bobcat": 6, "scowl": 4, "holding_mug": 1, "white_t-shirt": 3, "spilled_drink": 9, "business_attire": 1, "teal_shirt": 4, "invalid_background": 1}, "extra_evidence": {"black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "bobcat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5752}, "business_attire": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5642}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4794}, "holding_container": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7583}, "holding_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.915}, "jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6655}, "scowl": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5546}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "siamese": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6206}, "spilled_drink": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6346}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6992}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7585}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5953}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7323}}, "structural": [], "t1": 2.07, "t2": 9.07, "t3": 15.62, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=136 entity=0 copyright_filtered=1 generic_char_to_general=0 unknown_type=1"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 164, "n_selected": 21, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.1429, "R": 0.2143, "F1": 0.1714, "leaf_P": 0.0556, "leaf_R": 0.1, "leaf_F1": 0.0714, "n_leaf_sel": 18, "n_leaf_gt": 10, "ret_P": 0.0488, "sel_given_ret": 0.375, "over_sel": 1.5, "why": {"explicit": 21}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 25, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 61, "attempts_by_n_local": {"25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "43": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1429, "gen_R": 0.2143, "gen_F1": 0.1714, "missed": ["clothed", "clothing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "extra": ["black_hair", "blue_hair", "braided_hair", "crying_laughing", "gorilla", "grinning_at_viewer", "hand_above_head", "holding_arms", "laugh", "loincloth", "monkey", "neutral_expression", "raised_arms", "raised_hand", "smug_grin", "spread_arms", "ursine", "wide_grin"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["ape", "bear", "black_hair", "blue_hair", "braided_hair", "crying_laughing", "dancing", "gorilla", "grinning_at_viewer", "hand_above_head", "holding_arms", "laugh", "loincloth", "monkey", "neutral_expression", "raised_arms", "raised_hand", "smug_grin", "spread_arms", "ursine", "wide_grin"], "stage3_selected": ["ape", "bear", "black_hair", "blue_hair", "braided_hair", "crying_laughing", "dancing", "gorilla", "grinning_at_viewer", "hand_above_head", "holding_arms", "laugh", "loincloth", "monkey", "neutral_expression", "raised_arms", "raised_hand", "smug_grin", "spread_arms", "ursine", "wide_grin"], "stage3_selected_scores": {"black_hair": 0.3428, "blue_hair": 0.3344, "bear": 0.5551, "braided_hair": 0.3274, "ursine": 0.4202, "loincloth": 0.5634, "dancing": 0.5523, "monkey": 0.7547, "laugh": 0.5229, "ape": 0.9763, "raised_arms": 0.5407, "raised_hand": 0.3752, "gorilla": 0.8287, "spread_arms": 0.3987, "smug_grin": 0.3655, "neutral_expression": 0.4019, "crying_laughing": 0.3837, "hand_above_head": 0.374, "holding_arms": 0.3945, "grinning_at_viewer": 0.4379, "wide_grin": 0.5248}, "stage3_selected_ranks": {"black_hair": 132, "blue_hair": 145, "bear": 15, "braided_hair": 154, "ursine": 59, "loincloth": 13, "dancing": 16, "monkey": 6, "laugh": 25, "ape": 1, "raised_arms": 21, "raised_hand": 102, "gorilla": 5, "spread_arms": 80, "smug_grin": 110, "neutral_expression": 77, "crying_laughing": 89, "hand_above_head": 103, "holding_arms": 82, "grinning_at_viewer": 43, "wide_grin": 23}, "stage3_selected_phrase_ranks": {"black_hair": 7, "blue_hair": 9, "bear": 1, "braided_hair": 8, "ursine": 9, "loincloth": 1, "dancing": 1, "monkey": 6, "laugh": 1, "ape": 1, "raised_arms": 1, "raised_hand": 8, "gorilla": 1, "spread_arms": 5, "smug_grin": 4, "neutral_expression": 6, "crying_laughing": 2, "hand_above_head": 10, "holding_arms": 7, "grinning_at_viewer": 3, "wide_grin": 1}, "extra_evidence": {"black_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3428}, "blue_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3344}, "braided_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3274}, "crying_laughing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "gorilla": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8287}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4379}, "hand_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.374}, "holding_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3945}, "laugh": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5229}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "neutral_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3752}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3655}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "ursine": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}}, "structural": [], "t1": 2.66, "t2": 9.17, "t3": 18.78, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=163 entity=5 copyright_filtered=4 generic_char_to_general=1 unknown_type=3"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 138, "n_selected": 23, "n_implied": 0, "n_structural": 0, "ret_R": 0.7143, "P": 0.3478, "R": 0.5714, "F1": 0.4324, "leaf_P": 0.2632, "leaf_R": 0.5556, "leaf_F1": 0.3571, "n_leaf_sel": 19, "n_leaf_gt": 9, "ret_P": 0.0725, "sel_given_ret": 0.8, "over_sel": 1.64, "why": {"explicit": 23}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 21, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 96, "attempts_by_n_local": {"21": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3478, "gen_R": 0.5714, "gen_F1": 0.4324, "missed": ["blush", "clothed", "clothing", "lagomorph", "leporid", "mammal"], "extra": ["coat", "confident", "domestic_rabbit", "embrace", "eyes", "facing_each_other", "fur_coat", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "pink_blush", "red_coat", "setting", "white_coat"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["anthro", "blue_eyes", "coat", "confident", "domestic_rabbit", "duo", "embrace", "eyes", "facing_each_other", "fur_coat", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "pink_blush", "plushie", "rabbit", "red_coat", "romantic", "romantic_couple", "setting", "teal_eyes", "white_coat"], "stage3_selected": ["anthro", "blue_eyes", "coat", "confident", "domestic_rabbit", "duo", "embrace", "eyes", "facing_each_other", "fur_coat", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "pink_blush", "plushie", "rabbit", "red_coat", "romantic", "romantic_couple", "setting", "teal_eyes", "white_coat"], "stage3_selected_scores": {"anthro": 0.4179, "duo": 0.3624, "blue_eyes": 0.6147, "rabbit": 0.5936, "embrace": 0.4198, "romantic": 0.5301, "romantic_couple": 0.5437, "coat": 0.638, "plushie": 0.7452, "teal_eyes": 0.6281, "lab_coat": 0.5158, "domestic_rabbit": 0.4128, "inanimate_object": 0.4648, "pink_blush": 0.476, "confident": 0.5158, "holding_plushie": 0.7791, "fur_coat": 0.4905, "white_coat": 0.5252, "red_coat": 0.5206, "holding_toy": 0.5853, "facing_each_other": 0.4299, "setting": 0.5566, "eyes": 0.8767}, "stage3_selected_ranks": {"anthro": 122, "duo": 136, "blue_eyes": 12, "rabbit": 13, "embrace": 117, "romantic": 29, "romantic_couple": 22, "coat": 7, "plushie": 3, "teal_eyes": 8, "lab_coat": 42, "domestic_rabbit": 129, "inanimate_object": 76, "pink_blush": 67, "confident": 41, "holding_plushie": 2, "fur_coat": 54, "white_coat": 33, "red_coat": 36, "holding_toy": 14, "facing_each_other": 104, "setting": 18, "eyes": 1}, "stage3_selected_phrase_ranks": {"anthro": 8, "duo": 3, "blue_eyes": 1, "rabbit": 1, "embrace": 9, "romantic": 2, "romantic_couple": 1, "coat": 1, "plushie": 1, "teal_eyes": 1, "lab_coat": 5, "domestic_rabbit": 9, "inanimate_object": 9, "pink_blush": 3, "confident": 7, "holding_plushie": 1, "fur_coat": 9, "white_coat": 3, "red_coat": 4, "holding_toy": 4, "facing_each_other": 4, "setting": 1, "eyes": 1}, "extra_evidence": {"coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "domestic_rabbit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4198}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "facing_each_other": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4299}, "fur_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4905}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "holding_toy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5853}, "inanimate_object": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "lab_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "pink_blush": {"source": "stage3", "why": "explicit", "retrieval_score": 0.476}, "red_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5206}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5566}, "white_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5252}}, "structural": [], "t1": 2.67, "t2": 8.92, "t3": 16.87, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=136 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 117, "n_selected": 9, "n_implied": 0, "n_structural": 0, "ret_R": 0.75, "P": 0.2222, "R": 0.5, "F1": 0.3077, "leaf_P": 0.2222, "leaf_R": 0.5, "leaf_F1": 0.3077, "n_leaf_sel": 9, "n_leaf_gt": 4, "ret_P": 0.0256, "sel_given_ret": 0.6667, "over_sel": 2.25, "why": {"explicit": 10}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 15, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 51, "attempts_by_n_local": {"15": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2222, "gen_R": 0.5, "gen_F1": 0.3077, "missed": ["smile", "solo"], "extra": ["big_eyes", "elemental_creature", "floating", "nose", "spotted_back", "tan_face", "toothy_smile"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["big_eyes", "elemental_creature", "floating", "nose", "red_nose", "spotted_back", "tan_body", "tan_face", "toothy_smile"], "stage3_selected": ["big_eyes", "elemental_creature", "floating", "nose", "red_nose", "spotted_back", "tan_body", "tan_face", "toothy_smile", "white_background"], "stage3_selected_scores": {"white_background": 0.6069, "tan_body": 0.658, "elemental_creature": 0.4515, "big_eyes": 0.6933, "red_nose": 0.7473, "floating": 0.6452, "tan_face": 0.6956, "spotted_back": 0.7, "toothy_smile": 0.43, "nose": 0.8611}, "stage3_selected_ranks": {"white_background": 32, "tan_body": 15, "elemental_creature": 89, "big_eyes": 7, "red_nose": 3, "floating": 20, "tan_face": 6, "spotted_back": 4, "toothy_smile": 101, "nose": 2}, "stage3_selected_phrase_ranks": {"white_background": 1, "tan_body": 6, "elemental_creature": 7, "big_eyes": 1, "red_nose": 1, "floating": 1, "tan_face": 1, "spotted_back": 1, "toothy_smile": 10, "nose": 1}, "extra_evidence": {"big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6933}, "elemental_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4515}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8611}, "spotted_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7}, "tan_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6956}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}}, "structural": [], "t1": 1.87, "t2": 8.0, "t3": 12.79, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=0 copyright_filtered=2 generic_char_to_general=0 unknown_type=4"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 172, "n_selected": 29, "n_implied": 0, "n_structural": 0, "ret_R": 0.3636, "P": 0.2069, "R": 0.2727, "F1": 0.2353, "leaf_P": 0.1154, "leaf_R": 0.25, "leaf_F1": 0.1579, "n_leaf_sel": 26, "n_leaf_gt": 12, "ret_P": 0.0465, "sel_given_ret": 0.75, "over_sel": 1.32, "why": {"explicit": 29}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 54, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "47": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2069, "gen_R": 0.2727, "gen_F1": 0.2353, "missed": ["anthro", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "topless"], "extra": ["actual_fur", "big_biceps", "blue_shorts", "chest_fur", "confident", "countershade_belly", "countershade_body", "ear_tuft", "fluffy_fur", "glistening_eyes", "hand_on_another's_head", "hand_on_arm", "heterochromia", "hotpants", "multicolored_fur", "muscular_arms", "muscular_legs", "muscular_thighs", "posed", "striped_fur", "white_chest", "white_neck", "yellow_bottomwear"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["actual_fur", "big_biceps", "blue_eyes", "blue_shorts", "chest_fur", "confident", "countershade_belly", "countershade_body", "ear_tuft", "fluffy_fur", "glistening_eyes", "hand_on_another's_head", "hand_on_arm", "hand_on_head", "heterochromia", "hotpants", "multicolored_fur", "muscular_arms", "muscular_legs", "muscular_thighs", "posed", "shorts", "striped_fur", "stripes", "tiger", "tuft", "white_chest", "white_neck", "yellow_bottomwear"], "stage3_selected": ["actual_fur", "big_biceps", "blue_eyes", "blue_shorts", "chest_fur", "confident", "countershade_belly", "countershade_body", "ear_tuft", "fluffy_fur", "glistening_eyes", "hand_on_another's_head", "hand_on_arm", "hand_on_head", "heterochromia", "hotpants", "multicolored_fur", "muscular_arms", "muscular_legs", "muscular_thighs", "posed", "shorts", "striped_fur", "stripes", "tiger", "tuft", "white_chest", "white_neck", "yellow_bottomwear"], "stage3_selected_scores": {"blue_eyes": 0.5709, "tuft": 0.4886, "multicolored_fur": 0.455, "stripes": 0.4671, "shorts": 0.5777, "tiger": 0.5962, "striped_fur": 0.6375, "heterochromia": 0.3966, "hand_on_head": 0.5925, "glistening_eyes": 0.4737, "ear_tuft": 0.5887, "hotpants": 0.5175, "muscular_thighs": 0.7068, "muscular_arms": 0.7946, "big_biceps": 0.6929, "muscular_legs": 0.7889, "hand_on_arm": 0.6075, "blue_shorts": 0.6037, "chest_fur": 0.4596, "yellow_bottomwear": 0.6518, "confident": 0.5044, "white_chest": 0.9198, "hand_on_another's_head": 0.4936, "countershade_body": 0.8755, "fluffy_fur": 0.6692, "countershade_belly": 0.8305, "white_neck": 0.8394, "posed": 0.4331, "actual_fur": 0.4532}, "stage3_selected_ranks": {"blue_eyes": 61, "tuft": 85, "multicolored_fur": 105, "stripes": 100, "shorts": 60, "tiger": 50, "striped_fur": 37, "heterochromia": 133, "hand_on_head": 53, "glistening_eyes": 96, "ear_tuft": 55, "hotpants": 72, "muscular_thighs": 21, "muscular_arms": 14, "big_biceps": 24, "muscular_legs": 15, "hand_on_arm": 47, "blue_shorts": 48, "chest_fur": 102, "yellow_bottomwear": 33, "confident": 76, "white_chest": 2, "hand_on_another's_head": 82, "countershade_body": 3, "fluffy_fur": 28, "countershade_belly": 10, "white_neck": 6, "posed": 117, "actual_fur": 106}, "stage3_selected_phrase_ranks": {"blue_eyes": 2, "tuft": 4, "multicolored_fur": 4, "stripes": 3, "shorts": 1, "tiger": 1, "striped_fur": 2, "heterochromia": 8, "hand_on_head": 2, "glistening_eyes": 3, "ear_tuft": 2, "hotpants": 7, "muscular_thighs": 3, "muscular_arms": 1, "big_biceps": 7, "muscular_legs": 2, "hand_on_arm": 1, "blue_shorts": 3, "chest_fur": 10, "yellow_bottomwear": 1, "confident": 2, "white_chest": 1, "hand_on_another's_head": 8, "countershade_body": 1, "fluffy_fur": 1, "countershade_belly": 2, "white_neck": 4, "posed": 10, "actual_fur": 5}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4532}, "big_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6929}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6037}, "chest_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5044}, "countershade_belly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8305}, "countershade_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8755}, "ear_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5887}, "fluffy_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6692}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4737}, "hand_on_another's_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4936}, "hand_on_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6075}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3966}, "hotpants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "multicolored_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.455}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7946}, "muscular_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7889}, "muscular_thighs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4331}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6375}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9198}, "white_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8394}, "yellow_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}}, "structural": [], "t1": 1.36, "t2": 1.52, "t3": 10.75, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=167 entity=3 copyright_filtered=3 generic_char_to_general=0 unknown_type=2"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 192, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.6923, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 17, "n_leaf_gt": 6, "ret_P": 0.0469, "sel_given_ret": 0.0, "over_sel": 1.38, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 22, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 5, "calls_with_selection": 4, "calls_exhausted_retries": 1, "attempts_total": 7, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 88, "attempts_by_n_local": {"22": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 5, "parse_ok": 2, "parse_fail": 0, "errors": 3}, "3": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.42857142857142855, "call_exhaustion_rate": 0.2}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "armor", "caprine_demon", "confession", "creepy", "darkness", "frown", "gecko", "laying_on_ground", "light", "lighting", "lying_on_ground", "medieval", "medieval_clothing", "multiple_images", "on_ground", "pointy_speech_bubble", "sad"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "armor", "caprine_demon", "confession", "creepy", "darkness", "frown", "gecko", "laying_on_ground", "light", "lighting", "lying_on_ground", "medieval", "medieval_clothing", "multiple_images", "on_ground", "pointy_speech_bubble", "sad"], "stage3_selected": ["3_panel_comic", "armor", "caprine_demon", "confession", "creepy", "darkness", "frown", "gecko", "laying_on_ground", "light", "lighting", "lying_on_ground", "medieval", "medieval_clothing", "multiple_images", "on_ground", "pointy_speech_bubble", "sad"], "stage3_selected_scores": {"armor": 0.5048, "light": 0.7781, "frown": 0.4831, "on_ground": 0.6688, "multiple_images": 0.5337, "lighting": 0.6777, "sad": 0.5454, "creepy": 0.5133, "caprine_demon": 0.5623, "gecko": 0.6351, "pointy_speech_bubble": 0.5911, "lying_on_ground": 0.7873, "medieval": 0.5251, "darkness": 0.8346, "laying_on_ground": 0.7028, "3_panel_comic": 0.6184, "medieval_clothing": 0.4985, "confession": 0.4425}, "stage3_selected_ranks": {"armor": 145, "light": 4, "frown": 167, "on_ground": 17, "multiple_images": 108, "lighting": 14, "sad": 93, "creepy": 132, "caprine_demon": 75, "gecko": 29, "pointy_speech_bubble": 52, "lying_on_ground": 3, "medieval": 117, "darkness": 2, "laying_on_ground": 11, "3_panel_comic": 33, "medieval_clothing": 155, "confession": 182}, "stage3_selected_phrase_ranks": {"armor": 7, "light": 1, "frown": 7, "on_ground": 3, "multiple_images": 8, "lighting": 3, "sad": 6, "creepy": 5, "caprine_demon": 6, "gecko": 3, "pointy_speech_bubble": 10, "lying_on_ground": 1, "medieval": 4, "darkness": 1, "laying_on_ground": 2, "3_panel_comic": 2, "medieval_clothing": 8, "confession": 7}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6184}, "armor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5048}, "caprine_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5623}, "confession": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4425}, "creepy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5133}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4831}, "gecko": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6351}, "laying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7028}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "lighting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6777}, "lying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7873}, "medieval": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5251}, "medieval_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4985}, "multiple_images": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5337}, "on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6688}, "pointy_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5911}, "sad": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5454}}, "structural": [], "t1": 2.47, "t2": 1.73, "t3": 25.85, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=183 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 207, "n_selected": 33, "n_implied": 0, "n_structural": 0, "ret_R": 0.7143, "P": 0.2727, "R": 0.6429, "F1": 0.383, "leaf_P": 0.1852, "leaf_R": 0.4167, "leaf_F1": 0.2564, "n_leaf_sel": 27, "n_leaf_gt": 12, "ret_P": 0.0483, "sel_given_ret": 0.9, "over_sel": 2.36, "why": {"explicit": 33}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 26, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 5, "calls_with_selection": 5, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 5, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 100, "attempts_by_n_local": {"26": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "30": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2727, "gen_R": 0.6429, "gen_F1": 0.383, "missed": ["angry", "bed", "duo", "eyes_closed", "eyeshadow"], "extra": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "bed_sheet", "bedding", "bedroom", "english_text", "expressions", "eyes", "lipstick", "looking_down_at_another", "lying_on_bed", "mascara", "pajamas", "relaxed_expression", "resting", "sitting_on_bed", "sleeping_together", "speech_bubble", "text_box", "under_blanket", "watermark", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_eyes", "english_text", "expressions", "eyes", "green_eyes", "hair", "lipstick", "looking_down_at_another", "lying", "lying_on_bed", "makeup", "mascara", "pajamas", "purple_hair", "relaxed_expression", "resting", "sitting_on_bed", "sleeping", "sleeping_together", "speech_bubble", "text", "text_box", "under_blanket", "watermark", "zzz"], "stage3_selected": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_eyes", "english_text", "expressions", "eyes", "green_eyes", "hair", "lipstick", "looking_down_at_another", "lying", "lying_on_bed", "makeup", "mascara", "pajamas", "purple_hair", "relaxed_expression", "resting", "sitting_on_bed", "sleeping", "sleeping_together", "speech_bubble", "text", "text_box", "under_blanket", "watermark", "zzz"], "stage3_selected_scores": {"hair": 0.5936, "text": 0.5939, "blue_eyes": 0.595, "lying": 0.436, "green_eyes": 0.5933, "blonde_hair": 0.5872, "speech_bubble": 0.4006, "purple_hair": 0.5591, "makeup": 0.5894, "watermark": 0.5996, "lipstick": 0.4782, "bedroom": 0.4873, "sleeping": 0.5987, "bedding": 0.3861, "annoyed": 0.5677, "bed_sheet": 0.3933, "mascara": 0.4405, "lying_on_bed": 0.4059, "text_box": 0.3881, "zzz": 0.4122, "pajamas": 0.4043, "sitting_on_bed": 0.3765, "resting": 0.5015, "annoyed_expression": 0.7219, "expressions": 0.5393, "looking_down_at_another": 0.4471, "sleeping_together": 0.5063, "3rd_party_watermark": 0.3932, "under_blanket": 0.4251, "applying_makeup": 0.4698, "relaxed_expression": 0.5026, "eyes": 0.8955, "english_text": 0.4128}, "stage3_selected_ranks": {"hair": 8, "text": 7, "blue_eyes": 6, "lying": 79, "green_eyes": 9, "blonde_hair": 11, "speech_bubble": 121, "purple_hair": 14, "makeup": 10, "watermark": 4, "lipstick": 48, "bedroom": 42, "sleeping": 5, "bedding": 150, "annoyed": 13, "bed_sheet": 133, "mascara": 72, "lying_on_bed": 111, "text_box": 145, "zzz": 106, "pajamas": 113, "sitting_on_bed": 158, "resting": 33, "annoyed_expression": 2, "expressions": 19, "looking_down_at_another": 66, "sleeping_together": 31, "3rd_party_watermark": 134, "under_blanket": 87, "applying_makeup": 55, "relaxed_expression": 32, "eyes": 1, "english_text": 104}, "stage3_selected_phrase_ranks": {"hair": 1, "text": 1, "blue_eyes": 1, "lying": 7, "green_eyes": 1, "blonde_hair": 1, "speech_bubble": 5, "purple_hair": 1, "makeup": 1, "watermark": 1, "lipstick": 2, "bedroom": 1, "sleeping": 1, "bedding": 8, "annoyed": 2, "bed_sheet": 5, "mascara": 8, "lying_on_bed": 4, "text_box": 8, "zzz": 8, "pajamas": 4, "sitting_on_bed": 10, "resting": 1, "annoyed_expression": 1, "expressions": 3, "looking_down_at_another": 6, "sleeping_together": 2, "3rd_party_watermark": 3, "under_blanket": 7, "applying_makeup": 4, "relaxed_expression": 6, "eyes": 1, "english_text": 4}, "extra_evidence": {"3rd_party_watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "annoyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5677}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4698}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5393}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4782}, "looking_down_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4471}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "mascara": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4405}, "pajamas": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4043}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5026}, "resting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sitting_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5063}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4006}, "text_box": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3881}, "under_blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4251}, "watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5996}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4122}}, "structural": [], "t1": 2.66, "t2": 2.02, "t3": 22.55, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=210 entity=0 copyright_filtered=0 generic_char_to_general=0 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 106, "n_selected": 24, "n_implied": 0, "n_structural": 0, "ret_R": 0.3636, "P": 0.2083, "R": 0.2273, "F1": 0.2174, "leaf_P": 0.1111, "leaf_R": 0.1538, "leaf_F1": 0.129, "n_leaf_sel": 18, "n_leaf_gt": 13, "ret_P": 0.0755, "sel_given_ret": 0.625, "over_sel": 1.09, "why": {"explicit": 25}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 14, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 59, "attempts_by_n_local": {"14": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "49": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2083, "gen_R": 0.2273, "gen_F1": 0.2174, "missed": ["anthro", "canid", "canine", "clothed", "clothing", "fingers", "fur", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["curled_hair", "determined", "dire_wolf", "electric_guitar", "flowing_hair", "holding_guitar", "long_hair", "pastel_background", "playing_guitar", "playing_music", "relaxed_expression", "teal_background", "torn_body", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_pants", "torn_topwear", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["bass_guitar", "claws", "curled_hair", "determined", "dire_wolf", "electric_guitar", "flowing_hair", "guitar", "holding_guitar", "long_hair", "musical_instrument", "pastel_background", "playing_guitar", "playing_music", "relaxed_expression", "teal_background", "torn_body", "torn_bottomwear", "torn_clothing", "torn_jeans", "torn_leggings", "torn_pants", "torn_topwear", "wolf"], "stage3_selected": ["bass_guitar", "claws", "curled_hair", "determined", "dire_wolf", "electric_guitar", "flowing_hair", "guitar", "holding_guitar", "long_hair", "musical_instrument", "pastel_background", "playing_guitar", "playing_music", "relaxed_expression", "teal_background", "torn_body", "torn_bottomwear", "torn_clothing", "torn_jeans", "torn_leggings", "torn_pants", "torn_topwear", "transparent_background", "wolf"], "stage3_selected_scores": {"claws": 0.5499, "wolf": 0.5686, "long_hair": 0.4035, "torn_clothing": 0.3948, "transparent_background": 0.4423, "musical_instrument": 0.8616, "torn_bottomwear": 0.4251, "curled_hair": 0.3666, "guitar": 0.9787, "torn_pants": 0.4557, "torn_topwear": 0.3786, "playing_music": 0.889, "playing_guitar": 0.9493, "torn_jeans": 0.4782, "dire_wolf": 0.4338, "teal_background": 0.4412, "electric_guitar": 0.8828, "bass_guitar": 0.9285, "flowing_hair": 0.5463, "determined": 0.4438, "holding_guitar": 0.8595, "torn_leggings": 0.3983, "relaxed_expression": 0.4178, "torn_body": 0.3879, "pastel_background": 0.5448}, "stage3_selected_ranks": {"claws": 18, "wolf": 12, "long_hair": 79, "torn_clothing": 83, "transparent_background": 52, "musical_instrument": 10, "torn_bottomwear": 64, "curled_hair": 97, "guitar": 1, "torn_pants": 43, "torn_topwear": 91, "playing_music": 4, "playing_guitar": 2, "torn_jeans": 30, "dire_wolf": 60, "teal_background": 54, "electric_guitar": 5, "bass_guitar": 3, "flowing_hair": 19, "determined": 50, "holding_guitar": 11, "torn_leggings": 81, "relaxed_expression": 71, "torn_body": 87, "pastel_background": 20}, "stage3_selected_phrase_ranks": {"claws": 1, "wolf": 1, "long_hair": 3, "torn_clothing": 6, "transparent_background": 8, "musical_instrument": 8, "torn_bottomwear": 3, "curled_hair": 4, "guitar": 1, "torn_pants": 2, "torn_topwear": 9, "playing_music": 3, "playing_guitar": 1, "torn_jeans": 1, "dire_wolf": 5, "teal_background": 9, "electric_guitar": 4, "bass_guitar": 2, "flowing_hair": 1, "determined": 2, "holding_guitar": 10, "torn_leggings": 5, "relaxed_expression": 2, "torn_body": 8, "pastel_background": 1}, "extra_evidence": {"curled_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3666}, "determined": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4438}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4338}, "electric_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8828}, "flowing_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5463}, "holding_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8595}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4035}, "pastel_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5448}, "playing_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9493}, "playing_music": {"source": "stage3", "why": "explicit", "retrieval_score": 0.889}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4178}, "teal_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4412}, "torn_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3879}, "torn_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4251}, "torn_jeans": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4782}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3983}, "torn_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4557}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3786}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5686}}, "structural": [], "t1": 1.39, "t2": 1.17, "t3": 7.99, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=109 entity=2 copyright_filtered=0 generic_char_to_general=0 unknown_type=2"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 151, "n_selected": 25, "n_implied": 0, "n_structural": 0, "ret_R": 0.5652, "P": 0.32, "R": 0.3478, "F1": 0.3333, "leaf_P": 0.2632, "leaf_R": 0.3333, "leaf_F1": 0.2941, "n_leaf_sel": 19, "n_leaf_gt": 15, "ret_P": 0.0861, "sel_given_ret": 0.6154, "over_sel": 1.09, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 19, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 103, "attempts_by_n_local": {"19": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "31": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.32, "gen_R": 0.3478, "gen_F1": 0.3333, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "duo", "fox", "lagomorph", "leporid", "looking_at_another", "mammal", "pants", "standing", "topwear"], "extra": ["actual_fur", "black_bottomwear", "black_claws", "black_markings", "black_nose", "black_pants", "blue_overalls", "ear_markings", "grey_claws", "grey_fur", "grey_shirt", "open_mouth", "two_tone_fur", "white_markings", "white_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["actual_fur", "black_bottomwear", "black_claws", "black_markings", "black_nose", "black_pants", "blue_overalls", "claws", "crossed_arms", "ear_markings", "facial_markings", "fur", "grey_background", "grey_claws", "grey_fur", "grey_shirt", "open_mouth", "overalls", "rabbit", "shirt", "two_tone_fur", "white_markings", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["actual_fur", "black_bottomwear", "black_claws", "black_markings", "black_nose", "black_pants", "blue_overalls", "claws", "crossed_arms", "ear_markings", "facial_markings", "fur", "grey_background", "grey_claws", "grey_fur", "grey_shirt", "open_mouth", "overalls", "rabbit", "shirt", "simple_background", "two_tone_fur", "white_markings", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"fur": 0.653, "simple_background": 0.4159, "open_mouth": 0.6329, "claws": 0.6303, "shirt": 0.7482, "black_nose": 0.4892, "rabbit": 0.651, "grey_fur": 0.4914, "two_tone_fur": 0.4866, "grey_background": 0.6783, "facial_markings": 0.6944, "black_claws": 0.5069, "crossed_arms": 0.7284, "white_topwear": 0.767, "black_bottomwear": 0.7383, "white_shirt": 0.8197, "black_markings": 0.6011, "white_markings": 0.5974, "overalls": 0.8775, "black_pants": 0.8329, "ear_markings": 0.5953, "grey_claws": 0.5165, "grey_shirt": 0.6922, "white_t-shirt": 0.7504, "blue_overalls": 0.9203, "actual_fur": 0.4837}, "stage3_selected_ranks": {"fur": 40, "simple_background": 153, "open_mouth": 48, "claws": 50, "shirt": 18, "black_nose": 132, "rabbit": 41, "grey_fur": 131, "two_tone_fur": 136, "grey_background": 30, "facial_markings": 27, "black_claws": 125, "crossed_arms": 21, "white_topwear": 6, "black_bottomwear": 19, "white_shirt": 4, "black_markings": 63, "white_markings": 65, "overalls": 2, "black_pants": 3, "ear_markings": 66, "grey_claws": 121, "grey_shirt": 28, "white_t-shirt": 15, "blue_overalls": 1, "actual_fur": 137}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 8, "open_mouth": 1, "claws": 1, "shirt": 1, "black_nose": 8, "rabbit": 1, "grey_fur": 7, "two_tone_fur": 9, "grey_background": 1, "facial_markings": 1, "black_claws": 10, "crossed_arms": 1, "white_topwear": 3, "black_bottomwear": 5, "white_shirt": 1, "black_markings": 9, "white_markings": 8, "overalls": 1, "black_pants": 1, "ear_markings": 10, "grey_claws": 8, "grey_shirt": 4, "white_t-shirt": 10, "blue_overalls": 1, "actual_fur": 10}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4837}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7383}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5069}, "black_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6011}, "black_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4892}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "blue_overalls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9203}, "ear_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5953}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5165}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4914}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6922}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6329}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4866}, "white_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5974}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7504}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.767}}, "structural": [], "t1": 2.11, "t2": 1.39, "t3": 26.59, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=151 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 202, "n_selected": 58, "n_implied": 0, "n_structural": 0, "ret_R": 0.5455, "P": 0.0862, "R": 0.4545, "F1": 0.1449, "leaf_P": 0.08, "leaf_R": 0.5714, "leaf_F1": 0.1404, "n_leaf_sel": 50, "n_leaf_gt": 7, "ret_P": 0.0297, "sel_given_ret": 0.8333, "over_sel": 5.27, "why": {"explicit": 59}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 25, "bootstrap_anchor_selected": 22, "bootstrap_anchor_kept": 12, "bootstrap_reranked": true, "calls_total": 5, "calls_with_selection": 5, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 5, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 142, "attempts_by_n_local": {"25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "26": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 2, "char_F1": 0.0, "gen_P": 0.0893, "gen_R": 0.4545, "gen_F1": 0.1493, "missed": ["blue_nose", "canid", "canine", "mammal", "solo", "white_body"], "extra": ["2d_animation", "action_pose", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "blue_stripes", "blue_tail", "blushing_profusely", "canid_humanoid", "canine_humanoid", "character_request", "colorful", "cool_colors", "curved_tail", "eyes", "fan_character", "female_humanoid", "fennec_humanoid", "fluffy_fur", "glistening_fur", "heterochromia", "humanoid", "jumper", "jumping", "long_mouth", "long_tail", "midair", "mouth_closed", "paw_pose", "pink_blush", "pink_mouth", "pink_stripes", "pixel_animation", "pointed_tail", "pose", "sparkling_character", "striped_back", "striped_fur", "stripes", "tail", "tail_fluff", "tanuki_humanoid", "teeth", "thick_fur", "thong_straps", "tongue", "two_tone_fur", "two_tone_tail", "unnamed_character", "white_stripes", "yellow_stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["2d_animation", "action_pose", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "blue_eyes", "blue_stripes", "blue_tail", "blushing_profusely", "canid_humanoid", "canine_humanoid", "character_request", "colorful", "cool_colors", "curved_tail", "eyes", "fan_character", "female_humanoid", "fennec_humanoid", "fluffy_fur", "fur", "glistening_fur", "heterochromia", "humanoid", "jumper", "jumping", "long_mouth", "long_tail", "midair", "mouth_closed", "open_mouth", "paw_pose", "pink_blush", "pink_mouth", "pink_stripes", "pixel_animation", "pointed_tail", "pose", "purple_body", "sparkling_character", "striped_back", "striped_fur", "stripes", "tail", "tail_fluff", "tanuki_humanoid", "teeth", "thick_fur", "thong_straps", "tongue", "two_tone_fur", "two_tone_tail", "unnamed_character", "white_fur", "white_stripes", "yellow_stripes"], "stage3_selected": ["2d_animation", "action_pose", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "blue_eyes", "blue_stripes", "blue_tail", "blushing_profusely", "canid_humanoid", "canine_humanoid", "character_request", "colorful", "cool_colors", "curved_tail", "eyes", "fan_character", "female_humanoid", "fennec_humanoid", "fluffy_fur", "fur", "glistening_fur", "heterochromia", "humanoid", "jumper", "jumping", "long_mouth", "long_tail", "midair", "mouth_closed", "open_mouth", "paw_pose", "pink_blush", "pink_mouth", "pink_stripes", "pixel_animation", "pointed_tail", "pose", "purple_body", "simple_background", "sparkling_character", "striped_back", "striped_fur", "stripes", "tail", "tail_fluff", "tanuki_humanoid", "teeth", "thick_fur", "thong_straps", "tongue", "two_tone_fur", "two_tone_tail", "unnamed_character", "white_fur", "white_stripes", "yellow_stripes"], "stage3_selected_scores": {"fur": 0.539, "simple_background": 0.5542, "open_mouth": 0.5607, "tongue": 0.3123, "tail": 0.557, "teeth": 0.3117, "blue_eyes": 0.5536, "white_fur": 0.5404, "humanoid": 0.6815, "animal_humanoid": 0.6341, "stripes": 0.5404, "pose": 0.5685, "fan_character": 0.3979, "purple_body": 0.5187, "two_tone_fur": 0.4047, "long_tail": 0.4245, "striped_fur": 0.4464, "canid_humanoid": 0.8979, "canine_humanoid": 0.9364, "heterochromia": 0.3888, "two_tone_tail": 0.3868, "blue_tail": 0.4201, "mouth_closed": 0.3904, "action_pose": 0.5585, "jumping": 0.5525, "glistening_fur": 0.4031, "blue_stripes": 0.456, "cool_colors": 0.3987, "colorful": 0.3364, "white_stripes": 0.4388, "midair": 0.3386, "blushing_profusely": 0.3099, "long_mouth": 0.3994, "character_request": 0.3172, "thong_straps": 0.2907, "paw_pose": 0.5376, "yellow_stripes": 0.4136, "pink_blush": 0.39, "pointed_tail": 0.4396, "tanuki_humanoid": 0.792, "pink_stripes": 0.4577, "female_humanoid": 0.5503, "fluffy_fur": 0.429, "curved_tail": 0.4918, "pink_mouth": 0.3819, "striped_back": 0.4522, "unnamed_character": 0.3244, "tail_fluff": 0.3827, "sparkling_character": 0.3052, "thick_fur": 0.3073, "fennec_humanoid": 0.8082, "jumper": 0.3959, "actual_fur": 0.4175, "animated_comic": 0.3974, "animated_png": 0.4385, "eyes": 0.6909, "animated": 0.3689, "2d_animation": 0.3569, "pixel_animation": 0.3451}, "stage3_selected_ranks": {"fur": 30, "simple_background": 22, "open_mouth": 18, "tongue": 191, "tail": 21, "teeth": 192, "blue_eyes": 24, "white_fur": 29, "humanoid": 12, "animal_humanoid": 13, "stripes": 28, "pose": 15, "fan_character": 117, "purple_body": 34, "two_tone_fur": 107, "long_tail": 77, "striped_fur": 56, "canid_humanoid": 2, "canine_humanoid": 1, "heterochromia": 130, "two_tone_tail": 136, "blue_tail": 87, "mouth_closed": 127, "action_pose": 20, "jumping": 25, "glistening_fur": 108, "blue_stripes": 48, "cool_colors": 115, "colorful": 170, "white_stripes": 62, "midair": 168, "blushing_profusely": 193, "long_mouth": 113, "character_request": 186, "thong_straps": 208, "paw_pose": 31, "yellow_stripes": 95, "pink_blush": 129, "pointed_tail": 60, "tanuki_humanoid": 7, "pink_stripes": 46, "female_humanoid": 26, "fluffy_fur": 71, "curved_tail": 38, "pink_mouth": 141, "striped_back": 52, "unnamed_character": 181, "tail_fluff": 140, "sparkling_character": 201, "thick_fur": 198, "fennec_humanoid": 6, "jumper": 121, "actual_fur": 90, "animated_comic": 119, "animated_png": 64, "eyes": 11, "animated": 146, "2d_animation": 153, "pixel_animation": 164}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 1, "open_mouth": 1, "tongue": 7, "tail": 1, "teeth": 8, "blue_eyes": 1, "white_fur": 1, "humanoid": 1, "animal_humanoid": 2, "stripes": 1, "pose": 1, "fan_character": 2, "purple_body": 1, "two_tone_fur": 8, "long_tail": 9, "striped_fur": 5, "canid_humanoid": 2, "canine_humanoid": 1, "heterochromia": 3, "two_tone_tail": 9, "blue_tail": 2, "mouth_closed": 9, "action_pose": 1, "jumping": 1, "glistening_fur": 9, "blue_stripes": 2, "cool_colors": 4, "colorful": 9, "white_stripes": 4, "midair": 5, "blushing_profusely": 9, "long_mouth": 6, "character_request": 7, "thong_straps": 8, "paw_pose": 2, "yellow_stripes": 5, "pink_blush": 6, "pointed_tail": 5, "tanuki_humanoid": 7, "pink_stripes": 1, "female_humanoid": 8, "fluffy_fur": 3, "curved_tail": 1, "pink_mouth": 10, "striped_back": 4, "unnamed_character": 6, "tail_fluff": 9, "sparkling_character": 9, "thick_fur": 6, "fennec_humanoid": 6, "jumper": 2, "actual_fur": 2, "animated_comic": 2, "animated_png": 1, "eyes": 1, "animated": 4, "2d_animation": 5, "pixel_animation": 6}, "extra_evidence": {"2d_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3569}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5585}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4175}, "animal_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6341}, "animated": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3689}, "animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3974}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4385}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.456}, "blue_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4201}, "blushing_profusely": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3099}, "canid_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8979}, "canine_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9364}, "character_request": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3172}, "colorful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3364}, "cool_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4918}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6909}, "fan_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3979}, "female_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5503}, "fennec_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "fluffy_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.429}, "glistening_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4031}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3888}, "humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6815}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3959}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5525}, "long_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3994}, "long_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4245}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3386}, "mouth_closed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3904}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5376}, "pink_blush": {"source": "stage3", "why": "explicit", "retrieval_score": 0.39}, "pink_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3819}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4577}, "pixel_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3451}, "pointed_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4396}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5685}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3052}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4522}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4464}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5404}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.557}, "tail_fluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3827}, "tanuki_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.792}, "teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3117}, "thick_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3073}, "thong_straps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2907}, "tongue": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3123}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4047}, "two_tone_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3868}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3244}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4388}, "yellow_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4136}}, "structural": [], "t1": 1.97, "t2": 1.87, "t3": 27.9, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=206 entity=2 copyright_filtered=2 generic_char_to_general=4 unknown_type=11"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_enabled_explicit_skiprewrite.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:14:29.871641", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 13}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 11, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0909, "R": 0.0833, "F1": 0.087, "leaf_P": 0.1429, "leaf_R": 0.1111, "leaf_F1": 0.125, "n_leaf_sel": 7, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 0.92, "why": {"explicit": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 4, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 27, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0909, "gen_R": 0.0833, "gen_F1": 0.087, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "translucent_clothing", "white_clothing", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "translucent_clothing", "white_clothing", "white_t-shirt", "white_topwear"], "stage3_selected": ["alpha_channel", "face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "translucent_clothing", "transparent_background", "white_clothing", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "solo_focus": 0.4386, "alpha_channel": 0.4024, "translucent": 0.7484, "white_clothing": 0.6182, "necktie": 0.8082, "transparent_background": 0.5794, "translucent_clothing": 0.6249, "white_topwear": 0.6638, "white_t-shirt": 0.6584, "focused": 0.6608, "face_focus": 0.4971}, "stage3_selected_ranks": {"shirt": 1, "solo_focus": 64, "alpha_channel": 71, "translucent": 10, "white_clothing": 25, "necktie": 3, "transparent_background": 39, "translucent_clothing": 24, "white_topwear": 13, "white_t-shirt": 15, "focused": 14, "face_focus": 56}, "stage3_selected_phrase_ranks": {"shirt": 1, "solo_focus": 7, "alpha_channel": 10, "translucent": 1, "white_clothing": 7, "necktie": 1, "transparent_background": 4, "translucent_clothing": 3, "white_topwear": 1, "white_t-shirt": 2, "focused": 1, "face_focus": 6}, "extra_evidence": {"face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "solo_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "translucent": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7484}, "translucent_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6249}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.54, "t3": 14.76, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 17, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.1176, "R": 0.1429, "F1": 0.129, "leaf_P": 0.1176, "leaf_R": 0.2, "leaf_F1": 0.1481, "n_leaf_sel": 17, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 0.6667, "over_sel": 1.21, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 22, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 63, "attempts_by_n_local": {"22": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1176, "gen_R": 0.1429, "gen_F1": 0.129, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "bright", "extended_arms", "grin", "hand_above_head", "high_waisted_bottomwear", "larger_anthro", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "smaller_anthro", "smug_grin", "spread_arms", "toothy_grin", "toothy_smile"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "bright", "extended_arms", "grin", "hand_above_head", "high_waisted_bottomwear", "larger_anthro", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "smaller_anthro", "smug_grin", "spread_arms", "toothy_grin", "toothy_smile"], "stage3_selected": ["animated_comic", "bright", "extended_arms", "grin", "hand_above_head", "high_waisted_bottomwear", "larger_anthro", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "simple_background", "smaller_anthro", "smug_grin", "spread_arms", "toothy_grin", "toothy_smile"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "looking_at_viewer": 0.5612, "grin": 0.7174, "larger_anthro": 0.4912, "smaller_anthro": 0.5391, "raised_arms": 0.8636, "spread_arms": 0.605, "extended_arms": 0.6595, "smug_grin": 0.4997, "toothy_grin": 0.6248, "partially_clothed_anthro": 0.4576, "high_waisted_bottomwear": 0.4614, "hand_above_head": 0.6835, "bright": 0.5873, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "looking_at_viewer": 22, "grin": 3, "larger_anthro": 72, "smaller_anthro": 36, "raised_arms": 1, "spread_arms": 14, "extended_arms": 8, "smug_grin": 64, "toothy_grin": 13, "partially_clothed_anthro": 100, "high_waisted_bottomwear": 96, "hand_above_head": 4, "bright": 16, "relaxed_expression": 48, "toothy_smile": 23, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "looking_at_viewer": 2, "grin": 1, "larger_anthro": 3, "smaller_anthro": 2, "raised_arms": 1, "spread_arms": 6, "extended_arms": 6, "smug_grin": 5, "toothy_grin": 2, "partially_clothed_anthro": 10, "high_waisted_bottomwear": 9, "hand_above_head": 3, "bright": 1, "relaxed_expression": 1, "toothy_smile": 3, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "extended_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6595}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "hand_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6835}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "larger_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4912}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "smaller_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5391}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.605}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}}, "structural": [], "t1": 0.0, "t2": 9.87, "t3": 14.0, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 13, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0769, "R": 0.0714, "F1": 0.0741, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 13, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 0.93, "why": {"explicit": 13}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 26, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0769, "gen_R": 0.0714, "gen_F1": 0.0741, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "exposed_back", "fantasy", "intimate", "netherland_dwarf_rabbit", "partial_line_speech_bubble", "patting", "petting", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "exposed_back", "fantasy", "intimate", "netherland_dwarf_rabbit", "partial_line_speech_bubble", "patting", "petting", "romantic", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "stage3_selected": ["cheek_to_cheek", "exposed_back", "fantasy", "intimate", "netherland_dwarf_rabbit", "partial_line_speech_bubble", "patting", "petting", "romantic", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "stage3_selected_scores": {"romantic": 0.5683, "wide_eyed": 0.5498, "fantasy": 0.5458, "petting": 0.7411, "intimate": 0.5697, "netherland_dwarf_rabbit": 0.5541, "patting": 0.6189, "shocked_face": 0.616, "exposed_back": 0.4752, "partial_line_speech_bubble": 0.5962, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "setting": 1.0}, "stage3_selected_ranks": {"romantic": 43, "wide_eyed": 50, "fantasy": 52, "petting": 12, "intimate": 42, "netherland_dwarf_rabbit": 45, "patting": 17, "shocked_face": 18, "exposed_back": 72, "partial_line_speech_bubble": 24, "smiling_at_another": 49, "cheek_to_cheek": 29, "setting": 2}, "stage3_selected_phrase_ranks": {"romantic": 6, "wide_eyed": 7, "fantasy": 9, "petting": 2, "intimate": 5, "netherland_dwarf_rabbit": 6, "patting": 6, "shocked_face": 1, "exposed_back": 9, "partial_line_speech_bubble": 6, "smiling_at_another": 10, "cheek_to_cheek": 8, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "exposed_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4752}, "fantasy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5458}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "netherland_dwarf_rabbit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5541}, "partial_line_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5962}, "patting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6189}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5498}}, "structural": [], "t1": 0.0, "t2": 8.5, "t3": 18.12, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 4, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 4, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.0, "why": {"explicit": 5}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 3, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 14, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["round_eyes", "round_nose", "stylized_empty_eyes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["round_eyes", "round_nose", "stylized_empty_eyes", "toony"], "stage3_selected": ["round_eyes", "round_nose", "simple_background", "stylized_empty_eyes", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "toony": 0.763, "round_eyes": 0.6227, "round_nose": 0.555, "stylized_empty_eyes": 0.6448}, "stage3_selected_ranks": {"simple_background": 10, "toony": 6, "round_eyes": 30, "round_nose": 59, "stylized_empty_eyes": 25}, "stage3_selected_phrase_ranks": {"simple_background": 5, "toony": 3, "round_eyes": 1, "round_nose": 10, "stylized_empty_eyes": 1}, "extra_evidence": {"round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "stylized_empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6448}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.51, "t3": 7.99, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1111, "R": 0.0909, "F1": 0.1, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 15, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.82, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 10, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 43, "attempts_by_n_local": {"10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1111, "gen_R": 0.0909, "gen_F1": 0.1, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_markings", "striped_shirt"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "shorts", "striped_markings", "striped_shirt"], "stage3_selected": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "shorts", "striped_markings", "striped_shirt"], "stage3_selected_scores": {"fur": 0.7785, "grey_fur": 0.5932, "pose": 0.847, "shorts": 0.9202, "happy": 0.457, "neck_tuft": 0.5847, "raised_arm": 0.6005, "striped_markings": 0.5966, "blue_bottomwear": 0.6293, "playful": 0.6803, "blue_shorts": 0.7288, "playing": 0.5527, "striped_shirt": 0.6015, "raised_finger": 0.595, "hand_gesture": 0.6065, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334, "dolphin_shorts": 0.7425}, "stage3_selected_ranks": {"fur": 8, "grey_fur": 48, "pose": 3, "shorts": 1, "happy": 87, "neck_tuft": 51, "raised_arm": 42, "striped_markings": 46, "blue_bottomwear": 32, "playful": 18, "blue_shorts": 14, "playing": 58, "striped_shirt": 40, "raised_finger": 47, "hand_gesture": 39, "hand_over_head": 56, "hand_on_own_head": 66, "dolphin_shorts": 10}, "stage3_selected_phrase_ranks": {"fur": 1, "grey_fur": 8, "pose": 1, "shorts": 1, "happy": 5, "neck_tuft": 9, "raised_arm": 2, "striped_markings": 8, "blue_bottomwear": 2, "playful": 1, "blue_shorts": 1, "playing": 2, "striped_shirt": 5, "raised_finger": 3, "hand_gesture": 1, "hand_over_head": 5, "hand_on_own_head": 6, "dolphin_shorts": 7}, "extra_evidence": {"blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "dolphin_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7425}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5932}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "striped_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6015}}, "structural": [], "t1": 0.0, "t2": 0.81, "t3": 11.43, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 35, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.1429, "R": 0.3846, "F1": 0.2083, "leaf_P": 0.069, "leaf_R": 0.3333, "leaf_F1": 0.1143, "n_leaf_sel": 29, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.625, "over_sel": 2.69, "why": {"explicit": 35}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 79, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.1471, "gen_R": 0.3846, "gen_F1": 0.2128, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "4_panel_comic", "caprine_demon", "confusion", "creepy_face", "dark", "dark_skin", "dark_theme", "darkness", "expressionless", "frown", "goat_demon", "group", "holding_character", "horror_(theme)", "light", "light_beam", "lighter", "lizard_(divinity)", "lizard_tail", "lizardman", "monitor_lizard", "nightmare", "one_page_comic", "red_light", "scared", "scary", "shocked_face", "speech_bubble", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "4_panel_comic", "caprine_demon", "confusion", "creepy_face", "dark", "dark_skin", "dark_theme", "darkness", "expressionless", "frown", "goat", "goat_demon", "group", "holding_character", "horror_(theme)", "human", "light", "light_beam", "lighter", "lizard", "lizard_(divinity)", "lizard_tail", "lizardman", "monitor_lizard", "nightmare", "one_page_comic", "red_light", "reptile", "scalie", "scared", "scary", "shocked_face", "speech_bubble", "unnamed_character"], "stage3_selected": ["3_panel_comic", "4_panel_comic", "caprine_demon", "confusion", "creepy_face", "dark", "dark_skin", "dark_theme", "darkness", "expressionless", "frown", "goat", "goat_demon", "group", "holding_character", "horror_(theme)", "human", "light", "light_beam", "lighter", "lizard", "lizard_(divinity)", "lizard_tail", "lizardman", "monitor_lizard", "nightmare", "one_page_comic", "red_light", "reptile", "scalie", "scared", "scary", "shocked_face", "speech_bubble", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "scalie": 0.4916, "human": 0.6298, "reptile": 0.5077, "speech_bubble": 0.551, "lizard": 0.8035, "goat": 0.777, "light": 0.5737, "frown": 0.4011, "dark_skin": 0.4239, "scared": 0.4465, "confusion": 0.4487, "dark": 0.5246, "holding_character": 0.3838, "caprine_demon": 0.5314, "goat_demon": 0.5311, "light_beam": 0.5797, "lizardman": 0.5549, "monitor_lizard": 0.5353, "darkness": 0.7908, "dark_theme": 0.5093, "scary": 0.4653, "lighter": 0.567, "horror_(theme)": 0.4557, "lizard_tail": 0.5169, "nightmare": 0.4738, "shocked_face": 0.4417, "unnamed_character": 0.4967, "3_panel_comic": 0.4885, "red_light": 0.5752, "4_panel_comic": 0.4744, "one_page_comic": 0.4707, "lizard_(divinity)": 0.5263, "creepy_face": 0.4758, "expressionless": 0.4291}, "stage3_selected_ranks": {"group": 110, "scalie": 53, "human": 6, "reptile": 46, "speech_bubble": 23, "lizard": 1, "goat": 3, "light": 15, "frown": 119, "dark_skin": 105, "scared": 95, "confusion": 90, "dark": 37, "holding_character": 124, "caprine_demon": 32, "goat_demon": 33, "light_beam": 12, "lizardman": 20, "monitor_lizard": 28, "darkness": 2, "dark_theme": 43, "scary": 79, "lighter": 18, "horror_(theme)": 83, "lizard_tail": 41, "nightmare": 72, "shocked_face": 98, "unnamed_character": 50, "3_panel_comic": 56, "red_light": 14, "4_panel_comic": 70, "one_page_comic": 75, "lizard_(divinity)": 36, "creepy_face": 68, "expressionless": 104}, "stage3_selected_phrase_ranks": {"group": 5, "scalie": 10, "human": 1, "reptile": 9, "speech_bubble": 1, "lizard": 1, "goat": 1, "light": 4, "frown": 8, "dark_skin": 7, "scared": 10, "confusion": 4, "dark": 3, "holding_character": 9, "caprine_demon": 7, "goat_demon": 7, "light_beam": 2, "lizardman": 4, "monitor_lizard": 4, "darkness": 1, "dark_theme": 4, "scary": 10, "lighter": 6, "horror_(theme)": 7, "lizard_tail": 8, "nightmare": 8, "shocked_face": 5, "unnamed_character": 1, "3_panel_comic": 2, "red_light": 3, "4_panel_comic": 4, "one_page_comic": 5, "lizard_(divinity)": 5, "creepy_face": 6, "expressionless": 3}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "4_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4744}, "caprine_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5314}, "confusion": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4487}, "creepy_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4758}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5246}, "dark_skin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4239}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "expressionless": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4291}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5311}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "holding_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3838}, "horror_(theme)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4557}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "light_beam": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5797}, "lighter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.567}, "lizard_(divinity)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5263}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5169}, "lizardman": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5549}, "monitor_lizard": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5353}, "nightmare": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4738}, "one_page_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4707}, "red_light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5752}, "scared": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4465}, "scary": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4653}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4417}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.75, "t3": 29.36, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"style_or_meta\"}, {\"i\": 43, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"style_or_meta\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 40}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 30, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.2667, "R": 0.5714, "F1": 0.3636, "leaf_P": 0.1852, "leaf_R": 0.4167, "leaf_F1": 0.2564, "n_leaf_sel": 27, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 1.0, "over_sel": 2.14, "why": {"explicit": 30}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 73, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2667, "gen_R": 0.5714, "gen_F1": 0.3636, "missed": ["angry", "duo", "eyes_closed", "lying", "purple_hair", "text"], "extra": ["animated_comic", "annoyed_expression", "bed_covers", "bed_sheet", "bedroom", "blue_lipstick", "clown_makeup", "creating_art", "green_eyebrows", "half-length_portrait", "lidded_eyes", "lipstick_on_face", "long_hair", "lying_on_bed", "pillow", "portuguese_text", "purple_legs", "red_lipstick", "sleeping_together", "sleepover", "sleepwear", "under_blanket"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["animated_comic", "annoyed_expression", "bed", "bed_covers", "bed_sheet", "bedroom", "blonde_hair", "blue_eyes", "blue_lipstick", "clown_makeup", "creating_art", "eyeshadow", "green_eyebrows", "green_eyes", "hair", "half-length_portrait", "lidded_eyes", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "pillow", "portuguese_text", "purple_legs", "red_lipstick", "sleeping", "sleeping_together", "sleepover", "sleepwear", "under_blanket"], "stage3_selected": ["animated_comic", "annoyed_expression", "bed", "bed_covers", "bed_sheet", "bedroom", "blonde_hair", "blue_eyes", "blue_lipstick", "clown_makeup", "creating_art", "eyeshadow", "green_eyebrows", "green_eyes", "hair", "half-length_portrait", "lidded_eyes", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "pillow", "portuguese_text", "purple_legs", "red_lipstick", "sleeping", "sleeping_together", "sleepover", "sleepwear", "under_blanket"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "long_hair": 0.3962, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "pillow": 0.441, "eyeshadow": 0.4274, "bedroom": 0.4116, "sleeping": 0.4995, "half-length_portrait": 0.352, "bed_sheet": 0.4395, "lidded_eyes": 0.3759, "lying_on_bed": 0.412, "red_lipstick": 0.4186, "blue_lipstick": 0.3955, "annoyed_expression": 0.3458, "bed_covers": 0.4011, "creating_art": 0.3914, "green_eyebrows": 0.4683, "lipstick_on_face": 0.3766, "sleeping_together": 0.408, "sleepover": 0.5015, "under_blanket": 0.3449, "sleepwear": 0.4346, "clown_makeup": 0.4004, "purple_legs": 0.6369, "animated_comic": 0.3497, "portuguese_text": 0.3865}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "long_hair": 48, "blonde_hair": 96, "bed": 15, "makeup": 11, "pillow": 27, "eyeshadow": 32, "bedroom": 39, "sleeping": 20, "half-length_portrait": 87, "bed_sheet": 28, "lidded_eyes": 63, "lying_on_bed": 38, "red_lipstick": 34, "blue_lipstick": 50, "annoyed_expression": 97, "bed_covers": 44, "creating_art": 51, "green_eyebrows": 23, "lipstick_on_face": 62, "sleeping_together": 41, "sleepover": 19, "under_blanket": 99, "sleepwear": 30, "clown_makeup": 45, "purple_legs": 3, "animated_comic": 90, "portuguese_text": 54}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "long_hair": 3, "blonde_hair": 8, "bed": 1, "makeup": 1, "pillow": 3, "eyeshadow": 3, "bedroom": 7, "sleeping": 2, "half-length_portrait": 8, "bed_sheet": 4, "lidded_eyes": 6, "lying_on_bed": 6, "red_lipstick": 4, "blue_lipstick": 10, "annoyed_expression": 9, "bed_covers": 9, "creating_art": 1, "green_eyebrows": 2, "lipstick_on_face": 4, "sleeping_together": 1, "sleepover": 1, "under_blanket": 10, "sleepwear": 5, "clown_makeup": 6, "purple_legs": 1, "animated_comic": 1, "portuguese_text": 1}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3497}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blue_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3955}, "clown_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "creating_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3914}, "green_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4683}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.352}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3759}, "lipstick_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3766}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3962}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "portuguese_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3865}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "sleepover": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4346}, "under_blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}}, "structural": [], "t1": 0.0, "t2": 1.57, "t3": 15.25, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1667, "R": 0.1364, "F1": 0.15, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 14, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.6, "over_sel": 0.82, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 14, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 49, "attempts_by_n_local": {"14": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}}, "attempt_failure_rate": 0.25, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 0.1364, "gen_F1": 0.15, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["action_pose", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "long_claws", "maned_wolf", "official_art", "orange_background", "pose", "rock_creature", "torn_leggings", "torn_stockings", "torn_topwear", "yellow_background"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["action_pose", "canine", "claws", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "long_claws", "maned_wolf", "official_art", "orange_background", "pose", "rock_creature", "torn_clothing", "torn_leggings", "torn_stockings", "torn_topwear", "yellow_background"], "stage3_selected": ["action_pose", "canine", "claws", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "long_claws", "maned_wolf", "official_art", "orange_background", "pose", "rock_creature", "torn_clothing", "torn_leggings", "torn_stockings", "torn_topwear", "yellow_background"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "pose": 0.8955, "torn_clothing": 0.5179, "yellow_background": 0.6256, "official_art": 0.5154, "orange_background": 0.6518, "action_pose": 0.6726, "color_swatch": 0.7261, "maned_wolf": 0.664, "torn_topwear": 0.4853, "long_claws": 0.6953, "torn_stockings": 0.4869, "dire_wolf": 0.6438, "rock_creature": 0.5385, "floating_head": 0.4571, "torn_leggings": 0.4969, "gold_claws": 0.6234}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "pose": 2, "torn_clothing": 73, "yellow_background": 42, "official_art": 74, "orange_background": 24, "action_pose": 19, "color_swatch": 9, "maned_wolf": 21, "torn_topwear": 88, "long_claws": 15, "torn_stockings": 84, "dire_wolf": 30, "rock_creature": 69, "floating_head": 100, "torn_leggings": 79, "gold_claws": 43}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "pose": 1, "torn_clothing": 4, "yellow_background": 10, "official_art": 3, "orange_background": 2, "action_pose": 2, "color_swatch": 1, "maned_wolf": 1, "torn_topwear": 10, "long_claws": 2, "torn_stockings": 8, "dire_wolf": 4, "rock_creature": 4, "floating_head": 5, "torn_leggings": 6, "gold_claws": 3}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6438}, "floating_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4571}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "official_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5154}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "rock_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5385}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_stockings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4869}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6256}}, "structural": [], "t1": 0.0, "t2": 1.1, "t3": 24.61, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 38}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.38.why\n Field required [type=missing, input_value={'i': 38}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.2692, "R": 0.3043, "F1": 0.2857, "leaf_P": 0.0556, "leaf_R": 0.0667, "leaf_F1": 0.0606, "n_leaf_sel": 18, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.875, "over_sel": 1.13, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 12, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 44, "attempts_by_n_local": {"12": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2692, "gen_R": 0.3043, "gen_F1": 0.2857, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing"], "extra": ["4_claws", "black_bottomwear", "black_fur", "black_pants", "black_topwear", "cheek_markings", "claws_out", "grey_shirt", "grey_theme", "grey_tongue", "grey_topwear", "grey_tuft", "half-closed_eyes", "partially_clothed_anthro", "retractable_claws", "white_dress_shirt", "white_fur", "white_markings", "white_shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "black_bottomwear", "black_fur", "black_pants", "black_topwear", "cheek_markings", "claws", "claws_out", "facial_markings", "fur", "grey_background", "grey_shirt", "grey_theme", "grey_tongue", "grey_topwear", "grey_tuft", "half-closed_eyes", "pants", "partially_clothed_anthro", "retractable_claws", "shirt", "topwear", "white_dress_shirt", "white_fur", "white_markings", "white_shirt"], "stage3_selected": ["4_claws", "black_bottomwear", "black_fur", "black_pants", "black_topwear", "cheek_markings", "claws", "claws_out", "facial_markings", "fur", "grey_background", "grey_shirt", "grey_theme", "grey_tongue", "grey_topwear", "grey_tuft", "half-closed_eyes", "pants", "partially_clothed_anthro", "retractable_claws", "shirt", "topwear", "white_dress_shirt", "white_fur", "white_markings", "white_shirt"], "stage3_selected_scores": {"fur": 0.7011, "claws": 0.6687, "white_fur": 0.5776, "topwear": 0.6085, "shirt": 0.7037, "black_fur": 0.5396, "half-closed_eyes": 0.4928, "pants": 0.7123, "grey_background": 0.5635, "facial_markings": 0.9015, "black_topwear": 0.5595, "black_bottomwear": 0.5638, "white_shirt": 0.5609, "white_markings": 0.7013, "black_pants": 0.5822, "grey_topwear": 0.5774, "grey_shirt": 0.6272, "claws_out": 0.5754, "partially_clothed_anthro": 0.5361, "grey_theme": 0.5547, "cheek_markings": 0.719, "grey_tongue": 0.5557, "4_claws": 0.6123, "white_dress_shirt": 0.5611, "retractable_claws": 0.565, "grey_tuft": 0.5545}, "stage3_selected_ranks": {"fur": 16, "claws": 19, "white_fur": 44, "topwear": 30, "shirt": 13, "black_fur": 75, "half-closed_eyes": 86, "pants": 10, "grey_background": 58, "facial_markings": 1, "black_topwear": 65, "black_bottomwear": 57, "white_shirt": 63, "white_markings": 15, "black_pants": 39, "grey_topwear": 45, "grey_shirt": 25, "claws_out": 46, "partially_clothed_anthro": 76, "grey_theme": 69, "cheek_markings": 7, "grey_tongue": 68, "4_claws": 28, "white_dress_shirt": 62, "retractable_claws": 56, "grey_tuft": 70}, "stage3_selected_phrase_ranks": {"fur": 1, "claws": 1, "white_fur": 3, "topwear": 8, "shirt": 1, "black_fur": 10, "half-closed_eyes": 8, "pants": 1, "grey_background": 3, "facial_markings": 1, "black_topwear": 10, "black_bottomwear": 6, "white_shirt": 4, "white_markings": 8, "black_pants": 5, "grey_topwear": 1, "grey_shirt": 7, "claws_out": 7, "partially_clothed_anthro": 7, "grey_theme": 8, "cheek_markings": 5, "grey_tongue": 7, "4_claws": 3, "white_dress_shirt": 3, "retractable_claws": 6, "grey_tuft": 9}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6123}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5638}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5396}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5822}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "cheek_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.719}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5754}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "grey_tongue": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5557}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "grey_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5545}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5361}, "retractable_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.565}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5776}, "white_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7013}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5609}}, "structural": [], "t1": 0.0, "t2": 0.96, "t3": 14.14, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 34, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0294, "R": 0.0909, "F1": 0.0444, "leaf_P": 0.0303, "leaf_R": 0.1429, "leaf_F1": 0.05, "n_leaf_sel": 33, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 3.09, "why": {"explicit": 36}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 97, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0294, "gen_R": 0.0909, "gen_F1": 0.0444, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "action_pose", "anime_eyes", "big_eyes", "blue_fur", "blue_markings", "colored_line_art", "curved_tail", "featureless_arms", "hand_on_floor", "happy", "irregular_speech_bubble", "jumper", "jumping", "light_legs", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "pose", "posed", "purple_stripes", "raised_paw", "smiling_at_viewer", "striped_arms", "striped_face", "striped_neck", "tail_between_legs", "tan_stripes", "what_has_science_done"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "action_pose", "anime_eyes", "big_eyes", "blue_fur", "blue_markings", "blue_nose", "colored_line_art", "curved_tail", "featureless_arms", "hand_on_floor", "happy", "irregular_speech_bubble", "jumper", "jumping", "light_legs", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "pose", "posed", "purple_stripes", "raised_paw", "smiling_at_viewer", "striped_arms", "striped_face", "striped_neck", "tail_between_legs", "tan_stripes", "what_has_science_done"], "stage3_selected": [">:)", ">:d", "abstract_background", "action_pose", "anime_eyes", "big_eyes", "blue_fur", "blue_markings", "blue_nose", "colored_line_art", "curved_tail", "featureless_arms", "hand_on_floor", "happy", "irregular_speech_bubble", "jumper", "jumping", "light_legs", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "pose", "posed", "purple_stripes", "raised_paw", "simple_background", "smiling_at_viewer", "striped_arms", "striped_face", "striped_neck", "tail_between_legs", "tan_stripes", "what_has_science_done"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "pose": 0.6068, "happy": 0.36, "abstract_background": 0.3542, "smiling_at_viewer": 0.451, "blue_nose": 0.5017, "big_eyes": 0.4619, "multiple_poses": 0.4969, "blue_markings": 0.4512, "action_pose": 0.4596, "what_has_science_done": 0.2437, "jumping": 0.5295, "playful": 0.6118, "tail_between_legs": 0.351, "striped_arms": 0.6782, "purple_stripes": 0.4704, "midair": 0.43, "raised_paw": 0.3182, "paw_pose": 0.3304, "striped_face": 0.7013, "pink_stripes": 0.5235, "curved_tail": 0.5875, "striped_neck": 0.711, "anime_eyes": 0.4019, "light_legs": 0.6973, ">:d": 0.3455, ">:)": 0.3449, "colored_line_art": 0.3343, "irregular_speech_bubble": 0.3253, "tan_stripes": 0.7126, "hand_on_floor": 0.497, "featureless_arms": 0.3094, "posed": 0.4337, "jumper": 0.4127, "nose": 0.8806}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "pose": 19, "happy": 101, "abstract_background": 107, "smiling_at_viewer": 62, "blue_nose": 45, "big_eyes": 57, "multiple_poses": 48, "blue_markings": 61, "action_pose": 58, "what_has_science_done": 161, "jumping": 35, "playful": 17, "tail_between_legs": 109, "striped_arms": 10, "purple_stripes": 56, "midair": 71, "raised_paw": 134, "paw_pose": 122, "striped_face": 7, "pink_stripes": 37, "curved_tail": 22, "striped_neck": 6, "anime_eyes": 84, "light_legs": 8, ">:d": 111, ">:)": 113, "colored_line_art": 119, "irregular_speech_bubble": 127, "tan_stripes": 5, "hand_on_floor": 47, "featureless_arms": 138, "posed": 69, "jumper": 81, "nose": 2}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "pose": 1, "happy": 3, "abstract_background": 8, "smiling_at_viewer": 8, "blue_nose": 3, "big_eyes": 4, "multiple_poses": 5, "blue_markings": 6, "action_pose": 3, "what_has_science_done": 10, "jumping": 1, "playful": 1, "tail_between_legs": 5, "striped_arms": 6, "purple_stripes": 4, "midair": 2, "raised_paw": 9, "paw_pose": 7, "striped_face": 5, "pink_stripes": 1, "curved_tail": 3, "striped_neck": 4, "anime_eyes": 7, "light_legs": 1, ">:d": 8, ">:)": 9, "colored_line_art": 8, "irregular_speech_bubble": 7, "tan_stripes": 3, "hand_on_floor": 10, "featureless_arms": 9, "posed": 10, "jumper": 4, "nose": 1}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4619}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4512}, "colored_line_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3343}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "featureless_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3094}, "hand_on_floor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.497}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "irregular_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3253}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "light_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6973}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8806}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3304}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6782}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7013}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.711}, "tail_between_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.351}, "tan_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7126}, "what_has_science_done": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2437}}, "structural": [], "t1": 0.0, "t2": 1.37, "t3": 15.92, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p3.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:18:18.566871", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 16}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0556, "R": 0.0833, "F1": 0.0667, "leaf_P": 0.0769, "leaf_R": 0.1111, "leaf_F1": 0.0909, "n_leaf_sel": 13, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 1.5, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 4, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 25, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0556, "gen_R": 0.0833, "gen_F1": 0.0667, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["black_bottomwear", "black_necktie", "button_(fastener)", "clasped_hands", "focused", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "translucent", "trio_focus", "white_dress_shirt", "white_outline", "white_shirt", "white_tank_top", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "black_bottomwear", "black_necktie", "button_(fastener)", "clasped_hands", "focused", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "translucent", "trio_focus", "white_dress_shirt", "white_outline", "white_shirt", "white_tank_top", "white_topwear"], "stage3_selected": ["alpha_channel", "black_bottomwear", "black_necktie", "button_(fastener)", "clasped_hands", "focused", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "translucent", "transparent_background", "trio_focus", "white_dress_shirt", "white_outline", "white_shirt", "white_tank_top", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "alpha_channel": 0.4024, "translucent": 0.7484, "tank_top": 0.7545, "necktie": 0.8082, "transparent_background": 0.5794, "white_topwear": 0.6638, "black_bottomwear": 0.6065, "white_shirt": 0.6481, "button_(fastener)": 0.5695, "white_outline": 0.573, "white_tank_top": 0.6359, "trio_focus": 0.533, "clasped_hands": 0.5876, "black_necktie": 0.625, "white_dress_shirt": 0.6255, "focused": 0.6608, "sparkling_character": 0.5249, "holding_topwear": 0.595}, "stage3_selected_ranks": {"shirt": 1, "alpha_channel": 71, "translucent": 10, "tank_top": 8, "necktie": 3, "transparent_background": 39, "white_topwear": 13, "black_bottomwear": 31, "white_shirt": 16, "button_(fastener)": 47, "white_outline": 44, "white_tank_top": 18, "trio_focus": 51, "clasped_hands": 37, "black_necktie": 23, "white_dress_shirt": 22, "focused": 14, "sparkling_character": 52, "holding_topwear": 36}, "stage3_selected_phrase_ranks": {"shirt": 1, "alpha_channel": 10, "translucent": 1, "tank_top": 7, "necktie": 1, "transparent_background": 4, "white_topwear": 1, "black_bottomwear": 8, "white_shirt": 3, "button_(fastener)": 8, "white_outline": 9, "white_tank_top": 4, "trio_focus": 3, "clasped_hands": 5, "black_necktie": 6, "white_dress_shirt": 5, "focused": 1, "sparkling_character": 1, "holding_topwear": 3}, "extra_evidence": {"black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "button_(fastener)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "clasped_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5876}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "holding_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5249}, "tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7545}, "translucent": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7484}, "trio_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.533}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6255}, "white_outline": {"source": "stage3", "why": "explicit", "retrieval_score": 0.573}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6481}, "white_tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.58, "t3": 11.22, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.0714, "R": 0.0714, "F1": 0.0714, "leaf_P": 0.0714, "leaf_R": 0.1, "leaf_F1": 0.0833, "n_leaf_sel": 14, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 0.3333, "over_sel": 1.0, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 22, "bootstrap_anchor_selected": 11, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 3, "calls_exhausted_retries": 1, "attempts_total": 6, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 66, "attempts_by_n_local": {"22": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 1, "parse_fail": 0, "errors": 3}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.5, "call_exhaustion_rate": 0.25}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0714, "gen_R": 0.0714, "gen_F1": 0.0714, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "mammal", "primate"], "extra": ["bright", "grin", "high_waisted_bottomwear", "holding_face", "hunter", "larger_anthro", "petting", "raised_arm", "raised_arms", "relaxed_expression", "toony_expression", "toothy_grin", "toothy_smile"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["bright", "grin", "high_waisted_bottomwear", "holding_face", "hunter", "larger_anthro", "male", "petting", "raised_arm", "raised_arms", "relaxed_expression", "toony_expression", "toothy_grin", "toothy_smile"], "stage3_selected": ["bright", "grin", "high_waisted_bottomwear", "holding_face", "hunter", "larger_anthro", "male", "petting", "raised_arm", "raised_arms", "relaxed_expression", "simple_background", "toony_expression", "toothy_grin", "toothy_smile"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "grin": 0.7174, "raised_arm": 0.7205, "larger_anthro": 0.4912, "raised_arms": 0.8636, "petting": 0.4897, "toothy_grin": 0.6248, "hunter": 0.3114, "high_waisted_bottomwear": 0.4614, "holding_face": 0.5182, "bright": 0.5873, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "toony_expression": 0.4675}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "grin": 3, "raised_arm": 2, "larger_anthro": 72, "raised_arms": 1, "petting": 73, "toothy_grin": 13, "hunter": 173, "high_waisted_bottomwear": 96, "holding_face": 47, "bright": 16, "relaxed_expression": 48, "toothy_smile": 23, "toony_expression": 89}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "grin": 1, "raised_arm": 2, "larger_anthro": 3, "raised_arms": 1, "petting": 2, "toothy_grin": 2, "hunter": 6, "high_waisted_bottomwear": 9, "holding_face": 5, "bright": 1, "relaxed_expression": 1, "toothy_smile": 3, "toony_expression": 6}, "extra_evidence": {"bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "holding_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5182}, "hunter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3114}, "larger_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4912}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4897}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}}, "structural": [], "t1": 0.0, "t2": 9.87, "t3": 50.78, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 49, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 38}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 38}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 49, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 13, \"why\": \"strong_implied\"}, {\"i\": 17, \"why\": \"strong_implied\"}, {\"i\": 22, \"why\": \"strong_implied\"}, {\"i\": 24, \"why\": \"strong_implied\"}, {\"i\": 26, \"why\": \"strong_implied\"}, {\"i\": 30, \"why\": \"strong_implied\"}, {\"i\": 32, \"why\": \"strong_implied\"}, {\"i\": 34, \"why\": \"strong_implied\"}, {\"i\": 36, \"why\": \"strong_implied\"}, {\"i\": 38, \"why\": \"strong_implied\"}, {\"i\": 40}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 40}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"strong_implied\"}, {\"i\": 4, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"strong_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 49, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: gave up after 3 attempts"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 7, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.5, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 1, "dupe_indices_total": 4, "kept_total": 27, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["bored_expression", "cheek_to_cheek", "facial_expressions", "passionate", "setting", "smiling_at_another", "spitting"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["bored_expression", "cheek_to_cheek", "facial_expressions", "passionate", "setting", "smiling_at_another", "spitting"], "stage3_selected": ["bored_expression", "cheek_to_cheek", "facial_expressions", "passionate", "setting", "smiling_at_another", "spitting"], "stage3_selected_scores": {"passionate": 0.5508, "spitting": 0.5399, "bored_expression": 0.5856, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "facial_expressions": 0.5874, "setting": 1.0}, "stage3_selected_ranks": {"passionate": 48, "spitting": 54, "bored_expression": 36, "smiling_at_another": 49, "cheek_to_cheek": 29, "facial_expressions": 33, "setting": 2}, "stage3_selected_phrase_ranks": {"passionate": 9, "spitting": 10, "bored_expression": 9, "smiling_at_another": 10, "cheek_to_cheek": 8, "facial_expressions": 8, "setting": 1}, "extra_evidence": {"bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5856}, "cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "facial_expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5874}, "passionate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "spitting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5399}}, "structural": [], "t1": 0.0, "t2": 8.58, "t3": 9.63, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 4, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 4, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.0, "why": {"explicit": 5}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 4, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 19, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["air_creature", "creature_print", "round_eyes", "round_nose"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["air_creature", "creature_print", "round_eyes", "round_nose"], "stage3_selected": ["air_creature", "creature_print", "round_eyes", "round_nose", "simple_background"], "stage3_selected_scores": {"simple_background": 0.7167, "creature_print": 0.5405, "air_creature": 0.6051, "round_eyes": 0.6227, "round_nose": 0.555}, "stage3_selected_ranks": {"simple_background": 10, "creature_print": 68, "air_creature": 36, "round_eyes": 30, "round_nose": 59}, "stage3_selected_phrase_ranks": {"simple_background": 5, "creature_print": 9, "air_creature": 1, "round_eyes": 1, "round_nose": 10}, "extra_evidence": {"air_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6051}, "creature_print": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5405}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}}, "structural": [], "t1": 0.0, "t2": 8.51, "t3": 6.65, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 15, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1333, "R": 0.0909, "F1": 0.1081, "leaf_P": 0.0714, "leaf_R": 0.0833, "leaf_F1": 0.0769, "n_leaf_sel": 14, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.68, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 10, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 39, "attempts_by_n_local": {"10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1333, "gen_R": 0.0909, "gen_F1": 0.1081, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_shorts", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_finger", "striped_markings", "striped_topwear"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "striped_topwear"], "stage3_selected": ["blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "happy", "neck_tuft", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "striped_topwear"], "stage3_selected_scores": {"fur": 0.7785, "pose": 0.847, "shorts": 0.9202, "happy": 0.457, "neck_tuft": 0.5847, "striped_markings": 0.5966, "hand_on_face": 0.5657, "striped_topwear": 0.5988, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "hand_gesture": 0.6065, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"fur": 8, "pose": 3, "shorts": 1, "happy": 87, "neck_tuft": 51, "striped_markings": 46, "hand_on_face": 55, "striped_topwear": 43, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "hand_gesture": 39, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"fur": 1, "pose": 1, "shorts": 1, "happy": 5, "neck_tuft": 9, "striped_markings": 8, "hand_on_face": 4, "striped_topwear": 6, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "hand_gesture": 1, "hand_on_own_head": 6}, "extra_evidence": {"blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5657}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "striped_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5988}}, "structural": [], "t1": 0.0, "t2": 0.74, "t3": 5.7, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.1875, "R": 0.2308, "F1": 0.2069, "leaf_P": 0.1429, "leaf_R": 0.3333, "leaf_F1": 0.2, "n_leaf_sel": 14, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.375, "over_sel": 1.23, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 86, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.2, "gen_R": 0.2308, "gen_F1": 0.2143, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["bright_light", "clothed_human", "dark_theme", "darkness", "duo", "group", "light", "mature_human", "one_page_comic", "pointy_speech_bubble", "speech_bubble", "thought_bubble", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["bright_light", "clothed_human", "dark_theme", "darkness", "duo", "goat", "group", "human", "light", "lizard", "mature_human", "one_page_comic", "pointy_speech_bubble", "speech_bubble", "thought_bubble", "unnamed_character"], "stage3_selected": ["bright_light", "clothed_human", "dark_theme", "darkness", "duo", "goat", "group", "human", "light", "lizard", "mature_human", "one_page_comic", "pointy_speech_bubble", "speech_bubble", "thought_bubble", "unnamed_character"], "stage3_selected_scores": {"duo": 0.4015, "group": 0.4182, "human": 0.6298, "speech_bubble": 0.551, "lizard": 0.8035, "goat": 0.777, "light": 0.5737, "thought_bubble": 0.441, "pointy_speech_bubble": 0.4817, "darkness": 0.7908, "dark_theme": 0.5093, "mature_human": 0.4229, "unnamed_character": 0.4967, "one_page_comic": 0.4707, "bright_light": 0.5857, "clothed_human": 0.4818}, "stage3_selected_ranks": {"duo": 118, "group": 110, "human": 6, "speech_bubble": 23, "lizard": 1, "goat": 3, "light": 15, "thought_bubble": 100, "pointy_speech_bubble": 61, "darkness": 2, "dark_theme": 43, "mature_human": 107, "unnamed_character": 50, "one_page_comic": 75, "bright_light": 10, "clothed_human": 60}, "stage3_selected_phrase_ranks": {"duo": 6, "group": 5, "human": 1, "speech_bubble": 1, "lizard": 1, "goat": 1, "light": 4, "thought_bubble": 10, "pointy_speech_bubble": 6, "darkness": 1, "dark_theme": 4, "mature_human": 8, "unnamed_character": 1, "one_page_comic": 5, "bright_light": 1, "clothed_human": 5}, "extra_evidence": {"bright_light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5857}, "clothed_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4818}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "duo": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4015}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "mature_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4229}, "one_page_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4707}, "pointy_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4817}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "thought_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.55, "t3": 31.41, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"strong_implied\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"strong_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"strong_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"strong_implied\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 34, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.1765, "R": 0.4286, "F1": 0.25, "leaf_P": 0.1, "leaf_R": 0.25, "leaf_F1": 0.1429, "n_leaf_sel": 30, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.75, "over_sel": 2.43, "why": {"explicit": 34}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 73, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1765, "gen_R": 0.4286, "gen_F1": 0.25, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "hair", "lying", "purple_hair", "text"], "extra": ["8_eyes", "annoyed_expression", "bed_covers", "bedding", "bedroom", "blanket", "blue_background", "blue_lipstick", "green_background", "heterochromia", "lipstick_on_face", "long_hair", "lying_on_bed", "palette", "pillow", "purple_arms", "purple_background", "purple_belly", "purple_eyelids", "purple_feet", "purple_legs", "purple_text", "red_lipstick", "relaxed_expression", "sleep_fetish", "sleeping_together", "sleepwear", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["8_eyes", "annoyed_expression", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyes", "blue_lipstick", "green_background", "green_eyes", "heterochromia", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "palette", "pillow", "purple_arms", "purple_background", "purple_belly", "purple_eyelids", "purple_feet", "purple_legs", "purple_text", "red_lipstick", "relaxed_expression", "sleep_fetish", "sleeping", "sleeping_together", "sleepwear", "zzz"], "stage3_selected": ["8_eyes", "annoyed_expression", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyes", "blue_lipstick", "green_background", "green_eyes", "heterochromia", "lipstick_on_face", "long_hair", "lying_on_bed", "makeup", "palette", "pillow", "purple_arms", "purple_background", "purple_belly", "purple_eyelids", "purple_feet", "purple_legs", "purple_text", "red_lipstick", "relaxed_expression", "sleep_fetish", "sleeping", "sleeping_together", "sleepwear", "zzz"], "stage3_selected_scores": {"blue_eyes": 0.5401, "green_eyes": 0.538, "long_hair": 0.3962, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "pillow": 0.441, "bedroom": 0.4116, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "heterochromia": 0.3472, "purple_background": 0.3682, "green_background": 0.3352, "blanket": 0.3523, "lying_on_bed": 0.412, "zzz": 0.367, "red_lipstick": 0.4186, "blue_lipstick": 0.3955, "annoyed_expression": 0.3458, "bed_covers": 0.4011, "purple_text": 0.5604, "purple_belly": 0.5813, "lipstick_on_face": 0.3766, "sleeping_together": 0.408, "purple_feet": 0.6039, "palette": 0.3097, "purple_eyelids": 0.405, "sleepwear": 0.4346, "purple_legs": 0.6369, "purple_arms": 0.6358, "8_eyes": 0.3787, "relaxed_expression": 0.388, "sleep_fetish": 0.3748}, "stage3_selected_ranks": {"blue_eyes": 12, "green_eyes": 14, "long_hair": 48, "blonde_hair": 96, "bed": 15, "makeup": 11, "pillow": 27, "bedroom": 39, "blue_background": 105, "sleeping": 20, "bedding": 33, "heterochromia": 95, "purple_background": 67, "green_background": 114, "blanket": 86, "lying_on_bed": 38, "zzz": 68, "red_lipstick": 34, "blue_lipstick": 50, "annoyed_expression": 97, "bed_covers": 44, "purple_text": 10, "purple_belly": 9, "lipstick_on_face": 62, "sleeping_together": 41, "purple_feet": 7, "palette": 123, "purple_eyelids": 43, "sleepwear": 30, "purple_legs": 3, "purple_arms": 4, "8_eyes": 60, "relaxed_expression": 53, "sleep_fetish": 64}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "green_eyes": 1, "long_hair": 3, "blonde_hair": 8, "bed": 1, "makeup": 1, "pillow": 3, "bedroom": 7, "blue_background": 10, "sleeping": 2, "bedding": 5, "heterochromia": 7, "purple_background": 5, "green_background": 10, "blanket": 10, "lying_on_bed": 6, "zzz": 8, "red_lipstick": 4, "blue_lipstick": 10, "annoyed_expression": 9, "bed_covers": 9, "purple_text": 3, "purple_belly": 7, "lipstick_on_face": 4, "sleeping_together": 1, "purple_feet": 5, "palette": 4, "purple_eyelids": 3, "sleepwear": 5, "purple_legs": 1, "purple_arms": 2, "8_eyes": 4, "relaxed_expression": 2, "sleep_fetish": 7}, "extra_evidence": {"8_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3787}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "blue_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3955}, "green_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3352}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lipstick_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3766}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3962}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3097}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "purple_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6358}, "purple_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3682}, "purple_belly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5813}, "purple_eyelids": {"source": "stage3", "why": "explicit", "retrieval_score": 0.405}, "purple_feet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6039}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "purple_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5604}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "sleep_fetish": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3748}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4346}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}}, "structural": [], "t1": 0.0, "t2": 1.37, "t3": 20.22, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1154, "R": 0.1364, "F1": 0.125, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 21, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.6, "over_sel": 1.18, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 14, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 54, "attempts_by_n_local": {"14": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1154, "gen_R": 0.1364, "gen_F1": 0.125, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "extra": ["4_toes", "acoustic_guitar", "action_pose", "big_claws", "black_claws", "claws_out", "digitigrade", "dire_wolf", "finger_claws", "full-length_portrait", "glowing_claws", "gold_claws", "heel_claw", "long_claws", "portrait", "pose", "rottweiler", "toony", "torn_leggings", "torn_topwear", "unknown_species", "wolf", "yellow_background"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_toes", "acoustic_guitar", "action_pose", "big_claws", "black_claws", "canine", "claws", "claws_out", "digitigrade", "dire_wolf", "finger_claws", "full-length_portrait", "glowing_claws", "gold_claws", "heel_claw", "long_claws", "musical_instrument", "portrait", "pose", "rottweiler", "toony", "torn_leggings", "torn_topwear", "unknown_species", "wolf", "yellow_background"], "stage3_selected": ["4_toes", "acoustic_guitar", "action_pose", "big_claws", "black_claws", "canine", "claws", "claws_out", "digitigrade", "dire_wolf", "finger_claws", "full-length_portrait", "glowing_claws", "gold_claws", "heel_claw", "long_claws", "musical_instrument", "portrait", "pose", "rottweiler", "toony", "torn_leggings", "torn_topwear", "unknown_species", "wolf", "yellow_background"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "portrait": 0.6102, "4_toes": 0.695, "pose": 0.8955, "digitigrade": 0.7162, "full-length_portrait": 0.6018, "finger_claws": 0.7339, "toony": 0.4807, "black_claws": 0.6961, "unknown_species": 0.7852, "musical_instrument": 0.3362, "yellow_background": 0.6256, "action_pose": 0.6726, "rottweiler": 0.5579, "torn_topwear": 0.4853, "long_claws": 0.6953, "claws_out": 0.7061, "big_claws": 0.7461, "dire_wolf": 0.6438, "heel_claw": 0.5951, "glowing_claws": 0.5992, "torn_leggings": 0.4969, "gold_claws": 0.6234, "acoustic_guitar": 0.3382}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "wolf": 23, "portrait": 47, "4_toes": 16, "pose": 2, "digitigrade": 10, "full-length_portrait": 54, "finger_claws": 8, "toony": 90, "black_claws": 14, "unknown_species": 5, "musical_instrument": 117, "yellow_background": 42, "action_pose": 19, "rottweiler": 66, "torn_topwear": 88, "long_claws": 15, "claws_out": 12, "big_claws": 7, "dire_wolf": 30, "heel_claw": 57, "glowing_claws": 56, "torn_leggings": 79, "gold_claws": 43, "acoustic_guitar": 115}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "wolf": 2, "portrait": 7, "4_toes": 9, "pose": 1, "digitigrade": 5, "full-length_portrait": 10, "finger_claws": 4, "toony": 5, "black_claws": 4, "unknown_species": 1, "musical_instrument": 9, "yellow_background": 10, "action_pose": 2, "rottweiler": 8, "torn_topwear": 10, "long_claws": 2, "claws_out": 6, "big_claws": 1, "dire_wolf": 4, "heel_claw": 10, "glowing_claws": 9, "torn_leggings": 6, "gold_claws": 3, "acoustic_guitar": 7}, "extra_evidence": {"4_toes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.695}, "acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3382}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6961}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7061}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6438}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7339}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6018}, "glowing_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "heel_claw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5951}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6102}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "rottweiler": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5579}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "unknown_species": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7852}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6256}}, "structural": [], "t1": 0.0, "t2": 1.08, "t3": 11.63, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.375, "R": 0.2609, "F1": 0.3077, "leaf_P": 0.1818, "leaf_R": 0.1333, "leaf_F1": 0.1538, "n_leaf_sel": 11, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.75, "over_sel": 0.7, "why": {"explicit": 17}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 12, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 54, "attempts_by_n_local": {"12": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.375, "gen_R": 0.2609, "gen_F1": 0.3077, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing"], "extra": ["arm_markings", "black_pants", "flat_colors", "grey_shirt", "grey_theme", "grey_topwear", "hand_on_face", "multicolored_fur", "partially_clothed_anthro", "white_dress_shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["arm_markings", "black_pants", "claws", "facial_markings", "flat_colors", "fur", "grey_shirt", "grey_theme", "grey_topwear", "hand_on_face", "multicolored_fur", "pants", "partially_clothed_anthro", "shirt", "topwear", "white_dress_shirt"], "stage3_selected": ["arm_markings", "black_pants", "claws", "facial_markings", "flat_colors", "fur", "grey_shirt", "grey_theme", "grey_topwear", "hand_on_face", "multicolored_fur", "pants", "partially_clothed_anthro", "shirt", "simple_background", "topwear", "white_dress_shirt"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "topwear": 0.6085, "shirt": 0.7037, "multicolored_fur": 0.5716, "pants": 0.7123, "facial_markings": 0.9015, "flat_colors": 0.4857, "hand_on_face": 0.4889, "arm_markings": 0.7489, "black_pants": 0.5822, "grey_topwear": 0.5774, "grey_shirt": 0.6272, "partially_clothed_anthro": 0.5361, "grey_theme": 0.5547, "white_dress_shirt": 0.5611}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "topwear": 30, "shirt": 13, "multicolored_fur": 50, "pants": 10, "facial_markings": 1, "flat_colors": 88, "hand_on_face": 87, "arm_markings": 4, "black_pants": 39, "grey_topwear": 45, "grey_shirt": 25, "partially_clothed_anthro": 76, "grey_theme": 69, "white_dress_shirt": 62}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "topwear": 8, "shirt": 1, "multicolored_fur": 4, "pants": 1, "facial_markings": 1, "flat_colors": 3, "hand_on_face": 2, "arm_markings": 3, "black_pants": 5, "grey_topwear": 1, "grey_shirt": 7, "partially_clothed_anthro": 7, "grey_theme": 8, "white_dress_shirt": 3}, "extra_evidence": {"arm_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7489}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5822}, "flat_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4857}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4889}, "multicolored_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5716}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5361}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}}, "structural": [], "t1": 0.0, "t2": 0.9, "t3": 8.88, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0385, "R": 0.0909, "F1": 0.0541, "leaf_P": 0.04, "leaf_R": 0.1429, "leaf_F1": 0.0625, "n_leaf_sel": 25, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 2.36, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 11, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 94, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0385, "gen_R": 0.0909, "gen_F1": 0.0541, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": ["asexual_pride_colors", "blue_markings", "blue_paws", "colorful_background", "curved_tail", "eyes", "full-length_portrait", "jumper", "jumping", "nose", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes", "pushmi-pullyu", "raised_paw", "smiling_at_viewer", "striped_arms", "striped_back", "striped_legs", "stripes", "white_stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["asexual_pride_colors", "blue_markings", "blue_nose", "blue_paws", "colorful_background", "curved_tail", "eyes", "full-length_portrait", "jumper", "jumping", "nose", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes", "pushmi-pullyu", "raised_paw", "smiling_at_viewer", "striped_arms", "striped_back", "striped_legs", "stripes", "white_stripes"], "stage3_selected": ["asexual_pride_colors", "blue_markings", "blue_nose", "blue_paws", "colorful_background", "curved_tail", "eyes", "full-length_portrait", "jumper", "jumping", "nose", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes", "pushmi-pullyu", "raised_paw", "smiling_at_viewer", "striped_arms", "striped_back", "striped_legs", "stripes", "white_stripes"], "stage3_selected_scores": {"pawpads": 0.327, "stripes": 0.6073, "pose": 0.6068, "full-length_portrait": 0.4491, "smiling_at_viewer": 0.451, "blue_nose": 0.5017, "blue_markings": 0.4512, "jumping": 0.5295, "playful": 0.6118, "white_stripes": 0.6473, "playing": 0.3818, "striped_legs": 0.6654, "striped_arms": 0.6782, "purple_stripes": 0.4704, "raised_paw": 0.3182, "blue_paws": 0.5068, "pink_stripes": 0.5235, "curved_tail": 0.5875, "striped_back": 0.7273, "colorful_background": 0.3196, "asexual_pride_colors": 0.2701, "posed": 0.4337, "pushmi-pullyu": 0.2496, "jumper": 0.4127, "nose": 0.8806, "eyes": 0.97}, "stage3_selected_ranks": {"pawpads": 125, "stripes": 18, "pose": 19, "full-length_portrait": 63, "smiling_at_viewer": 62, "blue_nose": 45, "blue_markings": 61, "jumping": 35, "playful": 17, "white_stripes": 14, "playing": 91, "striped_legs": 12, "striped_arms": 10, "purple_stripes": 56, "raised_paw": 134, "blue_paws": 43, "pink_stripes": 37, "curved_tail": 22, "striped_back": 4, "colorful_background": 131, "asexual_pride_colors": 151, "posed": 69, "pushmi-pullyu": 157, "jumper": 81, "nose": 2, "eyes": 1}, "stage3_selected_phrase_ranks": {"pawpads": 8, "stripes": 10, "pose": 1, "full-length_portrait": 9, "smiling_at_viewer": 8, "blue_nose": 3, "blue_markings": 6, "jumping": 1, "playful": 1, "white_stripes": 9, "playing": 2, "striped_legs": 8, "striped_arms": 6, "purple_stripes": 4, "raised_paw": 9, "blue_paws": 2, "pink_stripes": 1, "curved_tail": 3, "striped_back": 2, "colorful_background": 9, "asexual_pride_colors": 9, "posed": 10, "pushmi-pullyu": 6, "jumper": 4, "nose": 1, "eyes": 1}, "extra_evidence": {"asexual_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2701}, "blue_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4512}, "blue_paws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5068}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3196}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8806}, "pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.327}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3818}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "pushmi-pullyu": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2496}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6782}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7273}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6654}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6473}}, "structural": [], "t1": 0.0, "t2": 1.35, "t3": 17.51, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p5.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:19:21.725022", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 13}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 10, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.1, "R": 0.0833, "F1": 0.0909, "leaf_P": 0.1111, "leaf_R": 0.1111, "leaf_F1": 0.1111, "n_leaf_sel": 9, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 0.83, "why": {"explicit": 11}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 28, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1, "gen_R": 0.0833, "gen_F1": 0.0909, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["depth_of_field", "face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "white_clothing", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "depth_of_field", "face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "white_clothing", "white_topwear"], "stage3_selected": ["alpha_channel", "depth_of_field", "face_focus", "focused", "necktie", "shirt", "solo_focus", "translucent", "transparent_background", "white_clothing", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "solo_focus": 0.4386, "alpha_channel": 0.4024, "translucent": 0.7484, "white_clothing": 0.6182, "necktie": 0.8082, "transparent_background": 0.5794, "white_topwear": 0.6638, "depth_of_field": 0.5407, "focused": 0.6608, "face_focus": 0.4971}, "stage3_selected_ranks": {"shirt": 1, "solo_focus": 64, "alpha_channel": 71, "translucent": 10, "white_clothing": 25, "necktie": 3, "transparent_background": 39, "white_topwear": 13, "depth_of_field": 49, "focused": 14, "face_focus": 56}, "stage3_selected_phrase_ranks": {"shirt": 1, "solo_focus": 7, "alpha_channel": 10, "translucent": 1, "white_clothing": 7, "necktie": 1, "transparent_background": 4, "white_topwear": 1, "depth_of_field": 2, "focused": 1, "face_focus": 6}, "extra_evidence": {"depth_of_field": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "solo_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "translucent": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7484}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.2, "t3": 8.15, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 23, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.1304, "R": 0.2143, "F1": 0.1622, "leaf_P": 0.0909, "leaf_R": 0.2, "leaf_F1": 0.125, "n_leaf_sel": 22, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 1.64, "why": {"explicit": 24}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 22, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 84, "attempts_by_n_local": {"22": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1304, "gen_R": 0.2143, "gen_F1": 0.1622, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "bent_arm", "breaking_the_fourth_wall", "bright", "expressions", "extended_arms", "grin", "high_waisted_bottomwear", "holding_arms", "humor", "larger_anthro", "larger_male", "magic_user", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "smaller_anthro", "spread_arms", "toony_expression", "toothy_smile"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "bent_arm", "breaking_the_fourth_wall", "bright", "expressions", "extended_arms", "grin", "group", "high_waisted_bottomwear", "holding_arms", "humor", "larger_anthro", "larger_male", "looking_at_viewer", "magic_user", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "smaller_anthro", "spread_arms", "toony_expression", "toothy_smile"], "stage3_selected": ["animated_comic", "bent_arm", "breaking_the_fourth_wall", "bright", "expressions", "extended_arms", "grin", "group", "high_waisted_bottomwear", "holding_arms", "humor", "larger_anthro", "larger_male", "looking_at_viewer", "magic_user", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "simple_background", "smaller_anthro", "spread_arms", "toony_expression", "toothy_smile"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "looking_at_viewer": 0.5612, "group": 0.4099, "larger_male": 0.4954, "grin": 0.7174, "humor": 0.3977, "larger_anthro": 0.4912, "magic_user": 0.3882, "smaller_anthro": 0.5391, "bent_arm": 0.6739, "raised_arms": 0.8636, "spread_arms": 0.605, "extended_arms": 0.6595, "breaking_the_fourth_wall": 0.4209, "partially_clothed_anthro": 0.4576, "expressions": 0.4746, "high_waisted_bottomwear": 0.4614, "holding_arms": 0.5774, "bright": 0.5873, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "toony_expression": 0.4675, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "looking_at_viewer": 22, "group": 132, "larger_male": 68, "grin": 3, "humor": 137, "larger_anthro": 72, "magic_user": 143, "smaller_anthro": 36, "bent_arm": 6, "raised_arms": 1, "spread_arms": 14, "extended_arms": 8, "breaking_the_fourth_wall": 119, "partially_clothed_anthro": 100, "expressions": 86, "high_waisted_bottomwear": 96, "holding_arms": 19, "bright": 16, "relaxed_expression": 48, "toothy_smile": 23, "toony_expression": 89, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "looking_at_viewer": 2, "group": 6, "larger_male": 4, "grin": 1, "humor": 5, "larger_anthro": 3, "magic_user": 3, "smaller_anthro": 2, "bent_arm": 3, "raised_arms": 1, "spread_arms": 6, "extended_arms": 6, "breaking_the_fourth_wall": 5, "partially_clothed_anthro": 10, "expressions": 5, "high_waisted_bottomwear": 9, "holding_arms": 9, "bright": 1, "relaxed_expression": 1, "toothy_smile": 3, "toony_expression": 6, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "bent_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6739}, "breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4209}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4746}, "extended_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6595}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "holding_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "humor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3977}, "larger_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4912}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "magic_user": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3882}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "smaller_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5391}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.605}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}}, "structural": [], "t1": 0.0, "t2": 9.46, "t3": 17.51, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.43, "why": {"explicit": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 5, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 22, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "embrace", "fantasy", "intimate", "setting", "shocked_face"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "embrace", "fantasy", "intimate", "setting", "shocked_face"], "stage3_selected": ["cheek_to_cheek", "embrace", "fantasy", "intimate", "setting", "shocked_face"], "stage3_selected_scores": {"embrace": 0.5734, "fantasy": 0.5458, "intimate": 0.5697, "shocked_face": 0.616, "cheek_to_cheek": 0.5892, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "fantasy": 52, "intimate": 42, "shocked_face": 18, "cheek_to_cheek": 29, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "fantasy": 9, "intimate": 5, "shocked_face": 1, "cheek_to_cheek": 8, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "fantasy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5458}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}}, "structural": [], "t1": 0.0, "t2": 8.17, "t3": 9.73, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 3, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 3, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 0.75, "why": {"explicit": 4}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 3, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 14, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["round_eyes", "round_nose", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["round_eyes", "round_nose", "toony"], "stage3_selected": ["round_eyes", "round_nose", "simple_background", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "toony": 0.763, "round_eyes": 0.6227, "round_nose": 0.555}, "stage3_selected_ranks": {"simple_background": 10, "toony": 6, "round_eyes": 30, "round_nose": 59}, "stage3_selected_phrase_ranks": {"simple_background": 5, "toony": 3, "round_eyes": 1, "round_nose": 10}, "extra_evidence": {"round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.11, "t3": 8.39, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 14, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 0.0, "over_sel": 0.73, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 10, "bootstrap_anchor_selected": 5, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 35, "attempts_by_n_local": {"10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["action_pose", "blue_bottomwear", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "blue_bottomwear", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger"], "stage3_selected": ["action_pose", "blue_bottomwear", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger"], "stage3_selected_scores": {"grey_fur": 0.5932, "pose": 0.847, "happy": 0.457, "neck_tuft": 0.5847, "raised_arm": 0.6005, "blue_bottomwear": 0.6293, "action_pose": 0.6359, "playful": 0.6803, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "muscle_tone": 0.5438, "hand_gesture": 0.6065, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334, "dolphin_shorts": 0.7425}, "stage3_selected_ranks": {"grey_fur": 48, "pose": 3, "happy": 87, "neck_tuft": 51, "raised_arm": 42, "blue_bottomwear": 32, "action_pose": 31, "playful": 18, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "muscle_tone": 62, "hand_gesture": 39, "hand_over_head": 56, "hand_on_own_head": 66, "dolphin_shorts": 10}, "stage3_selected_phrase_ranks": {"grey_fur": 8, "pose": 1, "happy": 5, "neck_tuft": 9, "raised_arm": 2, "blue_bottomwear": 2, "action_pose": 8, "playful": 1, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "muscle_tone": 6, "hand_gesture": 1, "hand_over_head": 5, "hand_on_own_head": 6, "dolphin_shorts": 7}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "dolphin_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7425}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5932}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "muscle_tone": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5438}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}}, "structural": [], "t1": 0.0, "t2": 0.84, "t3": 8.15, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 36, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.1389, "R": 0.3846, "F1": 0.2041, "leaf_P": 0.0606, "leaf_R": 0.3333, "leaf_F1": 0.1026, "n_leaf_sel": 33, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.625, "over_sel": 2.77, "why": {"explicit": 36}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 15, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 78, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.1429, "gen_R": 0.3846, "gen_F1": 0.2083, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "4_panel_comic", "breaking_the_fourth_wall", "chart", "confusion", "dark_skin", "dark_theme", "darkner", "darkness", "dim_lighting", "directed_motion_outline", "elemental_creature", "flesh_creature", "frown", "goat_demon", "goo_creature", "group", "holding_character", "light_bulb", "lighter", "lizard_tail", "lizardman", "mature_human", "nightmare", "sad", "scared", "shocked_face", "speech_bubble", "sweatdrop_(iconography)", "unnamed_character", "young_human"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "4_panel_comic", "breaking_the_fourth_wall", "chart", "confusion", "dark_skin", "dark_theme", "darkner", "darkness", "dim_lighting", "directed_motion_outline", "elemental_creature", "flesh_creature", "frown", "goat", "goat_demon", "goo_creature", "group", "holding_character", "human", "light_bulb", "lighter", "lizard", "lizard_tail", "lizardman", "mature_human", "nightmare", "reptile", "sad", "scalie", "scared", "shocked_face", "speech_bubble", "sweatdrop_(iconography)", "unnamed_character", "young_human"], "stage3_selected": ["3_panel_comic", "4_panel_comic", "breaking_the_fourth_wall", "chart", "confusion", "dark_skin", "dark_theme", "darkner", "darkness", "dim_lighting", "directed_motion_outline", "elemental_creature", "flesh_creature", "frown", "goat", "goat_demon", "goo_creature", "group", "holding_character", "human", "light_bulb", "lighter", "lizard", "lizard_tail", "lizardman", "mature_human", "nightmare", "reptile", "sad", "scalie", "scared", "shocked_face", "speech_bubble", "sweatdrop_(iconography)", "unnamed_character", "young_human"], "stage3_selected_scores": {"group": 0.4182, "scalie": 0.4916, "human": 0.6298, "reptile": 0.5077, "speech_bubble": 0.551, "lizard": 0.8035, "goat": 0.777, "elemental_creature": 0.4767, "frown": 0.4011, "dark_skin": 0.4239, "goo_creature": 0.4755, "scared": 0.4465, "sad": 0.4777, "confusion": 0.4487, "holding_character": 0.3838, "darkner": 0.5762, "goat_demon": 0.5311, "lizardman": 0.5549, "chart": 0.3778, "breaking_the_fourth_wall": 0.453, "darkness": 0.7908, "dark_theme": 0.5093, "lighter": 0.567, "light_bulb": 0.5412, "flesh_creature": 0.4794, "directed_motion_outline": 0.4318, "lizard_tail": 0.5169, "nightmare": 0.4738, "shocked_face": 0.4417, "mature_human": 0.4229, "dim_lighting": 0.5529, "unnamed_character": 0.4967, "3_panel_comic": 0.4885, "4_panel_comic": 0.4744, "sweatdrop_(iconography)": 0.4447, "young_human": 0.4202}, "stage3_selected_ranks": {"group": 110, "scalie": 53, "human": 6, "reptile": 46, "speech_bubble": 23, "lizard": 1, "goat": 3, "elemental_creature": 66, "frown": 119, "dark_skin": 105, "goo_creature": 69, "scared": 95, "sad": 64, "confusion": 90, "holding_character": 124, "darkner": 13, "goat_demon": 33, "lizardman": 20, "chart": 127, "breaking_the_fourth_wall": 84, "darkness": 2, "dark_theme": 43, "lighter": 18, "light_bulb": 25, "flesh_creature": 62, "directed_motion_outline": 103, "lizard_tail": 41, "nightmare": 72, "shocked_face": 98, "mature_human": 107, "dim_lighting": 22, "unnamed_character": 50, "3_panel_comic": 56, "4_panel_comic": 70, "sweatdrop_(iconography)": 96, "young_human": 109}, "stage3_selected_phrase_ranks": {"group": 5, "scalie": 10, "human": 1, "reptile": 9, "speech_bubble": 1, "lizard": 1, "goat": 1, "elemental_creature": 7, "frown": 8, "dark_skin": 7, "goo_creature": 5, "scared": 10, "sad": 6, "confusion": 4, "holding_character": 9, "darkner": 2, "goat_demon": 7, "lizardman": 4, "chart": 9, "breaking_the_fourth_wall": 2, "darkness": 1, "dark_theme": 4, "lighter": 6, "light_bulb": 9, "flesh_creature": 4, "directed_motion_outline": 9, "lizard_tail": 8, "nightmare": 8, "shocked_face": 5, "mature_human": 8, "dim_lighting": 8, "unnamed_character": 1, "3_panel_comic": 2, "4_panel_comic": 4, "sweatdrop_(iconography)": 8, "young_human": 9}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "4_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4744}, "breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.453}, "chart": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3778}, "confusion": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4487}, "dark_skin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4239}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5762}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "dim_lighting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5529}, "directed_motion_outline": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4318}, "elemental_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4767}, "flesh_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4794}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5311}, "goo_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4755}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "holding_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3838}, "light_bulb": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5412}, "lighter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.567}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5169}, "lizardman": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5549}, "mature_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4229}, "nightmare": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4738}, "sad": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4777}, "scared": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4465}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4417}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "sweatdrop_(iconography)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4447}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "young_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}}, "structural": [], "t1": 0.0, "t2": 1.67, "t3": 15.44, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.2692, "R": 0.5, "F1": 0.35, "leaf_P": 0.1429, "leaf_R": 0.25, "leaf_F1": 0.1818, "n_leaf_sel": 21, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 1.86, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 2, "kept_total": 72, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.2, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2692, "gen_R": 0.5, "gen_F1": 0.35, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "lying", "purple_hair", "text"], "extra": ["amber_eyes", "animated_comic", "bed_covers", "bed_sheet", "bedding", "bedroom", "blanket", "blue_hair", "bored_expression", "heterochromia", "lipstick_(object)", "lying_on_bed", "palette", "pillow", "purple_legs", "red_lipstick", "stylized_text", "under_blanket", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["amber_eyes", "animated_comic", "bed", "bed_covers", "bed_sheet", "bedding", "bedroom", "blanket", "blonde_hair", "blue_eyes", "blue_hair", "bored_expression", "green_eyes", "hair", "heterochromia", "lipstick_(object)", "lying_on_bed", "makeup", "palette", "pillow", "purple_legs", "red_lipstick", "sleeping", "stylized_text", "under_blanket", "zzz"], "stage3_selected": ["amber_eyes", "animated_comic", "bed", "bed_covers", "bed_sheet", "bedding", "bedroom", "blanket", "blonde_hair", "blue_eyes", "blue_hair", "bored_expression", "green_eyes", "hair", "heterochromia", "lipstick_(object)", "lying_on_bed", "makeup", "palette", "pillow", "purple_legs", "red_lipstick", "sleeping", "stylized_text", "under_blanket", "zzz"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "bed": 0.5296, "blue_hair": 0.3597, "makeup": 0.544, "pillow": 0.441, "bedroom": 0.4116, "sleeping": 0.4995, "bedding": 0.4246, "heterochromia": 0.3472, "amber_eyes": 0.3528, "bed_sheet": 0.4395, "blanket": 0.3523, "lying_on_bed": 0.412, "zzz": 0.367, "red_lipstick": 0.4186, "bed_covers": 0.4011, "bored_expression": 0.3613, "palette": 0.3097, "under_blanket": 0.3449, "purple_legs": 0.6369, "stylized_text": 0.3452, "lipstick_(object)": 0.363, "animated_comic": 0.3497}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "bed": 15, "blue_hair": 77, "makeup": 11, "pillow": 27, "bedroom": 39, "sleeping": 20, "bedding": 33, "heterochromia": 95, "amber_eyes": 85, "bed_sheet": 28, "blanket": 86, "lying_on_bed": 38, "zzz": 68, "red_lipstick": 34, "bed_covers": 44, "bored_expression": 76, "palette": 123, "under_blanket": 99, "purple_legs": 3, "stylized_text": 98, "lipstick_(object)": 75, "animated_comic": 90}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "bed": 1, "blue_hair": 5, "makeup": 1, "pillow": 3, "bedroom": 7, "sleeping": 2, "bedding": 5, "heterochromia": 7, "amber_eyes": 7, "bed_sheet": 4, "blanket": 10, "lying_on_bed": 6, "zzz": 8, "red_lipstick": 4, "bed_covers": 9, "bored_expression": 5, "palette": 4, "under_blanket": 10, "purple_legs": 1, "stylized_text": 5, "lipstick_(object)": 8, "animated_comic": 1}, "extra_evidence": {"amber_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3528}, "animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3497}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3597}, "bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3097}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "stylized_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3452}, "under_blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}}, "structural": [], "t1": 0.0, "t2": 1.43, "t3": 20.24, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 54, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"strong_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 8, \"why\": \"strong_implied\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 10, \"why\": \"strong_implied\"}, {\"i\": 12, \"why\": \"strong_implied\"}, {\"i\": 14, \"why\": \"style_or_meta\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 32}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1481, "R": 0.1818, "F1": 0.1633, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 23, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 1.23, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 14, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 62, "attempts_by_n_local": {"14": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 3, "parse_ok": 1, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1481, "gen_R": 0.1818, "gen_F1": 0.1633, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["action_pose", "big_claws", "black_claws", "bright_colors", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "heel_claw", "maned_wolf", "microphone_stand", "official_art", "orange_background", "pose", "rock_creature", "sharp_teeth", "smirking_at_viewer", "tail_ring", "torn_leggings", "torn_shorts", "warm_colors", "yellow_background", "yellow_sclera"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["action_pose", "big_claws", "black_claws", "bright_colors", "canine", "claws", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "heel_claw", "maned_wolf", "microphone_stand", "musical_instrument", "official_art", "orange_background", "pose", "rock_creature", "sharp_teeth", "smirking_at_viewer", "tail_ring", "torn_clothing", "torn_leggings", "torn_shorts", "warm_colors", "yellow_background", "yellow_sclera"], "stage3_selected": ["action_pose", "big_claws", "black_claws", "bright_colors", "canine", "claws", "color_swatch", "dire_wolf", "floating_head", "gold_claws", "heel_claw", "maned_wolf", "microphone_stand", "musical_instrument", "official_art", "orange_background", "pose", "rock_creature", "sharp_teeth", "smirking_at_viewer", "tail_ring", "torn_clothing", "torn_leggings", "torn_shorts", "warm_colors", "yellow_background", "yellow_sclera"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "pose": 0.8955, "sharp_teeth": 0.5486, "torn_clothing": 0.5179, "yellow_sclera": 0.606, "black_claws": 0.6961, "musical_instrument": 0.3362, "yellow_background": 0.6256, "official_art": 0.5154, "orange_background": 0.6518, "action_pose": 0.6726, "color_swatch": 0.7261, "tail_ring": 0.6036, "maned_wolf": 0.664, "warm_colors": 0.6979, "big_claws": 0.7461, "dire_wolf": 0.6438, "rock_creature": 0.5385, "smirking_at_viewer": 0.6312, "torn_shorts": 0.4967, "floating_head": 0.4571, "heel_claw": 0.5951, "torn_leggings": 0.4969, "gold_claws": 0.6234, "bright_colors": 0.6275, "microphone_stand": 0.4287}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "pose": 2, "sharp_teeth": 67, "torn_clothing": 73, "yellow_sclera": 48, "black_claws": 14, "musical_instrument": 117, "yellow_background": 42, "official_art": 74, "orange_background": 24, "action_pose": 19, "color_swatch": 9, "tail_ring": 52, "maned_wolf": 21, "warm_colors": 13, "big_claws": 7, "dire_wolf": 30, "rock_creature": 69, "smirking_at_viewer": 39, "torn_shorts": 80, "floating_head": 100, "heel_claw": 57, "torn_leggings": 79, "gold_claws": 43, "bright_colors": 41, "microphone_stand": 106}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "pose": 1, "sharp_teeth": 3, "torn_clothing": 4, "yellow_sclera": 5, "black_claws": 4, "musical_instrument": 9, "yellow_background": 10, "official_art": 3, "orange_background": 2, "action_pose": 2, "color_swatch": 1, "tail_ring": 7, "maned_wolf": 1, "warm_colors": 2, "big_claws": 1, "dire_wolf": 4, "rock_creature": 4, "smirking_at_viewer": 10, "torn_shorts": 7, "floating_head": 5, "heel_claw": 10, "torn_leggings": 6, "gold_claws": 3, "bright_colors": 6, "microphone_stand": 10}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6961}, "bright_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6275}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6438}, "floating_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4571}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "heel_claw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5951}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "microphone_stand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4287}, "official_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5154}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "rock_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5385}, "sharp_teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5486}, "smirking_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6312}, "tail_ring": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6036}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "warm_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6979}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6256}, "yellow_sclera": {"source": "stage3", "why": "explicit", "retrieval_score": 0.606}}, "structural": [], "t1": 0.0, "t2": 1.01, "t3": 32.46, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 40, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.38.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.38.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 38}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.38.why\n Field required [type=missing, input_value={'i': 38}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 9, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.5556, "R": 0.2174, "F1": 0.3125, "leaf_P": 0.5, "leaf_R": 0.2667, "leaf_F1": 0.3478, "n_leaf_sel": 8, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 0.39, "why": {"explicit": 10}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 12, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 58, "attempts_by_n_local": {"12": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.5556, "gen_R": 0.2174, "gen_F1": 0.3125, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing", "topwear"], "extra": ["black_topwear", "blue_pants", "grey_theme", "grey_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["black_topwear", "blue_pants", "claws", "facial_markings", "fur", "grey_theme", "grey_topwear", "pants", "shirt"], "stage3_selected": ["black_topwear", "blue_pants", "claws", "facial_markings", "fur", "grey_theme", "grey_topwear", "pants", "shirt", "simple_background"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "shirt": 0.7037, "pants": 0.7123, "facial_markings": 0.9015, "black_topwear": 0.5595, "blue_pants": 0.5806, "grey_topwear": 0.5774, "grey_theme": 0.5547}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "shirt": 13, "pants": 10, "facial_markings": 1, "black_topwear": 65, "blue_pants": 41, "grey_topwear": 45, "grey_theme": 69}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "shirt": 1, "pants": 1, "facial_markings": 1, "black_topwear": 10, "blue_pants": 7, "grey_topwear": 1, "grey_theme": 8}, "extra_evidence": {"black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "blue_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5806}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}}, "structural": [], "t1": 0.0, "t2": 0.9, "t3": 23.07, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 25, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 0.0, "over_sel": 2.45, "why": {"explicit": 29}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 88, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "colorful_background", "curved_eyebrows", "curved_tail", "eyes", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "jumping", "nose", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "smiling_at_viewer", "sparkling_eyes", "striped_legs", "stripes", "white_stripes", "wimple"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "colorful_background", "curved_eyebrows", "curved_tail", "eyes", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "jumping", "nose", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "smiling_at_viewer", "sparkling_eyes", "striped_legs", "stripes", "white_stripes", "wimple"], "stage3_selected": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "colorful_background", "curved_eyebrows", "curved_tail", "eyes", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "jumping", "nose", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "simple_background", "smiling_at_viewer", "sparkling_eyes", "striped_legs", "stripes", "transparent_background", "white_stripes", "wimple"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "transparent_background": 0.3162, "smiling_at_viewer": 0.451, "big_eyes": 0.4619, "action_pose": 0.4596, "jumping": 0.5295, "playful": 0.6118, "white_stripes": 0.6473, "striped_legs": 0.6654, "purple_stripes": 0.4704, "iconography": 0.3229, "paw_pose": 0.3304, "sparkling_eyes": 0.3343, "curved_eyebrows": 0.5269, "pink_stripes": 0.5235, "curved_tail": 0.5875, ">:d": 0.3455, "wimple": 0.3001, ">:)": 0.3449, "colorful_background": 0.3196, "irregular_speech_bubble": 0.3253, "hand_on_floor": 0.497, "jumper": 0.4127, "nose": 0.8806, "eyes": 0.97}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "stripes": 18, "pose": 19, "happy": 101, "transparent_background": 136, "smiling_at_viewer": 62, "big_eyes": 57, "action_pose": 58, "jumping": 35, "playful": 17, "white_stripes": 14, "striped_legs": 12, "purple_stripes": 56, "iconography": 130, "paw_pose": 122, "sparkling_eyes": 120, "curved_eyebrows": 36, "pink_stripes": 37, "curved_tail": 22, ">:d": 111, "wimple": 139, ">:)": 113, "colorful_background": 131, "irregular_speech_bubble": 127, "hand_on_floor": 47, "jumper": 81, "nose": 2, "eyes": 1}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "stripes": 10, "pose": 1, "happy": 3, "transparent_background": 10, "smiling_at_viewer": 8, "big_eyes": 4, "action_pose": 3, "jumping": 1, "playful": 1, "white_stripes": 9, "striped_legs": 8, "purple_stripes": 4, "iconography": 9, "paw_pose": 7, "sparkling_eyes": 6, "curved_eyebrows": 8, "pink_stripes": 1, "curved_tail": 3, ">:d": 8, "wimple": 10, ">:)": 9, "colorful_background": 9, "irregular_speech_bubble": 7, "hand_on_floor": 10, "jumper": 4, "nose": 1, "eyes": 1}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4619}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3196}, "curved_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5269}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "hand_on_floor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.497}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "iconography": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3229}, "irregular_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3253}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8806}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3304}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "sparkling_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3343}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6654}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6473}, "wimple": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3001}}, "structural": [], "t1": 0.0, "t2": 1.36, "t3": 13.98, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k1_cw0p7.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:20:33.612526", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 10}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 12, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 8, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 1.0, "why": {"explicit": 13}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 4, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 21, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["anthro_focus", "button_(fastener)", "domestic_cat", "necktie", "shirt", "tank_top", "translucent", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro_focus", "button_(fastener)", "domestic_cat", "necktie", "shirt", "tank_top", "translucent", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["anthro_focus", "button_(fastener)", "domestic_cat", "necktie", "shirt", "tank_top", "translucent", "transparent_background", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "domestic_cat": 0.5375, "translucent": 0.7484, "white_clothing": 0.6182, "tank_top": 0.7545, "necktie": 0.8082, "transparent_background": 0.5794, "white_topwear": 0.6638, "white_shirt": 0.6481, "anthro_focus": 0.4219, "button_(fastener)": 0.5695, "waiter": 0.6136, "white_t-shirt": 0.6584}, "stage3_selected_ranks": {"shirt": 1, "domestic_cat": 50, "translucent": 10, "white_clothing": 25, "tank_top": 8, "necktie": 3, "transparent_background": 39, "white_topwear": 13, "white_shirt": 16, "anthro_focus": 68, "button_(fastener)": 47, "waiter": 28, "white_t-shirt": 15}, "stage3_selected_phrase_ranks": {"shirt": 1, "domestic_cat": 10, "translucent": 1, "white_clothing": 7, "tank_top": 7, "necktie": 1, "transparent_background": 4, "white_topwear": 1, "white_shirt": 3, "anthro_focus": 9, "button_(fastener)": 8, "waiter": 6, "white_t-shirt": 2}, "extra_evidence": {"anthro_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4219}, "button_(fastener)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "domestic_cat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5375}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7545}, "translucent": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7484}, "waiter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6136}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6481}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.27, "t3": 8.52, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 21, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.0952, "R": 0.1429, "F1": 0.1143, "leaf_P": 0.0952, "leaf_R": 0.2, "leaf_F1": 0.129, "n_leaf_sel": 21, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 0.6667, "over_sel": 1.5, "why": {"explicit": 22}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 22, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 78, "attempts_by_n_local": {"22": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0952, "gen_R": 0.1429, "gen_F1": 0.1143, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["bent_arm", "bright", "character_name", "clenched_teeth", "extended_arms", "flat_colors", "grin", "larger_anthro", "larger_human", "magic_user", "night", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "skimpy", "smaller_anthro", "smug_grin", "toony_expression", "toothy_smile"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["bent_arm", "bright", "character_name", "clenched_teeth", "extended_arms", "flat_colors", "grin", "larger_anthro", "larger_human", "looking_at_viewer", "magic_user", "male", "night", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "skimpy", "smaller_anthro", "smug_grin", "toony_expression", "toothy_smile"], "stage3_selected": ["bent_arm", "bright", "character_name", "clenched_teeth", "extended_arms", "flat_colors", "grin", "larger_anthro", "larger_human", "looking_at_viewer", "magic_user", "male", "night", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "simple_background", "skimpy", "smaller_anthro", "smug_grin", "toony_expression", "toothy_smile"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "looking_at_viewer": 0.5612, "grin": 0.7174, "skimpy": 0.4928, "night": 0.5122, "clenched_teeth": 0.4797, "larger_anthro": 0.4912, "magic_user": 0.3882, "flat_colors": 0.3754, "smaller_anthro": 0.5391, "bent_arm": 0.6739, "raised_arms": 0.8636, "character_name": 0.4713, "larger_human": 0.2794, "extended_arms": 0.6595, "smug_grin": 0.4997, "partially_clothed_anthro": 0.4576, "bright": 0.5873, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "toony_expression": 0.4675}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "looking_at_viewer": 22, "grin": 3, "skimpy": 71, "night": 52, "clenched_teeth": 78, "larger_anthro": 72, "magic_user": 143, "flat_colors": 147, "smaller_anthro": 36, "bent_arm": 6, "raised_arms": 1, "character_name": 88, "larger_human": 179, "extended_arms": 8, "smug_grin": 64, "partially_clothed_anthro": 100, "bright": 16, "relaxed_expression": 48, "toothy_smile": 23, "toony_expression": 89}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "looking_at_viewer": 2, "grin": 1, "skimpy": 5, "night": 6, "clenched_teeth": 7, "larger_anthro": 3, "magic_user": 3, "flat_colors": 9, "smaller_anthro": 2, "bent_arm": 3, "raised_arms": 1, "character_name": 5, "larger_human": 9, "extended_arms": 6, "smug_grin": 5, "partially_clothed_anthro": 10, "bright": 1, "relaxed_expression": 1, "toothy_smile": 3, "toony_expression": 6}, "extra_evidence": {"bent_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6739}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "character_name": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4713}, "clenched_teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4797}, "extended_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6595}, "flat_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3754}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "larger_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4912}, "larger_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2794}, "magic_user": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3882}, "night": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5122}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "skimpy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "smaller_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5391}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}}, "structural": [], "t1": 0.0, "t2": 9.54, "t3": 15.39, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.43, "why": {"explicit": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 1, "kept_total": 24, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "facial_expressions", "intimate", "passionate", "setting", "smiling_at_another"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "facial_expressions", "intimate", "passionate", "setting", "smiling_at_another"], "stage3_selected": ["cheek_to_cheek", "facial_expressions", "intimate", "passionate", "setting", "smiling_at_another"], "stage3_selected_scores": {"intimate": 0.5697, "passionate": 0.5508, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "facial_expressions": 0.5874, "setting": 1.0}, "stage3_selected_ranks": {"intimate": 42, "passionate": 48, "smiling_at_another": 49, "cheek_to_cheek": 29, "facial_expressions": 33, "setting": 2}, "stage3_selected_phrase_ranks": {"intimate": 5, "passionate": 9, "smiling_at_another": 10, "cheek_to_cheek": 8, "facial_expressions": 8, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "facial_expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5874}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "passionate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}}, "structural": [], "t1": 0.0, "t2": 8.26, "t3": 12.74, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 5, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 5, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.25, "why": {"explicit": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 8, "bootstrap_anchor_selected": 5, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 21, "attempts_by_n_local": {"8": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["air_creature", "colorful_background", "round_eyes", "thick_eyelashes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["air_creature", "colorful_background", "round_eyes", "thick_eyelashes", "toony"], "stage3_selected": ["air_creature", "colorful_background", "round_eyes", "simple_background", "thick_eyelashes", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "toony": 0.763, "thick_eyelashes": 0.536, "air_creature": 0.6051, "round_eyes": 0.6227, "colorful_background": 0.7328}, "stage3_selected_ranks": {"simple_background": 10, "toony": 6, "thick_eyelashes": 69, "air_creature": 36, "round_eyes": 30, "colorful_background": 9}, "stage3_selected_phrase_ranks": {"simple_background": 5, "toony": 3, "thick_eyelashes": 10, "air_creature": 1, "round_eyes": 1, "colorful_background": 4}, "extra_evidence": {"air_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6051}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7328}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.536}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.23, "t3": 10.4, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 15, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1333, "R": 0.0909, "F1": 0.1081, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 12, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.68, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 10, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 41, "attempts_by_n_local": {"10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1333, "gen_R": 0.0909, "gen_F1": 0.1081, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_bottomwear", "blue_shorts", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_finger", "striped_markings", "tan_fur"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "tan_fur"], "stage3_selected": ["blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "tan_fur"], "stage3_selected_scores": {"fur": 0.7785, "pose": 0.847, "tan_fur": 0.6279, "shorts": 0.9202, "striped_markings": 0.5966, "blue_bottomwear": 0.6293, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "muscle_tone": 0.5438, "hand_gesture": 0.6065, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"fur": 8, "pose": 3, "tan_fur": 33, "shorts": 1, "striped_markings": 46, "blue_bottomwear": 32, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "muscle_tone": 62, "hand_gesture": 39, "hand_over_head": 56, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"fur": 1, "pose": 1, "tan_fur": 4, "shorts": 1, "striped_markings": 8, "blue_bottomwear": 2, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "muscle_tone": 6, "hand_gesture": 1, "hand_over_head": 5, "hand_on_own_head": 6}, "extra_evidence": {"blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "muscle_tone": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5438}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "tan_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6279}}, "structural": [], "t1": 0.0, "t2": 0.85, "t3": 14.98, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 16, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.1875, "R": 0.2308, "F1": 0.2069, "leaf_P": 0.1875, "leaf_R": 0.5, "leaf_F1": 0.2727, "n_leaf_sel": 16, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.375, "over_sel": 1.23, "why": {"explicit": 16}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 65, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.2, "gen_R": 0.2308, "gen_F1": 0.2143, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "4_panel_comic", "bright_light", "dark", "dark_theme", "darkness", "frown", "group", "light_beam", "motion_lines", "surrounded", "unnamed_character", "yelling"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "4_panel_comic", "bright_light", "dark", "dark_theme", "darkness", "frown", "goat", "group", "human", "light_beam", "lizard", "motion_lines", "surrounded", "unnamed_character", "yelling"], "stage3_selected": ["3_panel_comic", "4_panel_comic", "bright_light", "dark", "dark_theme", "darkness", "frown", "goat", "group", "human", "light_beam", "lizard", "motion_lines", "surrounded", "unnamed_character", "yelling"], "stage3_selected_scores": {"group": 0.4182, "human": 0.6298, "lizard": 0.8035, "motion_lines": 0.4524, "goat": 0.777, "frown": 0.4011, "yelling": 0.4996, "dark": 0.5246, "light_beam": 0.5797, "darkness": 0.7908, "dark_theme": 0.5093, "surrounded": 0.5229, "unnamed_character": 0.4967, "3_panel_comic": 0.4885, "4_panel_comic": 0.4744, "bright_light": 0.5857}, "stage3_selected_ranks": {"group": 110, "human": 6, "lizard": 1, "motion_lines": 86, "goat": 3, "frown": 119, "yelling": 49, "dark": 37, "light_beam": 12, "darkness": 2, "dark_theme": 43, "surrounded": 38, "unnamed_character": 50, "3_panel_comic": 56, "4_panel_comic": 70, "bright_light": 10}, "stage3_selected_phrase_ranks": {"group": 5, "human": 1, "lizard": 1, "motion_lines": 5, "goat": 1, "frown": 8, "yelling": 1, "dark": 3, "light_beam": 2, "darkness": 1, "dark_theme": 4, "surrounded": 3, "unnamed_character": 1, "3_panel_comic": 2, "4_panel_comic": 4, "bright_light": 1}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "4_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4744}, "bright_light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5857}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5246}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light_beam": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5797}, "motion_lines": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4524}, "surrounded": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5229}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "yelling": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4996}}, "structural": [], "t1": 0.0, "t2": 1.55, "t3": 11.69, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 40, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.175, "R": 0.5, "F1": 0.2593, "leaf_P": 0.1429, "leaf_R": 0.4167, "leaf_F1": 0.2128, "n_leaf_sel": 35, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 2.86, "why": {"explicit": 40}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 11, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 79, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.175, "gen_R": 0.5, "gen_F1": 0.2593, "missed": ["angry", "bed", "duo", "eyes_closed", "lying", "purple_hair", "text"], "extra": ["animated_comic", "annoyed_expression", "bed_sheet", "bedding", "bedroom", "blue_lipstick", "creating_art", "digital_drawing_(artwork)", "gem", "glistening_eyes", "green_eyebrows", "hair_dye", "hair_sticks", "half-length_portrait", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "modelling", "no_irises", "on_bed", "pink_lipstick", "portuguese_text", "purple_hands", "purple_legs", "purple_pupils", "red_lipstick", "relaxed_expression", "romantic_ambiance", "sleepover", "sparkling_character", "stylized_text", "sweatdrop_(iconography)"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["animated_comic", "annoyed_expression", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_eyes", "blue_lipstick", "creating_art", "digital_drawing_(artwork)", "eyeshadow", "gem", "glistening_eyes", "green_eyebrows", "green_eyes", "hair", "hair_dye", "hair_sticks", "half-length_portrait", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "modelling", "no_irises", "on_bed", "pink_lipstick", "portuguese_text", "purple_hands", "purple_legs", "purple_pupils", "red_lipstick", "relaxed_expression", "romantic_ambiance", "sleeping", "sleepover", "sparkling_character", "stylized_text", "sweatdrop_(iconography)"], "stage3_selected": ["animated_comic", "annoyed_expression", "bed_sheet", "bedding", "bedroom", "blonde_hair", "blue_eyes", "blue_lipstick", "creating_art", "digital_drawing_(artwork)", "eyeshadow", "gem", "glistening_eyes", "green_eyebrows", "green_eyes", "hair", "hair_dye", "hair_sticks", "half-length_portrait", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "modelling", "no_irises", "on_bed", "pink_lipstick", "portuguese_text", "purple_hands", "purple_legs", "purple_pupils", "red_lipstick", "relaxed_expression", "romantic_ambiance", "sleeping", "sleepover", "sparkling_character", "stylized_text", "sweatdrop_(iconography)"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "digital_drawing_(artwork)": 0.3798, "makeup": 0.544, "on_bed": 0.4468, "eyeshadow": 0.4274, "lipstick": 0.4386, "bedroom": 0.4116, "sleeping": 0.4995, "bedding": 0.4246, "glistening_eyes": 0.3639, "gem": 0.2689, "half-length_portrait": 0.352, "bed_sheet": 0.4395, "romantic_ambiance": 0.3904, "lidded_eyes": 0.3759, "lying_on_bed": 0.412, "red_lipstick": 0.4186, "pink_lipstick": 0.4125, "blue_lipstick": 0.3955, "annoyed_expression": 0.3458, "creating_art": 0.3914, "no_irises": 0.3837, "purple_pupils": 0.3814, "green_eyebrows": 0.4683, "hair_sticks": 0.5043, "sweatdrop_(iconography)": 0.3415, "sleepover": 0.5015, "sparkling_character": 0.3657, "purple_hands": 0.6346, "purple_legs": 0.6369, "stylized_text": 0.3452, "lipstick_(object)": 0.363, "modelling": 0.2674, "relaxed_expression": 0.388, "hair_dye": 0.3543, "animated_comic": 0.3497, "portuguese_text": 0.3865}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "digital_drawing_(artwork)": 58, "makeup": 11, "on_bed": 26, "eyeshadow": 32, "lipstick": 29, "bedroom": 39, "sleeping": 20, "bedding": 33, "glistening_eyes": 74, "gem": 146, "half-length_portrait": 87, "bed_sheet": 28, "romantic_ambiance": 52, "lidded_eyes": 63, "lying_on_bed": 38, "red_lipstick": 34, "pink_lipstick": 37, "blue_lipstick": 50, "annoyed_expression": 97, "creating_art": 51, "no_irises": 55, "purple_pupils": 57, "green_eyebrows": 23, "hair_sticks": 18, "sweatdrop_(iconography)": 104, "sleepover": 19, "sparkling_character": 72, "purple_hands": 5, "purple_legs": 3, "stylized_text": 98, "lipstick_(object)": 75, "modelling": 148, "relaxed_expression": 53, "hair_dye": 84, "animated_comic": 90, "portuguese_text": 54}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "digital_drawing_(artwork)": 4, "makeup": 1, "on_bed": 2, "eyeshadow": 3, "lipstick": 2, "bedroom": 7, "sleeping": 2, "bedding": 5, "glistening_eyes": 4, "gem": 6, "half-length_portrait": 8, "bed_sheet": 4, "romantic_ambiance": 1, "lidded_eyes": 6, "lying_on_bed": 6, "red_lipstick": 4, "pink_lipstick": 5, "blue_lipstick": 10, "annoyed_expression": 9, "creating_art": 1, "no_irises": 2, "purple_pupils": 6, "green_eyebrows": 2, "hair_sticks": 2, "sweatdrop_(iconography)": 8, "sleepover": 1, "sparkling_character": 2, "purple_hands": 1, "purple_legs": 1, "stylized_text": 5, "lipstick_(object)": 8, "modelling": 8, "relaxed_expression": 2, "hair_dye": 6, "animated_comic": 1, "portuguese_text": 1}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3497}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blue_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3955}, "creating_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3914}, "digital_drawing_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3798}, "gem": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2689}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3639}, "green_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4683}, "hair_dye": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3543}, "hair_sticks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5043}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.352}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3759}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "modelling": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2674}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4468}, "pink_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4125}, "portuguese_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3865}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6346}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "purple_pupils": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3814}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "romantic_ambiance": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3904}, "sleepover": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3657}, "stylized_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3452}, "sweatdrop_(iconography)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3415}}, "structural": [], "t1": 0.0, "t2": 1.4, "t3": 11.1, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 19, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.2105, "R": 0.1818, "F1": 0.1951, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 16, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 0.86, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 14, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 48, "attempts_by_n_local": {"14": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2105, "gen_R": 0.1818, "gen_F1": 0.1951, "missed": ["anthro", "bass_guitar", "canine", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["action_pose", "claws_out", "digitigrade", "floating_head", "gold_claws", "model_sheet", "orange_background", "pinup", "pose", "toe_claws", "toony", "torn_leggings", "torn_shorts", "torn_stockings", "unknown_species"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["action_pose", "canid", "claws", "claws_out", "digitigrade", "floating_head", "gold_claws", "model_sheet", "musical_instrument", "orange_background", "pinup", "pose", "toe_claws", "toony", "torn_clothing", "torn_leggings", "torn_shorts", "torn_stockings", "unknown_species"], "stage3_selected": ["action_pose", "canid", "claws", "claws_out", "digitigrade", "floating_head", "gold_claws", "model_sheet", "musical_instrument", "orange_background", "pinup", "pose", "toe_claws", "toony", "torn_clothing", "torn_leggings", "torn_shorts", "torn_stockings", "unknown_species"], "stage3_selected_scores": {"canid": 0.6217, "claws": 0.898, "pose": 0.8955, "toe_claws": 0.8175, "digitigrade": 0.7162, "torn_clothing": 0.5179, "pinup": 0.7901, "toony": 0.4807, "model_sheet": 0.6636, "unknown_species": 0.7852, "musical_instrument": 0.3362, "orange_background": 0.6518, "action_pose": 0.6726, "claws_out": 0.7061, "torn_stockings": 0.4869, "torn_shorts": 0.4967, "floating_head": 0.4571, "torn_leggings": 0.4969, "gold_claws": 0.6234}, "stage3_selected_ranks": {"canid": 44, "claws": 1, "pose": 2, "toe_claws": 3, "digitigrade": 10, "torn_clothing": 73, "pinup": 4, "toony": 90, "model_sheet": 22, "unknown_species": 5, "musical_instrument": 117, "orange_background": 24, "action_pose": 19, "claws_out": 12, "torn_stockings": 84, "torn_shorts": 80, "floating_head": 100, "torn_leggings": 79, "gold_claws": 43}, "stage3_selected_phrase_ranks": {"canid": 5, "claws": 1, "pose": 1, "toe_claws": 2, "digitigrade": 5, "torn_clothing": 4, "pinup": 2, "toony": 5, "model_sheet": 1, "unknown_species": 1, "musical_instrument": 9, "orange_background": 2, "action_pose": 2, "claws_out": 6, "torn_stockings": 8, "torn_shorts": 7, "floating_head": 5, "torn_leggings": 6, "gold_claws": 3}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7061}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "floating_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4571}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "model_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6636}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "pinup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7901}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "toe_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8175}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "torn_stockings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4869}, "unknown_species": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7852}}, "structural": [], "t1": 0.0, "t2": 1.09, "t3": 13.4, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 19, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.3158, "R": 0.2609, "F1": 0.2857, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 12, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.75, "over_sel": 0.83, "why": {"explicit": 20}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 12, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 47, "attempts_by_n_local": {"12": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3158, "gen_R": 0.2609, "gen_F1": 0.2857, "missed": ["anthro", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing", "topwear"], "extra": ["5_claws", "black_bottomwear", "black_topwear", "blue_pants", "cheek_markings", "dress_shirt", "eye_markings", "grey_shirt", "grey_topwear", "grey_tuft", "tan_fur", "white_dress_shirt", "yellow_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["5_claws", "black_bottomwear", "black_topwear", "blue_pants", "bottomwear", "cheek_markings", "claws", "dress_shirt", "eye_markings", "facial_markings", "fur", "grey_shirt", "grey_topwear", "grey_tuft", "pants", "shirt", "tan_fur", "white_dress_shirt", "yellow_topwear"], "stage3_selected": ["5_claws", "black_bottomwear", "black_topwear", "blue_pants", "bottomwear", "cheek_markings", "claws", "dress_shirt", "eye_markings", "facial_markings", "fur", "grey_shirt", "grey_topwear", "grey_tuft", "pants", "shirt", "simple_background", "tan_fur", "white_dress_shirt", "yellow_topwear"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "bottomwear": 0.5813, "shirt": 0.7037, "pants": 0.7123, "tan_fur": 0.5593, "facial_markings": 0.9015, "black_topwear": 0.5595, "black_bottomwear": 0.5638, "dress_shirt": 0.6075, "blue_pants": 0.5806, "grey_topwear": 0.5774, "eye_markings": 0.743, "yellow_topwear": 0.5617, "grey_shirt": 0.6272, "cheek_markings": 0.719, "white_dress_shirt": 0.5611, "grey_tuft": 0.5545, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "bottomwear": 40, "shirt": 13, "pants": 10, "tan_fur": 66, "facial_markings": 1, "black_topwear": 65, "black_bottomwear": 57, "dress_shirt": 31, "blue_pants": 41, "grey_topwear": 45, "eye_markings": 5, "yellow_topwear": 61, "grey_shirt": 25, "cheek_markings": 7, "white_dress_shirt": 62, "grey_tuft": 70, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "bottomwear": 6, "shirt": 1, "pants": 1, "tan_fur": 5, "facial_markings": 1, "black_topwear": 10, "black_bottomwear": 6, "dress_shirt": 9, "blue_pants": 7, "grey_topwear": 1, "eye_markings": 4, "yellow_topwear": 8, "grey_shirt": 7, "cheek_markings": 5, "white_dress_shirt": 3, "grey_tuft": 9, "5_claws": 1}, "extra_evidence": {"5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5638}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "blue_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5806}, "cheek_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.719}, "dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6075}, "eye_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.743}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "grey_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5545}, "tan_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5593}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "yellow_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5617}}, "structural": [], "t1": 0.0, "t2": 0.95, "t3": 7.55, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 22, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 21, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 0.0, "over_sel": 2.0, "why": {"explicit": 23}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 18, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 111, "attempts_by_n_local": {"18": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "blue_pawpads", "curved_tail", "encouragement", "eyes", "happy", "jumper", "jumping", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "blue_pawpads", "curved_tail", "encouragement", "eyes", "happy", "jumper", "jumping", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes"], "stage3_selected": [">:)", ">:d", "action_pose", "big_eyes", "blue_fur", "blue_pawpads", "curved_tail", "encouragement", "eyes", "happy", "jumper", "jumping", "midair", "multiple_poses", "nose", "paw_pose", "pink_stripes", "playful", "playing", "pose", "posed", "purple_stripes", "transparent_background"], "stage3_selected_scores": {"blue_fur": 0.4171, "pose": 0.6068, "happy": 0.36, "transparent_background": 0.3162, "big_eyes": 0.4619, "multiple_poses": 0.4969, "blue_pawpads": 0.4264, "action_pose": 0.4596, "jumping": 0.5295, "playful": 0.6118, "playing": 0.3818, "purple_stripes": 0.4704, "midair": 0.43, "paw_pose": 0.3304, "pink_stripes": 0.5235, "encouragement": 0.2536, "curved_tail": 0.5875, ">:d": 0.3455, ">:)": 0.3449, "posed": 0.4337, "jumper": 0.4127, "nose": 0.8806, "eyes": 0.97}, "stage3_selected_ranks": {"blue_fur": 77, "pose": 19, "happy": 101, "transparent_background": 136, "big_eyes": 57, "multiple_poses": 48, "blue_pawpads": 73, "action_pose": 58, "jumping": 35, "playful": 17, "playing": 91, "purple_stripes": 56, "midair": 71, "paw_pose": 122, "pink_stripes": 37, "encouragement": 155, "curved_tail": 22, ">:d": 111, ">:)": 113, "posed": 69, "jumper": 81, "nose": 2, "eyes": 1}, "stage3_selected_phrase_ranks": {"blue_fur": 9, "pose": 1, "happy": 3, "transparent_background": 10, "big_eyes": 4, "multiple_poses": 5, "blue_pawpads": 7, "action_pose": 3, "jumping": 1, "playful": 1, "playing": 2, "purple_stripes": 4, "midair": 2, "paw_pose": 7, "pink_stripes": 1, "encouragement": 4, "curved_tail": 3, ">:d": 8, ">:)": 9, "posed": 10, "jumper": 4, "nose": 1, "eyes": 1}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4619}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "encouragement": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2536}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8806}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3304}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3818}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}}, "structural": [], "t1": 0.0, "t2": 1.52, "t3": 33.46, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p3.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:21:37.143171", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 16}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 11, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 1.17, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 35, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["business_suit", "button_(fastener)", "depth_of_field", "face_focus", "focused", "necktie", "shirt", "sparkling_character", "topwear", "translucent_hair", "trio_focus", "white_dress_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["business_suit", "button_(fastener)", "depth_of_field", "face_focus", "focused", "necktie", "shirt", "sparkling_character", "topwear", "translucent_hair", "trio_focus", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["business_suit", "button_(fastener)", "depth_of_field", "face_focus", "focused", "necktie", "shirt", "sparkling_character", "topwear", "translucent_hair", "transparent_background", "trio_focus", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"topwear": 0.7782, "shirt": 0.9019, "necktie": 0.8082, "transparent_background": 0.5794, "translucent_hair": 0.6309, "white_topwear": 0.6638, "button_(fastener)": 0.5695, "depth_of_field": 0.5407, "business_suit": 0.6178, "trio_focus": 0.533, "white_t-shirt": 0.6584, "white_dress_shirt": 0.6255, "focused": 0.6608, "face_focus": 0.4971, "sparkling_character": 0.5249}, "stage3_selected_ranks": {"topwear": 4, "shirt": 1, "necktie": 3, "transparent_background": 39, "translucent_hair": 19, "white_topwear": 13, "button_(fastener)": 47, "depth_of_field": 49, "business_suit": 26, "trio_focus": 51, "white_t-shirt": 15, "white_dress_shirt": 22, "focused": 14, "face_focus": 56, "sparkling_character": 52}, "stage3_selected_phrase_ranks": {"topwear": 3, "shirt": 1, "necktie": 1, "transparent_background": 4, "translucent_hair": 2, "white_topwear": 1, "button_(fastener)": 8, "depth_of_field": 2, "business_suit": 4, "trio_focus": 3, "white_t-shirt": 2, "white_dress_shirt": 5, "focused": 1, "face_focus": 6, "sparkling_character": 1}, "extra_evidence": {"business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "button_(fastener)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "depth_of_field": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5249}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7782}, "translucent_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6309}, "trio_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.533}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6255}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.14, "t3": 12.79, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.2143, "R": 0.2143, "F1": 0.2143, "leaf_P": 0.1538, "leaf_R": 0.2, "leaf_F1": 0.1739, "n_leaf_sel": 13, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 1.0, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 43, "bootstrap_anchor_selected": 16, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 71, "attempts_by_n_local": {"43": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2143, "gen_R": 0.2143, "gen_F1": 0.2143, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "bright", "expressions", "extended_arms", "grin", "larger_male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "toony_expression", "toothy_grin"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "bright", "expressions", "extended_arms", "grin", "group", "larger_male", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "toony_expression", "toothy_grin"], "stage3_selected": ["animated_comic", "bright", "expressions", "extended_arms", "grin", "group", "larger_male", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arms", "relaxed_expression", "simple_background", "toony_expression", "toothy_grin"], "stage3_selected_scores": {"male": 0.6605, "simple_background": 0.5361, "looking_at_viewer": 0.5612, "group": 0.4099, "larger_male": 0.4954, "grin": 0.7174, "raised_arms": 0.8636, "extended_arms": 0.6595, "toothy_grin": 0.6248, "partially_clothed_anthro": 0.4576, "expressions": 0.4746, "bright": 0.5873, "relaxed_expression": 0.5175, "toony_expression": 0.4675, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "simple_background": 38, "looking_at_viewer": 22, "group": 132, "larger_male": 68, "grin": 3, "raised_arms": 1, "extended_arms": 8, "toothy_grin": 13, "partially_clothed_anthro": 100, "expressions": 86, "bright": 16, "relaxed_expression": 48, "toony_expression": 89, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "simple_background": 3, "looking_at_viewer": 2, "group": 6, "larger_male": 4, "grin": 1, "raised_arms": 1, "extended_arms": 6, "toothy_grin": 2, "partially_clothed_anthro": 10, "expressions": 5, "bright": 1, "relaxed_expression": 1, "toony_expression": 6, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4746}, "extended_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6595}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}}, "structural": [], "t1": 0.0, "t2": 9.54, "t3": 20.7, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 9, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 9, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.64, "why": {"explicit": 9}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 28, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "embrace", "expressions", "fantasy", "intimate", "romeo_(leobo)", "setting", "shocked_face", "smiling_at_another"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "embrace", "expressions", "fantasy", "intimate", "romeo_(leobo)", "setting", "shocked_face", "smiling_at_another"], "stage3_selected": ["cheek_to_cheek", "embrace", "expressions", "fantasy", "intimate", "romeo_(leobo)", "setting", "shocked_face", "smiling_at_another"], "stage3_selected_scores": {"embrace": 0.5734, "fantasy": 0.5458, "intimate": 0.5697, "expressions": 0.8783, "shocked_face": 0.616, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "romeo_(leobo)": 0.5212, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "fantasy": 52, "intimate": 42, "expressions": 3, "shocked_face": 18, "smiling_at_another": 49, "cheek_to_cheek": 29, "romeo_(leobo)": 58, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "fantasy": 9, "intimate": 5, "expressions": 2, "shocked_face": 1, "smiling_at_another": 10, "cheek_to_cheek": 8, "romeo_(leobo)": 2, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "fantasy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5458}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "romeo_(leobo)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5212}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}}, "structural": [], "t1": 0.0, "t2": 8.19, "t3": 9.42, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 5, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 5, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.25, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 6, "bootstrap_anchor_kept": 1, "bootstrap_reranked": true, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 20, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["background_shot", "blowup_background", "creature_print", "round_eyes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["background_shot", "blowup_background", "creature_print", "round_eyes", "toony"], "stage3_selected": ["background_shot", "blowup_background", "creature_print", "round_eyes", "simple_background", "toony", "transparent_background"], "stage3_selected_scores": {"simple_background": 0.7167, "transparent_background": 0.7422, "toony": 0.763, "creature_print": 0.5405, "blowup_background": 0.7653, "round_eyes": 0.6227, "background_shot": 0.7129}, "stage3_selected_ranks": {"simple_background": 10, "transparent_background": 8, "toony": 6, "creature_print": 68, "blowup_background": 4, "round_eyes": 30, "background_shot": 11}, "stage3_selected_phrase_ranks": {"simple_background": 5, "transparent_background": 3, "toony": 3, "creature_print": 9, "blowup_background": 1, "round_eyes": 1, "background_shot": 6}, "extra_evidence": {"background_shot": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7129}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7653}, "creature_print": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5405}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.17, "t3": 10.41, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 20, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1, "R": 0.0909, "F1": 0.0952, "leaf_P": 0.0667, "leaf_R": 0.0833, "leaf_F1": 0.0741, "n_leaf_sel": 15, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.91, "why": {"explicit": 20}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 41, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1, "gen_R": 0.0909, "gen_F1": 0.0952, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["action_pose", "black_nose", "blue_bottomwear", "blue_shorts", "gym_bottomwear", "hand_gesture", "hand_holding", "hand_on_own_head", "hand_over_head", "playful", "playing", "pose", "raised_finger", "striped_markings", "striped_shirt", "striped_tail", "striped_topwear", "white_shorts"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "black_nose", "blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_holding", "hand_on_own_head", "hand_over_head", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "striped_shirt", "striped_tail", "striped_topwear", "white_shorts"], "stage3_selected": ["action_pose", "black_nose", "blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_holding", "hand_on_own_head", "hand_over_head", "playful", "playing", "pose", "raised_finger", "shorts", "striped_markings", "striped_shirt", "striped_tail", "striped_topwear", "white_shorts"], "stage3_selected_scores": {"fur": 0.7785, "black_nose": 0.6008, "pose": 0.847, "shorts": 0.9202, "striped_markings": 0.5966, "striped_tail": 0.59, "hand_holding": 0.5161, "blue_bottomwear": 0.6293, "action_pose": 0.6359, "striped_topwear": 0.5988, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "striped_shirt": 0.6015, "raised_finger": 0.595, "hand_gesture": 0.6065, "white_shorts": 0.621, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"fur": 8, "black_nose": 41, "pose": 3, "shorts": 1, "striped_markings": 46, "striped_tail": 50, "hand_holding": 72, "blue_bottomwear": 32, "action_pose": 31, "striped_topwear": 43, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "striped_shirt": 40, "raised_finger": 47, "hand_gesture": 39, "white_shorts": 35, "hand_over_head": 56, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"fur": 1, "black_nose": 6, "pose": 1, "shorts": 1, "striped_markings": 8, "striped_tail": 10, "hand_holding": 9, "blue_bottomwear": 2, "action_pose": 8, "striped_topwear": 6, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "striped_shirt": 5, "raised_finger": 3, "hand_gesture": 1, "white_shorts": 4, "hand_over_head": 5, "hand_on_own_head": 6}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "black_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6008}, "blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_holding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5161}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "striped_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6015}, "striped_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.59}, "striped_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5988}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.621}}, "structural": [], "t1": 0.0, "t2": 0.78, "t3": 9.98, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.0, "over_sel": 0.46, "why": {"explicit": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 38, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 2, "calls_exhausted_retries": 1, "attempts_total": 5, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 57, "attempts_by_n_local": {"38": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 1, "parse_fail": 0, "errors": 3}}, "attempt_failure_rate": 0.6, "call_exhaustion_rate": 0.3333333333333333}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["dark", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["dark", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "stage3_selected": ["dark", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "speech_bubble": 0.551, "light": 0.5737, "dark": 0.5246, "darkness": 0.7908, "unnamed_character": 0.4967}, "stage3_selected_ranks": {"group": 110, "speech_bubble": 23, "light": 15, "dark": 37, "darkness": 2, "unnamed_character": 50}, "stage3_selected_phrase_ranks": {"group": 5, "speech_bubble": 1, "light": 4, "dark": 3, "darkness": 1, "unnamed_character": 1}, "extra_evidence": {"dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5246}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.51, "t3": 32.49, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 34, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 57}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 57}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"style_or_meta\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 55}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 55}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"style_or_meta\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 55}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 55}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.2308, "R": 0.4286, "F1": 0.3, "leaf_P": 0.1739, "leaf_R": 0.3333, "leaf_F1": 0.2286, "n_leaf_sel": 23, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.75, "over_sel": 1.86, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 18, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 80, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.3333333333333333, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2308, "gen_R": 0.4286, "gen_F1": 0.3, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "hair", "lying", "purple_hair", "text"], "extra": ["2_frame_animation", "animated_png", "annoyed_expression", "bed_covers", "bedding", "blanket", "blue_background", "blue_eyebrows", "clown_makeup", "lipstick_(object)", "lying_on_bed", "on_pillow", "purple_background", "purple_face", "purple_text", "red_lipstick", "relaxed_expression", "sleeping_together", "sleepover", "two_tone_hair"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["2_frame_animation", "animated_png", "annoyed_expression", "bed", "bed_covers", "bedding", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "clown_makeup", "green_eyes", "lipstick_(object)", "lying_on_bed", "makeup", "on_pillow", "purple_background", "purple_face", "purple_text", "red_lipstick", "relaxed_expression", "sleeping", "sleeping_together", "sleepover", "two_tone_hair"], "stage3_selected": ["2_frame_animation", "animated_png", "annoyed_expression", "bed", "bed_covers", "bedding", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "clown_makeup", "green_eyes", "lipstick_(object)", "lying_on_bed", "makeup", "on_pillow", "purple_background", "purple_face", "purple_text", "red_lipstick", "relaxed_expression", "sleeping", "sleeping_together", "sleepover", "two_tone_hair"], "stage3_selected_scores": {"blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "two_tone_hair": 0.3447, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "purple_background": 0.3682, "blanket": 0.3523, "lying_on_bed": 0.412, "red_lipstick": 0.4186, "blue_eyebrows": 0.4608, "annoyed_expression": 0.3458, "bed_covers": 0.4011, "purple_text": 0.5604, "purple_face": 0.5961, "on_pillow": 0.4104, "sleeping_together": 0.408, "sleepover": 0.5015, "clown_makeup": 0.4004, "lipstick_(object)": 0.363, "relaxed_expression": 0.388, "animated_png": 0.2978, "2_frame_animation": 0.3038}, "stage3_selected_ranks": {"blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "bed": 15, "makeup": 11, "two_tone_hair": 100, "blue_background": 105, "sleeping": 20, "bedding": 33, "purple_background": 67, "blanket": 86, "lying_on_bed": 38, "red_lipstick": 34, "blue_eyebrows": 24, "annoyed_expression": 97, "bed_covers": 44, "purple_text": 10, "purple_face": 8, "on_pillow": 40, "sleeping_together": 41, "sleepover": 19, "clown_makeup": 45, "lipstick_(object)": 75, "relaxed_expression": 53, "animated_png": 130, "2_frame_animation": 127}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "bed": 1, "makeup": 1, "two_tone_hair": 10, "blue_background": 10, "sleeping": 2, "bedding": 5, "purple_background": 5, "blanket": 10, "lying_on_bed": 6, "red_lipstick": 4, "blue_eyebrows": 2, "annoyed_expression": 9, "bed_covers": 9, "purple_text": 3, "purple_face": 2, "on_pillow": 8, "sleeping_together": 1, "sleepover": 1, "clown_makeup": 6, "lipstick_(object)": 8, "relaxed_expression": 2, "animated_png": 3, "2_frame_animation": 2}, "extra_evidence": {"2_frame_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3038}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2978}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "blue_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4608}, "clown_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "on_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4104}, "purple_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3682}, "purple_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5961}, "purple_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5604}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "sleepover": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "two_tone_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3447}}, "structural": [], "t1": 0.0, "t2": 1.32, "t3": 37.11, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 16, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 53, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 17, \"why\": \"strong_implied\"}, {\"i\": 18, \"why\": \"strong_implied\"}, {\"i\": 20, \"why\": \"strong_implied\"}, {\"i\": 22, \"why\": \"strong_implied\"}, {\"i\": 28, \"why\": \"strong_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"style_or_meta\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 41, \"why\": \"other\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 44, \"why\": \"other\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 49, \"why\": \"other\"}, {\"i\": 51, \"why\": \"other\"}, {\"i\": 52}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 52}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 16, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"style_or_meta\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 24, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1667, "R": 0.1818, "F1": 0.1739, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 20, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 1.09, "why": {"explicit": 24}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 28, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 11, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 53, "attempts_by_n_local": {"28": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 0.1818, "gen_F1": 0.1739, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["4_toes", "action_pose", "color_swatch", "dewclaw", "digitigrade", "glowing_claws", "maned_wolf", "multiple_images", "multiple_poses", "orange_background", "pose", "reaction_image", "signature", "t-pose", "toony", "toony_expression", "torn_leggings", "torn_topwear", "wolf", "yellow_sclera"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_toes", "action_pose", "canine", "claws", "color_swatch", "dewclaw", "digitigrade", "glowing_claws", "maned_wolf", "multiple_images", "multiple_poses", "musical_instrument", "orange_background", "pose", "reaction_image", "signature", "t-pose", "toony", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "wolf", "yellow_sclera"], "stage3_selected": ["4_toes", "action_pose", "canine", "claws", "color_swatch", "dewclaw", "digitigrade", "glowing_claws", "maned_wolf", "multiple_images", "multiple_poses", "musical_instrument", "orange_background", "pose", "reaction_image", "signature", "t-pose", "toony", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "wolf", "yellow_sclera"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "signature": 0.6489, "4_toes": 0.695, "pose": 0.8955, "digitigrade": 0.7162, "torn_clothing": 0.5179, "yellow_sclera": 0.606, "toony": 0.4807, "multiple_images": 0.5786, "musical_instrument": 0.3362, "multiple_poses": 0.7533, "orange_background": 0.6518, "reaction_image": 0.4564, "action_pose": 0.6726, "color_swatch": 0.7261, "maned_wolf": 0.664, "torn_topwear": 0.4853, "dewclaw": 0.6939, "glowing_claws": 0.5992, "t-pose": 0.6738, "torn_leggings": 0.4969, "toony_expression": 0.4641}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "wolf": 23, "signature": 26, "4_toes": 16, "pose": 2, "digitigrade": 10, "torn_clothing": 73, "yellow_sclera": 48, "toony": 90, "multiple_images": 60, "musical_instrument": 117, "multiple_poses": 6, "orange_background": 24, "reaction_image": 102, "action_pose": 19, "color_swatch": 9, "maned_wolf": 21, "torn_topwear": 88, "dewclaw": 17, "glowing_claws": 56, "t-pose": 18, "torn_leggings": 79, "toony_expression": 98}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "wolf": 2, "signature": 3, "4_toes": 9, "pose": 1, "digitigrade": 5, "torn_clothing": 4, "yellow_sclera": 5, "toony": 5, "multiple_images": 1, "musical_instrument": 9, "multiple_poses": 3, "orange_background": 2, "reaction_image": 6, "action_pose": 2, "color_swatch": 1, "maned_wolf": 1, "torn_topwear": 10, "dewclaw": 10, "glowing_claws": 9, "t-pose": 4, "torn_leggings": 6, "toony_expression": 4}, "extra_evidence": {"4_toes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.695}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "dewclaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6939}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "glowing_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "multiple_images": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5786}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7533}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "reaction_image": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4564}, "signature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6489}, "t-pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6738}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4641}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}, "yellow_sclera": {"source": "stage3", "why": "explicit", "retrieval_score": 0.606}}, "structural": [], "t1": 0.0, "t2": 1.06, "t3": 10.28, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 17, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.2941, "R": 0.2174, "F1": 0.25, "leaf_P": 0.1818, "leaf_R": 0.1333, "leaf_F1": 0.1538, "n_leaf_sel": 11, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 0.74, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 59, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2941, "gen_R": 0.2174, "gen_F1": 0.25, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "standing"], "extra": ["4_claws", "5_claws", "black_bottomwear", "black_pants", "flat_colors", "grey_shirt", "grey_theme", "t-shirt", "white_bottomwear", "white_dress_shirt", "white_shirt", "white_t-shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "5_claws", "black_bottomwear", "black_pants", "claws", "facial_markings", "flat_colors", "fur", "grey_shirt", "grey_theme", "shirt", "t-shirt", "topwear", "white_bottomwear", "white_dress_shirt", "white_shirt", "white_t-shirt"], "stage3_selected": ["4_claws", "5_claws", "black_bottomwear", "black_pants", "claws", "facial_markings", "flat_colors", "fur", "grey_shirt", "grey_theme", "shirt", "simple_background", "t-shirt", "topwear", "white_bottomwear", "white_dress_shirt", "white_shirt", "white_t-shirt"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "topwear": 0.6085, "shirt": 0.7037, "facial_markings": 0.9015, "t-shirt": 0.6707, "flat_colors": 0.4857, "black_bottomwear": 0.5638, "white_shirt": 0.5609, "black_pants": 0.5822, "white_bottomwear": 0.5331, "grey_shirt": 0.6272, "grey_theme": 0.5547, "white_t-shirt": 0.531, "4_claws": 0.6123, "white_dress_shirt": 0.5611, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "topwear": 30, "shirt": 13, "facial_markings": 1, "t-shirt": 18, "flat_colors": 88, "black_bottomwear": 57, "white_shirt": 63, "black_pants": 39, "white_bottomwear": 77, "grey_shirt": 25, "grey_theme": 69, "white_t-shirt": 79, "4_claws": 28, "white_dress_shirt": 62, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "topwear": 8, "shirt": 1, "facial_markings": 1, "t-shirt": 2, "flat_colors": 3, "black_bottomwear": 6, "white_shirt": 4, "black_pants": 5, "white_bottomwear": 8, "grey_shirt": 7, "grey_theme": 8, "white_t-shirt": 10, "4_claws": 3, "white_dress_shirt": 3, "5_claws": 1}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6123}, "5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5638}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5822}, "flat_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4857}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6707}, "white_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5331}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5609}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.531}}, "structural": [], "t1": 0.0, "t2": 0.9, "t3": 11.83, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 29, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0345, "R": 0.0909, "F1": 0.05, "leaf_P": 0.0385, "leaf_R": 0.1429, "leaf_F1": 0.0606, "n_leaf_sel": 26, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 2.64, "why": {"explicit": 30}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 16, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 102, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0345, "gen_R": 0.0909, "gen_F1": 0.05, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": ["action_pose", "anime_eyes", "asexual_pride_colors", "blue_fur", "blue_pawpads", "blue_paws", "curved_tail", "eyes", "facial_stripes", "happy", "jumper", "jumping", "light_nose", "midair", "pink_background", "pink_stripes", "playful", "pose", "posed", "pride_colors", "purple_background", "smiling_at_viewer", "sparkling_background", "sparkling_eyes", "striped_back", "striped_neck", "stripes", "wave"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["action_pose", "anime_eyes", "asexual_pride_colors", "blue_fur", "blue_nose", "blue_pawpads", "blue_paws", "curved_tail", "eyes", "facial_stripes", "happy", "jumper", "jumping", "light_nose", "midair", "pink_background", "pink_stripes", "playful", "pose", "posed", "pride_colors", "purple_background", "smiling_at_viewer", "sparkling_background", "sparkling_eyes", "striped_back", "striped_neck", "stripes", "wave"], "stage3_selected": ["action_pose", "anime_eyes", "asexual_pride_colors", "blue_fur", "blue_nose", "blue_pawpads", "blue_paws", "curved_tail", "eyes", "facial_stripes", "happy", "jumper", "jumping", "light_nose", "midair", "pink_background", "pink_stripes", "playful", "pose", "posed", "pride_colors", "purple_background", "simple_background", "smiling_at_viewer", "sparkling_background", "sparkling_eyes", "striped_back", "striped_neck", "stripes", "wave"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "pink_background": 0.4277, "smiling_at_viewer": 0.451, "purple_background": 0.4514, "blue_nose": 0.5017, "blue_pawpads": 0.4264, "pride_colors": 0.2723, "action_pose": 0.4596, "jumping": 0.5295, "playful": 0.6118, "wave": 0.3749, "midair": 0.43, "sparkling_eyes": 0.3343, "blue_paws": 0.5068, "pink_stripes": 0.5235, "curved_tail": 0.5875, "striped_back": 0.7273, "light_nose": 0.6477, "striped_neck": 0.711, "anime_eyes": 0.4019, "facial_stripes": 0.6662, "sparkling_background": 0.4342, "asexual_pride_colors": 0.2701, "posed": 0.4337, "jumper": 0.4127, "eyes": 0.97}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "stripes": 18, "pose": 19, "happy": 101, "pink_background": 72, "smiling_at_viewer": 62, "purple_background": 60, "blue_nose": 45, "blue_pawpads": 73, "pride_colors": 148, "action_pose": 58, "jumping": 35, "playful": 17, "wave": 92, "midair": 71, "sparkling_eyes": 120, "blue_paws": 43, "pink_stripes": 37, "curved_tail": 22, "striped_back": 4, "light_nose": 13, "striped_neck": 6, "anime_eyes": 84, "facial_stripes": 11, "sparkling_background": 67, "asexual_pride_colors": 151, "posed": 69, "jumper": 81, "eyes": 1}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "stripes": 10, "pose": 1, "happy": 3, "pink_background": 1, "smiling_at_viewer": 8, "purple_background": 6, "blue_nose": 3, "blue_pawpads": 7, "pride_colors": 6, "action_pose": 3, "jumping": 1, "playful": 1, "wave": 5, "midair": 2, "sparkling_eyes": 6, "blue_paws": 2, "pink_stripes": 1, "curved_tail": 3, "striped_back": 2, "light_nose": 2, "striped_neck": 4, "anime_eyes": 7, "facial_stripes": 7, "sparkling_background": 1, "asexual_pride_colors": 9, "posed": 10, "jumper": 4, "eyes": 1}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "asexual_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2701}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "blue_paws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5068}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "facial_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6662}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "light_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6477}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "pink_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4277}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2723}, "purple_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4514}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "sparkling_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4342}, "sparkling_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3343}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7273}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.711}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "wave": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3749}}, "structural": [], "t1": 0.0, "t2": 1.34, "t3": 17.18, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p5.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:23:00.566821", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 18}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0556, "R": 0.0833, "F1": 0.0667, "leaf_P": 0.0833, "leaf_R": 0.1111, "leaf_F1": 0.0952, "n_leaf_sel": 12, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 1.5, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 31, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0556, "gen_R": 0.0833, "gen_F1": 0.0667, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["anthro_focus", "black_necktie", "business_suit", "depth_of_field", "face_focus", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "topwear", "translucent_hair", "white_clothing", "white_dress_shirt", "white_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "anthro_focus", "black_necktie", "business_suit", "depth_of_field", "face_focus", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "topwear", "translucent_hair", "white_clothing", "white_dress_shirt", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["alpha_channel", "anthro_focus", "black_necktie", "business_suit", "depth_of_field", "face_focus", "holding_topwear", "necktie", "shirt", "sparkling_character", "tank_top", "topwear", "translucent_hair", "transparent_background", "white_clothing", "white_dress_shirt", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"topwear": 0.7782, "shirt": 0.9019, "alpha_channel": 0.4024, "white_clothing": 0.6182, "tank_top": 0.7545, "necktie": 0.8082, "transparent_background": 0.5794, "translucent_hair": 0.6309, "white_topwear": 0.6638, "white_shirt": 0.6481, "anthro_focus": 0.4219, "depth_of_field": 0.5407, "business_suit": 0.6178, "white_t-shirt": 0.6584, "black_necktie": 0.625, "white_dress_shirt": 0.6255, "face_focus": 0.4971, "sparkling_character": 0.5249, "holding_topwear": 0.595}, "stage3_selected_ranks": {"topwear": 4, "shirt": 1, "alpha_channel": 71, "white_clothing": 25, "tank_top": 8, "necktie": 3, "transparent_background": 39, "translucent_hair": 19, "white_topwear": 13, "white_shirt": 16, "anthro_focus": 68, "depth_of_field": 49, "business_suit": 26, "white_t-shirt": 15, "black_necktie": 23, "white_dress_shirt": 22, "face_focus": 56, "sparkling_character": 52, "holding_topwear": 36}, "stage3_selected_phrase_ranks": {"topwear": 3, "shirt": 1, "alpha_channel": 10, "white_clothing": 7, "tank_top": 7, "necktie": 1, "transparent_background": 4, "translucent_hair": 2, "white_topwear": 1, "white_shirt": 3, "anthro_focus": 9, "depth_of_field": 2, "business_suit": 4, "white_t-shirt": 2, "black_necktie": 6, "white_dress_shirt": 5, "face_focus": 6, "sparkling_character": 1, "holding_topwear": 3}, "extra_evidence": {"anthro_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4219}, "black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "depth_of_field": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "holding_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5249}, "tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7545}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7782}, "translucent_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6309}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6255}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6481}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.18, "t3": 10.72, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.1111, "R": 0.1429, "F1": 0.125, "leaf_P": 0.1111, "leaf_R": 0.2, "leaf_F1": 0.1429, "n_leaf_sel": 18, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 0.6667, "over_sel": 1.29, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 43, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 85, "attempts_by_n_local": {"43": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1111, "gen_R": 0.1429, "gen_F1": 0.125, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "beast_(disambiguation)", "bright", "expressions", "extended_arms", "grin", "high_waisted_bottomwear", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "skimpy", "the_end", "toothy_grin"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "beast_(disambiguation)", "bright", "expressions", "extended_arms", "grin", "high_waisted_bottomwear", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "skimpy", "the_end", "toothy_grin"], "stage3_selected": ["animated_comic", "beast_(disambiguation)", "bright", "expressions", "extended_arms", "grin", "high_waisted_bottomwear", "looking_at_viewer", "male", "partially_clothed_anthro", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "skimpy", "the_end", "toothy_grin"], "stage3_selected_scores": {"male": 0.6605, "looking_at_viewer": 0.5612, "grin": 0.7174, "skimpy": 0.4928, "raised_arm": 0.7205, "raised_arms": 0.8636, "raised_hand": 0.6751, "simple_shading": 0.5424, "extended_arms": 0.6595, "toothy_grin": 0.6248, "partially_clothed_anthro": 0.4576, "expressions": 0.4746, "high_waisted_bottomwear": 0.4614, "bright": 0.5873, "beast_(disambiguation)": 0.338, "the_end": 0.4455, "relaxed_expression": 0.5175, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "looking_at_viewer": 22, "grin": 3, "skimpy": 71, "raised_arm": 2, "raised_arms": 1, "raised_hand": 5, "simple_shading": 33, "extended_arms": 8, "toothy_grin": 13, "partially_clothed_anthro": 100, "expressions": 86, "high_waisted_bottomwear": 96, "bright": 16, "beast_(disambiguation)": 162, "the_end": 105, "relaxed_expression": 48, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "looking_at_viewer": 2, "grin": 1, "skimpy": 5, "raised_arm": 2, "raised_arms": 1, "raised_hand": 4, "simple_shading": 1, "extended_arms": 6, "toothy_grin": 2, "partially_clothed_anthro": 10, "expressions": 5, "high_waisted_bottomwear": 9, "bright": 1, "beast_(disambiguation)": 9, "the_end": 1, "relaxed_expression": 1, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "beast_(disambiguation)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.338}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4746}, "extended_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6595}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6751}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "simple_shading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5424}, "skimpy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "the_end": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4455}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}}, "structural": [], "t1": 0.0, "t2": 9.52, "t3": 21.17, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 8, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 8, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.57, "why": {"explicit": 8}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 25, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "petting", "setting", "shocked_face"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "petting", "setting", "shocked_face"], "stage3_selected": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "petting", "setting", "shocked_face"], "stage3_selected_scores": {"embrace": 0.5734, "petting": 0.7411, "intimate": 0.5697, "expressions": 0.8783, "shocked_face": 0.616, "caressing_face": 0.5887, "cheek_to_cheek": 0.5892, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "petting": 12, "intimate": 42, "expressions": 3, "shocked_face": 18, "caressing_face": 30, "cheek_to_cheek": 29, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "petting": 2, "intimate": 5, "expressions": 2, "shocked_face": 1, "caressing_face": 2, "cheek_to_cheek": 8, "setting": 1}, "extra_evidence": {"caressing_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5887}, "cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}}, "structural": [], "t1": 0.0, "t2": 8.14, "t3": 6.19, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 3, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 3, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 0.75, "why": {"explicit": 4}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 1, "bootstrap_reranked": true, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 21, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["creature_print", "round_eyes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["creature_print", "round_eyes", "toony"], "stage3_selected": ["creature_print", "round_eyes", "simple_background", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "toony": 0.763, "creature_print": 0.5405, "round_eyes": 0.6227}, "stage3_selected_ranks": {"simple_background": 10, "toony": 6, "creature_print": 68, "round_eyes": 30}, "stage3_selected_phrase_ranks": {"simple_background": 5, "toony": 3, "creature_print": 9, "round_eyes": 1}, "extra_evidence": {"creature_print": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5405}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.15, "t3": 9.36, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 11, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1818, "R": 0.0909, "F1": 0.1212, "leaf_P": 0.1111, "leaf_R": 0.0833, "leaf_F1": 0.0952, "n_leaf_sel": 9, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.5, "why": {"explicit": 11}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 38, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1818, "gen_R": 0.0909, "gen_F1": 0.1212, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_bottomwear", "blue_shorts", "hand_gesture", "hand_on_own_head", "happy", "playful", "playing", "pose", "raised_finger"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_bottomwear", "blue_shorts", "fur", "hand_gesture", "hand_on_own_head", "happy", "playful", "playing", "pose", "raised_finger", "shorts"], "stage3_selected": ["blue_bottomwear", "blue_shorts", "fur", "hand_gesture", "hand_on_own_head", "happy", "playful", "playing", "pose", "raised_finger", "shorts"], "stage3_selected_scores": {"fur": 0.7785, "pose": 0.847, "shorts": 0.9202, "happy": 0.457, "blue_bottomwear": 0.6293, "playful": 0.6803, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "hand_gesture": 0.6065, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"fur": 8, "pose": 3, "shorts": 1, "happy": 87, "blue_bottomwear": 32, "playful": 18, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "hand_gesture": 39, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"fur": 1, "pose": 1, "shorts": 1, "happy": 5, "blue_bottomwear": 2, "playful": 1, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "hand_gesture": 1, "hand_on_own_head": 6}, "extra_evidence": {"blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}}, "structural": [], "t1": 0.0, "t2": 0.73, "t3": 9.2, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 7, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.0, "over_sel": 0.54, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 38, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 83, "attempts_by_n_local": {"38": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 5, "parse_ok": 2, "parse_fail": 0, "errors": 3}}, "attempt_failure_rate": 0.5, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "dark_theme", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "dark_theme", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "stage3_selected": ["3_panel_comic", "dark_theme", "darkness", "group", "light", "speech_bubble", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "speech_bubble": 0.551, "light": 0.5737, "darkness": 0.7908, "dark_theme": 0.5093, "unnamed_character": 0.4967, "3_panel_comic": 0.4885}, "stage3_selected_ranks": {"group": 110, "speech_bubble": 23, "light": 15, "darkness": 2, "dark_theme": 43, "unnamed_character": 50, "3_panel_comic": 56}, "stage3_selected_phrase_ranks": {"group": 5, "speech_bubble": 1, "light": 4, "darkness": 1, "dark_theme": 4, "unnamed_character": 1, "3_panel_comic": 2}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.52, "t3": 27.31, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"style_or_meta\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"style_or_meta\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 35}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"style_or_meta\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"style_or_meta\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"style_or_meta\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"style_or_meta\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 40, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.175, "R": 0.5, "F1": 0.2593, "leaf_P": 0.0938, "leaf_R": 0.25, "leaf_F1": 0.1364, "n_leaf_sel": 32, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 2.86, "why": {"explicit": 40}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 13, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 96, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.1795, "gen_R": 0.5, "gen_F1": 0.2642, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "lying", "purple_hair", "text"], "extra": ["animated_comic", "animated_png", "applying_makeup", "bed_covers", "bed_sheet", "bedding", "blanket", "blue_background", "clown_makeup", "heterochromia", "lipstick", "lipstick_(object)", "lipstick_on_face", "lying_on_bed", "on_bed", "on_pillow", "palette", "pillow", "purple_text", "red_lipstick", "red_pillow", "relaxed_expression", "restricted_palette", "running_mascara", "sleeping_together", "sleepover", "sleepwear", "stylized_text", "two_tone_hair", "unnamed_character", "x_eyes", "yellow_eyeshadow", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["animated_comic", "animated_png", "applying_makeup", "bed", "bed_covers", "bed_sheet", "bedding", "blanket", "blonde_hair", "blue_background", "blue_eyes", "clown_makeup", "green_eyes", "hair", "heterochromia", "lipstick", "lipstick_(object)", "lipstick_on_face", "lying_on_bed", "makeup", "on_bed", "on_pillow", "palette", "pillow", "purple_text", "red_lipstick", "red_pillow", "relaxed_expression", "restricted_palette", "running_mascara", "sleeping", "sleeping_together", "sleepover", "sleepwear", "stylized_text", "two_tone_hair", "unnamed_character", "x_eyes", "yellow_eyeshadow", "zzz"], "stage3_selected": ["animated_comic", "animated_png", "applying_makeup", "bed", "bed_covers", "bed_sheet", "bedding", "blanket", "blonde_hair", "blue_background", "blue_eyes", "clown_makeup", "green_eyes", "hair", "heterochromia", "lipstick", "lipstick_(object)", "lipstick_on_face", "lying_on_bed", "makeup", "on_bed", "on_pillow", "palette", "pillow", "purple_text", "red_lipstick", "red_pillow", "relaxed_expression", "restricted_palette", "running_mascara", "sleeping", "sleeping_together", "sleepover", "sleepwear", "stylized_text", "two_tone_hair", "unnamed_character", "x_eyes", "yellow_eyeshadow", "zzz"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "pillow": 0.441, "two_tone_hair": 0.3447, "on_bed": 0.4468, "lipstick": 0.4386, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "heterochromia": 0.3472, "bed_sheet": 0.4395, "blanket": 0.3523, "restricted_palette": 0.2856, "lying_on_bed": 0.412, "zzz": 0.367, "red_lipstick": 0.4186, "running_mascara": 0.3589, "bed_covers": 0.4011, "x_eyes": 0.3583, "purple_text": 0.5604, "on_pillow": 0.4104, "lipstick_on_face": 0.3766, "sleeping_together": 0.408, "unnamed_character": 0.3132, "palette": 0.3097, "sleepover": 0.5015, "sleepwear": 0.4346, "yellow_eyeshadow": 0.4068, "clown_makeup": 0.4004, "stylized_text": 0.3452, "lipstick_(object)": 0.363, "applying_makeup": 0.431, "red_pillow": 0.396, "relaxed_expression": 0.388, "animated_comic": 0.3497, "animated_png": 0.2978}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "bed": 15, "makeup": 11, "pillow": 27, "two_tone_hair": 100, "on_bed": 26, "lipstick": 29, "blue_background": 105, "sleeping": 20, "bedding": 33, "heterochromia": 95, "bed_sheet": 28, "blanket": 86, "restricted_palette": 138, "lying_on_bed": 38, "zzz": 68, "red_lipstick": 34, "running_mascara": 79, "bed_covers": 44, "x_eyes": 80, "purple_text": 10, "on_pillow": 40, "lipstick_on_face": 62, "sleeping_together": 41, "unnamed_character": 119, "palette": 123, "sleepover": 19, "sleepwear": 30, "yellow_eyeshadow": 42, "clown_makeup": 45, "stylized_text": 98, "lipstick_(object)": 75, "applying_makeup": 31, "red_pillow": 49, "relaxed_expression": 53, "animated_comic": 90, "animated_png": 130}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "bed": 1, "makeup": 1, "pillow": 3, "two_tone_hair": 10, "on_bed": 2, "lipstick": 2, "blue_background": 10, "sleeping": 2, "bedding": 5, "heterochromia": 7, "bed_sheet": 4, "blanket": 10, "restricted_palette": 9, "lying_on_bed": 6, "zzz": 8, "red_lipstick": 4, "running_mascara": 10, "bed_covers": 9, "x_eyes": 7, "purple_text": 3, "on_pillow": 8, "lipstick_on_face": 4, "sleeping_together": 1, "unnamed_character": 6, "palette": 4, "sleepover": 1, "sleepwear": 5, "yellow_eyeshadow": 7, "clown_makeup": 6, "stylized_text": 5, "lipstick_(object)": 8, "applying_makeup": 1, "red_pillow": 10, "relaxed_expression": 2, "animated_comic": 1, "animated_png": 3}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3497}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2978}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.431}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "clown_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lipstick_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3766}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4468}, "on_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4104}, "palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3097}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "purple_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5604}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "red_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.396}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "restricted_palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2856}, "running_mascara": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3589}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "sleepover": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4346}, "stylized_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3452}, "two_tone_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3447}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3132}, "x_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3583}, "yellow_eyeshadow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4068}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}}, "structural": [], "t1": 0.0, "t2": 1.38, "t3": 22.68, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 24, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1667, "R": 0.1818, "F1": 0.1739, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 19, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 1.09, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 28, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 58, "attempts_by_n_local": {"28": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 0.1818, "gen_F1": 0.1739, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["action_pose", "blowup_background", "bright_colors", "canis", "digitigrade", "green_background", "long_claws", "maned_wolf", "multiple_poses", "notched_ear", "orange_background", "pose", "toony", "toony_expression", "torn_leggings", "torn_topwear", "unknown_species", "warm_colors", "wolf", "yellow_sclera"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["action_pose", "blowup_background", "bright_colors", "canine", "canis", "claws", "digitigrade", "green_background", "long_claws", "maned_wolf", "multiple_poses", "musical_instrument", "notched_ear", "orange_background", "pose", "toony", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "unknown_species", "warm_colors", "wolf", "yellow_sclera"], "stage3_selected": ["action_pose", "blowup_background", "blurred_background", "bright_colors", "canine", "canis", "claws", "digitigrade", "gradient_background", "green_background", "long_claws", "maned_wolf", "multiple_poses", "musical_instrument", "notched_ear", "orange_background", "pose", "toony", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "unknown_species", "warm_colors", "wolf", "yellow_sclera"], "stage3_selected_scores": {"canine": 0.6203, "canis": 0.6493, "claws": 0.898, "wolf": 0.6635, "pose": 0.8955, "digitigrade": 0.7162, "torn_clothing": 0.5179, "yellow_sclera": 0.606, "gradient_background": 0.636, "toony": 0.4807, "unknown_species": 0.7852, "green_background": 0.6349, "notched_ear": 0.4855, "blurred_background": 0.6454, "musical_instrument": 0.3362, "multiple_poses": 0.7533, "orange_background": 0.6518, "action_pose": 0.6726, "maned_wolf": 0.664, "warm_colors": 0.6979, "torn_topwear": 0.4853, "long_claws": 0.6953, "blowup_background": 0.6301, "torn_leggings": 0.4969, "bright_colors": 0.6275, "toony_expression": 0.4641}, "stage3_selected_ranks": {"canine": 45, "canis": 25, "claws": 1, "wolf": 23, "pose": 2, "digitigrade": 10, "torn_clothing": 73, "yellow_sclera": 48, "gradient_background": 36, "toony": 90, "unknown_species": 5, "green_background": 37, "notched_ear": 87, "blurred_background": 29, "musical_instrument": 117, "multiple_poses": 6, "orange_background": 24, "action_pose": 19, "maned_wolf": 21, "warm_colors": 13, "torn_topwear": 88, "long_claws": 15, "blowup_background": 40, "torn_leggings": 79, "bright_colors": 41, "toony_expression": 98}, "stage3_selected_phrase_ranks": {"canine": 6, "canis": 2, "claws": 1, "wolf": 2, "pose": 1, "digitigrade": 5, "torn_clothing": 4, "yellow_sclera": 5, "gradient_background": 7, "toony": 5, "unknown_species": 1, "green_background": 8, "notched_ear": 9, "blurred_background": 5, "musical_instrument": 9, "multiple_poses": 3, "orange_background": 2, "action_pose": 2, "maned_wolf": 1, "warm_colors": 2, "torn_topwear": 10, "long_claws": 2, "blowup_background": 9, "torn_leggings": 6, "bright_colors": 6, "toony_expression": 4}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6301}, "bright_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6275}, "canis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6493}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "green_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6349}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7533}, "notched_ear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4855}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4641}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "unknown_species": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7852}, "warm_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6979}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}, "yellow_sclera": {"source": "stage3", "why": "explicit", "retrieval_score": 0.606}}, "structural": [], "t1": 0.0, "t2": 1.09, "t3": 6.33, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 11, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.3636, "R": 0.1739, "F1": 0.2353, "leaf_P": 0.1429, "leaf_R": 0.0667, "leaf_F1": 0.0909, "n_leaf_sel": 7, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.5, "over_sel": 0.48, "why": {"explicit": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 60, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3636, "gen_R": 0.1739, "gen_F1": 0.2353, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "standing", "topwear"], "extra": ["big_claws", "black_topwear", "cheek_markings", "grey_shirt", "grey_topwear", "grey_tuft", "retractable_claws"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["big_claws", "black_topwear", "cheek_markings", "claws", "facial_markings", "fur", "grey_shirt", "grey_topwear", "grey_tuft", "retractable_claws", "shirt"], "stage3_selected": ["big_claws", "black_topwear", "cheek_markings", "claws", "facial_markings", "fur", "grey_shirt", "grey_topwear", "grey_tuft", "retractable_claws", "shirt", "simple_background"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "shirt": 0.7037, "facial_markings": 0.9015, "black_topwear": 0.5595, "grey_topwear": 0.5774, "grey_shirt": 0.6272, "big_claws": 0.569, "cheek_markings": 0.719, "retractable_claws": 0.565, "grey_tuft": 0.5545}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "shirt": 13, "facial_markings": 1, "black_topwear": 65, "grey_topwear": 45, "grey_shirt": 25, "big_claws": 52, "cheek_markings": 7, "retractable_claws": 56, "grey_tuft": 70}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "shirt": 1, "facial_markings": 1, "black_topwear": 10, "grey_topwear": 1, "grey_shirt": 7, "big_claws": 4, "cheek_markings": 5, "retractable_claws": 6, "grey_tuft": 9}, "extra_evidence": {"big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.569}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "cheek_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.719}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "grey_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5545}, "retractable_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.565}}, "structural": [], "t1": 0.0, "t2": 0.89, "t3": 13.33, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 25, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 24, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 0.0, "over_sel": 2.27, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 16, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 3, "calls_exhausted_retries": 1, "attempts_total": 7, "attempt_errors": 4, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 68, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 5, "parse_ok": 1, "parse_fail": 0, "errors": 4}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.5714285714285714, "call_exhaustion_rate": 0.25}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "blue_pawpads", "curved_tail", "dark_eyelids", "half-closed_eyes", "happy", "jumper", "jumping", "light_nose", "pink_stripes", "playful", "playing", "pose", "posed", "pride_colors", "purple_stripes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "striped_back", "striped_face", "stripes", "teal_fur", "white_stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "blue_pawpads", "curved_tail", "dark_eyelids", "half-closed_eyes", "happy", "jumper", "jumping", "light_nose", "pink_stripes", "playful", "playing", "pose", "posed", "pride_colors", "purple_stripes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "striped_back", "striped_face", "stripes", "teal_fur", "white_stripes"], "stage3_selected": [">:)", ">:d", "blue_pawpads", "curved_tail", "dark_eyelids", "half-closed_eyes", "happy", "jumper", "jumping", "light_nose", "pink_stripes", "playful", "playing", "pose", "posed", "pride_colors", "purple_stripes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "striped_back", "striped_face", "stripes", "teal_fur", "transparent_background", "white_stripes"], "stage3_selected_scores": {"half-closed_eyes": 0.3863, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "transparent_background": 0.3162, "smiling_at_viewer": 0.451, "blue_pawpads": 0.4264, "pride_colors": 0.2723, "jumping": 0.5295, "playful": 0.6118, "white_stripes": 0.6473, "playing": 0.3818, "purple_stripes": 0.4704, "six-stripe_rainbow_pride_colors": 0.263, "striped_face": 0.7013, "pink_stripes": 0.5235, "curved_tail": 0.5875, "dark_eyelids": 0.5778, "striped_back": 0.7273, "light_nose": 0.6477, ">:d": 0.3455, ">:)": 0.3449, "sketch_background": 0.3715, "posed": 0.4337, "jumper": 0.4127, "teal_fur": 0.4161}, "stage3_selected_ranks": {"half-closed_eyes": 89, "stripes": 18, "pose": 19, "happy": 101, "transparent_background": 136, "smiling_at_viewer": 62, "blue_pawpads": 73, "pride_colors": 148, "jumping": 35, "playful": 17, "white_stripes": 14, "playing": 91, "purple_stripes": 56, "six-stripe_rainbow_pride_colors": 152, "striped_face": 7, "pink_stripes": 37, "curved_tail": 22, "dark_eyelids": 24, "striped_back": 4, "light_nose": 13, ">:d": 111, ">:)": 113, "sketch_background": 93, "posed": 69, "jumper": 81, "teal_fur": 78}, "stage3_selected_phrase_ranks": {"half-closed_eyes": 10, "stripes": 10, "pose": 1, "happy": 3, "transparent_background": 10, "smiling_at_viewer": 8, "blue_pawpads": 7, "pride_colors": 6, "jumping": 1, "playful": 1, "white_stripes": 9, "playing": 2, "purple_stripes": 4, "six-stripe_rainbow_pride_colors": 10, "striped_face": 5, "pink_stripes": 1, "curved_tail": 3, "dark_eyelids": 6, "striped_back": 2, "light_nose": 2, ">:d": 8, ">:)": 9, "sketch_background": 6, "posed": 10, "jumper": 4, "teal_fur": 10}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "dark_eyelids": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5778}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3863}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "light_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6477}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3818}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2723}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "six-stripe_rainbow_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.263}, "sketch_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3715}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7273}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7013}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "teal_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4161}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6473}}, "structural": [], "t1": 0.0, "t2": 1.35, "t3": 46.06, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 48, \"why\": \"explicit\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"other\"}, {\"i\": 56, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 59}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 59}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 15, \"why\": \"other\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"other\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.38.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.38.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"other\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.38.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.38.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 15, \"why\": \"other\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"other\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.38.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.38.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: gave up after 3 attempts"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k2_cw0p7.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:24:26.505940", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 22}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0556, "R": 0.0833, "F1": 0.0667, "leaf_P": 0.0833, "leaf_R": 0.1111, "leaf_F1": 0.0952, "n_leaf_sel": 12, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 1.5, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 30, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0556, "gen_R": 0.0833, "gen_F1": 0.0667, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["black_necktie", "business_suit", "circle_eyebrows", "depth_of_field", "directed_motion_outline", "face_focus", "focused", "necktie", "shirt", "solo_focus", "t-shirt", "topwear", "translucent_clothing", "translucent_hair", "white_clothing", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "black_necktie", "business_suit", "circle_eyebrows", "depth_of_field", "directed_motion_outline", "face_focus", "focused", "necktie", "shirt", "solo_focus", "t-shirt", "topwear", "translucent_clothing", "translucent_hair", "white_clothing", "white_t-shirt", "white_topwear"], "stage3_selected": ["alpha_channel", "black_necktie", "business_suit", "circle_eyebrows", "depth_of_field", "directed_motion_outline", "face_focus", "focused", "necktie", "shirt", "solo_focus", "t-shirt", "topwear", "translucent_clothing", "translucent_hair", "transparent_background", "white_clothing", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"topwear": 0.7782, "shirt": 0.9019, "solo_focus": 0.4386, "alpha_channel": 0.4024, "white_clothing": 0.6182, "necktie": 0.8082, "transparent_background": 0.5794, "translucent_hair": 0.6309, "translucent_clothing": 0.6249, "t-shirt": 0.8329, "white_topwear": 0.6638, "depth_of_field": 0.5407, "business_suit": 0.6178, "circle_eyebrows": 0.6081, "white_t-shirt": 0.6584, "directed_motion_outline": 0.4578, "black_necktie": 0.625, "focused": 0.6608, "face_focus": 0.4971}, "stage3_selected_ranks": {"topwear": 4, "shirt": 1, "solo_focus": 64, "alpha_channel": 71, "white_clothing": 25, "necktie": 3, "transparent_background": 39, "translucent_hair": 19, "translucent_clothing": 24, "t-shirt": 2, "white_topwear": 13, "depth_of_field": 49, "business_suit": 26, "circle_eyebrows": 29, "white_t-shirt": 15, "directed_motion_outline": 62, "black_necktie": 23, "focused": 14, "face_focus": 56}, "stage3_selected_phrase_ranks": {"topwear": 3, "shirt": 1, "solo_focus": 7, "alpha_channel": 10, "white_clothing": 7, "necktie": 1, "transparent_background": 4, "translucent_hair": 2, "translucent_clothing": 3, "t-shirt": 2, "white_topwear": 1, "depth_of_field": 2, "business_suit": 4, "circle_eyebrows": 3, "white_t-shirt": 2, "directed_motion_outline": 5, "black_necktie": 6, "focused": 1, "face_focus": 6}, "extra_evidence": {"black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "circle_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6081}, "depth_of_field": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "directed_motion_outline": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4578}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "solo_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7782}, "translucent_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6249}, "translucent_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6309}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.37, "t3": 7.93, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 24, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.125, "R": 0.2143, "F1": 0.1579, "leaf_P": 0.087, "leaf_R": 0.2, "leaf_F1": 0.1212, "n_leaf_sel": 23, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 1.71, "why": {"explicit": 24}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 43, "bootstrap_anchor_selected": 23, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 97, "attempts_by_n_local": {"43": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.125, "gen_R": 0.2143, "gen_F1": 0.1579, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["bored_expression", "expression_sheet", "game_media", "grey_background", "grin", "hand_over_head", "high_waisted_bottomwear", "holding_arms", "home", "larger_male", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "smaller_male", "smug_grin", "spread_arms", "straight_arms", "toony_expression", "toothy_smile"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["bored_expression", "expression_sheet", "game_media", "grey_background", "grin", "group", "hand_over_head", "high_waisted_bottomwear", "holding_arms", "home", "larger_male", "looking_at_viewer", "male", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "smaller_male", "smug_grin", "spread_arms", "straight_arms", "toony_expression", "toothy_smile"], "stage3_selected": ["bored_expression", "expression_sheet", "game_media", "grey_background", "grin", "group", "hand_over_head", "high_waisted_bottomwear", "holding_arms", "home", "larger_male", "looking_at_viewer", "male", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "simple_shading", "smaller_male", "smug_grin", "spread_arms", "straight_arms", "toony_expression", "toothy_smile"], "stage3_selected_scores": {"male": 0.6605, "looking_at_viewer": 0.5612, "group": 0.4099, "larger_male": 0.4954, "smaller_male": 0.5477, "grin": 0.7174, "grey_background": 0.5501, "raised_arm": 0.7205, "raised_arms": 0.8636, "raised_hand": 0.6751, "simple_shading": 0.5424, "spread_arms": 0.605, "smug_grin": 0.4997, "expression_sheet": 0.3785, "high_waisted_bottomwear": 0.4614, "holding_arms": 0.5774, "straight_arms": 0.6554, "game_media": 0.353, "bored_expression": 0.5047, "hand_over_head": 0.5861, "home": 0.3434, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "toony_expression": 0.4675}, "stage3_selected_ranks": {"male": 7, "looking_at_viewer": 22, "group": 132, "larger_male": 68, "smaller_male": 27, "grin": 3, "grey_background": 26, "raised_arm": 2, "raised_arms": 1, "raised_hand": 5, "simple_shading": 33, "spread_arms": 14, "smug_grin": 64, "expression_sheet": 146, "high_waisted_bottomwear": 96, "holding_arms": 19, "straight_arms": 10, "game_media": 156, "bored_expression": 61, "hand_over_head": 18, "home": 159, "relaxed_expression": 48, "toothy_smile": 23, "toony_expression": 89}, "stage3_selected_phrase_ranks": {"male": 1, "looking_at_viewer": 2, "group": 6, "larger_male": 4, "smaller_male": 1, "grin": 1, "grey_background": 3, "raised_arm": 2, "raised_arms": 1, "raised_hand": 4, "simple_shading": 1, "spread_arms": 6, "smug_grin": 5, "expression_sheet": 8, "high_waisted_bottomwear": 9, "holding_arms": 9, "straight_arms": 2, "game_media": 5, "bored_expression": 2, "hand_over_head": 1, "home": 6, "relaxed_expression": 1, "toothy_smile": 3, "toony_expression": 6}, "extra_evidence": {"bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5047}, "expression_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3785}, "game_media": {"source": "stage3", "why": "explicit", "retrieval_score": 0.353}, "grey_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5501}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5861}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "holding_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "home": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3434}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6751}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "simple_shading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5424}, "smaller_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5477}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.605}, "straight_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6554}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}}, "structural": [], "t1": 0.0, "t2": 9.69, "t3": 25.46, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 9, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 9, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.64, "why": {"explicit": 9}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 26, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "kissing_cheek", "petting", "setting", "shocked_face"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "kissing_cheek", "petting", "setting", "shocked_face"], "stage3_selected": ["caressing_face", "cheek_to_cheek", "embrace", "expressions", "intimate", "kissing_cheek", "petting", "setting", "shocked_face"], "stage3_selected_scores": {"embrace": 0.5734, "petting": 0.7411, "intimate": 0.5697, "kissing_cheek": 0.5509, "expressions": 0.8783, "shocked_face": 0.616, "caressing_face": 0.5887, "cheek_to_cheek": 0.5892, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "petting": 12, "intimate": 42, "kissing_cheek": 47, "expressions": 3, "shocked_face": 18, "caressing_face": 30, "cheek_to_cheek": 29, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "petting": 2, "intimate": 5, "kissing_cheek": 8, "expressions": 2, "shocked_face": 1, "caressing_face": 2, "cheek_to_cheek": 8, "setting": 1}, "extra_evidence": {"caressing_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5887}, "cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "kissing_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5509}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}}, "structural": [], "t1": 0.0, "t2": 8.38, "t3": 8.25, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.5, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 16, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 22, "attempts_by_n_local": {"16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["grey_background", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["grey_background", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected": ["grey_background", "round_eyes", "round_nose", "simple_background", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "grey_background": 0.688, "toony": 0.763, "thick_eyelashes": 0.536, "round_eyes": 0.6227, "round_nose": 0.555, "stylized_empty_eyes": 0.6448}, "stage3_selected_ranks": {"simple_background": 10, "grey_background": 18, "toony": 6, "thick_eyelashes": 69, "round_eyes": 30, "round_nose": 59, "stylized_empty_eyes": 25}, "stage3_selected_phrase_ranks": {"simple_background": 5, "grey_background": 10, "toony": 3, "thick_eyelashes": 10, "round_eyes": 1, "round_nose": 10, "stylized_empty_eyes": 1}, "extra_evidence": {"grey_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.688}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "stylized_empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6448}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.536}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.34, "t3": 6.04, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 19, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1053, "R": 0.0909, "F1": 0.0976, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 16, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.86, "why": {"explicit": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 20, "bootstrap_anchor_selected": 9, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 46, "attempts_by_n_local": {"20": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1053, "gen_R": 0.0909, "gen_F1": 0.0976, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "grey_fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_arm", "raised_finger", "reaching_towards_viewer", "striped_markings", "thought_bubble"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "grey_fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_arm", "raised_finger", "reaching_towards_viewer", "shorts", "striped_markings", "thought_bubble"], "stage3_selected": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "grey_fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "playful", "playing", "pose", "raised_arm", "raised_finger", "reaching_towards_viewer", "shorts", "striped_markings", "thought_bubble"], "stage3_selected_scores": {"fur": 0.7785, "grey_fur": 0.5932, "pose": 0.847, "shorts": 0.9202, "raised_arm": 0.6005, "striped_markings": 0.5966, "thought_bubble": 0.5016, "blue_bottomwear": 0.6293, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "muscle_tone": 0.5438, "reaching_towards_viewer": 0.5248, "hand_gesture": 0.6065, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334, "dolphin_shorts": 0.7425}, "stage3_selected_ranks": {"fur": 8, "grey_fur": 48, "pose": 3, "shorts": 1, "raised_arm": 42, "striped_markings": 46, "thought_bubble": 76, "blue_bottomwear": 32, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "muscle_tone": 62, "reaching_towards_viewer": 70, "hand_gesture": 39, "hand_over_head": 56, "hand_on_own_head": 66, "dolphin_shorts": 10}, "stage3_selected_phrase_ranks": {"fur": 1, "grey_fur": 8, "pose": 1, "shorts": 1, "raised_arm": 2, "striped_markings": 8, "thought_bubble": 8, "blue_bottomwear": 2, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "muscle_tone": 6, "reaching_towards_viewer": 10, "hand_gesture": 1, "hand_over_head": 5, "hand_on_own_head": 6, "dolphin_shorts": 7}, "extra_evidence": {"blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "dolphin_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7425}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5932}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "muscle_tone": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5438}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "reaching_towards_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "thought_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5016}}, "structural": [], "t1": 0.0, "t2": 0.77, "t3": 10.15, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 7, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.0, "over_sel": 0.54, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 38, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 2, "calls_exhausted_retries": 1, "attempts_total": 7, "attempt_errors": 5, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 60, "attempts_by_n_local": {"38": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}, "60": {"attempts": 5, "parse_ok": 1, "parse_fail": 0, "errors": 4}}, "attempt_failure_rate": 0.7142857142857143, "call_exhaustion_rate": 0.3333333333333333}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["dark_theme", "group", "light", "nightmare", "speech_bubble", "surrounded", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["dark_theme", "group", "light", "nightmare", "speech_bubble", "surrounded", "unnamed_character"], "stage3_selected": ["dark_theme", "group", "light", "nightmare", "speech_bubble", "surrounded", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "speech_bubble": 0.551, "light": 0.5737, "dark_theme": 0.5093, "surrounded": 0.5229, "nightmare": 0.4738, "unnamed_character": 0.4967}, "stage3_selected_ranks": {"group": 110, "speech_bubble": 23, "light": 15, "dark_theme": 43, "surrounded": 38, "nightmare": 72, "unnamed_character": 50}, "stage3_selected_phrase_ranks": {"group": 5, "speech_bubble": 1, "light": 4, "dark_theme": 4, "surrounded": 3, "nightmare": 8, "unnamed_character": 1}, "extra_evidence": {"dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "nightmare": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4738}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "surrounded": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5229}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.69, "t3": 37.2, "t3s": 0.0, "err": null, "issues": ["Stage3 bootstrap_anchor: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"style_or_meta\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 40}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 35}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"style_or_meta\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: gave up after 3 attempts", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"style_or_meta\"}, {\"i\": 7, \"why\": \"style_or_meta\"}, {\"i\": 8, \"why\": \"style_or_meta\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"style_or_meta\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"style_or_meta\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"style_or_meta\"}, {\"i\": 27, \"why\": \"style_or_meta\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 23, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.3043, "R": 0.5, "F1": 0.3784, "leaf_P": 0.25, "leaf_R": 0.4167, "leaf_F1": 0.3125, "n_leaf_sel": 20, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 1.64, "why": {"explicit": 23}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 18, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 94, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.3333333333333333, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3043, "gen_R": 0.5, "gen_F1": 0.3784, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "lying", "purple_hair", "text"], "extra": ["animated_comic", "animated_png", "annoyed_expression", "bedroom", "eyes", "hair_sticks", "half-length_portrait", "heterochromia", "lipstick", "long_hair", "palette", "purple_arms", "purple_feet", "purple_legs", "purple_membrane", "red_lipstick"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["animated_comic", "animated_png", "annoyed_expression", "bed", "bedroom", "blonde_hair", "blue_eyes", "eyes", "green_eyes", "hair", "hair_sticks", "half-length_portrait", "heterochromia", "lipstick", "long_hair", "makeup", "palette", "purple_arms", "purple_feet", "purple_legs", "purple_membrane", "red_lipstick", "sleeping"], "stage3_selected": ["animated_comic", "animated_png", "annoyed_expression", "bed", "bedroom", "blonde_hair", "blue_eyes", "eyes", "green_eyes", "hair", "hair_sticks", "half-length_portrait", "heterochromia", "lipstick", "long_hair", "makeup", "palette", "purple_arms", "purple_feet", "purple_legs", "purple_membrane", "red_lipstick", "sleeping"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "long_hair": 0.3962, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "lipstick": 0.4386, "bedroom": 0.4116, "sleeping": 0.4995, "heterochromia": 0.3472, "half-length_portrait": 0.352, "red_lipstick": 0.4186, "annoyed_expression": 0.3458, "hair_sticks": 0.5043, "purple_feet": 0.6039, "palette": 0.3097, "purple_legs": 0.6369, "purple_membrane": 0.6335, "purple_arms": 0.6358, "animated_comic": 0.3497, "eyes": 0.9321, "animated_png": 0.2978}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "long_hair": 48, "blonde_hair": 96, "bed": 15, "makeup": 11, "lipstick": 29, "bedroom": 39, "sleeping": 20, "heterochromia": 95, "half-length_portrait": 87, "red_lipstick": 34, "annoyed_expression": 97, "hair_sticks": 18, "purple_feet": 7, "palette": 123, "purple_legs": 3, "purple_membrane": 6, "purple_arms": 4, "animated_comic": 90, "eyes": 1, "animated_png": 130}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "long_hair": 3, "blonde_hair": 8, "bed": 1, "makeup": 1, "lipstick": 2, "bedroom": 7, "sleeping": 2, "heterochromia": 7, "half-length_portrait": 8, "red_lipstick": 4, "annoyed_expression": 9, "hair_sticks": 2, "purple_feet": 5, "palette": 4, "purple_legs": 1, "purple_membrane": 4, "purple_arms": 2, "animated_comic": 1, "eyes": 1, "animated_png": 3}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3497}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2978}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9321}, "hair_sticks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5043}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.352}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3962}, "palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3097}, "purple_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6358}, "purple_feet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6039}, "purple_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "purple_membrane": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6335}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}}, "structural": [], "t1": 0.0, "t2": 1.5, "t3": 33.09, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 58, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 54, \"why\": \"explicit\"}, {\"i\": 58, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 40, \"why\": \"style_or_meta\"}, {\"i\": 44, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 46, \"why\": \"style_or_meta\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 21, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1429, "R": 0.1364, "F1": 0.1395, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 16, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.6, "over_sel": 0.95, "why": {"explicit": 22}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 28, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 51, "attempts_by_n_local": {"28": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}, "50": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1429, "gen_R": 0.1364, "gen_F1": 0.1395, "missed": ["anthro", "bass_guitar", "canid", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["acoustic_guitar", "action_pose", "bared_teeth", "clenched_teeth", "color_swatch", "digitigrade", "dire_wolf", "finger_claws", "gold_claws", "green_background", "long_claws", "maned_wolf", "mexican_wolf", "painted_claws", "pose", "torch", "torn_topwear", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["acoustic_guitar", "action_pose", "bared_teeth", "canine", "claws", "clenched_teeth", "color_swatch", "digitigrade", "dire_wolf", "finger_claws", "gold_claws", "green_background", "long_claws", "maned_wolf", "mexican_wolf", "painted_claws", "pose", "torch", "torn_clothing", "torn_topwear", "wolf"], "stage3_selected": ["acoustic_guitar", "action_pose", "bared_teeth", "canine", "claws", "clenched_teeth", "color_swatch", "digitigrade", "dire_wolf", "finger_claws", "gold_claws", "green_background", "long_claws", "maned_wolf", "mexican_wolf", "painted_claws", "pose", "textured_background", "torch", "torn_clothing", "torn_topwear", "wolf"], "stage3_selected_scores": {"canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "pose": 0.8955, "digitigrade": 0.7162, "torn_clothing": 0.5179, "clenched_teeth": 0.498, "finger_claws": 0.7339, "green_background": 0.6349, "action_pose": 0.6726, "color_swatch": 0.7261, "maned_wolf": 0.664, "torn_topwear": 0.4853, "textured_background": 0.6672, "long_claws": 0.6953, "bared_teeth": 0.5651, "torch": 0.6054, "dire_wolf": 0.6438, "painted_claws": 0.6018, "mexican_wolf": 0.6193, "gold_claws": 0.6234, "acoustic_guitar": 0.3382}, "stage3_selected_ranks": {"canine": 45, "claws": 1, "wolf": 23, "pose": 2, "digitigrade": 10, "torn_clothing": 73, "clenched_teeth": 78, "finger_claws": 8, "green_background": 37, "action_pose": 19, "color_swatch": 9, "maned_wolf": 21, "torn_topwear": 88, "textured_background": 20, "long_claws": 15, "bared_teeth": 64, "torch": 49, "dire_wolf": 30, "painted_claws": 55, "mexican_wolf": 46, "gold_claws": 43, "acoustic_guitar": 115}, "stage3_selected_phrase_ranks": {"canine": 6, "claws": 1, "wolf": 2, "pose": 1, "digitigrade": 5, "torn_clothing": 4, "clenched_teeth": 5, "finger_claws": 4, "green_background": 8, "action_pose": 2, "color_swatch": 1, "maned_wolf": 1, "torn_topwear": 10, "textured_background": 1, "long_claws": 2, "bared_teeth": 2, "torch": 1, "dire_wolf": 4, "painted_claws": 8, "mexican_wolf": 7, "gold_claws": 3, "acoustic_guitar": 7}, "extra_evidence": {"acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3382}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "bared_teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5651}, "clenched_teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.498}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6438}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7339}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "green_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6349}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "mexican_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6193}, "painted_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6018}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "torch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6054}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}}, "structural": [], "t1": 0.0, "t2": 1.06, "t3": 21.84, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"explicit\"}, {\"i\": 52, \"why\": \"explicit\"}, {\"i\": 53, \"why\": \"explicit\"}, {\"i\": 55, \"why\": \"explicit\"}, {\"i\": 56, \"why\": \"explicit\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 45}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 45}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"strong_implied\"}, {\"i\": 16, \"why\": \"strong_implied\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 21, \"why\": \"style_or_meta\"}, {\"i\": 29, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 32, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"other\"}, {\"i\": 39, \"why\": \"other\"}, {\"i\": 40, \"why\": \"other\"}, {\"i\": 41, \"why\": \"other\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 46, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.37.why\n Field required [type=missing, input_value={'i': 46, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 28, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.1786, "R": 0.2174, "F1": 0.1961, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 22, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 1.22, "why": {"explicit": 29}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 48, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1786, "gen_R": 0.2174, "gen_F1": 0.1961, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing", "topwear"], "extra": ["4_claws", "5_claws", "black_topwear", "blue_pants", "cargo_pants", "eye_markings", "gloves_(marking)", "grey_nose", "grey_shirt", "hand_on_another's_shoulder", "hand_on_cheek", "hand_on_face", "head_markings", "head_turned", "hip_markings", "partially_clothed_anthro", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_dress_shirt", "white_markings", "white_shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "5_claws", "black_topwear", "blue_pants", "cargo_pants", "claws", "eye_markings", "facial_markings", "fur", "gloves_(marking)", "grey_nose", "grey_shirt", "hand_on_another's_shoulder", "hand_on_cheek", "hand_on_face", "head_markings", "head_turned", "hip_markings", "pants", "partially_clothed_anthro", "shirt", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_dress_shirt", "white_markings", "white_shirt"], "stage3_selected": ["4_claws", "5_claws", "black_topwear", "blue_pants", "cargo_pants", "claws", "eye_markings", "facial_markings", "fur", "gloves_(marking)", "grey_nose", "grey_shirt", "hand_on_another's_shoulder", "hand_on_cheek", "hand_on_face", "head_markings", "head_turned", "hip_markings", "pants", "partially_clothed_anthro", "shirt", "simple_background", "snout_markings", "t-shirt", "tan_fur", "undershirt", "white_dress_shirt", "white_markings", "white_shirt"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "shirt": 0.7037, "pants": 0.7123, "tan_fur": 0.5593, "gloves_(marking)": 0.7072, "head_markings": 0.8393, "facial_markings": 0.9015, "t-shirt": 0.6707, "black_topwear": 0.5595, "white_shirt": 0.5609, "white_markings": 0.7013, "hand_on_face": 0.4889, "blue_pants": 0.5806, "eye_markings": 0.743, "grey_nose": 0.5885, "snout_markings": 0.7146, "hand_on_cheek": 0.4975, "grey_shirt": 0.6272, "head_turned": 0.4663, "undershirt": 0.6369, "partially_clothed_anthro": 0.5361, "cargo_pants": 0.625, "4_claws": 0.6123, "white_dress_shirt": 0.5611, "hand_on_another's_shoulder": 0.4582, "hip_markings": 0.7094, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "shirt": 13, "pants": 10, "tan_fur": 66, "gloves_(marking)": 12, "head_markings": 2, "facial_markings": 1, "t-shirt": 18, "black_topwear": 65, "white_shirt": 63, "white_markings": 15, "hand_on_face": 87, "blue_pants": 41, "eye_markings": 5, "grey_nose": 37, "snout_markings": 9, "hand_on_cheek": 83, "grey_shirt": 25, "head_turned": 93, "undershirt": 22, "partially_clothed_anthro": 76, "cargo_pants": 26, "4_claws": 28, "white_dress_shirt": 62, "hand_on_another's_shoulder": 94, "hip_markings": 11, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "shirt": 1, "pants": 1, "tan_fur": 5, "gloves_(marking)": 9, "head_markings": 2, "facial_markings": 1, "t-shirt": 2, "black_topwear": 10, "white_shirt": 4, "white_markings": 8, "hand_on_face": 2, "blue_pants": 7, "eye_markings": 4, "grey_nose": 1, "snout_markings": 7, "hand_on_cheek": 1, "grey_shirt": 7, "head_turned": 6, "undershirt": 4, "partially_clothed_anthro": 7, "cargo_pants": 3, "4_claws": 3, "white_dress_shirt": 3, "hand_on_another's_shoulder": 9, "hip_markings": 7, "5_claws": 1}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6123}, "5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "blue_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5806}, "cargo_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "eye_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.743}, "gloves_(marking)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7072}, "grey_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5885}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "hand_on_another's_shoulder": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4582}, "hand_on_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4975}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4889}, "head_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8393}, "head_turned": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4663}, "hip_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7094}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5361}, "snout_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7146}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6707}, "tan_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5593}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "white_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7013}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5609}}, "structural": [], "t1": 0.0, "t2": 0.91, "t3": 16.61, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 35, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0286, "R": 0.0909, "F1": 0.0435, "leaf_P": 0.0294, "leaf_R": 0.1429, "leaf_F1": 0.0488, "n_leaf_sel": 34, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 3.18, "why": {"explicit": 37}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 36, "bootstrap_anchor_selected": 16, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 85, "attempts_by_n_local": {"36": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.3333333333333333, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0286, "gen_R": 0.0909, "gen_F1": 0.0435, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "action_pose", "blowup_background", "blue_fur", "blue_pawpads", "brand_parody", "buzzing", "curved_tail", "encouragement", "facial_stripes", "featureless_chest", "full-length_portrait", "glistening_legs", "happy", "jumper", "lidded_eyes", "midair", "partially_clothed_anthro", "paw_pose", "pink_background", "pink_stripes", "playful", "playing", "plug", "pose", "posed", "purple_stripes", "raised_paw", "round_nose", "six-stripe_rainbow_pride_colors", "striped_arms", "striped_legs", "teal_fur"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "action_pose", "blowup_background", "blue_fur", "blue_nose", "blue_pawpads", "brand_parody", "buzzing", "curved_tail", "encouragement", "facial_stripes", "featureless_chest", "full-length_portrait", "glistening_legs", "happy", "jumper", "lidded_eyes", "midair", "partially_clothed_anthro", "paw_pose", "pink_background", "pink_stripes", "playful", "playing", "plug", "pose", "posed", "purple_stripes", "raised_paw", "round_nose", "six-stripe_rainbow_pride_colors", "striped_arms", "striped_legs", "teal_fur"], "stage3_selected": [">:)", ">:d", "action_pose", "blowup_background", "blue_fur", "blue_nose", "blue_pawpads", "brand_parody", "buzzing", "curved_tail", "encouragement", "facial_stripes", "featureless_chest", "full-length_portrait", "glistening_legs", "happy", "jumper", "lidded_eyes", "midair", "partially_clothed_anthro", "paw_pose", "pink_background", "pink_stripes", "playful", "playing", "plug", "pose", "posed", "purple_stripes", "raised_paw", "round_nose", "simple_background", "six-stripe_rainbow_pride_colors", "striped_arms", "striped_legs", "teal_fur", "transparent_background"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "pose": 0.6068, "happy": 0.36, "full-length_portrait": 0.4491, "transparent_background": 0.3162, "pink_background": 0.4277, "blue_nose": 0.5017, "blue_pawpads": 0.4264, "lidded_eyes": 0.515, "action_pose": 0.4596, "featureless_chest": 0.3249, "playful": 0.6118, "playing": 0.3818, "striped_legs": 0.6654, "striped_arms": 0.6782, "purple_stripes": 0.4704, "midair": 0.43, "six-stripe_rainbow_pride_colors": 0.263, "raised_paw": 0.3182, "paw_pose": 0.3304, "partially_clothed_anthro": 0.5986, "glistening_legs": 0.6306, "plug": 0.2954, "pink_stripes": 0.5235, "encouragement": 0.2536, "curved_tail": 0.5875, "blowup_background": 0.3828, "buzzing": 0.2749, ">:d": 0.3455, ">:)": 0.3449, "facial_stripes": 0.6662, "round_nose": 0.4996, "brand_parody": 0.3359, "posed": 0.4337, "jumper": 0.4127, "teal_fur": 0.4161}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "pose": 19, "happy": 101, "full-length_portrait": 63, "transparent_background": 136, "pink_background": 72, "blue_nose": 45, "blue_pawpads": 73, "lidded_eyes": 38, "action_pose": 58, "featureless_chest": 128, "playful": 17, "playing": 91, "striped_legs": 12, "striped_arms": 10, "purple_stripes": 56, "midair": 71, "six-stripe_rainbow_pride_colors": 152, "raised_paw": 134, "paw_pose": 122, "partially_clothed_anthro": 20, "glistening_legs": 16, "plug": 140, "pink_stripes": 37, "encouragement": 155, "curved_tail": 22, "blowup_background": 90, "buzzing": 146, ">:d": 111, ">:)": 113, "facial_stripes": 11, "round_nose": 46, "brand_parody": 118, "posed": 69, "jumper": 81, "teal_fur": 78}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "pose": 1, "happy": 3, "full-length_portrait": 9, "transparent_background": 10, "pink_background": 1, "blue_nose": 3, "blue_pawpads": 7, "lidded_eyes": 9, "action_pose": 3, "featureless_chest": 8, "playful": 1, "playing": 2, "striped_legs": 8, "striped_arms": 6, "purple_stripes": 4, "midair": 2, "six-stripe_rainbow_pride_colors": 10, "raised_paw": 9, "paw_pose": 7, "partially_clothed_anthro": 3, "glistening_legs": 2, "plug": 5, "pink_stripes": 1, "encouragement": 4, "curved_tail": 3, "blowup_background": 5, "buzzing": 6, ">:d": 8, ">:)": 9, "facial_stripes": 7, "round_nose": 9, "brand_parody": 6, "posed": 10, "jumper": 4, "teal_fur": 10}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3828}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "brand_parody": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3359}, "buzzing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2749}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "encouragement": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2536}, "facial_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6662}, "featureless_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3249}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "glistening_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6306}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.515}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5986}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3304}, "pink_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4277}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3818}, "plug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2954}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4996}, "six-stripe_rainbow_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.263}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6782}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6654}, "teal_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4161}}, "structural": [], "t1": 0.0, "t2": 1.37, "t3": 31.98, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 34, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 48, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 52, \"why\": \"explicit\"}, {\"i\": 54, \"why\": \"explicit\"}, {\"i\": 56, \"why\": \"explicit\"}, {\"i\": 58, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"other\"}, {\"i\": 17}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.38.why\n Field required [type=missing, input_value={'i': 17}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 34, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 48, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 52, \"why\": \"explicit\"}, {\"i\": 54, \"why\": \"explicit\"}, {\"i\": 56, \"why\": \"explicit\"}, {\"i\": 58, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"other\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"other\"}, {\"i\": 17}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.38.why\n Field required [type=missing, input_value={'i': 17}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p3.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:25:42.449243", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 19}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 11, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.92, "why": {"explicit": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 29, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["business_suit", "face_focus", "focused", "necktie", "shirt", "topwear", "translucent_clothing", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["business_suit", "face_focus", "focused", "necktie", "shirt", "topwear", "translucent_clothing", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["business_suit", "face_focus", "focused", "necktie", "shirt", "topwear", "translucent_clothing", "transparent_background", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"topwear": 0.7782, "shirt": 0.9019, "white_clothing": 0.6182, "necktie": 0.8082, "transparent_background": 0.5794, "translucent_clothing": 0.6249, "white_topwear": 0.6638, "business_suit": 0.6178, "white_t-shirt": 0.6584, "white_dress_shirt": 0.6255, "focused": 0.6608, "face_focus": 0.4971}, "stage3_selected_ranks": {"topwear": 4, "shirt": 1, "white_clothing": 25, "necktie": 3, "transparent_background": 39, "translucent_clothing": 24, "white_topwear": 13, "business_suit": 26, "white_t-shirt": 15, "white_dress_shirt": 22, "focused": 14, "face_focus": 56}, "stage3_selected_phrase_ranks": {"topwear": 3, "shirt": 1, "white_clothing": 7, "necktie": 1, "transparent_background": 4, "translucent_clothing": 3, "white_topwear": 1, "business_suit": 4, "white_t-shirt": 2, "white_dress_shirt": 5, "focused": 1, "face_focus": 6}, "extra_evidence": {"business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "focused": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6608}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7782}, "translucent_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6249}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6255}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.22, "t3": 9.46, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 37, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.0811, "R": 0.2143, "F1": 0.1176, "leaf_P": 0.0571, "leaf_R": 0.2, "leaf_F1": 0.0889, "n_leaf_sel": 35, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 2.64, "why": {"explicit": 37}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 64, "bootstrap_anchor_selected": 22, "bootstrap_anchor_kept": 22, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 92, "attempts_by_n_local": {"64": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 3, "char_F1": 0.0, "gen_P": 0.0882, "gen_R": 0.2143, "gen_F1": 0.125, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["background_character", "beckoning", "bent_arm", "breaking_the_fourth_wall", "bright", "come_on_(interjection)", "fran_(litterbox_comics)", "grin", "hand_over_head", "holding_arms", "interrupted", "larger_male", "left-handed", "partially_clothed_anthro", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "revealing_clothes", "rim_light", "simple_face", "simple_shading", "size_difference", "skimpy", "smaller_human", "smaller_version_at_source", "smug_grin", "spread_arms", "thick_eyelashes", "toony_expression", "toothy_grin", "toothy_smile", "unnamed_character"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["background_character", "beckoning", "bent_arm", "breaking_the_fourth_wall", "bright", "come_on_(interjection)", "fran_(litterbox_comics)", "grin", "group", "hand_over_head", "holding_arms", "interrupted", "larger_male", "left-handed", "looking_at_viewer", "male", "partially_clothed_anthro", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "revealing_clothes", "rim_light", "simple_face", "simple_shading", "size_difference", "skimpy", "smaller_human", "smaller_version_at_source", "smug_grin", "spread_arms", "thick_eyelashes", "toony_expression", "toothy_grin", "toothy_smile", "unnamed_character"], "stage3_selected": ["background_character", "beckoning", "bent_arm", "breaking_the_fourth_wall", "bright", "come_on_(interjection)", "fran_(litterbox_comics)", "grin", "group", "hand_over_head", "holding_arms", "interrupted", "larger_male", "left-handed", "looking_at_viewer", "male", "partially_clothed_anthro", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "revealing_clothes", "rim_light", "simple_face", "simple_shading", "size_difference", "skimpy", "smaller_human", "smaller_version_at_source", "smug_grin", "spread_arms", "thick_eyelashes", "toony_expression", "toothy_grin", "toothy_smile", "unnamed_character"], "stage3_selected_scores": {"male": 0.6605, "looking_at_viewer": 0.5612, "group": 0.4099, "size_difference": 0.5253, "larger_male": 0.4954, "grin": 0.7174, "skimpy": 0.4928, "raised_arm": 0.7205, "smaller_human": 0.3792, "bent_arm": 0.6739, "raised_arms": 0.8636, "raised_hand": 0.6751, "beckoning": 0.4425, "petting": 0.4897, "simple_shading": 0.5424, "spread_arms": 0.605, "thick_eyelashes": 0.5524, "smug_grin": 0.4997, "toothy_grin": 0.6248, "breaking_the_fourth_wall": 0.4209, "background_character": 0.5068, "partially_clothed_anthro": 0.4576, "holding_arms": 0.5774, "bright": 0.5873, "interrupted": 0.4586, "simple_face": 0.4943, "unnamed_character": 0.4965, "come_on_(interjection)": 0.3621, "left-handed": 0.4525, "hand_over_head": 0.5861, "fran_(litterbox_comics)": 0.4659, "revealing_clothes": 0.4777, "rim_light": 0.5332, "relaxed_expression": 0.5175, "toothy_smile": 0.5535, "toony_expression": 0.4675, "smaller_version_at_source": 0.5079}, "stage3_selected_ranks": {"male": 7, "looking_at_viewer": 22, "group": 132, "size_difference": 44, "larger_male": 68, "grin": 3, "skimpy": 71, "raised_arm": 2, "smaller_human": 145, "bent_arm": 6, "raised_arms": 1, "raised_hand": 5, "beckoning": 106, "petting": 73, "simple_shading": 33, "spread_arms": 14, "thick_eyelashes": 24, "smug_grin": 64, "toothy_grin": 13, "breaking_the_fourth_wall": 119, "background_character": 58, "partially_clothed_anthro": 100, "holding_arms": 19, "bright": 16, "interrupted": 99, "simple_face": 70, "unnamed_character": 66, "come_on_(interjection)": 153, "left-handed": 102, "hand_over_head": 18, "fran_(litterbox_comics)": 90, "revealing_clothes": 80, "rim_light": 39, "relaxed_expression": 48, "toothy_smile": 23, "toony_expression": 89, "smaller_version_at_source": 57}, "stage3_selected_phrase_ranks": {"male": 1, "looking_at_viewer": 2, "group": 6, "size_difference": 2, "larger_male": 4, "grin": 1, "skimpy": 5, "raised_arm": 2, "smaller_human": 9, "bent_arm": 3, "raised_arms": 1, "raised_hand": 4, "beckoning": 2, "petting": 2, "simple_shading": 1, "spread_arms": 6, "thick_eyelashes": 2, "smug_grin": 5, "toothy_grin": 2, "breaking_the_fourth_wall": 5, "background_character": 2, "partially_clothed_anthro": 10, "holding_arms": 9, "bright": 1, "interrupted": 1, "simple_face": 5, "unnamed_character": 2, "come_on_(interjection)": 3, "left-handed": 1, "hand_over_head": 1, "fran_(litterbox_comics)": 1, "revealing_clothes": 6, "rim_light": 2, "relaxed_expression": 1, "toothy_smile": 3, "toony_expression": 6, "smaller_version_at_source": 3}, "extra_evidence": {"background_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5068}, "beckoning": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4425}, "bent_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6739}, "breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4209}, "bright": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5873}, "come_on_(interjection)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3621}, "fran_(litterbox_comics)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4659}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5861}, "holding_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "interrupted": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4586}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "left-handed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4525}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4576}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4897}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6751}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "revealing_clothes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4777}, "rim_light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5332}, "simple_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4943}, "simple_shading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5424}, "size_difference": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5253}, "skimpy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "smaller_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3792}, "smaller_version_at_source": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5079}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.605}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5524}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}, "toothy_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5535}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4965}}, "structural": [], "t1": 0.0, "t2": 9.5, "t3": 15.97, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 14, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 1.0, "why": {"explicit": 14}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 28, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["caressing_face", "cheek_to_cheek", "cute_expression", "embrace", "expressions", "fantasy", "intimate", "petting", "romeo_(leobo)", "rubbing_cheek", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["caressing_face", "cheek_to_cheek", "cute_expression", "embrace", "expressions", "fantasy", "intimate", "petting", "romeo_(leobo)", "rubbing_cheek", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "stage3_selected": ["caressing_face", "cheek_to_cheek", "cute_expression", "embrace", "expressions", "fantasy", "intimate", "petting", "romeo_(leobo)", "rubbing_cheek", "setting", "shocked_face", "smiling_at_another", "wide_eyed"], "stage3_selected_scores": {"embrace": 0.5734, "wide_eyed": 0.5498, "fantasy": 0.5458, "cute_expression": 0.8106, "petting": 0.7411, "intimate": 0.5697, "expressions": 0.8783, "shocked_face": 0.616, "caressing_face": 0.5887, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "romeo_(leobo)": 0.5212, "rubbing_cheek": 0.6245, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "wide_eyed": 50, "fantasy": 52, "cute_expression": 6, "petting": 12, "intimate": 42, "expressions": 3, "shocked_face": 18, "caressing_face": 30, "smiling_at_another": 49, "cheek_to_cheek": 29, "romeo_(leobo)": 58, "rubbing_cheek": 16, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "wide_eyed": 7, "fantasy": 9, "cute_expression": 5, "petting": 2, "intimate": 5, "expressions": 2, "shocked_face": 1, "caressing_face": 2, "smiling_at_another": 10, "cheek_to_cheek": 8, "romeo_(leobo)": 2, "rubbing_cheek": 1, "setting": 1}, "extra_evidence": {"caressing_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5887}, "cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "cute_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8106}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "fantasy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5458}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "romeo_(leobo)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5212}, "rubbing_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6245}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5498}}, "structural": [], "t1": 0.0, "t2": 8.24, "t3": 11.02, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 12, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 12, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 3.0, "why": {"explicit": 13}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 24, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["air_creature", "cartoon_physics", "creature_print", "green_background", "grey_background", "larger_male", "outline_speech_bubble", "round_eyes", "round_nose", "sky_background", "the_fairly_oddparents", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["air_creature", "cartoon_physics", "creature_print", "green_background", "grey_background", "larger_male", "outline_speech_bubble", "round_eyes", "round_nose", "sky_background", "the_fairly_oddparents", "toony"], "stage3_selected": ["air_creature", "cartoon_physics", "creature_print", "green_background", "grey_background", "larger_male", "outline_speech_bubble", "round_eyes", "round_nose", "simple_background", "sky_background", "the_fairly_oddparents", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "larger_male": 0.6348, "grey_background": 0.688, "toony": 0.763, "green_background": 0.7077, "creature_print": 0.5405, "air_creature": 0.6051, "round_eyes": 0.6227, "the_fairly_oddparents": 0.6718, "outline_speech_bubble": 0.5811, "sky_background": 0.7602, "round_nose": 0.555, "cartoon_physics": 0.795}, "stage3_selected_ranks": {"simple_background": 10, "larger_male": 27, "grey_background": 18, "toony": 6, "green_background": 13, "creature_print": 68, "air_creature": 36, "round_eyes": 30, "the_fairly_oddparents": 20, "outline_speech_bubble": 46, "sky_background": 7, "round_nose": 59, "cartoon_physics": 3}, "stage3_selected_phrase_ranks": {"simple_background": 5, "larger_male": 3, "grey_background": 10, "toony": 3, "green_background": 7, "creature_print": 9, "air_creature": 1, "round_eyes": 1, "the_fairly_oddparents": 8, "outline_speech_bubble": 3, "sky_background": 2, "round_nose": 10, "cartoon_physics": 1}, "extra_evidence": {"air_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6051}, "cartoon_physics": {"source": "stage3", "why": "explicit", "retrieval_score": 0.795}, "creature_print": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5405}, "green_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7077}, "grey_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.688}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6348}, "outline_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5811}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "sky_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7602}, "the_fairly_oddparents": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6718}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.16, "t3": 11.81, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0556, "R": 0.0455, "F1": 0.05, "leaf_P": 0.0588, "leaf_R": 0.0833, "leaf_F1": 0.069, "n_leaf_sel": 17, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 0.5, "over_sel": 0.82, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 30, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 35, "attempts_by_n_local": {"30": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 1, "parse_fail": 0, "errors": 1}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.25, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0556, "gen_R": 0.0455, "gen_F1": 0.05, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_background", "striped_markings", "white_shorts"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_background", "striped_markings", "white_shorts"], "stage3_selected": ["blue_bottomwear", "blue_shorts", "dolphin_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_background", "striped_markings", "white_shorts"], "stage3_selected_scores": {"fur": 0.7785, "pose": 0.847, "neck_tuft": 0.5847, "raised_arm": 0.6005, "striped_markings": 0.5966, "blue_bottomwear": 0.6293, "hand_on_face": 0.5657, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "striped_background": 0.6234, "hand_gesture": 0.6065, "white_shorts": 0.621, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334, "dolphin_shorts": 0.7425}, "stage3_selected_ranks": {"fur": 8, "pose": 3, "neck_tuft": 51, "raised_arm": 42, "striped_markings": 46, "blue_bottomwear": 32, "hand_on_face": 55, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "striped_background": 34, "hand_gesture": 39, "white_shorts": 35, "hand_over_head": 56, "hand_on_own_head": 66, "dolphin_shorts": 10}, "stage3_selected_phrase_ranks": {"fur": 1, "pose": 1, "neck_tuft": 9, "raised_arm": 2, "striped_markings": 8, "blue_bottomwear": 2, "hand_on_face": 4, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "striped_background": 3, "hand_gesture": 1, "white_shorts": 4, "hand_over_head": 5, "hand_on_own_head": 6, "dolphin_shorts": 7}, "extra_evidence": {"blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "dolphin_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7425}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5657}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.621}}, "structural": [], "t1": 0.0, "t2": 0.75, "t3": 44.13, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 59, \"why\": \"weak_implied\"}, {\"i\": 60, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 32, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.37.why\n Field required [type=missing, input_value={'i': 38}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 5, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.6, "R": 0.2308, "F1": 0.3333, "leaf_P": 0.6, "leaf_R": 0.5, "leaf_F1": 0.5455, "n_leaf_sel": 5, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.375, "over_sel": 0.38, "why": {"explicit": 5}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 1, "calls_exhausted_retries": 2, "attempts_total": 7, "attempt_errors": 6, "attempt_parse_fail": 0, "attempt_parse_ok": 1, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 20, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 6, "parse_ok": 0, "parse_fail": 0, "errors": 6}}, "attempt_failure_rate": 0.8571428571428571, "call_exhaustion_rate": 0.6666666666666666}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.75, "gen_R": 0.2308, "gen_F1": 0.3529, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["darkness", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["darkness", "goat", "human", "lizard", "unnamed_character"], "stage3_selected": ["darkness", "goat", "human", "lizard", "unnamed_character"], "stage3_selected_scores": {"human": 0.6298, "lizard": 0.8035, "goat": 0.777, "darkness": 0.7908, "unnamed_character": 0.4967}, "stage3_selected_ranks": {"human": 6, "lizard": 1, "goat": 3, "darkness": 2, "unnamed_character": 50}, "stage3_selected_phrase_ranks": {"human": 1, "lizard": 1, "goat": 1, "darkness": 1, "unnamed_character": 1}, "extra_evidence": {"darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.6, "t3": 43.51, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"style_or_meta\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 35}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: gave up after 3 attempts", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 45}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"strong_implied\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"strong_implied\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"strong_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"strong_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"strong_implied\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 42}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 33, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.1818, "R": 0.4286, "F1": 0.2553, "leaf_P": 0.1071, "leaf_R": 0.25, "leaf_F1": 0.15, "n_leaf_sel": 28, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.75, "over_sel": 2.36, "why": {"explicit": 33}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 9, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 91, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.1875, "gen_R": 0.4286, "gen_F1": 0.2609, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "hair", "lying", "purple_hair", "text"], "extra": ["2_frame_animation", "anime_eyes", "annoyed_expression", "bed_covers", "bedding", "bedroom", "blanket", "blue_background", "blue_eyebrows", "bored_expression", "clown_makeup", "heterochromia", "lipstick", "lipstick_(object)", "lying_on_bed", "no_irises", "partial_line_speech_bubble", "purple_arms", "purple_face", "purple_hands", "red_lipstick", "sleeping_bag", "tired", "two_panel_image", "two_tone_hair", "unnamed_character", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["2_frame_animation", "anime_eyes", "annoyed_expression", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "bored_expression", "clown_makeup", "green_eyes", "heterochromia", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "no_irises", "partial_line_speech_bubble", "purple_arms", "purple_face", "purple_hands", "red_lipstick", "sleeping", "sleeping_bag", "tired", "two_panel_image", "two_tone_hair", "unnamed_character", "zzz"], "stage3_selected": ["2_frame_animation", "anime_eyes", "annoyed_expression", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "bored_expression", "clown_makeup", "green_eyes", "heterochromia", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "no_irises", "partial_line_speech_bubble", "purple_arms", "purple_face", "purple_hands", "red_lipstick", "sleeping", "sleeping_bag", "tired", "two_panel_image", "two_tone_hair", "unnamed_character", "zzz"], "stage3_selected_scores": {"blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "two_tone_hair": 0.3447, "lipstick": 0.4386, "bedroom": 0.4116, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "heterochromia": 0.3472, "blanket": 0.3523, "tired": 0.4965, "lying_on_bed": 0.412, "zzz": 0.367, "red_lipstick": 0.4186, "blue_eyebrows": 0.4608, "annoyed_expression": 0.3458, "bed_covers": 0.4011, "two_panel_image": 0.3289, "no_irises": 0.3837, "purple_face": 0.5961, "sleeping_bag": 0.4513, "anime_eyes": 0.3746, "unnamed_character": 0.3132, "bored_expression": 0.3613, "purple_hands": 0.6346, "clown_makeup": 0.4004, "partial_line_speech_bubble": 0.3487, "lipstick_(object)": 0.363, "purple_arms": 0.6358, "2_frame_animation": 0.3038}, "stage3_selected_ranks": {"blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "bed": 15, "makeup": 11, "two_tone_hair": 100, "lipstick": 29, "bedroom": 39, "blue_background": 105, "sleeping": 20, "bedding": 33, "heterochromia": 95, "blanket": 86, "tired": 21, "lying_on_bed": 38, "zzz": 68, "red_lipstick": 34, "blue_eyebrows": 24, "annoyed_expression": 97, "bed_covers": 44, "two_panel_image": 117, "no_irises": 55, "purple_face": 8, "sleeping_bag": 25, "anime_eyes": 65, "unnamed_character": 119, "bored_expression": 76, "purple_hands": 5, "clown_makeup": 45, "partial_line_speech_bubble": 92, "lipstick_(object)": 75, "purple_arms": 4, "2_frame_animation": 127}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "bed": 1, "makeup": 1, "two_tone_hair": 10, "lipstick": 2, "bedroom": 7, "blue_background": 10, "sleeping": 2, "bedding": 5, "heterochromia": 7, "blanket": 10, "tired": 3, "lying_on_bed": 6, "zzz": 8, "red_lipstick": 4, "blue_eyebrows": 2, "annoyed_expression": 9, "bed_covers": 9, "two_panel_image": 4, "no_irises": 2, "purple_face": 2, "sleeping_bag": 4, "anime_eyes": 6, "unnamed_character": 6, "bored_expression": 5, "purple_hands": 1, "clown_makeup": 6, "partial_line_speech_bubble": 4, "lipstick_(object)": 8, "purple_arms": 2, "2_frame_animation": 2}, "extra_evidence": {"2_frame_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3038}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3746}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "blue_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4608}, "bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "clown_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "partial_line_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3487}, "purple_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6358}, "purple_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5961}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6346}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "sleeping_bag": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4513}, "tired": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4965}, "two_panel_image": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3289}, "two_tone_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3447}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3132}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}}, "structural": [], "t1": 0.0, "t2": 1.42, "t3": 22.77, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 19, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.2105, "R": 0.1818, "F1": 0.1951, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 14, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.8, "over_sel": 0.86, "why": {"explicit": 21}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 42, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 56, "attempts_by_n_local": {"42": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2105, "gen_R": 0.1818, "gen_F1": 0.1951, "missed": ["anthro", "bass_guitar", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["acoustic_guitar", "action_pose", "big_claws", "black_claws", "digitigrade", "glowing_claws", "heel_claw", "lotus_pose", "portrait", "pose", "rottweiler", "toony_expression", "torn_shorts", "torn_topwear", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["acoustic_guitar", "action_pose", "big_claws", "black_claws", "canid", "canine", "claws", "digitigrade", "glowing_claws", "heel_claw", "lotus_pose", "portrait", "pose", "rottweiler", "toony_expression", "torn_clothing", "torn_shorts", "torn_topwear", "wolf"], "stage3_selected": ["acoustic_guitar", "action_pose", "big_claws", "black_claws", "canid", "canine", "claws", "digitigrade", "glowing_claws", "gradient_background", "heel_claw", "lotus_pose", "portrait", "pose", "rottweiler", "textured_background", "toony_expression", "torn_clothing", "torn_shorts", "torn_topwear", "wolf"], "stage3_selected_scores": {"canid": 0.6217, "canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "portrait": 0.6102, "pose": 0.8955, "digitigrade": 0.7162, "torn_clothing": 0.5179, "gradient_background": 0.636, "black_claws": 0.6961, "action_pose": 0.6726, "rottweiler": 0.5579, "torn_topwear": 0.4853, "textured_background": 0.6672, "big_claws": 0.7461, "torn_shorts": 0.4967, "heel_claw": 0.5951, "glowing_claws": 0.5992, "acoustic_guitar": 0.3382, "lotus_pose": 0.636, "toony_expression": 0.4641}, "stage3_selected_ranks": {"canid": 44, "canine": 45, "claws": 1, "wolf": 23, "portrait": 47, "pose": 2, "digitigrade": 10, "torn_clothing": 73, "gradient_background": 36, "black_claws": 14, "action_pose": 19, "rottweiler": 66, "torn_topwear": 88, "textured_background": 20, "big_claws": 7, "torn_shorts": 80, "heel_claw": 57, "glowing_claws": 56, "acoustic_guitar": 115, "lotus_pose": 35, "toony_expression": 98}, "stage3_selected_phrase_ranks": {"canid": 5, "canine": 6, "claws": 1, "wolf": 2, "portrait": 7, "pose": 1, "digitigrade": 5, "torn_clothing": 4, "gradient_background": 7, "black_claws": 4, "action_pose": 2, "rottweiler": 8, "torn_topwear": 10, "textured_background": 1, "big_claws": 1, "torn_shorts": 7, "heel_claw": 10, "glowing_claws": 9, "acoustic_guitar": 7, "lotus_pose": 9, "toony_expression": 4}, "extra_evidence": {"acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3382}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6961}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "glowing_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "heel_claw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5951}, "lotus_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.636}, "portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6102}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "rottweiler": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5579}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4641}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}}, "structural": [], "t1": 0.0, "t2": 1.01, "t3": 14.99, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 21, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.2381, "R": 0.2174, "F1": 0.2273, "leaf_P": 0.1667, "leaf_R": 0.2, "leaf_F1": 0.1818, "n_leaf_sel": 18, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 0.91, "why": {"explicit": 21}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 35, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 12, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 42, "attempts_by_n_local": {"35": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2381, "gen_R": 0.2174, "gen_F1": 0.2273, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "standing", "topwear"], "extra": ["5_claws", "black_claws", "black_pants", "black_topwear", "blue_pants", "cargo_pants", "clothed_anthro", "grey_claws", "grey_shirt", "grey_theme", "grey_topwear", "neck_tuft", "t-shirt", "white_bottomwear", "white_dress_shirt", "white_shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["5_claws", "black_claws", "black_pants", "black_topwear", "blue_pants", "cargo_pants", "claws", "clothed_anthro", "facial_markings", "fur", "grey_background", "grey_claws", "grey_shirt", "grey_theme", "grey_topwear", "neck_tuft", "shirt", "t-shirt", "white_bottomwear", "white_dress_shirt", "white_shirt"], "stage3_selected": ["5_claws", "black_claws", "black_pants", "black_topwear", "blue_pants", "cargo_pants", "claws", "clothed_anthro", "facial_markings", "fur", "grey_background", "grey_claws", "grey_shirt", "grey_theme", "grey_topwear", "neck_tuft", "shirt", "t-shirt", "white_bottomwear", "white_dress_shirt", "white_shirt"], "stage3_selected_scores": {"fur": 0.7011, "claws": 0.6687, "shirt": 0.7037, "grey_background": 0.5635, "neck_tuft": 0.5864, "facial_markings": 0.9015, "black_claws": 0.5947, "t-shirt": 0.6707, "black_topwear": 0.5595, "white_shirt": 0.5609, "black_pants": 0.5822, "blue_pants": 0.5806, "grey_topwear": 0.5774, "grey_claws": 0.5681, "white_bottomwear": 0.5331, "clothed_anthro": 0.5317, "grey_shirt": 0.6272, "cargo_pants": 0.625, "grey_theme": 0.5547, "white_dress_shirt": 0.5611, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "claws": 19, "shirt": 13, "grey_background": 58, "neck_tuft": 38, "facial_markings": 1, "black_claws": 34, "t-shirt": 18, "black_topwear": 65, "white_shirt": 63, "black_pants": 39, "blue_pants": 41, "grey_topwear": 45, "grey_claws": 54, "white_bottomwear": 77, "clothed_anthro": 78, "grey_shirt": 25, "cargo_pants": 26, "grey_theme": 69, "white_dress_shirt": 62, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "claws": 1, "shirt": 1, "grey_background": 3, "neck_tuft": 2, "facial_markings": 1, "black_claws": 2, "t-shirt": 2, "black_topwear": 10, "white_shirt": 4, "black_pants": 5, "blue_pants": 7, "grey_topwear": 1, "grey_claws": 9, "white_bottomwear": 8, "clothed_anthro": 9, "grey_shirt": 7, "cargo_pants": 3, "grey_theme": 8, "white_dress_shirt": 3, "5_claws": 1}, "extra_evidence": {"5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5947}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5822}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "blue_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5806}, "cargo_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5317}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5681}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5547}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5864}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6707}, "white_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5331}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5609}}, "structural": [], "t1": 0.0, "t2": 0.94, "t3": 14.03, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 35, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0286, "R": 0.0909, "F1": 0.0435, "leaf_P": 0.0303, "leaf_R": 0.1429, "leaf_F1": 0.05, "n_leaf_sel": 33, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 3.18, "why": {"explicit": 37}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 81, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0286, "gen_R": 0.0909, "gen_F1": 0.0435, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", "action_pose", "anime_eyes", "blue_background", "blue_fur", "blue_markings", "colorful_background", "curved_tail", "eyelids", "eyes", "full-length_portrait", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "midair", "model_sheet", "no_irises", "partially_clothed_anthro", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "raised_paw", "smiling_at_viewer", "sparkling_background", "striped_arms", "striped_legs", "stripes", "teal_fur", "white_stripes", "why"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", "action_pose", "anime_eyes", "blue_background", "blue_fur", "blue_markings", "blue_nose", "colorful_background", "curved_tail", "eyelids", "eyes", "full-length_portrait", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "midair", "model_sheet", "no_irises", "partially_clothed_anthro", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "raised_paw", "smiling_at_viewer", "sparkling_background", "striped_arms", "striped_legs", "stripes", "teal_fur", "white_stripes", "why"], "stage3_selected": [">:)", "action_pose", "anime_eyes", "blue_background", "blue_fur", "blue_markings", "blue_nose", "colorful_background", "curved_tail", "eyelids", "eyes", "full-length_portrait", "gradient_background", "hand_on_floor", "happy", "iconography", "irregular_speech_bubble", "jumper", "midair", "model_sheet", "no_irises", "partially_clothed_anthro", "paw_pose", "pink_stripes", "playful", "pose", "purple_stripes", "raised_paw", "simple_background", "smiling_at_viewer", "sparkling_background", "striped_arms", "striped_legs", "stripes", "teal_fur", "white_stripes", "why"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "full-length_portrait": 0.4491, "gradient_background": 0.4138, "blue_background": 0.4216, "model_sheet": 0.3669, "smiling_at_viewer": 0.451, "blue_nose": 0.5017, "blue_markings": 0.4512, "action_pose": 0.4596, "why": 0.2766, "playful": 0.6118, "white_stripes": 0.6473, "striped_legs": 0.6654, "striped_arms": 0.6782, "purple_stripes": 0.4704, "midair": 0.43, "iconography": 0.3229, "raised_paw": 0.3182, "eyelids": 0.5312, "paw_pose": 0.3304, "partially_clothed_anthro": 0.5986, "no_irises": 0.4942, "pink_stripes": 0.5235, "curved_tail": 0.5875, "anime_eyes": 0.4019, ">:)": 0.3449, "colorful_background": 0.3196, "irregular_speech_bubble": 0.3253, "sparkling_background": 0.4342, "hand_on_floor": 0.497, "jumper": 0.4127, "eyes": 0.97, "teal_fur": 0.4161}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "stripes": 18, "pose": 19, "happy": 101, "full-length_portrait": 63, "gradient_background": 80, "blue_background": 76, "model_sheet": 95, "smiling_at_viewer": 62, "blue_nose": 45, "blue_markings": 61, "action_pose": 58, "why": 144, "playful": 17, "white_stripes": 14, "striped_legs": 12, "striped_arms": 10, "purple_stripes": 56, "midair": 71, "iconography": 130, "raised_paw": 134, "eyelids": 34, "paw_pose": 122, "partially_clothed_anthro": 20, "no_irises": 49, "pink_stripes": 37, "curved_tail": 22, "anime_eyes": 84, ">:)": 113, "colorful_background": 131, "irregular_speech_bubble": 127, "sparkling_background": 67, "hand_on_floor": 47, "jumper": 81, "eyes": 1, "teal_fur": 78}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "stripes": 10, "pose": 1, "happy": 3, "full-length_portrait": 9, "gradient_background": 9, "blue_background": 3, "model_sheet": 1, "smiling_at_viewer": 8, "blue_nose": 3, "blue_markings": 6, "action_pose": 3, "why": 2, "playful": 1, "white_stripes": 9, "striped_legs": 8, "striped_arms": 6, "purple_stripes": 4, "midair": 2, "iconography": 9, "raised_paw": 9, "eyelids": 7, "paw_pose": 7, "partially_clothed_anthro": 3, "no_irises": 2, "pink_stripes": 1, "curved_tail": 3, "anime_eyes": 7, ">:)": 9, "colorful_background": 9, "irregular_speech_bubble": 7, "sparkling_background": 1, "hand_on_floor": 10, "jumper": 4, "eyes": 1, "teal_fur": 10}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4216}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4512}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3196}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "eyelids": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5312}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "hand_on_floor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.497}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "iconography": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3229}, "irregular_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3253}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "model_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3669}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4942}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5986}, "paw_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3304}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "purple_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4704}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "sparkling_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4342}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6782}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6654}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "teal_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4161}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6473}, "why": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2766}}, "structural": [], "t1": 0.0, "t2": 1.34, "t3": 26.27, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p5.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:27:01.085216", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 14}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 8, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 1.17, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 30, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["black_necktie", "business_suit", "dress_shirt", "face_focus", "necktie", "shirt", "solo_focus", "t-shirt", "tank_top", "topwear", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["black_necktie", "business_suit", "dress_shirt", "face_focus", "necktie", "shirt", "solo_focus", "t-shirt", "tank_top", "topwear", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["black_necktie", "business_suit", "dress_shirt", "face_focus", "necktie", "shirt", "solo_focus", "t-shirt", "tank_top", "topwear", "transparent_background", "white_clothing", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"topwear": 0.7782, "shirt": 0.9019, "solo_focus": 0.4386, "white_clothing": 0.6182, "tank_top": 0.7545, "necktie": 0.8082, "transparent_background": 0.5794, "t-shirt": 0.8329, "white_topwear": 0.6638, "dress_shirt": 0.763, "business_suit": 0.6178, "white_t-shirt": 0.6584, "black_necktie": 0.625, "white_dress_shirt": 0.6255, "face_focus": 0.4971}, "stage3_selected_ranks": {"topwear": 4, "shirt": 1, "solo_focus": 64, "white_clothing": 25, "tank_top": 8, "necktie": 3, "transparent_background": 39, "t-shirt": 2, "white_topwear": 13, "dress_shirt": 6, "business_suit": 26, "white_t-shirt": 15, "black_necktie": 23, "white_dress_shirt": 22, "face_focus": 56}, "stage3_selected_phrase_ranks": {"topwear": 3, "shirt": 1, "solo_focus": 7, "white_clothing": 7, "tank_top": 7, "necktie": 1, "transparent_background": 4, "t-shirt": 2, "white_topwear": 1, "dress_shirt": 2, "business_suit": 4, "white_t-shirt": 2, "black_necktie": 6, "white_dress_shirt": 5, "face_focus": 6}, "extra_evidence": {"black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}, "face_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4971}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "solo_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7545}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7782}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6255}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.21, "t3": 7.14, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 9, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.3333, "R": 0.2143, "F1": 0.2609, "leaf_P": 0.3333, "leaf_R": 0.3, "leaf_F1": 0.3158, "n_leaf_sel": 9, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 1.0, "over_sel": 0.64, "why": {"explicit": 9}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 64, "bootstrap_anchor_selected": 23, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 100, "attempts_by_n_local": {"64": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.2, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3333, "gen_R": 0.2143, "gen_F1": 0.2609, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "mammal", "primate"], "extra": ["animated_comic", "grin", "raised_arms", "skimpy", "thick_eyelashes", "toothy_grin"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["animated_comic", "grin", "group", "looking_at_viewer", "male", "raised_arms", "skimpy", "thick_eyelashes", "toothy_grin"], "stage3_selected": ["animated_comic", "grin", "group", "looking_at_viewer", "male", "raised_arms", "skimpy", "thick_eyelashes", "toothy_grin"], "stage3_selected_scores": {"male": 0.6605, "looking_at_viewer": 0.5612, "group": 0.4099, "grin": 0.7174, "skimpy": 0.4928, "raised_arms": 0.8636, "thick_eyelashes": 0.5524, "toothy_grin": 0.6248, "animated_comic": 0.4028}, "stage3_selected_ranks": {"male": 7, "looking_at_viewer": 22, "group": 132, "grin": 3, "skimpy": 71, "raised_arms": 1, "thick_eyelashes": 24, "toothy_grin": 13, "animated_comic": 133}, "stage3_selected_phrase_ranks": {"male": 1, "looking_at_viewer": 2, "group": 6, "grin": 1, "skimpy": 5, "raised_arms": 1, "thick_eyelashes": 2, "toothy_grin": 2, "animated_comic": 7}, "extra_evidence": {"animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4028}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "skimpy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5524}, "toothy_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6248}}, "structural": [], "t1": 0.0, "t2": 9.55, "t3": 36.27, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"other\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"style_or_meta\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 31, \"why\": \"style_or_meta\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 8, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.125, "R": 0.0714, "F1": 0.0909, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 8, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 0.57, "why": {"explicit": 8}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 28, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.125, "gen_R": 0.0714, "gen_F1": 0.0909, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "embrace", "expressions", "intimate", "setting", "shocked_face", "wide_eyed"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "embrace", "expressions", "intimate", "romantic", "setting", "shocked_face", "wide_eyed"], "stage3_selected": ["cheek_to_cheek", "embrace", "expressions", "intimate", "romantic", "setting", "shocked_face", "wide_eyed"], "stage3_selected_scores": {"embrace": 0.5734, "romantic": 0.5683, "wide_eyed": 0.5498, "intimate": 0.5697, "expressions": 0.8783, "shocked_face": 0.616, "cheek_to_cheek": 0.5892, "setting": 1.0}, "stage3_selected_ranks": {"embrace": 40, "romantic": 43, "wide_eyed": 50, "intimate": 42, "expressions": 3, "shocked_face": 18, "cheek_to_cheek": 29, "setting": 2}, "stage3_selected_phrase_ranks": {"embrace": 4, "romantic": 6, "wide_eyed": 7, "intimate": 5, "expressions": 2, "shocked_face": 1, "cheek_to_cheek": 8, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5734}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5498}}, "structural": [], "t1": 0.0, "t2": 8.19, "t3": 14.08, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 7, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 7, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.75, "why": {"explicit": 8}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 21, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["grey_background", "rosy_cheeks", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["grey_background", "rosy_cheeks", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected": ["grey_background", "rosy_cheeks", "round_eyes", "round_nose", "simple_background", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "grey_background": 0.688, "toony": 0.763, "thick_eyelashes": 0.536, "rosy_cheeks": 0.6019, "round_eyes": 0.6227, "round_nose": 0.555, "stylized_empty_eyes": 0.6448}, "stage3_selected_ranks": {"simple_background": 10, "grey_background": 18, "toony": 6, "thick_eyelashes": 69, "rosy_cheeks": 38, "round_eyes": 30, "round_nose": 59, "stylized_empty_eyes": 25}, "stage3_selected_phrase_ranks": {"simple_background": 5, "grey_background": 10, "toony": 3, "thick_eyelashes": 10, "rosy_cheeks": 8, "round_eyes": 1, "round_nose": 10, "stylized_empty_eyes": 1}, "extra_evidence": {"grey_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.688}, "rosy_cheeks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6019}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "stylized_empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6448}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.536}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.15, "t3": 7.65, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 17, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 16, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 0.0, "over_sel": 0.77, "why": {"explicit": 17}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 30, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 41, "attempts_by_n_local": {"30": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["action_pose", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_markings"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_markings"], "stage3_selected": ["action_pose", "blue_shorts", "dolphin_shorts", "grey_fur", "hand_gesture", "hand_on_face", "hand_on_own_head", "hand_over_head", "happy", "muscle_tone", "neck_tuft", "playful", "playing", "pose", "raised_arm", "raised_finger", "striped_markings"], "stage3_selected_scores": {"grey_fur": 0.5932, "pose": 0.847, "happy": 0.457, "neck_tuft": 0.5847, "raised_arm": 0.6005, "striped_markings": 0.5966, "hand_on_face": 0.5657, "action_pose": 0.6359, "playful": 0.6803, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "muscle_tone": 0.5438, "hand_gesture": 0.6065, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334, "dolphin_shorts": 0.7425}, "stage3_selected_ranks": {"grey_fur": 48, "pose": 3, "happy": 87, "neck_tuft": 51, "raised_arm": 42, "striped_markings": 46, "hand_on_face": 55, "action_pose": 31, "playful": 18, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "muscle_tone": 62, "hand_gesture": 39, "hand_over_head": 56, "hand_on_own_head": 66, "dolphin_shorts": 10}, "stage3_selected_phrase_ranks": {"grey_fur": 8, "pose": 1, "happy": 5, "neck_tuft": 9, "raised_arm": 2, "striped_markings": 8, "hand_on_face": 4, "action_pose": 8, "playful": 1, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "muscle_tone": 6, "hand_gesture": 1, "hand_over_head": 5, "hand_on_own_head": 6, "dolphin_shorts": 7}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "dolphin_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7425}, "grey_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5932}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5657}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.457}, "muscle_tone": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5438}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.847}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}}, "structural": [], "t1": 0.0, "t2": 0.82, "t3": 10.38, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 13, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.2308, "R": 0.2308, "F1": 0.2308, "leaf_P": 0.25, "leaf_R": 0.5, "leaf_F1": 0.3333, "n_leaf_sel": 12, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.375, "over_sel": 1.0, "why": {"explicit": 13}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 89, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.25, "gen_R": 0.2308, "gen_F1": 0.24, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["3_panel_comic", "4_panel_comic", "dark", "dark_theme", "darkness", "group", "light", "light_beam", "speech_bubble", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["3_panel_comic", "4_panel_comic", "dark", "dark_theme", "darkness", "goat", "group", "human", "light", "light_beam", "lizard", "speech_bubble", "unnamed_character"], "stage3_selected": ["3_panel_comic", "4_panel_comic", "dark", "dark_theme", "darkness", "goat", "group", "human", "light", "light_beam", "lizard", "speech_bubble", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "human": 0.6298, "speech_bubble": 0.551, "lizard": 0.8035, "goat": 0.777, "light": 0.5737, "dark": 0.5246, "light_beam": 0.5797, "darkness": 0.7908, "dark_theme": 0.5093, "unnamed_character": 0.4967, "3_panel_comic": 0.4885, "4_panel_comic": 0.4744}, "stage3_selected_ranks": {"group": 110, "human": 6, "speech_bubble": 23, "lizard": 1, "goat": 3, "light": 15, "dark": 37, "light_beam": 12, "darkness": 2, "dark_theme": 43, "unnamed_character": 50, "3_panel_comic": 56, "4_panel_comic": 70}, "stage3_selected_phrase_ranks": {"group": 5, "human": 1, "speech_bubble": 1, "lizard": 1, "goat": 1, "light": 4, "dark": 3, "light_beam": 2, "darkness": 1, "dark_theme": 4, "unnamed_character": 1, "3_panel_comic": 2, "4_panel_comic": 4}, "extra_evidence": {"3_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4885}, "4_panel_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4744}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5246}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5093}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5737}, "light_beam": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5797}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.551}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.52, "t3": 31.2, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"style_or_meta\"}, {\"i\": 43, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 42}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 28, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.25, "R": 0.5, "F1": 0.3333, "leaf_P": 0.1667, "leaf_R": 0.3333, "leaf_F1": 0.2222, "n_leaf_sel": 24, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 2.0, "why": {"explicit": 28}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 84, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.2593, "gen_R": 0.5, "gen_F1": 0.3415, "missed": ["angry", "duo", "eyes_closed", "eyeshadow", "lying", "purple_hair", "text"], "extra": ["2_frame_animation", "annoyed_expression", "applying_makeup", "blanket", "blue_background", "bored_expression", "clown_makeup", "eyes", "heart_before_text", "heterochromia", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "purple_text", "red_lipstick", "relaxed_expression", "sleepwear", "two_tone_hair", "unnamed_character", "yellow_eyeshadow"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["2_frame_animation", "annoyed_expression", "applying_makeup", "bed", "blanket", "blonde_hair", "blue_background", "blue_eyes", "bored_expression", "clown_makeup", "eyes", "green_eyes", "hair", "heart_before_text", "heterochromia", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "purple_text", "red_lipstick", "relaxed_expression", "sleeping", "sleepwear", "two_tone_hair", "unnamed_character", "yellow_eyeshadow"], "stage3_selected": ["2_frame_animation", "annoyed_expression", "applying_makeup", "bed", "blanket", "blonde_hair", "blue_background", "blue_eyes", "bored_expression", "clown_makeup", "eyes", "green_eyes", "hair", "heart_before_text", "heterochromia", "lidded_eyes", "lipstick", "lipstick_(object)", "lying_on_bed", "makeup", "purple_text", "red_lipstick", "relaxed_expression", "sleeping", "sleepwear", "two_tone_hair", "unnamed_character", "yellow_eyeshadow"], "stage3_selected_scores": {"hair": 0.5386, "blue_eyes": 0.5401, "green_eyes": 0.538, "blonde_hair": 0.346, "bed": 0.5296, "makeup": 0.544, "two_tone_hair": 0.3447, "lipstick": 0.4386, "blue_background": 0.3407, "sleeping": 0.4995, "heterochromia": 0.3472, "blanket": 0.3523, "lidded_eyes": 0.3759, "lying_on_bed": 0.412, "red_lipstick": 0.4186, "annoyed_expression": 0.3458, "purple_text": 0.5604, "heart_before_text": 0.3566, "unnamed_character": 0.3132, "bored_expression": 0.3613, "sleepwear": 0.4346, "yellow_eyeshadow": 0.4068, "clown_makeup": 0.4004, "lipstick_(object)": 0.363, "applying_makeup": 0.431, "relaxed_expression": 0.388, "eyes": 0.9321, "2_frame_animation": 0.3038}, "stage3_selected_ranks": {"hair": 13, "blue_eyes": 12, "green_eyes": 14, "blonde_hair": 96, "bed": 15, "makeup": 11, "two_tone_hair": 100, "lipstick": 29, "blue_background": 105, "sleeping": 20, "heterochromia": 95, "blanket": 86, "lidded_eyes": 63, "lying_on_bed": 38, "red_lipstick": 34, "annoyed_expression": 97, "purple_text": 10, "heart_before_text": 81, "unnamed_character": 119, "bored_expression": 76, "sleepwear": 30, "yellow_eyeshadow": 42, "clown_makeup": 45, "lipstick_(object)": 75, "applying_makeup": 31, "relaxed_expression": 53, "eyes": 1, "2_frame_animation": 127}, "stage3_selected_phrase_ranks": {"hair": 1, "blue_eyes": 1, "green_eyes": 1, "blonde_hair": 8, "bed": 1, "makeup": 1, "two_tone_hair": 10, "lipstick": 2, "blue_background": 10, "sleeping": 2, "heterochromia": 7, "blanket": 10, "lidded_eyes": 6, "lying_on_bed": 6, "red_lipstick": 4, "annoyed_expression": 9, "purple_text": 3, "heart_before_text": 3, "unnamed_character": 6, "bored_expression": 5, "sleepwear": 5, "yellow_eyeshadow": 7, "clown_makeup": 6, "lipstick_(object)": 8, "applying_makeup": 1, "relaxed_expression": 2, "eyes": 1, "2_frame_animation": 2}, "extra_evidence": {"2_frame_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3038}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3458}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.431}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "clown_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9321}, "heart_before_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3566}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3759}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "lipstick_(object)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.363}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "purple_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5604}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4346}, "two_tone_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3447}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3132}, "yellow_eyeshadow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4068}}, "structural": [], "t1": 0.0, "t2": 1.39, "t3": 20.45, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1111, "R": 0.1364, "F1": 0.1224, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 21, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 0.6, "over_sel": 1.23, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 42, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 59, "attempts_by_n_local": {"42": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1111, "gen_R": 0.1364, "gen_F1": 0.1224, "missed": ["anthro", "bass_guitar", "canine", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["4_toes", "action_pose", "big_claws", "black_claws", "canis", "color_swatch", "dewclaw", "digitigrade", "dire_wolf", "finger_claws", "flat_colors", "full-length_portrait", "gold_claws", "heel_claw", "long_claws", "orange_background", "portrait", "pose", "toony_expression", "torn_leggings", "torn_topwear", "warm_colors", "wolf", "yellow_background"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_toes", "action_pose", "big_claws", "black_claws", "canid", "canis", "claws", "color_swatch", "dewclaw", "digitigrade", "dire_wolf", "finger_claws", "flat_colors", "full-length_portrait", "gold_claws", "heel_claw", "long_claws", "orange_background", "portrait", "pose", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "warm_colors", "wolf", "yellow_background"], "stage3_selected": ["4_toes", "action_pose", "big_claws", "black_claws", "canid", "canis", "claws", "color_swatch", "dewclaw", "digitigrade", "dire_wolf", "finger_claws", "flat_colors", "full-length_portrait", "gold_claws", "heel_claw", "long_claws", "orange_background", "portrait", "pose", "toony_expression", "torn_clothing", "torn_leggings", "torn_topwear", "warm_colors", "wolf", "yellow_background"], "stage3_selected_scores": {"canid": 0.6217, "canis": 0.6493, "claws": 0.898, "wolf": 0.6635, "portrait": 0.6102, "4_toes": 0.695, "pose": 0.8955, "digitigrade": 0.7162, "full-length_portrait": 0.6018, "torn_clothing": 0.5179, "finger_claws": 0.7339, "black_claws": 0.6961, "flat_colors": 0.6046, "yellow_background": 0.6256, "orange_background": 0.6518, "action_pose": 0.6726, "color_swatch": 0.7261, "warm_colors": 0.6979, "torn_topwear": 0.4853, "long_claws": 0.6953, "dewclaw": 0.6939, "big_claws": 0.7461, "dire_wolf": 0.6438, "heel_claw": 0.5951, "torn_leggings": 0.4969, "gold_claws": 0.6234, "toony_expression": 0.4641}, "stage3_selected_ranks": {"canid": 44, "canis": 25, "claws": 1, "wolf": 23, "portrait": 47, "4_toes": 16, "pose": 2, "digitigrade": 10, "full-length_portrait": 54, "torn_clothing": 73, "finger_claws": 8, "black_claws": 14, "flat_colors": 50, "yellow_background": 42, "orange_background": 24, "action_pose": 19, "color_swatch": 9, "warm_colors": 13, "torn_topwear": 88, "long_claws": 15, "dewclaw": 17, "big_claws": 7, "dire_wolf": 30, "heel_claw": 57, "torn_leggings": 79, "gold_claws": 43, "toony_expression": 98}, "stage3_selected_phrase_ranks": {"canid": 5, "canis": 2, "claws": 1, "wolf": 2, "portrait": 7, "4_toes": 9, "pose": 1, "digitigrade": 5, "full-length_portrait": 10, "torn_clothing": 4, "finger_claws": 4, "black_claws": 4, "flat_colors": 9, "yellow_background": 10, "orange_background": 2, "action_pose": 2, "color_swatch": 1, "warm_colors": 2, "torn_topwear": 10, "long_claws": 2, "dewclaw": 10, "big_claws": 1, "dire_wolf": 4, "heel_claw": 10, "torn_leggings": 6, "gold_claws": 3, "toony_expression": 4}, "extra_evidence": {"4_toes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.695}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6961}, "canis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6493}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "dewclaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6939}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6438}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7339}, "flat_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6046}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6018}, "gold_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6234}, "heel_claw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5951}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6102}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4641}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "warm_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6979}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6256}}, "structural": [], "t1": 0.0, "t2": 1.08, "t3": 18.99, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 11, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.4545, "R": 0.2174, "F1": 0.2941, "leaf_P": 0.375, "leaf_R": 0.2, "leaf_F1": 0.2609, "n_leaf_sel": 8, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 0.48, "why": {"explicit": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 35, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 57, "attempts_by_n_local": {"35": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4545, "gen_R": 0.2174, "gen_F1": 0.2941, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "standing", "topwear"], "extra": ["5_claws", "black_topwear", "grey_claws", "grey_shirt", "grey_topwear", "white_dress_shirt"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["5_claws", "black_topwear", "claws", "facial_markings", "fur", "grey_background", "grey_claws", "grey_shirt", "grey_topwear", "shirt", "white_dress_shirt"], "stage3_selected": ["5_claws", "black_topwear", "claws", "facial_markings", "fur", "grey_background", "grey_claws", "grey_shirt", "grey_topwear", "shirt", "simple_background", "white_dress_shirt"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "shirt": 0.7037, "grey_background": 0.5635, "facial_markings": 0.9015, "black_topwear": 0.5595, "grey_topwear": 0.5774, "grey_claws": 0.5681, "grey_shirt": 0.6272, "white_dress_shirt": 0.5611, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "shirt": 13, "grey_background": 58, "facial_markings": 1, "black_topwear": 65, "grey_topwear": 45, "grey_claws": 54, "grey_shirt": 25, "white_dress_shirt": 62, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "shirt": 1, "grey_background": 3, "facial_markings": 1, "black_topwear": 10, "grey_topwear": 1, "grey_claws": 9, "grey_shirt": 7, "white_dress_shirt": 3, "5_claws": 1}, "extra_evidence": {"5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5681}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}}, "structural": [], "t1": 0.0, "t2": 0.89, "t3": 12.86, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 36, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0278, "R": 0.0909, "F1": 0.0426, "leaf_P": 0.0294, "leaf_R": 0.1429, "leaf_F1": 0.0488, "n_leaf_sel": 34, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 3.27, "why": {"explicit": 39}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 98, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.2, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0278, "gen_R": 0.0909, "gen_F1": 0.0426, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": [">:)", ">:d", "action_pose", "anime_eyes", "big_eyes", "blue_background", "blue_fur", "blue_paws", "colored_line_art", "colorful_background", "curved_tail", "eyes", "flat_colors", "full-length_portrait", "glistening_eyes", "hand_on_ground", "handpaw", "happy", "jumper", "jumping", "midair", "partially_clothed_anthro", "pink_stripes", "playful", "pose", "pride_colors", "purple_pupils", "simple_coloring", "simple_eyes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "sparkling_background", "striped_neck", "stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": [">:)", ">:d", "action_pose", "anime_eyes", "big_eyes", "blue_background", "blue_fur", "blue_nose", "blue_paws", "colored_line_art", "colorful_background", "curved_tail", "eyes", "flat_colors", "full-length_portrait", "glistening_eyes", "hand_on_ground", "handpaw", "happy", "jumper", "jumping", "midair", "partially_clothed_anthro", "pink_stripes", "playful", "pose", "pride_colors", "purple_pupils", "simple_coloring", "simple_eyes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "sparkling_background", "striped_neck", "stripes"], "stage3_selected": [">:)", ">:d", "abstract_background", "action_pose", "anime_eyes", "big_eyes", "blue_background", "blue_fur", "blue_nose", "blue_paws", "colored_line_art", "colorful_background", "curved_tail", "eyes", "flat_colors", "full-length_portrait", "glistening_eyes", "gradient_background", "hand_on_ground", "handpaw", "happy", "jumper", "jumping", "midair", "partially_clothed_anthro", "pink_stripes", "playful", "pose", "pride_colors", "purple_pupils", "simple_background", "simple_coloring", "simple_eyes", "six-stripe_rainbow_pride_colors", "sketch_background", "smiling_at_viewer", "sparkling_background", "striped_neck", "stripes"], "stage3_selected_scores": {"simple_background": 0.401, "blue_fur": 0.4171, "stripes": 0.6073, "pose": 0.6068, "happy": 0.36, "full-length_portrait": 0.4491, "abstract_background": 0.3542, "gradient_background": 0.4138, "blue_background": 0.4216, "glistening_eyes": 0.4811, "smiling_at_viewer": 0.451, "blue_nose": 0.5017, "big_eyes": 0.4619, "flat_colors": 0.4218, "pride_colors": 0.2723, "action_pose": 0.4596, "jumping": 0.5295, "simple_eyes": 0.3912, "handpaw": 0.3564, "playful": 0.6118, "midair": 0.43, "six-stripe_rainbow_pride_colors": 0.263, "simple_coloring": 0.4099, "partially_clothed_anthro": 0.5986, "hand_on_ground": 0.5793, "blue_paws": 0.5068, "pink_stripes": 0.5235, "purple_pupils": 0.5129, "curved_tail": 0.5875, "striped_neck": 0.711, "anime_eyes": 0.4019, ">:d": 0.3455, ">:)": 0.3449, "colorful_background": 0.3196, "colored_line_art": 0.3343, "sketch_background": 0.3715, "sparkling_background": 0.4342, "jumper": 0.4127, "eyes": 0.97}, "stage3_selected_ranks": {"simple_background": 85, "blue_fur": 77, "stripes": 18, "pose": 19, "happy": 101, "full-length_portrait": 63, "abstract_background": 107, "gradient_background": 80, "blue_background": 76, "glistening_eyes": 51, "smiling_at_viewer": 62, "blue_nose": 45, "big_eyes": 57, "flat_colors": 74, "pride_colors": 148, "action_pose": 58, "jumping": 35, "simple_eyes": 87, "handpaw": 104, "playful": 17, "midair": 71, "six-stripe_rainbow_pride_colors": 152, "simple_coloring": 82, "partially_clothed_anthro": 20, "hand_on_ground": 23, "blue_paws": 43, "pink_stripes": 37, "purple_pupils": 40, "curved_tail": 22, "striped_neck": 6, "anime_eyes": 84, ">:d": 111, ">:)": 113, "colorful_background": 131, "colored_line_art": 119, "sketch_background": 93, "sparkling_background": 67, "jumper": 81, "eyes": 1}, "stage3_selected_phrase_ranks": {"simple_background": 4, "blue_fur": 9, "stripes": 10, "pose": 1, "happy": 3, "full-length_portrait": 9, "abstract_background": 8, "gradient_background": 9, "blue_background": 3, "glistening_eyes": 3, "smiling_at_viewer": 8, "blue_nose": 3, "big_eyes": 4, "flat_colors": 2, "pride_colors": 6, "action_pose": 3, "jumping": 1, "simple_eyes": 5, "handpaw": 4, "playful": 1, "midair": 2, "six-stripe_rainbow_pride_colors": 10, "simple_coloring": 3, "partially_clothed_anthro": 3, "hand_on_ground": 5, "blue_paws": 2, "pink_stripes": 1, "purple_pupils": 2, "curved_tail": 3, "striped_neck": 4, "anime_eyes": 7, ">:d": 8, ">:)": 9, "colorful_background": 9, "colored_line_art": 8, "sketch_background": 6, "sparkling_background": 1, "jumper": 4, "eyes": 1}, "extra_evidence": {">:)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3449}, ">:d": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3455}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4619}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4216}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_paws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5068}, "colored_line_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3343}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3196}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "flat_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4218}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4811}, "hand_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5793}, "handpaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3564}, "happy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.36}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5986}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2723}, "purple_pupils": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5129}, "simple_coloring": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4099}, "simple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3912}, "six-stripe_rainbow_pride_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.263}, "sketch_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3715}, "smiling_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.451}, "sparkling_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4342}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.711}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}}, "structural": [], "t1": 0.0, "t2": 1.39, "t3": 27.24, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"strong_implied\"}, {\"i\": 2, \"why\": \"strong_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 8, \"why\": \"style_or_meta\"}, {\"i\": 9, \"why\": \"strong_implied\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"strong_implied\"}, {\"i\": 14, \"why\": \"other\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"strong_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"strong_implied\"}, {\"i\": 25, \"why\": \"style_or_meta\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"style_or_meta\"}, {\"i\": 30, \"why\": \"strong_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"style_or_meta\"}, {\"i\": 33, \"why\": \"style_or_meta\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 35}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
data/eval_results/eval_caption_cogvlm_n10_bootstrap_tune_k3_cw0p7.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T04:28:13.394549", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": true, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "explicit", "expand_implications": false, "infer_structural": false, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 14}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 72, "n_selected": 14, "n_implied": 0, "n_structural": 0, "ret_R": 0.0833, "P": 0.0714, "R": 0.0833, "F1": 0.0769, "leaf_P": 0.1111, "leaf_R": 0.1111, "leaf_F1": 0.1111, "n_leaf_sel": 9, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 1.0, "over_sel": 1.17, "why": {"explicit": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 4, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 28, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0714, "gen_R": 0.0833, "gen_F1": 0.0769, "missed": ["anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "extra": ["breaking_the_fourth_wall", "business_suit", "clasped_hands", "necktie", "pockets", "shirt", "solo_focus", "t-shirt", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["alpha_channel", "breaking_the_fourth_wall", "business_suit", "clasped_hands", "necktie", "pockets", "shirt", "solo_focus", "t-shirt", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["alpha_channel", "breaking_the_fourth_wall", "business_suit", "clasped_hands", "necktie", "pockets", "shirt", "solo_focus", "t-shirt", "transparent_background", "waiter", "white_clothing", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"shirt": 0.9019, "solo_focus": 0.4386, "alpha_channel": 0.4024, "white_clothing": 0.6182, "necktie": 0.8082, "transparent_background": 0.5794, "t-shirt": 0.8329, "white_topwear": 0.6638, "white_shirt": 0.6481, "pockets": 0.6138, "waiter": 0.6136, "business_suit": 0.6178, "breaking_the_fourth_wall": 0.4671, "clasped_hands": 0.5876, "white_t-shirt": 0.6584}, "stage3_selected_ranks": {"shirt": 1, "solo_focus": 64, "alpha_channel": 71, "white_clothing": 25, "necktie": 3, "transparent_background": 39, "t-shirt": 2, "white_topwear": 13, "white_shirt": 16, "pockets": 27, "waiter": 28, "business_suit": 26, "breaking_the_fourth_wall": 60, "clasped_hands": 37, "white_t-shirt": 15}, "stage3_selected_phrase_ranks": {"shirt": 1, "solo_focus": 7, "alpha_channel": 10, "white_clothing": 7, "necktie": 1, "transparent_background": 4, "t-shirt": 2, "white_topwear": 1, "white_shirt": 3, "pockets": 5, "waiter": 6, "business_suit": 4, "breaking_the_fourth_wall": 3, "clasped_hands": 5, "white_t-shirt": 2}, "extra_evidence": {"breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4671}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6178}, "clasped_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5876}, "necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8082}, "pockets": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6138}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9019}, "solo_focus": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "waiter": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6136}, "white_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6182}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6481}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6584}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6638}}, "structural": [], "t1": 0.0, "t2": 8.28, "t3": 9.27, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=71 entity=2 copyright_filtered=0 generic_char_to_general=1 unknown_type=0"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 177, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.2143, "P": 0.0769, "R": 0.1429, "F1": 0.1, "leaf_P": 0.04, "leaf_R": 0.1, "leaf_F1": 0.0571, "n_leaf_sel": 25, "n_leaf_gt": 10, "ret_P": 0.0169, "sel_given_ret": 0.6667, "over_sel": 1.86, "why": {"explicit": 26}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 64, "bootstrap_anchor_selected": 23, "bootstrap_anchor_kept": 11, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 105, "attempts_by_n_local": {"64": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.08, "gen_R": 0.1429, "gen_F1": 0.1026, "missed": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "hair", "haplorhine", "human", "male", "mammal", "primate"], "extra": ["arm_above_head", "bangs", "bent_arm", "breaking_the_fourth_wall", "expressionless", "expressions", "grin", "hand_over_head", "high_waisted_bottomwear", "larger_male", "looking_down_at_viewer", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "rim_light", "round_head", "simple_shading", "smaller_anthro", "smaller_version_at_source", "spread_arms", "toony_expression", "unnamed_character"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["arm_above_head", "bangs", "bent_arm", "breaking_the_fourth_wall", "expressionless", "expressions", "grin", "group", "hand_over_head", "high_waisted_bottomwear", "larger_male", "looking_at_viewer", "looking_down_at_viewer", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "rim_light", "round_head", "simple_shading", "smaller_anthro", "smaller_version_at_source", "spread_arms", "toony_expression", "unnamed_character"], "stage3_selected": ["arm_above_head", "bangs", "bent_arm", "breaking_the_fourth_wall", "expressionless", "expressions", "grin", "group", "hand_over_head", "high_waisted_bottomwear", "larger_male", "looking_at_viewer", "looking_down_at_viewer", "petting", "raised_arm", "raised_arms", "raised_hand", "relaxed_expression", "rim_light", "round_head", "simple_shading", "smaller_anthro", "smaller_version_at_source", "spread_arms", "toony_expression", "unnamed_character"], "stage3_selected_scores": {"looking_at_viewer": 0.5612, "group": 0.4099, "larger_male": 0.4954, "grin": 0.7174, "raised_arm": 0.7205, "bangs": 0.5123, "smaller_anthro": 0.5391, "bent_arm": 0.6739, "raised_arms": 0.8636, "raised_hand": 0.6751, "petting": 0.4897, "simple_shading": 0.5424, "spread_arms": 0.605, "looking_down_at_viewer": 0.5399, "breaking_the_fourth_wall": 0.4209, "expressions": 0.4746, "arm_above_head": 0.5916, "high_waisted_bottomwear": 0.4614, "round_head": 0.4202, "unnamed_character": 0.4965, "hand_over_head": 0.5861, "expressionless": 0.4946, "rim_light": 0.5332, "relaxed_expression": 0.5175, "toony_expression": 0.4675, "smaller_version_at_source": 0.5079}, "stage3_selected_ranks": {"looking_at_viewer": 22, "group": 132, "larger_male": 68, "grin": 3, "raised_arm": 2, "bangs": 51, "smaller_anthro": 36, "bent_arm": 6, "raised_arms": 1, "raised_hand": 5, "petting": 73, "simple_shading": 33, "spread_arms": 14, "looking_down_at_viewer": 35, "breaking_the_fourth_wall": 119, "expressions": 86, "arm_above_head": 15, "high_waisted_bottomwear": 96, "round_head": 121, "unnamed_character": 66, "hand_over_head": 18, "expressionless": 69, "rim_light": 39, "relaxed_expression": 48, "toony_expression": 89, "smaller_version_at_source": 57}, "stage3_selected_phrase_ranks": {"looking_at_viewer": 2, "group": 6, "larger_male": 4, "grin": 1, "raised_arm": 2, "bangs": 7, "smaller_anthro": 2, "bent_arm": 3, "raised_arms": 1, "raised_hand": 4, "petting": 2, "simple_shading": 1, "spread_arms": 6, "looking_down_at_viewer": 8, "breaking_the_fourth_wall": 5, "expressions": 5, "arm_above_head": 7, "high_waisted_bottomwear": 9, "round_head": 5, "unnamed_character": 2, "hand_over_head": 1, "expressionless": 3, "rim_light": 2, "relaxed_expression": 1, "toony_expression": 6, "smaller_version_at_source": 3}, "extra_evidence": {"arm_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5916}, "bangs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5123}, "bent_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6739}, "breaking_the_fourth_wall": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4209}, "expressionless": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4946}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4746}, "grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5861}, "high_waisted_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "larger_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4954}, "looking_down_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5399}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4897}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7205}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8636}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6751}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5175}, "rim_light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5332}, "round_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}, "simple_shading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5424}, "smaller_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5391}, "smaller_version_at_source": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5079}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.605}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4675}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4965}}, "structural": [], "t1": 0.0, "t2": 9.68, "t3": 29.07, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=153 entity=15 copyright_filtered=12 generic_char_to_general=2 unknown_type=5"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 72, "n_selected": 12, "n_implied": 0, "n_structural": 0, "ret_R": 0.0714, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 12, "n_leaf_gt": 9, "ret_P": 0.0139, "sel_given_ret": 0.0, "over_sel": 0.86, "why": {"explicit": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 1, "dupe_indices_total": 0, "kept_total": 27, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "4": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "extra": ["cheek_to_cheek", "expressions", "eyes_mostly_closed", "facial_expressions", "intimate", "padding", "petting", "setting", "shocked_face", "smiling_at_another", "toony_expression", "wide_eyed"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["cheek_to_cheek", "expressions", "eyes_mostly_closed", "facial_expressions", "intimate", "padding", "petting", "setting", "shocked_face", "smiling_at_another", "toony_expression", "wide_eyed"], "stage3_selected": ["cheek_to_cheek", "expressions", "eyes_mostly_closed", "facial_expressions", "intimate", "padding", "petting", "setting", "shocked_face", "smiling_at_another", "toony_expression", "wide_eyed"], "stage3_selected_scores": {"wide_eyed": 0.5498, "petting": 0.7411, "intimate": 0.5697, "eyes_mostly_closed": 0.6047, "expressions": 0.8783, "padding": 0.6318, "shocked_face": 0.616, "smiling_at_another": 0.5508, "cheek_to_cheek": 0.5892, "toony_expression": 0.5935, "facial_expressions": 0.5874, "setting": 1.0}, "stage3_selected_ranks": {"wide_eyed": 50, "petting": 12, "intimate": 42, "eyes_mostly_closed": 20, "expressions": 3, "padding": 14, "shocked_face": 18, "smiling_at_another": 49, "cheek_to_cheek": 29, "toony_expression": 27, "facial_expressions": 33, "setting": 2}, "stage3_selected_phrase_ranks": {"wide_eyed": 7, "petting": 2, "intimate": 5, "eyes_mostly_closed": 3, "expressions": 2, "padding": 4, "shocked_face": 1, "smiling_at_another": 10, "cheek_to_cheek": 8, "toony_expression": 4, "facial_expressions": 8, "setting": 1}, "extra_evidence": {"cheek_to_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8783}, "eyes_mostly_closed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6047}, "facial_expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5874}, "intimate": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "padding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6318}, "petting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7411}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 1.0}, "shocked_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.616}, "smiling_at_another": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5508}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5935}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5498}}, "structural": [], "t1": 0.0, "t2": 8.33, "t3": 9.5, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=64 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=1"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 68, "n_selected": 6, "n_implied": 0, "n_structural": 0, "ret_R": 0.0, "P": 0.0, "R": 0.0, "F1": 0.0, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 6, "n_leaf_gt": 4, "ret_P": 0.0, "sel_given_ret": 0.0, "over_sel": 1.5, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 24, "bootstrap_anchor_selected": 8, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 2, "calls_with_selection": 2, "calls_exhausted_retries": 0, "attempts_total": 2, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 2, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 20, "attempts_by_n_local": {"24": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0, "gen_R": 0.0, "gen_F1": 0.0, "missed": ["red_nose", "smile", "solo", "tan_body"], "extra": ["grey_background", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["grey_background", "round_eyes", "round_nose", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected": ["grey_background", "round_eyes", "round_nose", "simple_background", "stylized_empty_eyes", "thick_eyelashes", "toony"], "stage3_selected_scores": {"simple_background": 0.7167, "grey_background": 0.688, "toony": 0.763, "thick_eyelashes": 0.536, "round_eyes": 0.6227, "round_nose": 0.555, "stylized_empty_eyes": 0.6448}, "stage3_selected_ranks": {"simple_background": 10, "grey_background": 18, "toony": 6, "thick_eyelashes": 69, "round_eyes": 30, "round_nose": 59, "stylized_empty_eyes": 25}, "stage3_selected_phrase_ranks": {"simple_background": 5, "grey_background": 10, "toony": 3, "thick_eyelashes": 10, "round_eyes": 1, "round_nose": 10, "stylized_empty_eyes": 1}, "extra_evidence": {"grey_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.688}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6227}, "round_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.555}, "stylized_empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6448}, "thick_eyelashes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.536}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.763}}, "structural": [], "t1": 0.0, "t2": 8.29, "t3": 7.25, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=58 entity=5 copyright_filtered=10 generic_char_to_general=0 unknown_type=1"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 87, "n_selected": 18, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.1111, "R": 0.0909, "F1": 0.1, "leaf_P": 0.0625, "leaf_R": 0.0833, "leaf_F1": 0.0714, "n_leaf_sel": 16, "n_leaf_gt": 12, "ret_P": 0.023, "sel_given_ret": 1.0, "over_sel": 0.82, "why": {"explicit": 18}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 30, "bootstrap_anchor_selected": 10, "bootstrap_anchor_kept": 6, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 41, "attempts_by_n_local": {"30": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1111, "gen_R": 0.0909, "gen_F1": 0.1, "missed": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "solo", "stripes", "tiger", "topless", "tuft"], "extra": ["action_pose", "blue_bottomwear", "blue_shorts", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "neck_tuft", "open_shorts", "pattern_topwear", "playful", "playing", "raised_arm", "raised_finger", "striped_markings"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "neck_tuft", "open_shorts", "pattern_topwear", "playful", "playing", "raised_arm", "raised_finger", "shorts", "striped_markings"], "stage3_selected": ["action_pose", "blue_bottomwear", "blue_shorts", "fur", "gym_bottomwear", "hand_gesture", "hand_on_own_head", "hand_over_head", "muscle_tone", "neck_tuft", "open_shorts", "pattern_topwear", "playful", "playing", "raised_arm", "raised_finger", "shorts", "striped_markings"], "stage3_selected_scores": {"fur": 0.7785, "shorts": 0.9202, "neck_tuft": 0.5847, "raised_arm": 0.6005, "striped_markings": 0.5966, "blue_bottomwear": 0.6293, "pattern_topwear": 0.5972, "action_pose": 0.6359, "playful": 0.6803, "gym_bottomwear": 0.7858, "blue_shorts": 0.7288, "playing": 0.5527, "raised_finger": 0.595, "muscle_tone": 0.5438, "hand_gesture": 0.6065, "open_shorts": 0.7362, "hand_over_head": 0.5634, "hand_on_own_head": 0.5334}, "stage3_selected_ranks": {"fur": 8, "shorts": 1, "neck_tuft": 51, "raised_arm": 42, "striped_markings": 46, "blue_bottomwear": 32, "pattern_topwear": 44, "action_pose": 31, "playful": 18, "gym_bottomwear": 7, "blue_shorts": 14, "playing": 58, "raised_finger": 47, "muscle_tone": 62, "hand_gesture": 39, "open_shorts": 11, "hand_over_head": 56, "hand_on_own_head": 66}, "stage3_selected_phrase_ranks": {"fur": 1, "shorts": 1, "neck_tuft": 9, "raised_arm": 2, "striped_markings": 8, "blue_bottomwear": 2, "pattern_topwear": 7, "action_pose": 8, "playful": 1, "gym_bottomwear": 6, "blue_shorts": 1, "playing": 2, "raised_finger": 3, "muscle_tone": 6, "hand_gesture": 1, "open_shorts": 8, "hand_over_head": 5, "hand_on_own_head": 6}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6359}, "blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6293}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7288}, "gym_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7858}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6065}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5334}, "hand_over_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "muscle_tone": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5438}, "neck_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5847}, "open_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7362}, "pattern_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5972}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6803}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5527}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6005}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "striped_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5966}}, "structural": [], "t1": 0.0, "t2": 0.77, "t3": 11.22, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=89 entity=3 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 131, "n_selected": 7, "n_implied": 0, "n_structural": 0, "ret_R": 0.6154, "P": 0.4286, "R": 0.2308, "F1": 0.3, "leaf_P": 0.4286, "leaf_R": 0.5, "leaf_F1": 0.4615, "n_leaf_sel": 7, "n_leaf_gt": 6, "ret_P": 0.0611, "sel_given_ret": 0.375, "over_sel": 0.54, "why": {"explicit": 7}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 20, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 7, "attempt_errors": 4, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 91, "attempts_by_n_local": {"54": {"attempts": 3, "parse_ok": 1, "parse_fail": 0, "errors": 2}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.5714285714285714, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.5, "gen_R": 0.2308, "gen_F1": 0.3158, "missed": ["bovid", "caprine", "dialogue", "fur", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "extra": ["darkness", "group", "light_beam", "unnamed_character"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["darkness", "goat", "group", "human", "light_beam", "lizard", "unnamed_character"], "stage3_selected": ["darkness", "goat", "group", "human", "light_beam", "lizard", "unnamed_character"], "stage3_selected_scores": {"group": 0.4182, "human": 0.6298, "lizard": 0.8035, "goat": 0.777, "light_beam": 0.5797, "darkness": 0.7908, "unnamed_character": 0.4967}, "stage3_selected_ranks": {"group": 110, "human": 6, "lizard": 1, "goat": 3, "light_beam": 12, "darkness": 2, "unnamed_character": 50}, "stage3_selected_phrase_ranks": {"group": 5, "human": 1, "lizard": 1, "goat": 1, "light_beam": 2, "darkness": 1, "unnamed_character": 1}, "extra_evidence": {"darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7908}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4182}, "light_beam": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5797}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}}, "structural": [], "t1": 0.0, "t2": 1.69, "t3": 48.93, "t3s": 0.0, "err": null, "issues": ["Stage3 bootstrap_anchor: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"style_or_meta\"}, {\"i\": 39, \"why\": \"style_or_meta\"}, {\"i\": 40, \"why\": \"style_or_meta\"}, {\"i\": 41, \"why\": \"style_or_meta\"}, {\"i\": 42}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 42}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 bootstrap_anchor: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"style_or_meta\"}, {\"i\": 43}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 43}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 split: general=120 entity=6 copyright_filtered=5 generic_char_to_general=4 unknown_type=3", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"explicit\"}, {\"i\": 51}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 51}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"style_or_meta\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 37, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"style_or_meta\"}, {\"i\": 39, \"why\": \"style_or_meta\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"other\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 47}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 14, "n_retrieved": 148, "n_selected": 48, "n_implied": 0, "n_structural": 0, "ret_R": 0.5714, "P": 0.1458, "R": 0.5, "F1": 0.2258, "leaf_P": 0.0714, "leaf_R": 0.25, "leaf_F1": 0.1111, "n_leaf_sel": 42, "n_leaf_gt": 12, "ret_P": 0.0541, "sel_given_ret": 0.875, "over_sel": 3.43, "why": {"explicit": 48}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 16, "bootstrap_anchor_kept": 10, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 104, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1458, "gen_R": 0.5, "gen_F1": 0.2258, "missed": ["angry", "duo", "eyes_closed", "hair", "lying", "purple_hair", "text"], "extra": ["2_frame_animation", "animated_png", "anime_eyes", "applying_makeup", "bed_covers", "bedding", "bedroom", "blanket", "blue_background", "blue_eyebrows", "blue_lipstick", "empty_eyes", "gem", "hair_sticks", "heart_in_signature", "hearts_around_head", "heterochromia", "lidded_eyes", "lipstick", "long_hair", "lying_on_bed", "no_irises", "partial_line_speech_bubble", "pillow", "portuguese_text", "purple_ears", "purple_feet", "purple_hands", "red_hair", "red_lipstick", "running_makeup", "running_mascara", "sleeping_together", "stylized_text", "taking_picture", "tired", "two_panel_image", "waking_up", "x_eyes", "yellow_eyeshadow", "zzz"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["2_frame_animation", "animated_png", "anime_eyes", "applying_makeup", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "blue_lipstick", "empty_eyes", "eyeshadow", "gem", "green_eyes", "hair_sticks", "heart_in_signature", "hearts_around_head", "heterochromia", "lidded_eyes", "lipstick", "long_hair", "lying_on_bed", "makeup", "no_irises", "partial_line_speech_bubble", "pillow", "portuguese_text", "purple_ears", "purple_feet", "purple_hands", "red_hair", "red_lipstick", "running_makeup", "running_mascara", "sleeping", "sleeping_together", "stylized_text", "taking_picture", "tired", "two_panel_image", "waking_up", "x_eyes", "yellow_eyeshadow", "zzz"], "stage3_selected": ["2_frame_animation", "animated_png", "anime_eyes", "applying_makeup", "bed", "bed_covers", "bedding", "bedroom", "blanket", "blonde_hair", "blue_background", "blue_eyebrows", "blue_eyes", "blue_lipstick", "empty_eyes", "eyeshadow", "gem", "green_eyes", "hair_sticks", "heart_in_signature", "hearts_around_head", "heterochromia", "lidded_eyes", "lipstick", "long_hair", "lying_on_bed", "makeup", "no_irises", "partial_line_speech_bubble", "pillow", "portuguese_text", "purple_ears", "purple_feet", "purple_hands", "red_hair", "red_lipstick", "running_makeup", "running_mascara", "sleeping", "sleeping_together", "stylized_text", "taking_picture", "tired", "two_panel_image", "waking_up", "x_eyes", "yellow_eyeshadow", "zzz"], "stage3_selected_scores": {"blue_eyes": 0.5401, "green_eyes": 0.538, "long_hair": 0.3962, "blonde_hair": 0.346, "bed": 0.5296, "red_hair": 0.359, "makeup": 0.544, "pillow": 0.441, "eyeshadow": 0.4274, "lipstick": 0.4386, "bedroom": 0.4116, "blue_background": 0.3407, "sleeping": 0.4995, "bedding": 0.4246, "heterochromia": 0.3472, "gem": 0.2689, "blanket": 0.3523, "empty_eyes": 0.3793, "tired": 0.4965, "lidded_eyes": 0.3759, "lying_on_bed": 0.412, "zzz": 0.367, "red_lipstick": 0.4186, "hearts_around_head": 0.3017, "purple_ears": 0.5084, "waking_up": 0.3545, "running_makeup": 0.4137, "blue_eyebrows": 0.4608, "running_mascara": 0.3589, "blue_lipstick": 0.3955, "taking_picture": 0.3117, "bed_covers": 0.4011, "x_eyes": 0.3583, "two_panel_image": 0.3289, "no_irises": 0.3837, "heart_in_signature": 0.3039, "sleeping_together": 0.408, "anime_eyes": 0.3746, "hair_sticks": 0.5043, "purple_feet": 0.6039, "purple_hands": 0.6346, "yellow_eyeshadow": 0.4068, "partial_line_speech_bubble": 0.3487, "stylized_text": 0.3452, "applying_makeup": 0.431, "portuguese_text": 0.3865, "2_frame_animation": 0.3038, "animated_png": 0.2978}, "stage3_selected_ranks": {"blue_eyes": 12, "green_eyes": 14, "long_hair": 48, "blonde_hair": 96, "bed": 15, "red_hair": 78, "makeup": 11, "pillow": 27, "eyeshadow": 32, "lipstick": 29, "bedroom": 39, "blue_background": 105, "sleeping": 20, "bedding": 33, "heterochromia": 95, "gem": 146, "blanket": 86, "empty_eyes": 59, "tired": 21, "lidded_eyes": 63, "lying_on_bed": 38, "zzz": 68, "red_lipstick": 34, "hearts_around_head": 128, "purple_ears": 16, "waking_up": 83, "running_makeup": 35, "blue_eyebrows": 24, "running_mascara": 79, "blue_lipstick": 50, "taking_picture": 120, "bed_covers": 44, "x_eyes": 80, "two_panel_image": 117, "no_irises": 55, "heart_in_signature": 126, "sleeping_together": 41, "anime_eyes": 65, "hair_sticks": 18, "purple_feet": 7, "purple_hands": 5, "yellow_eyeshadow": 42, "partial_line_speech_bubble": 92, "stylized_text": 98, "applying_makeup": 31, "portuguese_text": 54, "2_frame_animation": 127, "animated_png": 130}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "green_eyes": 1, "long_hair": 3, "blonde_hair": 8, "bed": 1, "red_hair": 5, "makeup": 1, "pillow": 3, "eyeshadow": 3, "lipstick": 2, "bedroom": 7, "blue_background": 10, "sleeping": 2, "bedding": 5, "heterochromia": 7, "gem": 6, "blanket": 10, "empty_eyes": 3, "tired": 3, "lidded_eyes": 6, "lying_on_bed": 6, "zzz": 8, "red_lipstick": 4, "hearts_around_head": 10, "purple_ears": 9, "waking_up": 7, "running_makeup": 2, "blue_eyebrows": 2, "running_mascara": 10, "blue_lipstick": 10, "taking_picture": 8, "bed_covers": 9, "x_eyes": 7, "two_panel_image": 4, "no_irises": 2, "heart_in_signature": 9, "sleeping_together": 1, "anime_eyes": 6, "hair_sticks": 2, "purple_feet": 5, "purple_hands": 1, "yellow_eyeshadow": 7, "partial_line_speech_bubble": 4, "stylized_text": 5, "applying_makeup": 1, "portuguese_text": 1, "2_frame_animation": 2, "animated_png": 3}, "extra_evidence": {"2_frame_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3038}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2978}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3746}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.431}, "bed_covers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4011}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4116}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3523}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3407}, "blue_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4608}, "blue_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3955}, "empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3793}, "gem": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2689}, "hair_sticks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5043}, "heart_in_signature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3039}, "hearts_around_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3017}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3472}, "lidded_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3759}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4386}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3962}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.412}, "no_irises": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "partial_line_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3487}, "pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "portuguese_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3865}, "purple_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5084}, "purple_feet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6039}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6346}, "red_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.359}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4186}, "running_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4137}, "running_mascara": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3589}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.408}, "stylized_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3452}, "taking_picture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3117}, "tired": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4965}, "two_panel_image": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3289}, "waking_up": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3545}, "x_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3583}, "yellow_eyeshadow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4068}, "zzz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.367}}, "structural": [], "t1": 0.0, "t2": 1.55, "t3": 16.33, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=145 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 115, "n_selected": 27, "n_implied": 0, "n_structural": 0, "ret_R": 0.2273, "P": 0.1852, "R": 0.2273, "F1": 0.2041, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 20, "n_leaf_gt": 13, "ret_P": 0.0435, "sel_given_ret": 1.0, "over_sel": 1.23, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 42, "bootstrap_anchor_selected": 13, "bootstrap_anchor_kept": 10, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 56, "attempts_by_n_local": {"42": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "50": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1852, "gen_R": 0.2273, "gen_F1": 0.2041, "missed": ["anthro", "bass_guitar", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail"], "extra": ["acoustic_guitar", "action_pose", "big_claws", "claws_out", "clenched_teeth", "color_swatch", "digitigrade", "finger_claws", "full-length_portrait", "green_background", "long_claws", "maned_wolf", "orange_background", "portrait", "pose", "toony", "toony_expression", "torn_shorts", "torn_stockings", "torn_topwear", "warm_colors", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["acoustic_guitar", "action_pose", "big_claws", "canid", "canine", "claws", "claws_out", "clenched_teeth", "color_swatch", "digitigrade", "finger_claws", "full-length_portrait", "green_background", "long_claws", "maned_wolf", "musical_instrument", "orange_background", "portrait", "pose", "toony", "toony_expression", "torn_clothing", "torn_shorts", "torn_stockings", "torn_topwear", "warm_colors", "wolf"], "stage3_selected": ["acoustic_guitar", "action_pose", "big_claws", "canid", "canine", "claws", "claws_out", "clenched_teeth", "color_swatch", "digitigrade", "finger_claws", "full-length_portrait", "green_background", "long_claws", "maned_wolf", "musical_instrument", "orange_background", "portrait", "pose", "toony", "toony_expression", "torn_clothing", "torn_shorts", "torn_stockings", "torn_topwear", "warm_colors", "wolf"], "stage3_selected_scores": {"canid": 0.6217, "canine": 0.6203, "claws": 0.898, "wolf": 0.6635, "portrait": 0.6102, "pose": 0.8955, "digitigrade": 0.7162, "full-length_portrait": 0.6018, "torn_clothing": 0.5179, "clenched_teeth": 0.498, "finger_claws": 0.7339, "toony": 0.4807, "green_background": 0.6349, "musical_instrument": 0.3362, "orange_background": 0.6518, "action_pose": 0.6726, "color_swatch": 0.7261, "maned_wolf": 0.664, "warm_colors": 0.6979, "torn_topwear": 0.4853, "long_claws": 0.6953, "claws_out": 0.7061, "big_claws": 0.7461, "torn_stockings": 0.4869, "torn_shorts": 0.4967, "acoustic_guitar": 0.3382, "toony_expression": 0.4641}, "stage3_selected_ranks": {"canid": 44, "canine": 45, "claws": 1, "wolf": 23, "portrait": 47, "pose": 2, "digitigrade": 10, "full-length_portrait": 54, "torn_clothing": 73, "clenched_teeth": 78, "finger_claws": 8, "toony": 90, "green_background": 37, "musical_instrument": 117, "orange_background": 24, "action_pose": 19, "color_swatch": 9, "maned_wolf": 21, "warm_colors": 13, "torn_topwear": 88, "long_claws": 15, "claws_out": 12, "big_claws": 7, "torn_stockings": 84, "torn_shorts": 80, "acoustic_guitar": 115, "toony_expression": 98}, "stage3_selected_phrase_ranks": {"canid": 5, "canine": 6, "claws": 1, "wolf": 2, "portrait": 7, "pose": 1, "digitigrade": 5, "full-length_portrait": 10, "torn_clothing": 4, "clenched_teeth": 5, "finger_claws": 4, "toony": 5, "green_background": 8, "musical_instrument": 9, "orange_background": 2, "action_pose": 2, "color_swatch": 1, "maned_wolf": 1, "warm_colors": 2, "torn_topwear": 10, "long_claws": 2, "claws_out": 6, "big_claws": 1, "torn_stockings": 8, "torn_shorts": 7, "acoustic_guitar": 7, "toony_expression": 4}, "extra_evidence": {"acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3382}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6726}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7461}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7061}, "clenched_teeth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.498}, "color_swatch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7261}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7339}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6018}, "green_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6349}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6953}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.664}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6518}, "portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6102}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4641}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4967}, "torn_stockings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4869}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4853}, "warm_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6979}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6635}}, "structural": [], "t1": 0.0, "t2": 1.12, "t3": 8.68, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=110 entity=2 copyright_filtered=6 generic_char_to_general=0 unknown_type=0"]}
10
+ {"id": 2021552, "n_gt": 23, "n_retrieved": 93, "n_selected": 32, "n_implied": 0, "n_structural": 0, "ret_R": 0.3478, "P": 0.1562, "R": 0.2174, "F1": 0.1818, "leaf_P": 0.0, "leaf_R": 0.0, "leaf_F1": 0.0, "n_leaf_sel": 23, "n_leaf_gt": 15, "ret_P": 0.086, "sel_given_ret": 0.625, "over_sel": 1.39, "why": {"explicit": 33}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 35, "bootstrap_anchor_selected": 12, "bootstrap_anchor_kept": 12, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 51, "attempts_by_n_local": {"35": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "33": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1562, "gen_R": 0.2174, "gen_F1": 0.1818, "missed": ["anthro", "bottomwear", "canid", "canine", "clothed", "clothing", "crossed_arms", "duo", "fox", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "rabbit", "standing", "topwear"], "extra": ["4_claws", "5_claws", "black_claws", "black_pants", "black_topwear", "cargo_pants", "dress_shirt", "eye_markings", "gloves_(marking)", "grey_claws", "grey_clothing", "grey_nose", "grey_shirt", "grey_topwear", "grey_tuft", "hand_markings", "hand_on_cheek", "hip_markings", "partially_clothed_anthro", "snout_markings", "t-shirt", "tan_fur", "two_tone_fur", "undershirt", "white_dress_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "lagomorph", "leporid", "looking_at_another", "mammal", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "5_claws", "black_claws", "black_pants", "black_topwear", "cargo_pants", "claws", "dress_shirt", "eye_markings", "facial_markings", "fur", "gloves_(marking)", "grey_claws", "grey_clothing", "grey_nose", "grey_shirt", "grey_topwear", "grey_tuft", "hand_markings", "hand_on_cheek", "hip_markings", "pants", "partially_clothed_anthro", "shirt", "snout_markings", "t-shirt", "tan_fur", "two_tone_fur", "undershirt", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["4_claws", "5_claws", "black_claws", "black_pants", "black_topwear", "cargo_pants", "claws", "dress_shirt", "eye_markings", "facial_markings", "fur", "gloves_(marking)", "grey_claws", "grey_clothing", "grey_nose", "grey_shirt", "grey_topwear", "grey_tuft", "hand_markings", "hand_on_cheek", "hip_markings", "pants", "partially_clothed_anthro", "shirt", "simple_background", "snout_markings", "t-shirt", "tan_fur", "two_tone_fur", "undershirt", "white_dress_shirt", "white_t-shirt", "white_topwear"], "stage3_selected_scores": {"fur": 0.7011, "simple_background": 0.4791, "claws": 0.6687, "shirt": 0.7037, "pants": 0.7123, "tan_fur": 0.5593, "two_tone_fur": 0.5581, "gloves_(marking)": 0.7072, "facial_markings": 0.9015, "black_claws": 0.5947, "t-shirt": 0.6707, "black_topwear": 0.5595, "white_topwear": 0.5694, "grey_clothing": 0.5671, "dress_shirt": 0.6075, "black_pants": 0.5822, "grey_topwear": 0.5774, "eye_markings": 0.743, "grey_nose": 0.5885, "grey_claws": 0.5681, "snout_markings": 0.7146, "hand_on_cheek": 0.4975, "grey_shirt": 0.6272, "undershirt": 0.6369, "partially_clothed_anthro": 0.5361, "cargo_pants": 0.625, "white_t-shirt": 0.531, "4_claws": 0.6123, "white_dress_shirt": 0.5611, "hip_markings": 0.7094, "grey_tuft": 0.5545, "hand_markings": 0.7033, "5_claws": 0.6236}, "stage3_selected_ranks": {"fur": 16, "simple_background": 89, "claws": 19, "shirt": 13, "pants": 10, "tan_fur": 66, "two_tone_fur": 67, "gloves_(marking)": 12, "facial_markings": 1, "black_claws": 34, "t-shirt": 18, "black_topwear": 65, "white_topwear": 51, "grey_clothing": 55, "dress_shirt": 31, "black_pants": 39, "grey_topwear": 45, "eye_markings": 5, "grey_nose": 37, "grey_claws": 54, "snout_markings": 9, "hand_on_cheek": 83, "grey_shirt": 25, "undershirt": 22, "partially_clothed_anthro": 76, "cargo_pants": 26, "white_t-shirt": 79, "4_claws": 28, "white_dress_shirt": 62, "hip_markings": 11, "grey_tuft": 70, "hand_markings": 14, "5_claws": 27}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 9, "claws": 1, "shirt": 1, "pants": 1, "tan_fur": 5, "two_tone_fur": 6, "gloves_(marking)": 9, "facial_markings": 1, "black_claws": 2, "t-shirt": 2, "black_topwear": 10, "white_topwear": 3, "grey_clothing": 5, "dress_shirt": 9, "black_pants": 5, "grey_topwear": 1, "eye_markings": 4, "grey_nose": 1, "grey_claws": 9, "snout_markings": 7, "hand_on_cheek": 1, "grey_shirt": 7, "undershirt": 4, "partially_clothed_anthro": 7, "cargo_pants": 3, "white_t-shirt": 10, "4_claws": 3, "white_dress_shirt": 3, "hip_markings": 7, "grey_tuft": 9, "hand_markings": 10, "5_claws": 1}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6123}, "5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5947}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5822}, "black_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5595}, "cargo_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.625}, "dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6075}, "eye_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.743}, "gloves_(marking)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7072}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5681}, "grey_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5671}, "grey_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5885}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6272}, "grey_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5774}, "grey_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5545}, "hand_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7033}, "hand_on_cheek": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4975}, "hip_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7094}, "partially_clothed_anthro": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5361}, "snout_markings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7146}, "t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6707}, "tan_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5593}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5581}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5611}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.531}, "white_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5694}}, "structural": [], "t1": 0.0, "t2": 0.9, "t3": 13.71, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=93 entity=1 copyright_filtered=0 generic_char_to_general=0 unknown_type=1"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 156, "n_selected": 26, "n_implied": 0, "n_structural": 0, "ret_R": 0.0909, "P": 0.0385, "R": 0.0909, "F1": 0.0541, "leaf_P": 0.0417, "leaf_R": 0.1429, "leaf_F1": 0.0645, "n_leaf_sel": 24, "n_leaf_gt": 7, "ret_P": 0.0064, "sel_given_ret": 1.0, "over_sel": 2.36, "why": {"explicit": 27}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 17, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 111, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "37": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.0385, "gen_R": 0.0909, "gen_F1": 0.0541, "missed": ["blue_eyes", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "extra": ["action_pose", "anime_eyes", "blowup_background", "blue_fur", "blue_pawpads", "curved_tail", "empty_eyes", "eyes", "handpaw", "holding_character", "jumper", "jumping", "midair", "multiple_poses", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "raised_paw", "small_ears", "sparkling_eyes", "striped_back", "suggestive_pose"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["action_pose", "anime_eyes", "blowup_background", "blue_fur", "blue_nose", "blue_pawpads", "curved_tail", "empty_eyes", "eyes", "handpaw", "holding_character", "jumper", "jumping", "midair", "multiple_poses", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "raised_paw", "small_ears", "sparkling_eyes", "striped_back", "suggestive_pose"], "stage3_selected": ["action_pose", "anime_eyes", "blowup_background", "blue_fur", "blue_nose", "blue_pawpads", "curved_tail", "empty_eyes", "eyes", "handpaw", "holding_character", "jumper", "jumping", "midair", "multiple_poses", "pawpads", "pink_stripes", "playful", "playing", "pose", "posed", "raised_paw", "simple_background", "small_ears", "sparkling_eyes", "striped_back", "suggestive_pose"], "stage3_selected_scores": {"simple_background": 0.401, "pawpads": 0.327, "blue_fur": 0.4171, "pose": 0.6068, "blue_nose": 0.5017, "multiple_poses": 0.4969, "blue_pawpads": 0.4264, "empty_eyes": 0.434, "action_pose": 0.4596, "jumping": 0.5295, "holding_character": 0.3418, "handpaw": 0.3564, "playful": 0.6118, "playing": 0.3818, "midair": 0.43, "raised_paw": 0.3182, "sparkling_eyes": 0.3343, "suggestive_pose": 0.5892, "pink_stripes": 0.5235, "small_ears": 0.5669, "curved_tail": 0.5875, "blowup_background": 0.3828, "striped_back": 0.7273, "anime_eyes": 0.4019, "posed": 0.4337, "jumper": 0.4127, "eyes": 0.97}, "stage3_selected_ranks": {"simple_background": 85, "pawpads": 125, "blue_fur": 77, "pose": 19, "blue_nose": 45, "multiple_poses": 48, "blue_pawpads": 73, "empty_eyes": 68, "action_pose": 58, "jumping": 35, "holding_character": 116, "handpaw": 104, "playful": 17, "playing": 91, "midair": 71, "raised_paw": 134, "sparkling_eyes": 120, "suggestive_pose": 21, "pink_stripes": 37, "small_ears": 25, "curved_tail": 22, "blowup_background": 90, "striped_back": 4, "anime_eyes": 84, "posed": 69, "jumper": 81, "eyes": 1}, "stage3_selected_phrase_ranks": {"simple_background": 4, "pawpads": 8, "blue_fur": 9, "pose": 1, "blue_nose": 3, "multiple_poses": 5, "blue_pawpads": 7, "empty_eyes": 5, "action_pose": 3, "jumping": 1, "holding_character": 6, "handpaw": 4, "playful": 1, "playing": 2, "midair": 2, "raised_paw": 9, "sparkling_eyes": 6, "suggestive_pose": 2, "pink_stripes": 1, "small_ears": 4, "curved_tail": 3, "blowup_background": 5, "striped_back": 2, "anime_eyes": 7, "posed": 10, "jumper": 4, "eyes": 1}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4596}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3828}, "blue_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4171}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4264}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5875}, "empty_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.434}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.97}, "handpaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3564}, "holding_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3418}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4127}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5295}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.43}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4969}, "pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.327}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5235}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6118}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3818}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6068}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "raised_paw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3182}, "small_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5669}, "sparkling_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3343}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7273}, "suggestive_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5892}}, "structural": [], "t1": 0.0, "t2": 1.35, "t3": 25.16, "t3s": 0.0, "err": null, "issues": ["Stage3 split: general=157 entity=2 copyright_filtered=2 generic_char_to_general=1 unknown_type=4"]}
data/eval_results/eval_caption_cogvlm_n10_e2e_structimp_baseline_default.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T05:37:47.646533", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "strong_implied", "expand_implications": true, "infer_structural": true, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 15}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 151, "n_selected": 64, "n_implied": 25, "n_structural": 5, "ret_R": 0.25, "P": 0.1562, "R": 0.8333, "F1": 0.2632, "leaf_P": 0.125, "leaf_R": 0.4444, "leaf_F1": 0.1951, "n_leaf_sel": 32, "n_leaf_gt": 9, "ret_P": 0.0199, "sel_given_ret": 3.3333, "over_sel": 5.33, "why": {"explicit": 33, "strong_implied": 1}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 78, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "34": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1562, "gen_R": 0.8333, "gen_F1": 0.2632, "missed": ["alpha_channel", "fingers"], "extra": ["beverage", "black_body", "black_clothing", "black_fur", "black_hands", "black_necktie", "black_shirt", "black_topwear", "black_vest", "blue_clothing", "blue_shirt", "blue_topwear", "bobtail_cat", "business_attire", "business_suit", "businesswear", "domestic_cat", "felis", "green_clothing", "green_topwear", "green_vest", "grey_clothing", "grey_shirt", "grey_topwear", "hair_bun", "holding_beverage", "holding_object", "lagomorph", "leporid", "looking_at_viewer", "necktie", "polo_shirt", "red_clothing", "red_topwear", "red_vest", "shirt", "sleeveless_shirt", "suit", "suit_jacket", "teal_shirt", "teal_topwear", "topwear", "undershirt", "vest", "white_arms", "white_clothing", "white_dress_shirt", "white_hands", "white_necktie", "white_shirt", "white_topwear", "yellow_clothing", "yellow_shirt", "yellow_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "beverage", "black_body", "black_clothing", "black_fur", "black_hands", "black_necktie", "black_shirt", "black_topwear", "black_vest", "blue_clothing", "blue_shirt", "blue_topwear", "bobtail_cat", "business_attire", "business_suit", "businesswear", "clothed", "clothing", "domestic_cat", "felid", "feline", "felis", "fur", "green_clothing", "green_topwear", "green_vest", "grey_clothing", "grey_shirt", "grey_topwear", "hair", "hair_bun", "holding_beverage", "holding_object", "lagomorph", "leporid", "looking_at_viewer", "male", "mammal", "necktie", "polo_shirt", "red_clothing", "red_topwear", "red_vest", "shirt", "sleeveless_shirt", "solo", "suit", "suit_jacket", "teal_shirt", "teal_topwear", "topwear", "undershirt", "vest", "white_arms", "white_clothing", "white_dress_shirt", "white_hands", "white_necktie", "white_shirt", "white_topwear", "yellow_clothing", "yellow_shirt", "yellow_topwear"], "stage3_selected": ["black_body", "black_fur", "black_hands", "black_necktie", "black_shirt", "black_vest", "blue_shirt", "blue_topwear", "bobtail_cat", "business_attire", "business_suit", "businesswear", "domestic_cat", "feline", "felis", "green_vest", "grey_shirt", "hair_bun", "holding_beverage", "leporid", "polo_shirt", "red_vest", "shirt", "sleeveless_shirt", "suit_jacket", "teal_shirt", "undershirt", "vest", "white_arms", "white_dress_shirt", "white_hands", "white_necktie", "white_shirt", "yellow_shirt"], "stage3_selected_scores": {"feline": 0.7023, "black_body": 0.5747, "shirt": 0.7976, "felis": 0.633, "domestic_cat": 0.6294, "leporid": 0.3887, "black_fur": 0.7157, "vest": 0.8387, "blue_topwear": 0.7266, "white_shirt": 0.737, "black_shirt": 0.7341, "blue_shirt": 0.7502, "hair_bun": 0.6912, "sleeveless_shirt": 0.7057, "grey_shirt": 0.7575, "business_suit": 0.5728, "yellow_shirt": 0.7542, "black_hands": 0.4561, "white_hands": 0.4517, "undershirt": 0.7585, "polo_shirt": 0.7082, "white_arms": 0.4277, "suit_jacket": 0.6849, "black_necktie": 0.7125, "black_vest": 0.7112, "businesswear": 0.5699, "white_dress_shirt": 0.6876, "green_vest": 0.7228, "business_attire": 0.5642, "bobtail_cat": 0.5721, "red_vest": 0.6636, "teal_shirt": 0.7466, "white_necktie": 0.6419, "holding_beverage": 0.7705}, "stage3_selected_ranks": {"feline": 45, "black_body": 94, "shirt": 5, "felis": 70, "domestic_cat": 73, "leporid": 154, "black_fur": 37, "vest": 3, "blue_topwear": 30, "white_shirt": 22, "black_shirt": 25, "blue_shirt": 18, "hair_bun": 52, "sleeveless_shirt": 43, "grey_shirt": 13, "business_suit": 97, "yellow_shirt": 15, "black_hands": 139, "white_hands": 140, "undershirt": 11, "polo_shirt": 42, "white_arms": 145, "suit_jacket": 54, "black_necktie": 39, "black_vest": 41, "businesswear": 99, "white_dress_shirt": 53, "green_vest": 33, "business_attire": 102, "bobtail_cat": 98, "red_vest": 59, "teal_shirt": 19, "white_necktie": 65, "holding_beverage": 9}, "stage3_selected_phrase_ranks": {"feline": 1, "black_body": 8, "shirt": 1, "felis": 3, "domestic_cat": 4, "leporid": 9, "black_fur": 1, "vest": 1, "blue_topwear": 10, "white_shirt": 4, "black_shirt": 8, "blue_shirt": 3, "hair_bun": 1, "sleeveless_shirt": 7, "grey_shirt": 1, "business_suit": 8, "yellow_shirt": 2, "black_hands": 6, "white_hands": 7, "undershirt": 3, "polo_shirt": 6, "white_arms": 10, "suit_jacket": 4, "black_necktie": 2, "black_vest": 9, "businesswear": 9, "white_dress_shirt": 5, "green_vest": 6, "business_attire": 1, "bobtail_cat": 7, "red_vest": 9, "teal_shirt": 4, "white_necktie": 10, "holding_beverage": 3}, "extra_evidence": {"beverage": {"source": "implied"}, "black_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5747}, "black_clothing": {"source": "implied"}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "black_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4561}, "black_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7125}, "black_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7341}, "black_topwear": {"source": "implied"}, "black_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7112}, "blue_clothing": {"source": "implied"}, "blue_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7502}, "blue_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7266}, "bobtail_cat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5721}, "business_attire": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5642}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5728}, "businesswear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5699}, "domestic_cat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6294}, "felis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "green_clothing": {"source": "implied"}, "green_topwear": {"source": "implied"}, "green_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7228}, "grey_clothing": {"source": "implied"}, "grey_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7575}, "grey_topwear": {"source": "implied"}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6912}, "holding_beverage": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7705}, "holding_object": {"source": "implied"}, "lagomorph": {"source": "implied"}, "leporid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3887}, "looking_at_viewer": {"source": "structural"}, "necktie": {"source": "implied"}, "polo_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7082}, "red_clothing": {"source": "implied"}, "red_topwear": {"source": "implied"}, "red_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6636}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "sleeveless_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7057}, "suit": {"source": "implied"}, "suit_jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6849}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "teal_topwear": {"source": "implied"}, "topwear": {"source": "implied"}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7585}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "white_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4277}, "white_clothing": {"source": "implied"}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6876}, "white_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4517}, "white_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6419}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.737}, "white_topwear": {"source": "implied"}, "yellow_clothing": {"source": "implied"}, "yellow_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7542}, "yellow_topwear": {"source": "implied"}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 4.0, "t2": 4.15, "t3": 39.94, "t3s": 3.58, "err": null, "issues": ["Stage3 split: general=154 entity=1 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 164, "n_selected": 43, "n_implied": 10, "n_structural": 6, "ret_R": 0.5714, "P": 0.2558, "R": 0.7857, "F1": 0.386, "leaf_P": 0.1379, "leaf_R": 0.4, "leaf_F1": 0.2051, "n_leaf_sel": 29, "n_leaf_gt": 10, "ret_P": 0.0488, "sel_given_ret": 1.375, "over_sel": 3.07, "why": {"explicit": 17, "strong_implied": 10}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 60, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "43": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2683, "gen_R": 0.7857, "gen_F1": 0.4, "missed": ["fur", "hair", "human"], "extra": ["anthro", "belly_dancer_outfit", "bottomwear", "cheek_to_cheek", "donkey_kong_(series)", "fist", "forearms", "gorilla", "grinning_at_viewer", "hand_above_head", "holding_arms", "humanoid", "kong", "loincloth", "monkey", "nintendo", "polar_bear", "raised_arms", "raised_hand", "relaxed_expression", "sighing", "smile", "smiling_at_viewer", "smirk", "smirking_at_viewer", "smug_eyes", "spread_arms", "surprised_expression", "trio", "ursine", "wide_grin", "yelling"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["anthro", "ape", "bear", "belly_dancer_outfit", "bottomwear", "cheek_to_cheek", "clothed", "clothing", "dancing", "donkey_kong_(series)", "fist", "forearms", "gorilla", "grinning_at_viewer", "group", "hand_above_head", "haplorhine", "holding_arms", "humanoid", "kong", "loincloth", "looking_at_viewer", "male", "mammal", "monkey", "nintendo", "polar_bear", "primate", "raised_arms", "raised_hand", "relaxed_expression", "sighing", "smile", "smiling_at_viewer", "smirk", "smirking_at_viewer", "smug_eyes", "spread_arms", "surprised_expression", "trio", "ursine", "wide_grin", "yelling"], "stage3_selected": ["ape", "bear", "belly_dancer_outfit", "cheek_to_cheek", "dancing", "fist", "forearms", "gorilla", "grinning_at_viewer", "hand_above_head", "holding_arms", "kong", "loincloth", "monkey", "polar_bear", "primate", "raised_arms", "raised_hand", "relaxed_expression", "sighing", "smirking_at_viewer", "smug_eyes", "spread_arms", "surprised_expression", "ursine", "wide_grin", "yelling"], "stage3_selected_scores": {"bear": 0.5551, "primate": 0.8894, "ursine": 0.4202, "loincloth": 0.5634, "dancing": 0.5523, "monkey": 0.7547, "ape": 0.9763, "raised_arms": 0.5407, "yelling": 0.3659, "gorilla": 0.8287, "spread_arms": 0.3987, "kong": 0.7493, "hand_above_head": 0.374, "grinning_at_viewer": 0.4379, "belly_dancer_outfit": 0.351, "relaxed_expression": 0.4112, "wide_grin": 0.5248, "polar_bear": 0.4316, "fist": 0.3108, "surprised_expression": 0.3615, "raised_hand": 0.3752, "forearms": 0.4005, "smirking_at_viewer": 0.4305, "holding_arms": 0.3945, "sighing": 0.3646, "cheek_to_cheek": 0.3672, "smug_eyes": 0.3425}, "stage3_selected_ranks": {"bear": 15, "primate": 2, "ursine": 59, "loincloth": 13, "dancing": 16, "monkey": 6, "ape": 1, "raised_arms": 21, "yelling": 109, "gorilla": 5, "spread_arms": 80, "kong": 7, "hand_above_head": 103, "grinning_at_viewer": 43, "belly_dancer_outfit": 122, "relaxed_expression": 72, "wide_grin": 23, "polar_bear": 47, "fist": 167, "surprised_expression": 115, "raised_hand": 102, "forearms": 78, "smirking_at_viewer": 48, "holding_arms": 82, "sighing": 111, "cheek_to_cheek": 108, "smug_eyes": 133}, "stage3_selected_phrase_ranks": {"bear": 1, "primate": 1, "ursine": 9, "loincloth": 1, "dancing": 1, "monkey": 6, "ape": 1, "raised_arms": 1, "yelling": 3, "gorilla": 1, "spread_arms": 5, "kong": 5, "hand_above_head": 10, "grinning_at_viewer": 3, "belly_dancer_outfit": 7, "relaxed_expression": 7, "wide_grin": 1, "polar_bear": 6, "fist": 10, "surprised_expression": 10, "raised_hand": 8, "forearms": 5, "smirking_at_viewer": 4, "holding_arms": 7, "sighing": 4, "cheek_to_cheek": 3, "smug_eyes": 5}, "extra_evidence": {"anthro": {"source": "structural"}, "belly_dancer_outfit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.351}, "bottomwear": {"source": "implied"}, "cheek_to_cheek": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3672}, "donkey_kong_(series)": {"source": "implied"}, "fist": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3108}, "forearms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4005}, "gorilla": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8287}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4379}, "hand_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.374}, "holding_arms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3945}, "humanoid": {"source": "structural"}, "kong": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7493}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "nintendo": {"source": "implied"}, "polar_bear": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4316}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_hand": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3752}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4112}, "sighing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3646}, "smile": {"source": "implied"}, "smiling_at_viewer": {"source": "implied"}, "smirk": {"source": "implied"}, "smirking_at_viewer": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4305}, "smug_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3425}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "surprised_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3615}, "trio": {"source": "structural"}, "ursine": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}, "yelling": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3659}}, "structural": ["trio", "anthro", "humanoid", "male", "clothed", "looking_at_viewer"], "t1": 1.58, "t2": 6.97, "t3": 9.74, "t3s": 4.05, "err": null, "issues": ["Stage3 split: general=163 entity=5 copyright_filtered=4 generic_char_to_general=1 unknown_type=3"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 133, "n_selected": 38, "n_implied": 10, "n_structural": 5, "ret_R": 0.7143, "P": 0.3421, "R": 0.9286, "F1": 0.5, "leaf_P": 0.2308, "leaf_R": 0.6667, "leaf_F1": 0.3429, "n_leaf_sel": 26, "n_leaf_gt": 9, "ret_P": 0.0752, "sel_given_ret": 1.3, "over_sel": 2.71, "why": {"explicit": 19, "strong_implied": 4}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 59, "attempts_by_n_local": {"60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.4, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3421, "gen_R": 0.9286, "gen_F1": 0.5, "missed": ["blue_eyes"], "extra": ["blue_hair", "coat", "comforting", "cuddling", "embrace", "eyes", "female", "fur_coat", "hair", "holding_object", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "looking_at_viewer", "pink_blush", "relationship", "romantic_ambiance", "round_ears", "setting", "teal_clothing", "topwear", "toy", "vest", "winter_coat"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["anthro", "blue_hair", "blush", "clothed", "clothing", "coat", "comforting", "cuddling", "duo", "embrace", "eyes", "female", "fur_coat", "hair", "holding_object", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "lagomorph", "leporid", "looking_at_viewer", "mammal", "pink_blush", "plushie", "rabbit", "relationship", "romantic", "romantic_ambiance", "romantic_couple", "round_ears", "setting", "teal_clothing", "teal_eyes", "topwear", "toy", "vest", "winter_coat"], "stage3_selected": ["blue_hair", "coat", "comforting", "cuddling", "embrace", "eyes", "fur_coat", "holding_plushie", "holding_toy", "inanimate_object", "lab_coat", "pink_blush", "plushie", "rabbit", "relationship", "romantic_ambiance", "romantic_couple", "round_ears", "setting", "teal_clothing", "teal_eyes", "vest", "winter_coat"], "stage3_selected_scores": {"rabbit": 0.5936, "blue_hair": 0.4198, "embrace": 0.4349, "coat": 0.638, "plushie": 0.7452, "vest": 0.5025, "cuddling": 0.4801, "teal_eyes": 0.6281, "lab_coat": 0.5158, "round_ears": 0.434, "inanimate_object": 0.4648, "pink_blush": 0.4648, "holding_plushie": 0.7791, "fur_coat": 0.4905, "winter_coat": 0.4757, "teal_clothing": 0.4335, "holding_toy": 0.5853, "setting": 0.5566, "eyes": 0.8767, "romantic_couple": 0.5617, "romantic_ambiance": 0.5277, "comforting": 0.4366, "relationship": 0.6203}, "stage3_selected_ranks": {"rabbit": 13, "blue_hair": 118, "embrace": 101, "coat": 7, "plushie": 3, "vest": 47, "cuddling": 63, "teal_eyes": 8, "lab_coat": 40, "round_ears": 102, "inanimate_object": 73, "pink_blush": 74, "holding_plushie": 2, "fur_coat": 53, "winter_coat": 66, "teal_clothing": 103, "holding_toy": 14, "setting": 20, "eyes": 1, "romantic_couple": 18, "romantic_ambiance": 30, "comforting": 100, "relationship": 9}, "stage3_selected_phrase_ranks": {"rabbit": 1, "blue_hair": 8, "embrace": 7, "coat": 1, "plushie": 1, "vest": 6, "cuddling": 4, "teal_eyes": 1, "lab_coat": 5, "round_ears": 10, "inanimate_object": 9, "pink_blush": 4, "holding_plushie": 1, "fur_coat": 9, "winter_coat": 10, "teal_clothing": 6, "holding_toy": 4, "setting": 1, "eyes": 1, "romantic_couple": 1, "romantic_ambiance": 3, "comforting": 9, "relationship": 1}, "extra_evidence": {"blue_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4198}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "comforting": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4366}, "cuddling": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4801}, "embrace": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4349}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "female": {"source": "structural"}, "fur_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4905}, "hair": {"source": "implied"}, "holding_object": {"source": "implied"}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "holding_toy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5853}, "inanimate_object": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "lab_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "looking_at_viewer": {"source": "structural"}, "pink_blush": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "relationship": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6203}, "romantic_ambiance": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5277}, "round_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.434}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5566}, "teal_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4335}, "topwear": {"source": "implied"}, "toy": {"source": "implied"}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5025}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4757}}, "structural": ["duo", "anthro", "female", "clothed", "looking_at_viewer"], "t1": 1.77, "t2": 6.15, "t3": 79.72, "t3s": 1.33, "err": null, "issues": ["Stage3 split: general=131 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"style_or_meta\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"strong_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"strong_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"style_or_meta\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 53}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"strong_implied\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"strong_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 56, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 58}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 58}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 124, "n_selected": 24, "n_implied": 2, "n_structural": 4, "ret_R": 0.75, "P": 0.1667, "R": 1.0, "F1": 0.2857, "leaf_P": 0.15, "leaf_R": 0.75, "leaf_F1": 0.25, "n_leaf_sel": 20, "n_leaf_gt": 4, "ret_P": 0.0242, "sel_given_ret": 1.3333, "over_sel": 6.0, "why": {"explicit": 15, "strong_implied": 5}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 71, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "10": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1667, "gen_R": 1.0, "gen_F1": 0.2857, "missed": [], "extra": ["ambiguous_gender", "eyes", "feral", "floating", "floating_limbs", "hovering", "midair", "red_spots", "round_eyes", "round_head", "smiling_at_each_other", "smiling_at_viewer", "smirk", "smirking_at_viewer", "spots", "spotted_face", "tan_face", "text", "unknown_species", "x_eyes"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["ambiguous_gender", "eyes", "feral", "floating", "floating_limbs", "hovering", "midair", "red_nose", "red_spots", "round_eyes", "round_head", "smile", "smiling_at_each_other", "smiling_at_viewer", "smirk", "smirking_at_viewer", "solo", "spots", "spotted_face", "tan_body", "tan_face", "text", "unknown_species", "x_eyes"], "stage3_selected": ["eyes", "floating", "floating_limbs", "hovering", "midair", "red_nose", "red_spots", "round_eyes", "round_head", "simple_background", "smile", "smiling_at_each_other", "smirking_at_viewer", "spots", "spotted_face", "tan_body", "tan_face", "unknown_species", "white_background", "x_eyes"], "stage3_selected_scores": {"simple_background": 0.5276, "smile": 0.6033, "white_background": 0.5412, "tan_body": 0.6726, "spots": 0.6278, "unknown_species": 0.5951, "red_nose": 0.7427, "midair": 0.4352, "x_eyes": 0.4592, "smirking_at_viewer": 0.4521, "red_spots": 0.6254, "round_eyes": 0.8854, "hovering": 0.6368, "floating_limbs": 0.4376, "eyes": 0.929, "floating": 0.6711, "tan_face": 0.7175, "spotted_face": 0.7275, "round_head": 0.6693, "smiling_at_each_other": 0.4455}, "stage3_selected_ranks": {"simple_background": 66, "smile": 45, "white_background": 62, "tan_body": 18, "spots": 37, "unknown_species": 49, "red_nose": 4, "midair": 114, "x_eyes": 99, "smirking_at_viewer": 101, "red_spots": 38, "round_eyes": 3, "hovering": 34, "floating_limbs": 111, "eyes": 1, "floating": 20, "tan_face": 8, "spotted_face": 6, "round_head": 23, "smiling_at_each_other": 108}, "stage3_selected_phrase_ranks": {"simple_background": 6, "smile": 2, "white_background": 4, "tan_body": 6, "spots": 9, "unknown_species": 1, "red_nose": 1, "midair": 7, "x_eyes": 6, "smirking_at_viewer": 8, "red_spots": 10, "round_eyes": 1, "hovering": 1, "floating_limbs": 6, "eyes": 1, "floating": 1, "tan_face": 1, "spotted_face": 1, "round_head": 3, "smiling_at_each_other": 10}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.929}, "feral": {"source": "structural"}, "floating": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6711}, "floating_limbs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "hovering": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6368}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4352}, "red_spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6254}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8854}, "round_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6693}, "smiling_at_each_other": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4455}, "smiling_at_viewer": {"source": "implied"}, "smirk": {"source": "implied"}, "smirking_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4521}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6278}, "spotted_face": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7275}, "tan_face": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7175}, "text": {"source": "structural"}, "unknown_species": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5951}, "x_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4592}}, "structural": ["solo", "feral", "ambiguous_gender", "text"], "t1": 2.04, "t2": 5.48, "t3": 11.81, "t3s": 2.65, "err": null, "issues": ["Stage3 split: general=130 entity=0 copyright_filtered=0 generic_char_to_general=0 unknown_type=4"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 207, "n_selected": 71, "n_implied": 23, "n_structural": 5, "ret_R": 0.4545, "P": 0.2535, "R": 0.8182, "F1": 0.3871, "leaf_P": 0.0732, "leaf_R": 0.25, "leaf_F1": 0.1132, "n_leaf_sel": 41, "n_leaf_gt": 12, "ret_P": 0.0483, "sel_given_ret": 1.8, "over_sel": 3.23, "why": {"explicit": 37, "strong_implied": 6}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 78, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "26": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2535, "gen_R": 0.8182, "gen_F1": 0.3871, "missed": ["chest_tuft", "hand_on_head", "muscular_male", "topless"], "extra": ["action_pose", "big_biceps", "big_muscles", "blue_body", "blue_bottomwear", "blue_clothing", "blue_countershading", "blue_shirt", "blue_shorts", "blue_tank_top", "blue_topwear", "cheek_tuft", "confident", "countershade_fur", "ear_tuft", "eyes", "facial_tuft", "flexing_bicep", "full-length_portrait", "fur_pattern", "grass", "hand_gesture", "hand_on_chest", "heterochromia", "looking_at_viewer", "muscular_arms", "muscular_thighs", "obliques", "plant", "portrait", "pose", "posed", "raised_hand", "raised_head", "relaxed_expression", "shirt", "siberian_tiger", "striped_body", "striped_fur", "striped_neck", "tan_bottomwear", "tan_clothing", "tan_shorts", "tank_top", "toony_expression", "topwear", "tufted_fur", "two_tone_chest", "two_tone_neck", "white_bottomwear", "white_chest", "white_clothing", "white_shorts"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["action_pose", "anthro", "big_biceps", "big_muscles", "blue_body", "blue_bottomwear", "blue_clothing", "blue_countershading", "blue_eyes", "blue_shirt", "blue_shorts", "blue_tank_top", "blue_topwear", "bottomwear", "cheek_tuft", "clothed", "clothing", "confident", "countershade_fur", "countershading", "ear_tuft", "eyes", "facial_tuft", "felid", "flexing_bicep", "full-length_portrait", "fur", "fur_pattern", "grass", "hand_gesture", "hand_on_chest", "heterochromia", "looking_at_viewer", "male", "mammal", "muscular", "muscular_anthro", "muscular_arms", "muscular_thighs", "obliques", "pantherine", "plant", "portrait", "pose", "posed", "raised_hand", "raised_head", "relaxed_expression", "shirt", "shorts", "siberian_tiger", "solo", "striped_body", "striped_fur", "striped_neck", "stripes", "tan_bottomwear", "tan_clothing", "tan_shorts", "tank_top", "tiger", "toony_expression", "topwear", "tuft", "tufted_fur", "two_tone_chest", "two_tone_neck", "white_bottomwear", "white_chest", "white_clothing", "white_shorts"], "stage3_selected": ["action_pose", "big_biceps", "big_muscles", "blue_bottomwear", "blue_countershading", "blue_eyes", "blue_shorts", "blue_tank_top", "cheek_tuft", "confident", "countershade_fur", "ear_tuft", "eyes", "flexing_bicep", "full-length_portrait", "fur_pattern", "grass", "hand_gesture", "hand_on_chest", "heterochromia", "muscular_anthro", "muscular_arms", "muscular_thighs", "obliques", "posed", "raised_hand", "raised_head", "relaxed_expression", "shorts", "siberian_tiger", "striped_fur", "striped_neck", "stripes", "tan_shorts", "tiger", "toony_expression", "tuft", "tufted_fur", "two_tone_chest", "two_tone_neck", "white_bottomwear", "white_chest", "white_shorts"], "stage3_selected_scores": {"blue_eyes": 0.5834, "tuft": 0.5054, "stripes": 0.4817, "shorts": 0.5928, "tiger": 0.6036, "cheek_tuft": 0.4887, "full-length_portrait": 0.4553, "striped_fur": 0.6535, "heterochromia": 0.4128, "ear_tuft": 0.6019, "blue_bottomwear": 0.5664, "hand_on_chest": 0.4215, "muscular_thighs": 0.7072, "countershade_fur": 0.673, "action_pose": 0.4803, "muscular_arms": 0.7914, "raised_hand": 0.7019, "big_biceps": 0.6865, "white_bottomwear": 0.5996, "blue_shorts": 0.6243, "flexing_bicep": 0.6599, "blue_countershading": 0.682, "confident": 0.5015, "white_chest": 0.9241, "hand_gesture": 0.5158, "tufted_fur": 0.488, "white_shorts": 0.6109, "two_tone_neck": 0.8256, "striped_neck": 0.686, "blue_tank_top": 0.5693, "siberian_tiger": 0.4911, "raised_head": 0.5209, "tan_shorts": 0.6103, "two_tone_chest": 0.8327, "relaxed_expression": 0.4837, "toony_expression": 0.4807, "eyes": 0.9775, "muscular_anthro": 0.5506, "grass": 0.4572, "big_muscles": 0.5544, "obliques": 0.6211, "fur_pattern": 0.4741, "posed": 0.442}, "stage3_selected_ranks": {"blue_eyes": 66, "tuft": 105, "stripes": 131, "shorts": 63, "tiger": 54, "cheek_tuft": 125, "full-length_portrait": 156, "striped_fur": 39, "heterochromia": 182, "ear_tuft": 56, "blue_bottomwear": 70, "hand_on_chest": 177, "muscular_thighs": 24, "countershade_fur": 31, "action_pose": 135, "muscular_arms": 14, "raised_hand": 25, "big_biceps": 26, "white_bottomwear": 58, "blue_shorts": 47, "flexing_bicep": 36, "blue_countershading": 29, "confident": 112, "white_chest": 2, "hand_gesture": 96, "tufted_fur": 126, "white_shorts": 50, "two_tone_neck": 10, "striped_neck": 27, "blue_tank_top": 69, "siberian_tiger": 122, "raised_head": 89, "tan_shorts": 51, "two_tone_chest": 7, "relaxed_expression": 129, "toony_expression": 132, "eyes": 1, "muscular_anthro": 73, "grass": 154, "big_muscles": 72, "obliques": 48, "fur_pattern": 137, "posed": 168}, "stage3_selected_phrase_ranks": {"blue_eyes": 2, "tuft": 4, "stripes": 2, "shorts": 1, "tiger": 1, "cheek_tuft": 7, "full-length_portrait": 9, "striped_fur": 2, "heterochromia": 8, "ear_tuft": 2, "blue_bottomwear": 10, "hand_on_chest": 10, "muscular_thighs": 3, "countershade_fur": 6, "action_pose": 7, "muscular_arms": 1, "raised_hand": 1, "big_biceps": 7, "white_bottomwear": 7, "blue_shorts": 3, "flexing_bicep": 5, "blue_countershading": 5, "confident": 1, "white_chest": 1, "hand_gesture": 5, "tufted_fur": 8, "white_shorts": 4, "two_tone_neck": 7, "striped_neck": 4, "blue_tank_top": 9, "siberian_tiger": 2, "raised_head": 10, "tan_shorts": 2, "two_tone_chest": 5, "relaxed_expression": 4, "toony_expression": 6, "eyes": 1, "muscular_anthro": 10, "grass": 9, "big_muscles": 9, "obliques": 9, "fur_pattern": 8, "posed": 10}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4803}, "big_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6865}, "big_muscles": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5544}, "blue_body": {"source": "implied"}, "blue_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5664}, "blue_clothing": {"source": "implied"}, "blue_countershading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.682}, "blue_shirt": {"source": "implied"}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6243}, "blue_tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5693}, "blue_topwear": {"source": "implied"}, "cheek_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4887}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "countershade_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.673}, "ear_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6019}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9775}, "facial_tuft": {"source": "implied"}, "flexing_bicep": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6599}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4553}, "fur_pattern": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4741}, "grass": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4572}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "hand_on_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4215}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "looking_at_viewer": {"source": "structural"}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7914}, "muscular_thighs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7072}, "obliques": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6211}, "plant": {"source": "implied"}, "portrait": {"source": "implied"}, "pose": {"source": "implied"}, "posed": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.442}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7019}, "raised_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5209}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4837}, "shirt": {"source": "implied"}, "siberian_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4911}, "striped_body": {"source": "implied"}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6535}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.686}, "tan_bottomwear": {"source": "implied"}, "tan_clothing": {"source": "implied"}, "tan_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6103}, "tank_top": {"source": "implied"}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4807}, "topwear": {"source": "implied"}, "tufted_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.488}, "two_tone_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8327}, "two_tone_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8256}, "white_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5996}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9241}, "white_clothing": {"source": "implied"}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6109}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 1.94, "t2": 1.76, "t3": 44.9, "t3s": 0.99, "err": null, "issues": ["Stage3 split: general=206 entity=3 copyright_filtered=3 generic_char_to_general=0 unknown_type=3"]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 187, "n_selected": 42, "n_implied": 11, "n_structural": 5, "ret_R": 0.6923, "P": 0.2143, "R": 0.6923, "F1": 0.3273, "leaf_P": 0.1034, "leaf_R": 0.5, "leaf_F1": 0.1714, "n_leaf_sel": 29, "n_leaf_gt": 6, "ret_P": 0.0481, "sel_given_ret": 1.0, "over_sel": 3.23, "why": {"explicit": 15, "strong_implied": 12}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 90, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "57": {"attempts": 3, "parse_ok": 1, "parse_fail": 0, "errors": 2}}, "attempt_failure_rate": 0.5, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2143, "gen_R": 0.6923, "gen_F1": 0.3273, "missed": ["dialogue", "fur", "white_body", "white_fur"], "extra": ["ambiguous_gender", "anthro", "attack", "bubble", "clothed", "clothing", "creepy", "darkness", "defeated", "elemental_creature", "evil_look", "eyemask", "fight", "group", "guardian", "hands_on_ground", "hands_on_hips", "human_to_humanoid", "kneeling", "kneeling_on_ground", "light", "looking_angry", "lying_on_ground", "medieval", "mineral_fauna", "monitor_lizard", "on_ground", "rock", "rock_creature", "species_transformation", "speech_bubble", "surrounded", "transformation"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["ambiguous_gender", "anthro", "attack", "bovid", "bubble", "caprine", "clothed", "clothing", "creepy", "darkness", "defeated", "elemental_creature", "evil_look", "eyemask", "fight", "goat", "group", "guardian", "hands_on_ground", "hands_on_hips", "human", "human_to_humanoid", "kneeling", "kneeling_on_ground", "light", "lizard", "looking_angry", "lying_on_ground", "mammal", "medieval", "mineral_fauna", "monitor_lizard", "on_ground", "reptile", "rock", "rock_creature", "scalie", "species_transformation", "speech_bubble", "surrounded", "text", "transformation"], "stage3_selected": ["attack", "bubble", "creepy", "darkness", "defeated", "evil_look", "eyemask", "fight", "goat", "guardian", "hands_on_ground", "hands_on_hips", "human", "human_to_humanoid", "kneeling_on_ground", "light", "lizard", "looking_angry", "lying_on_ground", "medieval", "monitor_lizard", "on_ground", "reptile", "rock_creature", "speech_bubble", "surrounded", "text"], "stage3_selected_scores": {"human": 0.6639, "speech_bubble": 0.7486, "lizard": 0.8361, "goat": 0.7748, "light": 0.7781, "on_ground": 0.6688, "bubble": 0.7514, "hands_on_hips": 0.5225, "defeated": 0.6149, "lying_on_ground": 0.7873, "darkness": 0.8346, "surrounded": 0.4808, "hands_on_ground": 0.5695, "eyemask": 0.5009, "kneeling_on_ground": 0.563, "text": 0.6006, "reptile": 0.5533, "fight": 0.5245, "creepy": 0.5133, "monitor_lizard": 0.5652, "attack": 0.5453, "medieval": 0.5251, "human_to_humanoid": 0.3295, "rock_creature": 0.5115, "evil_look": 0.564, "looking_angry": 0.492, "guardian": 0.456}, "stage3_selected_ranks": {"human": 19, "speech_bubble": 9, "lizard": 1, "goat": 5, "light": 4, "on_ground": 18, "bubble": 8, "hands_on_hips": 116, "defeated": 35, "lying_on_ground": 3, "darkness": 2, "surrounded": 162, "hands_on_ground": 62, "eyemask": 146, "kneeling_on_ground": 75, "text": 44, "reptile": 90, "fight": 115, "creepy": 127, "monitor_lizard": 69, "attack": 96, "medieval": 113, "human_to_humanoid": 187, "rock_creature": 130, "evil_look": 74, "looking_angry": 151, "guardian": 172}, "stage3_selected_phrase_ranks": {"human": 1, "speech_bubble": 1, "lizard": 1, "goat": 1, "light": 1, "on_ground": 3, "bubble": 1, "hands_on_hips": 7, "defeated": 2, "lying_on_ground": 1, "darkness": 1, "surrounded": 8, "hands_on_ground": 8, "eyemask": 7, "kneeling_on_ground": 9, "text": 5, "reptile": 8, "fight": 7, "creepy": 5, "monitor_lizard": 7, "attack": 5, "medieval": 4, "human_to_humanoid": 10, "rock_creature": 7, "evil_look": 5, "looking_angry": 6, "guardian": 8}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "anthro": {"source": "structural"}, "attack": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5453}, "bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7514}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "creepy": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5133}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "defeated": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6149}, "elemental_creature": {"source": "implied"}, "evil_look": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.564}, "eyemask": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5009}, "fight": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5245}, "group": {"source": "structural"}, "guardian": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.456}, "hands_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "hands_on_hips": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5225}, "human_to_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3295}, "kneeling": {"source": "implied"}, "kneeling_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.563}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "looking_angry": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.492}, "lying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7873}, "medieval": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5251}, "mineral_fauna": {"source": "implied"}, "monitor_lizard": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5652}, "on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6688}, "rock": {"source": "implied"}, "rock_creature": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5115}, "species_transformation": {"source": "implied"}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7486}, "surrounded": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4808}, "transformation": {"source": "implied"}}, "structural": ["group", "anthro", "ambiguous_gender", "clothed", "text"], "t1": 1.58, "t2": 1.46, "t3": 106.55, "t3s": 3.52, "err": null, "issues": ["Stage3 split: general=177 entity=4 copyright_filtered=6 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"other\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"other\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 48}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"other\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"explicit\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.33.why\n Field required [type=missing, input_value={'i': 34}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 15, "n_retrieved": 207, "n_selected": 38, "n_implied": 5, "n_structural": 5, "ret_R": 0.6667, "P": 0.2895, "R": 0.7333, "F1": 0.4151, "leaf_P": 0.2667, "leaf_R": 0.6667, "leaf_F1": 0.381, "n_leaf_sel": 30, "n_leaf_gt": 12, "ret_P": 0.0483, "sel_given_ret": 1.1, "over_sel": 2.53, "why": {"explicit": 28}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 69, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "29": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2895, "gen_R": 0.7333, "gen_F1": 0.4151, "missed": ["angry", "bed", "eyes_closed", "furniture"], "extra": ["3rd_party_watermark", "ambiguous_gender", "annoyed_expression", "applying_makeup", "auburn_hair", "bedroom", "clothed", "clothing", "comic", "digital_media_(artwork)", "english_text", "expressions", "eyeliner", "eyes", "half-closed_eyes", "heterochromia", "humanoid", "lipstick", "narrowed_eyes", "personal_grooming", "purple_eyes", "purple_hands", "red_lipstick", "relaxed_expression", "sleepwear", "watermark", "yellow_eyeshadow"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "furniture", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["3rd_party_watermark", "ambiguous_gender", "annoyed_expression", "applying_makeup", "auburn_hair", "bedroom", "blonde_hair", "blue_eyes", "clothed", "clothing", "comic", "digital_media_(artwork)", "duo", "english_text", "expressions", "eyeliner", "eyes", "eyeshadow", "green_eyes", "hair", "half-closed_eyes", "heterochromia", "humanoid", "lipstick", "lying", "makeup", "narrowed_eyes", "personal_grooming", "purple_eyes", "purple_hair", "purple_hands", "red_lipstick", "relaxed_expression", "sleeping", "sleepwear", "text", "watermark", "yellow_eyeshadow"], "stage3_selected": ["3rd_party_watermark", "annoyed_expression", "applying_makeup", "auburn_hair", "bedroom", "blonde_hair", "blue_eyes", "comic", "digital_media_(artwork)", "english_text", "expressions", "eyeliner", "eyes", "eyeshadow", "green_eyes", "half-closed_eyes", "heterochromia", "lipstick", "lying", "makeup", "purple_eyes", "purple_hair", "purple_hands", "red_lipstick", "relaxed_expression", "sleeping", "sleepwear", "yellow_eyeshadow"], "stage3_selected_scores": {"digital_media_(artwork)": 0.3347, "blue_eyes": 0.595, "lying": 0.436, "green_eyes": 0.5933, "comic": 0.3823, "blonde_hair": 0.5872, "half-closed_eyes": 0.3908, "purple_eyes": 0.4293, "purple_hair": 0.5591, "makeup": 0.5894, "eyeshadow": 0.4712, "lipstick": 0.4782, "bedroom": 0.4873, "sleeping": 0.5987, "heterochromia": 0.4, "eyeliner": 0.4399, "red_lipstick": 0.463, "annoyed_expression": 0.7219, "expressions": 0.5393, "auburn_hair": 0.399, "3rd_party_watermark": 0.3932, "purple_hands": 0.5404, "sleepwear": 0.444, "yellow_eyeshadow": 0.4499, "applying_makeup": 0.4698, "relaxed_expression": 0.5026, "eyes": 0.8955, "english_text": 0.4128}, "stage3_selected_ranks": {"digital_media_(artwork)": 192, "blue_eyes": 6, "lying": 77, "green_eyes": 9, "comic": 154, "blonde_hair": 11, "half-closed_eyes": 138, "purple_eyes": 79, "purple_hair": 14, "makeup": 10, "eyeshadow": 54, "lipstick": 48, "bedroom": 42, "sleeping": 5, "heterochromia": 122, "eyeliner": 71, "red_lipstick": 57, "annoyed_expression": 2, "expressions": 19, "auburn_hair": 123, "3rd_party_watermark": 134, "purple_hands": 17, "sleepwear": 67, "yellow_eyeshadow": 62, "applying_makeup": 55, "relaxed_expression": 32, "eyes": 1, "english_text": 101}, "stage3_selected_phrase_ranks": {"digital_media_(artwork)": 10, "blue_eyes": 1, "lying": 7, "green_eyes": 1, "comic": 10, "blonde_hair": 1, "half-closed_eyes": 9, "purple_eyes": 2, "purple_hair": 1, "makeup": 1, "eyeshadow": 3, "lipstick": 2, "bedroom": 1, "sleeping": 1, "heterochromia": 9, "eyeliner": 9, "red_lipstick": 5, "annoyed_expression": 1, "expressions": 3, "auburn_hair": 7, "3rd_party_watermark": 3, "purple_hands": 2, "sleepwear": 6, "yellow_eyeshadow": 6, "applying_makeup": 4, "relaxed_expression": 6, "eyes": 1, "english_text": 4}, "extra_evidence": {"3rd_party_watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "ambiguous_gender": {"source": "structural"}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4698}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.399}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3823}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5393}, "eyeliner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4399}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3908}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4}, "humanoid": {"source": "structural"}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4782}, "narrowed_eyes": {"source": "implied"}, "personal_grooming": {"source": "implied"}, "purple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4293}, "purple_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5404}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.463}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5026}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "watermark": {"source": "implied"}, "yellow_eyeshadow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4499}}, "structural": ["duo", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 2.05, "t2": 1.51, "t3": 20.15, "t3s": 0.77, "err": null, "issues": ["Stage3 split: general=209 entity=0 copyright_filtered=1 generic_char_to_general=0 unknown_type=4"]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 179, "n_selected": 89, "n_implied": 36, "n_structural": 4, "ret_R": 0.5, "P": 0.191, "R": 0.7727, "F1": 0.3063, "leaf_P": 0.087, "leaf_R": 0.3077, "leaf_F1": 0.1356, "n_leaf_sel": 46, "n_leaf_gt": 13, "ret_P": 0.0615, "sel_given_ret": 1.5455, "over_sel": 4.05, "why": {"explicit": 50}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 81, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "59": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.191, "gen_R": 0.7727, "gen_F1": 0.3063, "missed": ["fingers", "fur", "holding_musical_instrument", "holding_object", "music"], "extra": ["3_claws", "4_claws", "accessory", "acoustic_guitar", "action_pose", "auburn_hair", "big_claws", "big_hair", "big_tail", "blonde_hair", "bottomwear", "canis", "claws_out", "coywolf", "curled_hair", "denim", "denim_clothing", "digitigrade", "dire_wolf", "domestic_dog", "dreamcatcher", "ear_piercing", "ear_ring", "electric_guitar", "flowing_hair", "furgonomics", "holding_guitar", "holding_hair", "hybrid", "jeans", "jewelry", "jojo_pose", "leggings", "legwear", "long_claws", "male", "maned_wolf", "mexican_wolf", "mountain_dog", "neutral_expression", "notched_ear", "nub_tail", "pants", "personification", "piercing", "playing_guitar", "playing_music", "pointed_tail", "pose", "psychedelic", "ring", "ring_piercing", "shirt", "shocked_expression", "shorts", "spitz", "tail_accessory", "tail_jewelry", "tail_ring", "topless", "topwear", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_legwear", "torn_pants", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "wavy_hair", "wolf"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["3_claws", "4_claws", "accessory", "acoustic_guitar", "action_pose", "anthro", "auburn_hair", "bass_guitar", "big_claws", "big_hair", "big_tail", "blonde_hair", "bottomwear", "canid", "canine", "canis", "claws", "claws_out", "clothed", "clothing", "coywolf", "curled_hair", "denim", "denim_clothing", "digitigrade", "dire_wolf", "domestic_dog", "dreamcatcher", "ear_piercing", "ear_ring", "electric_guitar", "flowing_hair", "furgonomics", "guitar", "hair", "holding_guitar", "holding_hair", "hybrid", "jeans", "jewelry", "jojo_pose", "leggings", "legwear", "long_claws", "male", "mammal", "maned_wolf", "mexican_wolf", "mountain_dog", "musical_instrument", "neutral_expression", "notched_ear", "nub_tail", "pants", "personification", "piercing", "playing_guitar", "playing_music", "plucked_string_instrument", "pointed_tail", "pose", "psychedelic", "ring", "ring_piercing", "shirt", "shocked_expression", "shorts", "solo", "spade_tail", "spitz", "string_instrument", "tail", "tail_accessory", "tail_jewelry", "tail_ring", "topless", "topwear", "torn_bottomwear", "torn_clothing", "torn_jeans", "torn_leggings", "torn_legwear", "torn_pants", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "wavy_hair", "wolf"], "stage3_selected": ["3_claws", "4_claws", "acoustic_guitar", "action_pose", "auburn_hair", "bass_guitar", "big_claws", "big_hair", "big_tail", "blonde_hair", "claws_out", "coywolf", "curled_hair", "detailed_background", "digitigrade", "dire_wolf", "dreamcatcher", "ear_ring", "electric_guitar", "flowing_hair", "holding_guitar", "holding_hair", "jojo_pose", "long_claws", "maned_wolf", "mexican_wolf", "mountain_dog", "musical_instrument", "neutral_expression", "notched_ear", "nub_tail", "personification", "playing_guitar", "playing_music", "pointed_tail", "psychedelic", "shocked_expression", "spade_tail", "spitz", "string_instrument", "tail", "tail_ring", "torn_bottomwear", "torn_clothing", "torn_jeans", "torn_leggings", "torn_shirt", "torn_shorts", "wavy_hair", "wolf"], "stage3_selected_scores": {"tail": 0.5654, "wolf": 0.5777, "blonde_hair": 0.3817, "detailed_background": 0.4654, "digitigrade": 0.419, "ear_ring": 0.3919, "torn_clothing": 0.4129, "spitz": 0.444, "big_tail": 0.3836, "spade_tail": 0.6177, "notched_ear": 0.4314, "musical_instrument": 0.8463, "string_instrument": 0.8616, "torn_bottomwear": 0.436, "curled_hair": 0.3871, "action_pose": 0.5815, "tail_ring": 0.3933, "maned_wolf": 0.4595, "shocked_expression": 0.3892, "mountain_dog": 0.456, "wavy_hair": 0.3489, "playing_music": 0.873, "big_hair": 0.3626, "torn_shirt": 0.4046, "long_claws": 0.4361, "playing_guitar": 0.9316, "torn_jeans": 0.4822, "claws_out": 0.4376, "big_claws": 0.4295, "3_claws": 0.4376, "dire_wolf": 0.4474, "electric_guitar": 0.8663, "pointed_tail": 0.3765, "psychedelic": 0.39, "neutral_expression": 0.4846, "torn_shorts": 0.3993, "personification": 0.3494, "auburn_hair": 0.3765, "bass_guitar": 0.9118, "flowing_hair": 0.5667, "nub_tail": 0.4298, "4_claws": 0.4515, "mexican_wolf": 0.4378, "holding_guitar": 0.8442, "torn_leggings": 0.424, "dreamcatcher": 0.2735, "holding_hair": 0.3935, "jojo_pose": 0.4866, "acoustic_guitar": 0.8654, "coywolf": 0.4311}, "stage3_selected_ranks": {"tail": 21, "wolf": 16, "blonde_hair": 136, "detailed_background": 53, "digitigrade": 102, "ear_ring": 125, "torn_clothing": 106, "spitz": 75, "big_tail": 134, "spade_tail": 13, "notched_ear": 87, "musical_instrument": 10, "string_instrument": 8, "torn_bottomwear": 83, "curled_hair": 131, "action_pose": 15, "tail_ring": 124, "maned_wolf": 60, "shocked_expression": 130, "mountain_dog": 65, "wavy_hair": 164, "playing_music": 4, "big_hair": 154, "torn_shirt": 112, "long_claws": 82, "playing_guitar": 2, "torn_jeans": 43, "claws_out": 80, "big_claws": 95, "3_claws": 81, "dire_wolf": 73, "electric_guitar": 5, "pointed_tail": 140, "psychedelic": 127, "neutral_expression": 42, "torn_shorts": 117, "personification": 163, "auburn_hair": 141, "bass_guitar": 3, "flowing_hair": 20, "nub_tail": 93, "4_claws": 69, "mexican_wolf": 79, "holding_guitar": 11, "torn_leggings": 99, "dreamcatcher": 183, "holding_hair": 123, "jojo_pose": 40, "acoustic_guitar": 7, "coywolf": 88}, "stage3_selected_phrase_ranks": {"tail": 1, "wolf": 1, "blonde_hair": 6, "detailed_background": 6, "digitigrade": 10, "ear_ring": 4, "torn_clothing": 6, "spitz": 7, "big_tail": 9, "spade_tail": 1, "notched_ear": 4, "musical_instrument": 8, "string_instrument": 7, "torn_bottomwear": 3, "curled_hair": 5, "action_pose": 1, "tail_ring": 7, "maned_wolf": 3, "shocked_expression": 10, "mountain_dog": 6, "wavy_hair": 9, "playing_music": 3, "big_hair": 7, "torn_shirt": 7, "long_claws": 8, "playing_guitar": 1, "torn_jeans": 1, "claws_out": 6, "big_claws": 9, "3_claws": 7, "dire_wolf": 4, "electric_guitar": 4, "pointed_tail": 5, "psychedelic": 6, "neutral_expression": 6, "torn_shorts": 8, "personification": 7, "auburn_hair": 7, "bass_guitar": 2, "flowing_hair": 1, "nub_tail": 3, "4_claws": 4, "mexican_wolf": 5, "holding_guitar": 10, "torn_leggings": 5, "dreamcatcher": 8, "holding_hair": 4, "jojo_pose": 4, "acoustic_guitar": 5, "coywolf": 6}, "extra_evidence": {"3_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4515}, "accessory": {"source": "implied"}, "acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8654}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5815}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}, "big_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4295}, "big_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3626}, "big_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3836}, "blonde_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3817}, "bottomwear": {"source": "implied"}, "canis": {"source": "implied"}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "coywolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4311}, "curled_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3871}, "denim": {"source": "implied"}, "denim_clothing": {"source": "implied"}, "digitigrade": {"source": "stage3", "why": "explicit", "retrieval_score": 0.419}, "dire_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4474}, "domestic_dog": {"source": "implied"}, "dreamcatcher": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2735}, "ear_piercing": {"source": "implied"}, "ear_ring": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3919}, "electric_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8663}, "flowing_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5667}, "furgonomics": {"source": "implied"}, "holding_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8442}, "holding_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3935}, "hybrid": {"source": "implied"}, "jeans": {"source": "implied"}, "jewelry": {"source": "implied"}, "jojo_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4866}, "leggings": {"source": "implied"}, "legwear": {"source": "implied"}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4361}, "male": {"source": "structural"}, "maned_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4595}, "mexican_wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4378}, "mountain_dog": {"source": "stage3", "why": "explicit", "retrieval_score": 0.456}, "neutral_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4846}, "notched_ear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4314}, "nub_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4298}, "pants": {"source": "implied"}, "personification": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3494}, "piercing": {"source": "implied"}, "playing_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9316}, "playing_music": {"source": "stage3", "why": "explicit", "retrieval_score": 0.873}, "pointed_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}, "pose": {"source": "implied"}, "psychedelic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.39}, "ring": {"source": "implied"}, "ring_piercing": {"source": "implied"}, "shirt": {"source": "implied"}, "shocked_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3892}, "shorts": {"source": "implied"}, "spitz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "tail_accessory": {"source": "implied"}, "tail_jewelry": {"source": "implied"}, "tail_ring": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "topless": {"source": "structural"}, "topwear": {"source": "implied"}, "torn_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.436}, "torn_jeans": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4822}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.424}, "torn_legwear": {"source": "implied"}, "torn_pants": {"source": "implied"}, "torn_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4046}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3993}, "torn_topwear": {"source": "implied"}, "touching_hair": {"source": "implied"}, "wavy_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3489}, "wolf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5777}}, "structural": ["solo", "anthro", "male", "topless"], "t1": 1.05, "t2": 1.34, "t3": 47.01, "t3s": 2.72, "err": null, "issues": ["Stage3 split: general=179 entity=2 copyright_filtered=4 generic_char_to_general=0 unknown_type=2"]}
10
+ {"id": 2021552, "n_gt": 25, "n_retrieved": 151, "n_selected": 52, "n_implied": 19, "n_structural": 5, "ret_R": 0.6, "P": 0.4231, "R": 0.88, "F1": 0.5714, "leaf_P": 0.2308, "leaf_R": 0.4, "leaf_F1": 0.2927, "n_leaf_sel": 26, "n_leaf_gt": 15, "ret_P": 0.0993, "sel_given_ret": 1.4667, "over_sel": 2.08, "why": {"explicit": 14, "strong_implied": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 97, "attempts_by_n_local": {"60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "31": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4231, "gen_R": 0.88, "gen_F1": 0.5714, "missed": ["grey_background", "looking_at_another", "standing"], "extra": ["3_claws", "actual_fur", "ambiguous_gender", "arctic_fox", "black_bottomwear", "black_clothing", "black_pants", "blue_clothing", "blue_topwear", "cheek_markings", "cross_fox", "finger_claws", "fluffy_fur", "grey_clothing", "grey_shirt", "grey_topwear", "leg_markings", "looking_at_viewer", "open_mouth", "red_fox", "snout", "snout_markings", "spread_arms", "straight_arms", "toe_claws", "white_body", "white_clothing", "white_fur", "white_shirt", "white_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "head_markings", "lagomorph", "leporid", "looking_at_another", "mammal", "markings", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["3_claws", "actual_fur", "ambiguous_gender", "anthro", "arctic_fox", "black_bottomwear", "black_clothing", "black_pants", "blue_clothing", "blue_topwear", "bottomwear", "canid", "canine", "cheek_markings", "claws", "clothed", "clothing", "cross_fox", "crossed_arms", "duo", "facial_markings", "finger_claws", "fluffy_fur", "fox", "fur", "grey_clothing", "grey_shirt", "grey_topwear", "head_markings", "lagomorph", "leg_markings", "leporid", "looking_at_viewer", "mammal", "markings", "open_mouth", "overalls", "pants", "rabbit", "red_fox", "shirt", "snout", "snout_markings", "spread_arms", "straight_arms", "toe_claws", "topwear", "white_body", "white_clothing", "white_fur", "white_shirt", "white_topwear"], "stage3_selected": ["3_claws", "actual_fur", "arctic_fox", "black_bottomwear", "black_pants", "blue_clothing", "blue_topwear", "cheek_markings", "claws", "cross_fox", "crossed_arms", "facial_markings", "finger_claws", "fluffy_fur", "fur", "grey_shirt", "leg_markings", "leporid", "open_mouth", "overalls", "rabbit", "shirt", "simple_background", "snout_markings", "spread_arms", "straight_arms", "toe_claws", "white_fur", "white_shirt"], "stage3_selected_scores": {"fur": 0.653, "simple_background": 0.4159, "open_mouth": 0.6329, "claws": 0.6303, "shirt": 0.7482, "rabbit": 0.651, "facial_markings": 0.6944, "blue_clothing": 0.6538, "crossed_arms": 0.7284, "white_shirt": 0.8197, "overalls": 0.8775, "black_pants": 0.8329, "straight_arms": 0.5178, "actual_fur": 0.4837, "white_fur": 0.5148, "leporid": 0.5826, "toe_claws": 0.5549, "leg_markings": 0.6051, "finger_claws": 0.5344, "black_bottomwear": 0.7383, "blue_topwear": 0.6659, "arctic_fox": 0.4522, "spread_arms": 0.55, "snout_markings": 0.6213, "grey_shirt": 0.6922, "cross_fox": 0.4687, "3_claws": 0.5531, "cheek_markings": 0.6211, "fluffy_fur": 0.4964}, "stage3_selected_ranks": {"fur": 40, "simple_background": 153, "open_mouth": 48, "claws": 50, "shirt": 18, "rabbit": 41, "facial_markings": 27, "blue_clothing": 39, "crossed_arms": 21, "white_shirt": 4, "overalls": 2, "black_pants": 3, "straight_arms": 120, "actual_fur": 137, "white_fur": 123, "leporid": 76, "toe_claws": 91, "leg_markings": 61, "finger_claws": 109, "black_bottomwear": 19, "blue_topwear": 33, "arctic_fox": 144, "spread_arms": 96, "snout_markings": 54, "grey_shirt": 28, "cross_fox": 141, "3_claws": 92, "cheek_markings": 55, "fluffy_fur": 129}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 8, "open_mouth": 1, "claws": 1, "shirt": 1, "rabbit": 1, "facial_markings": 1, "blue_clothing": 8, "crossed_arms": 1, "white_shirt": 1, "overalls": 1, "black_pants": 1, "straight_arms": 9, "actual_fur": 10, "white_fur": 4, "leporid": 4, "toe_claws": 4, "leg_markings": 8, "finger_claws": 6, "black_bottomwear": 5, "blue_topwear": 4, "arctic_fox": 5, "spread_arms": 5, "snout_markings": 3, "grey_shirt": 4, "cross_fox": 4, "3_claws": 5, "cheek_markings": 4, "fluffy_fur": 5}, "extra_evidence": {"3_claws": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5531}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4837}, "ambiguous_gender": {"source": "structural"}, "arctic_fox": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4522}, "black_bottomwear": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7383}, "black_clothing": {"source": "implied"}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8329}, "blue_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6538}, "blue_topwear": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6659}, "cheek_markings": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6211}, "cross_fox": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4687}, "finger_claws": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5344}, "fluffy_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4964}, "grey_clothing": {"source": "implied"}, "grey_shirt": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6922}, "grey_topwear": {"source": "implied"}, "leg_markings": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6051}, "looking_at_viewer": {"source": "structural"}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6329}, "red_fox": {"source": "implied"}, "snout": {"source": "implied"}, "snout_markings": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6213}, "spread_arms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.55}, "straight_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5178}, "toe_claws": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5549}, "white_body": {"source": "implied"}, "white_clothing": {"source": "implied"}, "white_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5148}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_topwear": {"source": "implied"}}, "structural": ["duo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 4.24, "t2": 1.08, "t3": 67.32, "t3s": 0.84, "err": null, "issues": ["Stage3 split: general=151 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3"]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 204, "n_selected": 82, "n_implied": 23, "n_structural": 5, "ret_R": 0.6364, "P": 0.1341, "R": 1.0, "F1": 0.2366, "leaf_P": 0.0926, "leaf_R": 0.7143, "leaf_F1": 0.1639, "n_leaf_sel": 54, "n_leaf_gt": 7, "ret_P": 0.0343, "sel_given_ret": 1.5714, "over_sel": 7.45, "why": {"explicit": 31, "strong_implied": 24}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": false, "bootstrap_anchor_pool_size": 0, "bootstrap_anchor_selected": 0, "bootstrap_anchor_kept": 0, "bootstrap_reranked": false, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 91, "attempts_by_n_local": {"60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1341, "gen_R": 1.0, "gen_F1": 0.2366, "missed": [], "extra": ["action_pose", "actual_fur", "ambiguous_gender", "animal_humanoid", "anime_eyes", "anthro", "big_eyes", "big_mouth_(anatomy)", "blue_fingers", "blue_pawpads", "blue_paws", "blue_stripes", "blue_toes", "blue_tuft", "canid_humanoid", "canine_humanoid", "clothed", "clothing", "curved_tail", "expressions", "facial_markings", "facial_stripes", "fluffy_fur", "fox_humanoid", "glistening", "glistening_nose", "half-closed_eyes", "head_markings", "heterochromia", "humanoid", "jumper", "looking_at_viewer", "male", "male_humanoid", "mammal_humanoid", "markings", "membrane_(anatomy)", "midair", "multicolored_body", "multicolored_fur", "multicolored_tail", "muscular", "muscular_male", "narrowed_eyes", "open_smile", "paw_pose", "pawpads", "pink_ears", "pink_legs", "pink_stripes", "pose", "posed", "purple_membrane", "smile", "striped_face", "striped_neck", "stripes", "suina", "suina_humanoid", "tail", "tailed_humanoid", "tanuki_humanoid", "teal_fur", "tongue", "tongue_out", "toony_expression", "two_tone_tail", "walking", "white_tail", "x_eyes", "yellow_stripes"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["action_pose", "actual_fur", "ambiguous_gender", "animal_humanoid", "anime_eyes", "anthro", "big_eyes", "big_mouth_(anatomy)", "blue_eyes", "blue_fingers", "blue_nose", "blue_pawpads", "blue_paws", "blue_stripes", "blue_toes", "blue_tuft", "canid", "canid_humanoid", "canine", "canine_humanoid", "clothed", "clothing", "curved_tail", "expressions", "facial_markings", "facial_stripes", "fluffy_fur", "fox_humanoid", "fur", "glistening", "glistening_nose", "half-closed_eyes", "head_markings", "heterochromia", "humanoid", "jumper", "looking_at_viewer", "male", "male_humanoid", "mammal", "mammal_humanoid", "markings", "membrane_(anatomy)", "midair", "multicolored_body", "multicolored_fur", "multicolored_tail", "muscular", "muscular_male", "narrowed_eyes", "open_mouth", "open_smile", "paw_pose", "pawpads", "pink_ears", "pink_legs", "pink_stripes", "pose", "posed", "purple_body", "purple_membrane", "smile", "solo", "striped_face", "striped_neck", "stripes", "suina", "suina_humanoid", "tail", "tailed_humanoid", "tanuki_humanoid", "teal_fur", "tongue", "tongue_out", "toony_expression", "two_tone_tail", "walking", "white_body", "white_fur", "white_tail", "x_eyes", "yellow_stripes"], "stage3_selected": ["action_pose", "actual_fur", "animal_humanoid", "anime_eyes", "big_eyes", "big_mouth_(anatomy)", "blue_eyes", "blue_fingers", "blue_nose", "blue_pawpads", "blue_paws", "blue_stripes", "blue_toes", "blue_tuft", "curved_tail", "expressions", "facial_stripes", "fluffy_fur", "fox_humanoid", "glistening_nose", "half-closed_eyes", "heterochromia", "humanoid", "jumper", "male_humanoid", "midair", "multicolored_fur", "multicolored_tail", "muscular_male", "open_mouth", "open_smile", "paw_pose", "pink_ears", "pink_legs", "pink_stripes", "posed", "purple_body", "purple_membrane", "simple_background", "striped_face", "striped_neck", "suina_humanoid", "tail", "tailed_humanoid", "tanuki_humanoid", "teal_fur", "tongue_out", "toony_expression", "two_tone_tail", "walking", "white_body", "white_fur", "white_tail", "x_eyes", "yellow_stripes"], "stage3_selected_scores": {"simple_background": 0.5945, "open_mouth": 0.6005, "tail": 0.6105, "blue_eyes": 0.5993, "white_fur": 0.5992, "humanoid": 0.6698, "muscular_male": 0.31, "half-closed_eyes": 0.3627, "purple_body": 0.5638, "open_smile": 0.4866, "multicolored_tail": 0.4716, "white_tail": 0.5201, "heterochromia": 0.4421, "two_tone_tail": 0.5195, "blue_nose": 0.6031, "blue_pawpads": 0.4889, "glistening_nose": 0.4645, "blue_stripes": 0.6746, "x_eyes": 0.3998, "blue_paws": 0.4985, "tanuki_humanoid": 0.7573, "pink_stripes": 0.6819, "curved_tail": 0.6369, "male_humanoid": 0.5626, "striped_neck": 0.5948, "big_mouth_(anatomy)": 0.4805, "suina_humanoid": 0.5629, "blue_tuft": 0.5037, "purple_membrane": 0.5811, "actual_fur": 0.4562, "teal_fur": 0.4367, "white_body": 0.4872, "tongue_out": 0.3534, "multicolored_fur": 0.4993, "animal_humanoid": 0.6158, "fox_humanoid": 0.8203, "big_eyes": 0.4205, "walking": 0.3532, "action_pose": 0.6168, "pink_ears": 0.5254, "midair": 0.4197, "paw_pose": 0.5581, "yellow_stripes": 0.5954, "expressions": 0.4954, "striped_face": 0.5806, "tailed_humanoid": 0.5524, "fluffy_fur": 0.5593, "blue_fingers": 0.5077, "anime_eyes": 0.49, "blue_toes": 0.5148, "facial_stripes": 0.5968, "posed": 0.4482, "jumper": 0.4076, "pink_legs": 0.5284, "toony_expression": 0.4501}, "stage3_selected_ranks": {"simple_background": 36, "open_mouth": 28, "tail": 23, "blue_eyes": 29, "white_fur": 30, "humanoid": 15, "muscular_male": 202, "half-closed_eyes": 187, "purple_body": 49, "open_smile": 116, "multicolored_tail": 131, "white_tail": 82, "heterochromia": 158, "two_tone_tail": 83, "blue_nose": 26, "blue_pawpads": 109, "glistening_nose": 138, "blue_stripes": 14, "x_eyes": 177, "blue_paws": 102, "tanuki_humanoid": 8, "pink_stripes": 13, "curved_tail": 16, "male_humanoid": 52, "striped_neck": 35, "big_mouth_(anatomy)": 121, "suina_humanoid": 51, "blue_tuft": 96, "purple_membrane": 42, "actual_fur": 145, "teal_fur": 160, "white_body": 114, "tongue_out": 194, "multicolored_fur": 100, "animal_humanoid": 20, "fox_humanoid": 3, "big_eyes": 167, "walking": 195, "action_pose": 19, "pink_ears": 75, "midair": 168, "paw_pose": 56, "yellow_stripes": 34, "expressions": 103, "striped_face": 43, "tailed_humanoid": 59, "fluffy_fur": 54, "blue_fingers": 92, "anime_eyes": 107, "blue_toes": 87, "facial_stripes": 33, "posed": 154, "jumper": 173, "pink_legs": 73, "toony_expression": 152}, "stage3_selected_phrase_ranks": {"simple_background": 1, "open_mouth": 1, "tail": 1, "blue_eyes": 1, "white_fur": 1, "humanoid": 1, "muscular_male": 8, "half-closed_eyes": 6, "purple_body": 2, "open_smile": 2, "multicolored_tail": 9, "white_tail": 7, "heterochromia": 6, "two_tone_tail": 8, "blue_nose": 1, "blue_pawpads": 10, "glistening_nose": 8, "blue_stripes": 1, "x_eyes": 9, "blue_paws": 9, "tanuki_humanoid": 7, "pink_stripes": 1, "curved_tail": 1, "male_humanoid": 7, "striped_neck": 4, "big_mouth_(anatomy)": 7, "suina_humanoid": 6, "blue_tuft": 7, "purple_membrane": 1, "actual_fur": 3, "teal_fur": 7, "white_body": 8, "tongue_out": 10, "multicolored_fur": 6, "animal_humanoid": 2, "fox_humanoid": 3, "big_eyes": 6, "walking": 10, "action_pose": 1, "pink_ears": 4, "midair": 2, "paw_pose": 2, "yellow_stripes": 5, "expressions": 3, "striped_face": 5, "tailed_humanoid": 9, "fluffy_fur": 2, "blue_fingers": 5, "anime_eyes": 2, "blue_toes": 4, "facial_stripes": 3, "posed": 9, "jumper": 3, "pink_legs": 3, "toony_expression": 9}, "extra_evidence": {"action_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6168}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4562}, "ambiguous_gender": {"source": "structural"}, "animal_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6158}, "anime_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.49}, "anthro": {"source": "structural"}, "big_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4205}, "big_mouth_(anatomy)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4805}, "blue_fingers": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5077}, "blue_pawpads": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4889}, "blue_paws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4985}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6746}, "blue_toes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5148}, "blue_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5037}, "canid_humanoid": {"source": "implied"}, "canine_humanoid": {"source": "implied"}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6369}, "expressions": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4954}, "facial_markings": {"source": "implied"}, "facial_stripes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5968}, "fluffy_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5593}, "fox_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.8203}, "glistening": {"source": "implied"}, "glistening_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4645}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3627}, "head_markings": {"source": "implied"}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4421}, "humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6698}, "jumper": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4076}, "looking_at_viewer": {"source": "structural"}, "male": {"source": "implied"}, "male_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5626}, "mammal_humanoid": {"source": "implied"}, "markings": {"source": "implied"}, "membrane_(anatomy)": {"source": "implied"}, "midair": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4197}, "multicolored_body": {"source": "implied"}, "multicolored_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4993}, "multicolored_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4716}, "muscular": {"source": "implied"}, "muscular_male": {"source": "stage3", "why": "explicit", "retrieval_score": 0.31}, "narrowed_eyes": {"source": "implied"}, "open_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4866}, "paw_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5581}, "pawpads": {"source": "implied"}, "pink_ears": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5254}, "pink_legs": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5284}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6819}, "pose": {"source": "implied"}, "posed": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4482}, "purple_membrane": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5811}, "smile": {"source": "implied"}, "striped_face": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5806}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5948}, "stripes": {"source": "implied"}, "suina": {"source": "implied"}, "suina_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5629}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6105}, "tailed_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5524}, "tanuki_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7573}, "teal_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4367}, "tongue": {"source": "implied"}, "tongue_out": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3534}, "toony_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4501}, "two_tone_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5195}, "walking": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3532}, "white_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5201}, "x_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3998}, "yellow_stripes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5954}}, "structural": ["solo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 3.87, "t2": 1.63, "t3": 13.08, "t3s": 0.92, "err": null, "issues": ["Stage3 split: general=205 entity=4 copyright_filtered=2 generic_char_to_general=0 unknown_type=5"]}
data/eval_results/eval_caption_cogvlm_n10_e2e_structimp_bootstrap_k3_cw0p5.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-25T05:41:08.229247", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "eval_path": "data/eval_samples/e621_sfw_sample_1000_seed123_buffer10000_caption_evident.jsonl", "per_phrase_k": 2, "per_phrase_final_k": 10, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 4, "min_why": "strong_implied", "expand_implications": true, "infer_structural": true, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 17}
2
+ {"id": 3285630, "n_gt": 12, "n_retrieved": 133, "n_selected": 53, "n_implied": 25, "n_structural": 5, "ret_R": 0.4167, "P": 0.1887, "R": 0.8333, "F1": 0.3077, "leaf_P": 0.1667, "leaf_R": 0.4444, "leaf_F1": 0.2424, "n_leaf_sel": 24, "n_leaf_gt": 9, "ret_P": 0.0376, "sel_given_ret": 2.0, "over_sel": 4.42, "why": {"explicit": 24}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 53, "bootstrap_anchor_selected": 15, "bootstrap_anchor_kept": 10, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 75, "attempts_by_n_local": {"53": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "16": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1887, "gen_R": 0.8333, "gen_F1": 0.3077, "missed": ["alpha_channel", "fingers"], "extra": ["beer_mug", "beverage", "black_body", "black_fur", "business_attire", "business_suit", "businesswear", "container", "cup", "domestic_cat", "felis", "frown", "hair_bun", "holding_beverage", "holding_container", "holding_cup", "holding_mug", "holding_object", "long_hair", "looking_at_viewer", "mug", "red_clothing", "red_topwear", "red_vest", "scowl", "serious", "shirt", "siamese", "suit", "suit_jacket", "t-shirt", "teal_shirt", "teal_topwear", "topwear", "vest", "white_clothing", "white_dress_shirt", "white_shirt", "white_t-shirt", "white_topwear", "yellow_clothing", "yellow_shirt", "yellow_topwear"], "ground_truth_tags": ["alpha_channel", "anthro", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "beer_mug", "beverage", "black_body", "black_fur", "business_attire", "business_suit", "businesswear", "clothed", "clothing", "container", "cup", "domestic_cat", "felid", "feline", "felis", "frown", "fur", "hair", "hair_bun", "holding_beverage", "holding_container", "holding_cup", "holding_mug", "holding_object", "long_hair", "looking_at_viewer", "male", "mammal", "mug", "red_clothing", "red_topwear", "red_vest", "scowl", "serious", "shirt", "siamese", "solo", "suit", "suit_jacket", "t-shirt", "teal_shirt", "teal_topwear", "topwear", "vest", "white_clothing", "white_dress_shirt", "white_shirt", "white_t-shirt", "white_topwear", "yellow_clothing", "yellow_shirt", "yellow_topwear"], "stage3_selected": ["anthro", "beer_mug", "black_fur", "business_attire", "business_suit", "businesswear", "feline", "hair_bun", "holding_beverage", "holding_cup", "holding_mug", "long_hair", "red_vest", "scowl", "serious", "shirt", "siamese", "suit_jacket", "teal_shirt", "topwear", "vest", "white_dress_shirt", "white_t-shirt", "yellow_shirt"], "stage3_selected_scores": {"anthro": 0.4903, "topwear": 0.6992, "feline": 0.7023, "long_hair": 0.5079, "shirt": 0.7976, "black_fur": 0.7157, "vest": 0.8387, "holding_cup": 0.7652, "hair_bun": 0.4794, "siamese": 0.6206, "holding_beverage": 0.7705, "business_suit": 0.5728, "yellow_shirt": 0.7542, "serious": 0.5805, "scowl": 0.5546, "holding_mug": 0.915, "beer_mug": 0.6581, "white_t-shirt": 0.7323, "suit_jacket": 0.6849, "businesswear": 0.5699, "white_dress_shirt": 0.6876, "business_attire": 0.5642, "red_vest": 0.6636, "teal_shirt": 0.7466}, "stage3_selected_ranks": {"anthro": 129, "topwear": 49, "feline": 45, "long_hair": 123, "shirt": 5, "black_fur": 37, "vest": 3, "holding_cup": 10, "hair_bun": 132, "siamese": 75, "holding_beverage": 9, "business_suit": 95, "yellow_shirt": 15, "serious": 90, "scowl": 103, "holding_mug": 1, "beer_mug": 60, "white_t-shirt": 26, "suit_jacket": 53, "businesswear": 97, "white_dress_shirt": 52, "business_attire": 100, "red_vest": 59, "teal_shirt": 19}, "stage3_selected_phrase_ranks": {"anthro": 9, "topwear": 10, "feline": 1, "long_hair": 4, "shirt": 1, "black_fur": 1, "vest": 1, "holding_cup": 4, "hair_bun": 8, "siamese": 5, "holding_beverage": 3, "business_suit": 8, "yellow_shirt": 2, "serious": 2, "scowl": 4, "holding_mug": 1, "beer_mug": 5, "white_t-shirt": 3, "suit_jacket": 4, "businesswear": 9, "white_dress_shirt": 5, "business_attire": 1, "red_vest": 9, "teal_shirt": 4}, "extra_evidence": {"beer_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6581}, "beverage": {"source": "implied"}, "black_body": {"source": "implied"}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "business_attire": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5642}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5728}, "businesswear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5699}, "container": {"source": "implied"}, "cup": {"source": "implied"}, "domestic_cat": {"source": "implied"}, "felis": {"source": "implied"}, "frown": {"source": "implied"}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4794}, "holding_beverage": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7705}, "holding_container": {"source": "implied"}, "holding_cup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7652}, "holding_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.915}, "holding_object": {"source": "implied"}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5079}, "looking_at_viewer": {"source": "structural"}, "mug": {"source": "implied"}, "red_clothing": {"source": "implied"}, "red_topwear": {"source": "implied"}, "red_vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6636}, "scowl": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5546}, "serious": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5805}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "siamese": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6206}, "suit": {"source": "implied"}, "suit_jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6849}, "t-shirt": {"source": "implied"}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "teal_topwear": {"source": "implied"}, "topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6992}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "white_clothing": {"source": "implied"}, "white_dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6876}, "white_shirt": {"source": "implied"}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7323}, "white_topwear": {"source": "implied"}, "yellow_clothing": {"source": "implied"}, "yellow_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7542}, "yellow_topwear": {"source": "implied"}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 3.3, "t2": 4.18, "t3": 26.09, "t3s": 0.69, "err": null, "issues": ["Stage3 split: general=136 entity=0 copyright_filtered=1 generic_char_to_general=0 unknown_type=1"]}
3
+ {"id": 260449, "n_gt": 14, "n_retrieved": 160, "n_selected": 56, "n_implied": 14, "n_structural": 6, "ret_R": 0.5714, "P": 0.2143, "R": 0.8571, "F1": 0.3429, "leaf_P": 0.0526, "leaf_R": 0.2, "leaf_F1": 0.0833, "n_leaf_sel": 38, "n_leaf_gt": 10, "ret_P": 0.05, "sel_given_ret": 1.5, "over_sel": 4.0, "why": {"explicit": 38}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 67, "bootstrap_anchor_selected": 24, "bootstrap_anchor_kept": 8, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 80, "attempts_by_n_local": {"67": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "40": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2222, "gen_R": 0.8571, "gen_F1": 0.3529, "missed": ["fur", "human"], "extra": ["anthro", "arms_by_side", "auburn_hair", "belly_dancer", "belly_dancer_outfit", "big_forearms", "blank_expression", "blue_hair", "bottomwear", "braided_hair", "cheeky", "cloth", "donkey_kong_(series)", "forearms", "gorilla", "grinning_at_viewer", "hands_behind_head", "humanoid", "koala", "kong", "loincloth", "looking_up_at_viewer", "marsupial", "monkey", "neutral_expression", "nintendo", "one_eye_closed", "raised_arm", "raised_arms", "raised_calf", "relaxed_expression", "smug_eyes", "smug_grin", "spread_arms", "surprised_expression", "tickling", "topless", "trio", "ursine", "vocalization", "vombatiform", "wide_grin", "wink", "winking_at_viewer"], "ground_truth_tags": ["ape", "bear", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate"], "selected_tags": ["anthro", "ape", "arms_by_side", "auburn_hair", "bear", "belly_dancer", "belly_dancer_outfit", "big_forearms", "blank_expression", "blue_hair", "bottomwear", "braided_hair", "cheeky", "cloth", "clothed", "clothing", "dancing", "donkey_kong_(series)", "forearms", "gorilla", "grinning_at_viewer", "group", "hair", "hands_behind_head", "haplorhine", "humanoid", "koala", "kong", "loincloth", "looking_at_viewer", "looking_up_at_viewer", "male", "mammal", "marsupial", "monkey", "neutral_expression", "nintendo", "one_eye_closed", "primate", "raised_arm", "raised_arms", "raised_calf", "relaxed_expression", "smug_eyes", "smug_grin", "spread_arms", "surprised_expression", "tickling", "topless", "trio", "ursine", "vocalization", "vombatiform", "wide_grin", "wink", "winking_at_viewer"], "stage3_selected": ["ape", "arms_by_side", "auburn_hair", "bear", "belly_dancer", "belly_dancer_outfit", "big_forearms", "blank_expression", "blue_hair", "braided_hair", "cheeky", "cloth", "dancing", "gorilla", "grinning_at_viewer", "hands_behind_head", "koala", "kong", "loincloth", "looking_at_viewer", "looking_up_at_viewer", "male", "monkey", "neutral_expression", "primate", "raised_arm", "raised_arms", "raised_calf", "relaxed_expression", "smug_eyes", "smug_grin", "spread_arms", "surprised_expression", "tickling", "ursine", "vocalization", "wide_grin", "winking_at_viewer"], "stage3_selected_scores": {"male": 0.552, "looking_at_viewer": 0.5423, "blue_hair": 0.3344, "bear": 0.5551, "hands_behind_head": 0.3616, "raised_arm": 0.4867, "braided_hair": 0.3274, "primate": 0.8894, "ursine": 0.4202, "loincloth": 0.5634, "dancing": 0.5523, "monkey": 0.7547, "surprised_expression": 0.3615, "tickling": 0.3266, "ape": 0.9763, "raised_arms": 0.5407, "gorilla": 0.8287, "spread_arms": 0.3987, "winking_at_viewer": 0.4285, "raised_calf": 0.3378, "smug_grin": 0.3655, "koala": 0.4283, "cloth": 0.3233, "belly_dancer": 0.3594, "arms_by_side": 0.3623, "kong": 0.7493, "neutral_expression": 0.4019, "auburn_hair": 0.3439, "looking_up_at_viewer": 0.4099, "grinning_at_viewer": 0.4379, "vocalization": 0.331, "belly_dancer_outfit": 0.351, "big_forearms": 0.3837, "relaxed_expression": 0.4112, "cheeky": 0.3862, "blank_expression": 0.444, "smug_eyes": 0.3425, "wide_grin": 0.5248}, "stage3_selected_ranks": {"male": 16, "looking_at_viewer": 17, "blue_hair": 141, "bear": 14, "hands_behind_head": 111, "raised_arm": 28, "braided_hair": 150, "primate": 2, "ursine": 56, "loincloth": 12, "dancing": 15, "monkey": 6, "surprised_expression": 112, "tickling": 153, "ape": 1, "raised_arms": 19, "gorilla": 5, "spread_arms": 77, "winking_at_viewer": 48, "raised_calf": 136, "smug_grin": 107, "koala": 49, "cloth": 154, "belly_dancer": 115, "arms_by_side": 110, "kong": 7, "neutral_expression": 74, "auburn_hair": 126, "looking_up_at_viewer": 70, "grinning_at_viewer": 40, "vocalization": 145, "belly_dancer_outfit": 119, "big_forearms": 87, "relaxed_expression": 69, "cheeky": 85, "blank_expression": 38, "smug_eyes": 129, "wide_grin": 21}, "stage3_selected_phrase_ranks": {"male": 1, "looking_at_viewer": 1, "blue_hair": 9, "bear": 1, "hands_behind_head": 10, "raised_arm": 1, "braided_hair": 8, "primate": 1, "ursine": 9, "loincloth": 1, "dancing": 1, "monkey": 6, "surprised_expression": 10, "tickling": 10, "ape": 1, "raised_arms": 1, "gorilla": 3, "spread_arms": 5, "winking_at_viewer": 4, "raised_calf": 9, "smug_grin": 4, "koala": 7, "cloth": 9, "belly_dancer": 6, "arms_by_side": 9, "kong": 5, "neutral_expression": 6, "auburn_hair": 6, "looking_up_at_viewer": 10, "grinning_at_viewer": 3, "vocalization": 8, "belly_dancer_outfit": 7, "big_forearms": 9, "relaxed_expression": 7, "cheeky": 1, "blank_expression": 5, "smug_eyes": 5, "wide_grin": 1}, "extra_evidence": {"anthro": {"source": "structural"}, "arms_by_side": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3623}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3439}, "belly_dancer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3594}, "belly_dancer_outfit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.351}, "big_forearms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3837}, "blank_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "blue_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3344}, "bottomwear": {"source": "implied"}, "braided_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3274}, "cheeky": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3862}, "cloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3233}, "donkey_kong_(series)": {"source": "implied"}, "forearms": {"source": "implied"}, "gorilla": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8287}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4379}, "hands_behind_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3616}, "humanoid": {"source": "structural"}, "koala": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4283}, "kong": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7493}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "looking_up_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4099}, "marsupial": {"source": "implied"}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "neutral_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4019}, "nintendo": {"source": "implied"}, "one_eye_closed": {"source": "implied"}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4867}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_calf": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3378}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4112}, "smug_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3425}, "smug_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3655}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "surprised_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3615}, "tickling": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3266}, "topless": {"source": "structural"}, "trio": {"source": "structural"}, "ursine": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4202}, "vocalization": {"source": "stage3", "why": "explicit", "retrieval_score": 0.331}, "vombatiform": {"source": "implied"}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}, "wink": {"source": "implied"}, "winking_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4285}}, "structural": ["trio", "anthro", "humanoid", "male", "topless", "looking_at_viewer"], "t1": 4.22, "t2": 3.69, "t3": 38.67, "t3s": 2.81, "err": null, "issues": ["Stage3 split: general=160 entity=5 copyright_filtered=3 generic_char_to_general=1 unknown_type=3"]}
4
+ {"id": 1078019, "n_gt": 14, "n_retrieved": 133, "n_selected": 37, "n_implied": 10, "n_structural": 5, "ret_R": 0.7143, "P": 0.3784, "R": 1.0, "F1": 0.549, "leaf_P": 0.2917, "leaf_R": 0.7778, "leaf_F1": 0.4242, "n_leaf_sel": 24, "n_leaf_gt": 9, "ret_P": 0.0752, "sel_given_ret": 1.4, "over_sel": 2.64, "why": {"explicit": 19, "strong_implied": 3}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 55, "bootstrap_anchor_selected": 18, "bootstrap_anchor_kept": 7, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 73, "attempts_by_n_local": {"55": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "11": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3784, "gen_R": 1.0, "gen_F1": 0.549, "missed": [], "extra": ["ambiguous_gender", "coat", "comforting", "expressions", "eyes", "holding_object", "holding_plushie", "holding_toy", "looking_at_viewer", "passionate", "pink_blush", "red_clothing", "red_coat", "red_topwear", "relationship", "rosy_cheeks", "surprised_face", "teal_clothing", "teddy_bear", "topwear", "toy", "vest", "winter_coat"], "ground_truth_tags": ["anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "mammal", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["ambiguous_gender", "anthro", "blue_eyes", "blush", "clothed", "clothing", "coat", "comforting", "duo", "expressions", "eyes", "holding_object", "holding_plushie", "holding_toy", "lagomorph", "leporid", "looking_at_viewer", "mammal", "passionate", "pink_blush", "plushie", "rabbit", "red_clothing", "red_coat", "red_topwear", "relationship", "romantic", "romantic_couple", "rosy_cheeks", "surprised_face", "teal_clothing", "teal_eyes", "teddy_bear", "topwear", "toy", "vest", "winter_coat"], "stage3_selected": ["blue_eyes", "coat", "comforting", "expressions", "eyes", "holding_plushie", "holding_toy", "lagomorph", "passionate", "pink_blush", "plushie", "rabbit", "red_coat", "relationship", "romantic_couple", "rosy_cheeks", "surprised_face", "teal_clothing", "teal_eyes", "teddy_bear", "vest", "winter_coat"], "stage3_selected_scores": {"blue_eyes": 0.6147, "lagomorph": 0.5322, "rabbit": 0.5936, "romantic_couple": 0.5617, "coat": 0.638, "plushie": 0.7452, "vest": 0.5025, "teal_eyes": 0.6281, "teddy_bear": 0.5456, "rosy_cheeks": 0.4718, "pink_blush": 0.4648, "expressions": 0.5451, "holding_plushie": 0.7791, "winter_coat": 0.4757, "teal_clothing": 0.4335, "red_coat": 0.5206, "holding_toy": 0.5853, "relationship": 0.6203, "eyes": 0.8767, "passionate": 0.4293, "comforting": 0.4366, "surprised_face": 0.62}, "stage3_selected_ranks": {"blue_eyes": 12, "lagomorph": 28, "rabbit": 13, "romantic_couple": 18, "coat": 7, "plushie": 3, "vest": 47, "teal_eyes": 8, "teddy_bear": 22, "rosy_cheeks": 68, "pink_blush": 74, "expressions": 23, "holding_plushie": 2, "winter_coat": 66, "teal_clothing": 103, "red_coat": 36, "holding_toy": 14, "relationship": 9, "eyes": 1, "passionate": 109, "comforting": 100, "surprised_face": 10}, "stage3_selected_phrase_ranks": {"blue_eyes": 1, "lagomorph": 2, "rabbit": 1, "romantic_couple": 1, "coat": 1, "plushie": 1, "vest": 6, "teal_eyes": 1, "teddy_bear": 5, "rosy_cheeks": 2, "pink_blush": 4, "expressions": 2, "holding_plushie": 1, "winter_coat": 10, "teal_clothing": 6, "red_coat": 4, "holding_toy": 4, "relationship": 1, "eyes": 1, "passionate": 8, "comforting": 9, "surprised_face": 3}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "comforting": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4366}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5451}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "holding_object": {"source": "implied"}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "holding_toy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5853}, "looking_at_viewer": {"source": "structural"}, "passionate": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4293}, "pink_blush": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "red_clothing": {"source": "implied"}, "red_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5206}, "red_topwear": {"source": "implied"}, "relationship": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6203}, "rosy_cheeks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4718}, "surprised_face": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.62}, "teal_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4335}, "teddy_bear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5456}, "topwear": {"source": "implied"}, "toy": {"source": "implied"}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5025}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4757}}, "structural": ["duo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 2.0, "t2": 4.12, "t3": 48.93, "t3s": 3.01, "err": null, "issues": ["Stage3 split: general=131 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
5
+ {"id": 1624724, "n_gt": 4, "n_retrieved": 117, "n_selected": 21, "n_implied": 3, "n_structural": 4, "ret_R": 0.75, "P": 0.1905, "R": 1.0, "F1": 0.32, "leaf_P": 0.2222, "leaf_R": 1.0, "leaf_F1": 0.3636, "n_leaf_sel": 18, "n_leaf_gt": 4, "ret_P": 0.0256, "sel_given_ret": 1.3333, "over_sel": 5.25, "why": {"explicit": 13, "strong_implied": 3}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 45, "bootstrap_anchor_selected": 7, "bootstrap_anchor_kept": 3, "bootstrap_reranked": true, "calls_total": 3, "calls_with_selection": 3, "calls_exhausted_retries": 0, "attempts_total": 3, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 3, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 41, "attempts_by_n_local": {"45": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1905, "gen_R": 1.0, "gen_F1": 0.32, "missed": [], "extra": ["ambiguous_gender", "big_eyes", "big_nose", "elemental_creature", "eyes", "feral", "floatie", "floating", "inflatable", "looking_at_viewer", "looking_down", "looking_down_at_viewer", "spots", "spotted_arms", "spotted_shoulders", "tan_chest", "tan_face"], "ground_truth_tags": ["red_nose", "smile", "solo", "tan_body"], "selected_tags": ["ambiguous_gender", "big_eyes", "big_nose", "elemental_creature", "eyes", "feral", "floatie", "floating", "inflatable", "looking_at_viewer", "looking_down", "looking_down_at_viewer", "red_nose", "smile", "solo", "spots", "spotted_arms", "spotted_shoulders", "tan_body", "tan_chest", "tan_face"], "stage3_selected": ["big_eyes", "big_nose", "elemental_creature", "eyes", "floatie", "floating", "looking_down_at_viewer", "red_nose", "smile", "spots", "spotted_arms", "spotted_shoulders", "tan_body", "tan_chest", "tan_face", "white_background"], "stage3_selected_scores": {"smile": 0.5953, "white_background": 0.6069, "tan_body": 0.658, "spots": 0.6222, "big_eyes": 0.6933, "red_nose": 0.7473, "floating": 0.6452, "big_nose": 0.5562, "looking_down_at_viewer": 0.4663, "tan_face": 0.6956, "tan_chest": 0.6867, "floatie": 0.428, "eyes": 0.9241, "elemental_creature": 0.4515, "spotted_arms": 0.6641, "spotted_shoulders": 0.6126}, "stage3_selected_ranks": {"smile": 36, "white_background": 32, "tan_body": 15, "spots": 27, "big_eyes": 7, "red_nose": 3, "floating": 20, "big_nose": 47, "looking_down_at_viewer": 81, "tan_face": 6, "tan_chest": 8, "floatie": 103, "eyes": 1, "elemental_creature": 89, "spotted_arms": 13, "spotted_shoulders": 31}, "stage3_selected_phrase_ranks": {"smile": 2, "white_background": 1, "tan_body": 6, "spots": 7, "big_eyes": 1, "red_nose": 1, "floating": 1, "big_nose": 3, "looking_down_at_viewer": 7, "tan_face": 1, "tan_chest": 2, "floatie": 6, "eyes": 1, "elemental_creature": 7, "spotted_arms": 4, "spotted_shoulders": 9}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6933}, "big_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5562}, "elemental_creature": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4515}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9241}, "feral": {"source": "structural"}, "floatie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.428}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "inflatable": {"source": "implied"}, "looking_at_viewer": {"source": "structural"}, "looking_down": {"source": "implied"}, "looking_down_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4663}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6222}, "spotted_arms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6641}, "spotted_shoulders": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6126}, "tan_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6867}, "tan_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6956}}, "structural": ["solo", "feral", "ambiguous_gender", "looking_at_viewer"], "t1": 1.68, "t2": 4.97, "t3": 37.83, "t3s": 1.85, "err": null, "issues": ["Stage3 split: general=120 entity=0 copyright_filtered=2 generic_char_to_general=0 unknown_type=4"]}
6
+ {"id": 1325009, "n_gt": 22, "n_retrieved": 182, "n_selected": 77, "n_implied": 27, "n_structural": 4, "ret_R": 0.4091, "P": 0.2208, "R": 0.7727, "F1": 0.3434, "leaf_P": 0.0667, "leaf_R": 0.25, "leaf_F1": 0.1053, "n_leaf_sel": 45, "n_leaf_gt": 12, "ret_P": 0.0495, "sel_given_ret": 1.8889, "over_sel": 3.5, "why": {"explicit": 46}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 65, "bootstrap_anchor_selected": 22, "bootstrap_anchor_kept": 19, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 101, "attempts_by_n_local": {"65": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "58": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.2, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2208, "gen_R": 0.7727, "gen_F1": 0.3434, "missed": ["hand_on_head", "muscular", "muscular_anthro", "muscular_male", "topless"], "extra": ["animal_print", "big_biceps", "blue_bottomwear", "blue_clothing", "blue_shirt", "blue_shorts", "blue_tank_top", "blue_topwear", "confident", "countershade_arms", "countershade_body", "countershade_fur", "ear_tuft", "eyes", "fist", "flexing_bicep", "flexing_muscles", "glistening", "glistening_body", "glistening_fur", "half-length_portrait", "hand_on_own_arm", "hand_on_own_chest", "hand_on_own_head", "hotpants", "minishorts", "muscular_arms", "muscular_thighs", "obliques", "playful", "playing", "portrait", "pose", "posed", "raised_arm", "raised_fist", "raised_hand", "raised_head", "shirt", "siberian_tiger", "striped_back", "striped_body", "striped_fur", "tan_body", "tan_bottomwear", "tan_clothing", "tan_countershading", "tan_shorts", "tank_top", "tiger_print", "topwear", "tufted_fur", "two_tone_chest", "white_arms", "white_bottomwear", "white_chest", "white_clothing", "white_shorts", "yellow_bottomwear", "yellow_clothing"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["animal_print", "anthro", "big_biceps", "blue_bottomwear", "blue_clothing", "blue_eyes", "blue_shirt", "blue_shorts", "blue_tank_top", "blue_topwear", "bottomwear", "chest_tuft", "clothed", "clothing", "confident", "countershade_arms", "countershade_body", "countershade_fur", "countershading", "ear_tuft", "eyes", "felid", "fist", "flexing_bicep", "flexing_muscles", "fur", "glistening", "glistening_body", "glistening_fur", "half-length_portrait", "hand_on_own_arm", "hand_on_own_chest", "hand_on_own_head", "hotpants", "male", "mammal", "minishorts", "muscular_arms", "muscular_thighs", "obliques", "pantherine", "playful", "playing", "portrait", "pose", "posed", "raised_arm", "raised_fist", "raised_hand", "raised_head", "shirt", "shorts", "siberian_tiger", "solo", "striped_back", "striped_body", "striped_fur", "stripes", "tan_body", "tan_bottomwear", "tan_clothing", "tan_countershading", "tan_shorts", "tank_top", "tiger", "tiger_print", "topwear", "tuft", "tufted_fur", "two_tone_chest", "white_arms", "white_bottomwear", "white_chest", "white_clothing", "white_shorts", "yellow_bottomwear", "yellow_clothing"], "stage3_selected": ["big_biceps", "blue_eyes", "blue_shorts", "blue_tank_top", "chest_tuft", "confident", "countershade_arms", "countershade_body", "countershade_fur", "ear_tuft", "eyes", "fist", "flexing_bicep", "flexing_muscles", "glistening_fur", "half-length_portrait", "hand_on_own_arm", "hand_on_own_chest", "hand_on_own_head", "minishorts", "muscular_arms", "muscular_thighs", "obliques", "playful", "playing", "pose", "posed", "raised_fist", "raised_hand", "raised_head", "shorts", "siberian_tiger", "striped_back", "striped_body", "striped_fur", "tan_countershading", "tan_shorts", "tiger", "tiger_print", "tufted_fur", "two_tone_chest", "white_arms", "white_bottomwear", "white_chest", "white_shorts", "yellow_bottomwear"], "stage3_selected_scores": {"blue_eyes": 0.5823, "pose": 0.634, "chest_tuft": 0.4694, "shorts": 0.5913, "tiger": 0.6029, "striped_body": 0.6013, "striped_fur": 0.6525, "half-length_portrait": 0.4322, "ear_tuft": 0.601, "fist": 0.5487, "muscular_thighs": 0.7067, "tan_countershading": 0.7313, "countershade_fur": 0.6721, "obliques": 0.6208, "muscular_arms": 0.7911, "raised_hand": 0.701, "big_biceps": 0.6864, "white_bottomwear": 0.5985, "glistening_fur": 0.5187, "playful": 0.4415, "blue_shorts": 0.6229, "countershade_arms": 0.654, "playing": 0.3537, "flexing_bicep": 0.6594, "yellow_bottomwear": 0.6651, "confident": 0.5176, "white_arms": 0.812, "white_chest": 0.9239, "countershade_body": 0.8718, "tufted_fur": 0.487, "raised_fist": 0.5247, "white_shorts": 0.6098, "minishorts": 0.5392, "tiger_print": 0.4064, "striped_back": 0.7162, "blue_tank_top": 0.5681, "siberian_tiger": 0.4903, "hand_on_own_arm": 0.533, "raised_head": 0.5202, "tan_shorts": 0.6092, "hand_on_own_head": 0.5843, "two_tone_chest": 0.8326, "flexing_muscles": 0.5967, "hand_on_own_chest": 0.5692, "posed": 0.441, "eyes": 0.9773}, "stage3_selected_ranks": {"blue_eyes": 63, "pose": 44, "chest_tuft": 114, "shorts": 60, "tiger": 53, "striped_body": 54, "striped_fur": 38, "half-length_portrait": 133, "ear_tuft": 55, "fist": 72, "muscular_thighs": 23, "tan_countershading": 18, "countershade_fur": 30, "obliques": 47, "muscular_arms": 14, "raised_hand": 24, "big_biceps": 25, "white_bottomwear": 57, "glistening_fur": 84, "playful": 127, "blue_shorts": 46, "countershade_arms": 37, "playing": 164, "flexing_bicep": 35, "yellow_bottomwear": 34, "confident": 85, "white_arms": 13, "white_chest": 2, "countershade_body": 3, "tufted_fur": 104, "raised_fist": 79, "white_shorts": 49, "minishorts": 73, "tiger_print": 147, "striped_back": 20, "blue_tank_top": 67, "siberian_tiger": 101, "hand_on_own_arm": 76, "raised_head": 82, "tan_shorts": 50, "hand_on_own_head": 62, "two_tone_chest": 7, "flexing_muscles": 58, "hand_on_own_chest": 66, "posed": 128, "eyes": 1}, "stage3_selected_phrase_ranks": {"blue_eyes": 2, "pose": 1, "chest_tuft": 10, "shorts": 1, "tiger": 1, "striped_body": 3, "striped_fur": 2, "half-length_portrait": 6, "ear_tuft": 2, "fist": 5, "muscular_thighs": 3, "tan_countershading": 4, "countershade_fur": 6, "obliques": 9, "muscular_arms": 1, "raised_hand": 1, "big_biceps": 7, "white_bottomwear": 7, "glistening_fur": 3, "playful": 1, "blue_shorts": 3, "countershade_arms": 10, "playing": 3, "flexing_bicep": 5, "yellow_bottomwear": 1, "confident": 2, "white_arms": 10, "white_chest": 1, "countershade_body": 1, "tufted_fur": 8, "raised_fist": 8, "white_shorts": 4, "minishorts": 5, "tiger_print": 10, "striped_back": 1, "blue_tank_top": 9, "siberian_tiger": 2, "hand_on_own_arm": 7, "raised_head": 10, "tan_shorts": 2, "hand_on_own_head": 4, "two_tone_chest": 5, "flexing_muscles": 6, "hand_on_own_chest": 1, "posed": 10, "eyes": 1}, "extra_evidence": {"animal_print": {"source": "implied"}, "big_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6864}, "blue_bottomwear": {"source": "implied"}, "blue_clothing": {"source": "implied"}, "blue_shirt": {"source": "implied"}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6229}, "blue_tank_top": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5681}, "blue_topwear": {"source": "implied"}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5176}, "countershade_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.654}, "countershade_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8718}, "countershade_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6721}, "ear_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.601}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9773}, "fist": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5487}, "flexing_bicep": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6594}, "flexing_muscles": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5967}, "glistening": {"source": "implied"}, "glistening_body": {"source": "implied"}, "glistening_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5187}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4322}, "hand_on_own_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.533}, "hand_on_own_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5692}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5843}, "hotpants": {"source": "implied"}, "minishorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5392}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7911}, "muscular_thighs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7067}, "obliques": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6208}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4415}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3537}, "portrait": {"source": "implied"}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.634}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.441}, "raised_arm": {"source": "implied"}, "raised_fist": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5247}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.701}, "raised_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5202}, "shirt": {"source": "implied"}, "siberian_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4903}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7162}, "striped_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6013}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6525}, "tan_body": {"source": "implied"}, "tan_bottomwear": {"source": "implied"}, "tan_clothing": {"source": "implied"}, "tan_countershading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7313}, "tan_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6092}, "tank_top": {"source": "implied"}, "tiger_print": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4064}, "topwear": {"source": "implied"}, "tufted_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.487}, "two_tone_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8326}, "white_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.812}, "white_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5985}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9239}, "white_clothing": {"source": "implied"}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6098}, "yellow_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6651}, "yellow_clothing": {"source": "implied"}}, "structural": ["solo", "anthro", "male", "clothed"], "t1": 2.22, "t2": 1.35, "t3": 68.25, "t3s": 6.79, "err": null, "issues": ["Stage3 split: general=178 entity=2 copyright_filtered=3 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"style_or_meta\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
7
+ {"id": 1023509, "n_gt": 13, "n_retrieved": 187, "n_selected": 44, "n_implied": 9, "n_structural": 5, "ret_R": 0.6923, "P": 0.2273, "R": 0.7692, "F1": 0.3509, "leaf_P": 0.0968, "leaf_R": 0.5, "leaf_F1": 0.1622, "n_leaf_sel": 31, "n_leaf_gt": 6, "ret_P": 0.0481, "sel_given_ret": 1.1111, "over_sel": 3.38, "why": {"explicit": 15, "strong_implied": 15}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 66, "bootstrap_anchor_selected": 22, "bootstrap_anchor_kept": 2, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 6, "attempt_errors": 2, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 114, "attempts_by_n_local": {"66": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 4, "parse_ok": 2, "parse_fail": 0, "errors": 2}, "57": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.3333333333333333, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2273, "gen_R": 0.7692, "gen_F1": 0.3509, "missed": ["fur", "white_body", "white_fur"], "extra": ["ambiguous_gender", "anthro", "armor", "bubble", "clothed", "clothing", "creepy", "darkness", "domestic_goat", "elemental_creature", "english_text", "frown", "gecko", "goat_ears", "group", "guardian", "hunched_over", "iguanid", "light", "lizardman", "lying_on_ground", "medieval", "medieval_clothing", "mineral_fauna", "monitor_lizard", "on_ground", "pointy_speech_bubble", "question_mark", "rock", "rock_creature", "running_away", "sad", "speech_bubble", "sunlight"], "ground_truth_tags": ["bovid", "caprine", "dialogue", "fur", "goat", "human", "lizard", "mammal", "reptile", "scalie", "text", "white_body", "white_fur"], "selected_tags": ["ambiguous_gender", "anthro", "armor", "bovid", "bubble", "caprine", "clothed", "clothing", "creepy", "darkness", "dialogue", "domestic_goat", "elemental_creature", "english_text", "frown", "gecko", "goat", "goat_ears", "group", "guardian", "human", "hunched_over", "iguanid", "light", "lizard", "lizardman", "lying_on_ground", "mammal", "medieval", "medieval_clothing", "mineral_fauna", "monitor_lizard", "on_ground", "pointy_speech_bubble", "question_mark", "reptile", "rock", "rock_creature", "running_away", "sad", "scalie", "speech_bubble", "sunlight", "text"], "stage3_selected": ["armor", "bovid", "bubble", "creepy", "darkness", "dialogue", "domestic_goat", "english_text", "frown", "gecko", "goat", "goat_ears", "guardian", "human", "hunched_over", "iguanid", "light", "lizardman", "lying_on_ground", "medieval", "medieval_clothing", "monitor_lizard", "on_ground", "pointy_speech_bubble", "question_mark", "rock_creature", "running_away", "sad", "speech_bubble", "sunlight"], "stage3_selected_scores": {"dialogue": 0.7402, "human": 0.6639, "speech_bubble": 0.7486, "bovid": 0.5984, "goat": 0.7748, "light": 0.7781, "question_mark": 0.4775, "on_ground": 0.6688, "bubble": 0.7514, "sunlight": 0.6724, "lying_on_ground": 0.7873, "iguanid": 0.5909, "darkness": 0.8346, "guardian": 0.456, "english_text": 0.6161, "armor": 0.4701, "frown": 0.4831, "sad": 0.5454, "creepy": 0.5133, "gecko": 0.6351, "pointy_speech_bubble": 0.5911, "lizardman": 0.5348, "monitor_lizard": 0.5652, "medieval": 0.5251, "rock_creature": 0.5115, "domestic_goat": 0.5946, "hunched_over": 0.5668, "medieval_clothing": 0.4985, "running_away": 0.5225, "goat_ears": 0.5349}, "stage3_selected_ranks": {"dialogue": 10, "human": 19, "speech_bubble": 9, "bovid": 46, "goat": 5, "light": 4, "question_mark": 165, "on_ground": 18, "bubble": 8, "sunlight": 17, "lying_on_ground": 3, "iguanid": 50, "darkness": 2, "guardian": 172, "english_text": 33, "armor": 168, "frown": 159, "sad": 95, "creepy": 127, "gecko": 31, "pointy_speech_bubble": 49, "lizardman": 104, "monitor_lizard": 69, "medieval": 113, "rock_creature": 130, "domestic_goat": 47, "hunched_over": 67, "medieval_clothing": 149, "running_away": 117, "goat_ears": 103}, "stage3_selected_phrase_ranks": {"dialogue": 1, "human": 1, "speech_bubble": 1, "bovid": 3, "goat": 1, "light": 1, "question_mark": 5, "on_ground": 3, "bubble": 1, "sunlight": 4, "lying_on_ground": 1, "iguanid": 4, "darkness": 1, "guardian": 8, "english_text": 4, "armor": 6, "frown": 7, "sad": 6, "creepy": 5, "gecko": 3, "pointy_speech_bubble": 10, "lizardman": 9, "monitor_lizard": 7, "medieval": 4, "rock_creature": 7, "domestic_goat": 4, "hunched_over": 4, "medieval_clothing": 8, "running_away": 2, "goat_ears": 9}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "anthro": {"source": "structural"}, "armor": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4701}, "bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7514}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "creepy": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5133}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "domestic_goat": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5946}, "elemental_creature": {"source": "implied"}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6161}, "frown": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4831}, "gecko": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6351}, "goat_ears": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5349}, "group": {"source": "structural"}, "guardian": {"source": "stage3", "why": "explicit", "retrieval_score": 0.456}, "hunched_over": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5668}, "iguanid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5909}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "lizardman": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5348}, "lying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7873}, "medieval": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5251}, "medieval_clothing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4985}, "mineral_fauna": {"source": "implied"}, "monitor_lizard": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5652}, "on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6688}, "pointy_speech_bubble": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5911}, "question_mark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4775}, "rock": {"source": "implied"}, "rock_creature": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5115}, "running_away": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5225}, "sad": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5454}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7486}, "sunlight": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6724}}, "structural": ["group", "anthro", "ambiguous_gender", "clothed", "text"], "t1": 1.43, "t2": 1.37, "t3": 104.01, "t3s": 2.52, "err": null, "issues": ["Stage3 split: general=177 entity=4 copyright_filtered=6 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"other\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 58, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.36.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.36.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"style_or_meta\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"strong_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"style_or_meta\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"other\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 15, "n_retrieved": 224, "n_selected": 63, "n_implied": 9, "n_structural": 5, "ret_R": 0.8, "P": 0.1905, "R": 0.8, "F1": 0.3077, "leaf_P": 0.14, "leaf_R": 0.5833, "leaf_F1": 0.2258, "n_leaf_sel": 50, "n_leaf_gt": 12, "ret_P": 0.0536, "sel_given_ret": 1.0, "over_sel": 4.2, "why": {"explicit": 46, "strong_implied": 5}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 82, "bootstrap_anchor_selected": 28, "bootstrap_anchor_kept": 18, "bootstrap_reranked": true, "calls_total": 5, "calls_with_selection": 5, "calls_exhausted_retries": 0, "attempts_total": 8, "attempt_errors": 3, "attempt_parse_fail": 0, "attempt_parse_ok": 5, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 121, "attempts_by_n_local": {"82": {"attempts": 3, "parse_ok": 1, "parse_fail": 0, "errors": 2}, "60": {"attempts": 4, "parse_ok": 3, "parse_fail": 0, "errors": 1}, "45": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.375, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1905, "gen_R": 0.8, "gen_F1": 0.3077, "missed": ["angry", "eyeshadow", "sleeping"], "extra": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "arm_around_back", "artist_name", "atmosphere", "bed_sheet", "bedding", "bedroom", "black_lipstick", "blanket", "blue_background", "calm", "calves_up", "clothed", "clothing", "comic", "cool_colors", "digital_media_(artwork)", "expressionless", "eye_contact", "eyeliner", "eyes", "female", "half-closed_eyes", "hand_on_torso", "head_on_pillow", "heterochromia", "humanoid", "letters", "lipstick", "looking_at_another", "looking_away", "lying_on_bed", "mascara", "narrowed_eyes", "on_bed", "palette", "personal_grooming", "pink_lipstick", "playful", "raised_foot", "red_lipstick", "relaxed_expression", "resting", "resting_bitch_face", "sleeping_bag", "text_box", "under_blanket", "watermark"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "blue_eyes", "duo", "eyes_closed", "eyeshadow", "furniture", "green_eyes", "hair", "lying", "makeup", "purple_hair", "sleeping", "text"], "selected_tags": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "arm_around_back", "artist_name", "atmosphere", "bed", "bed_sheet", "bedding", "bedroom", "black_lipstick", "blanket", "blonde_hair", "blue_background", "blue_eyes", "calm", "calves_up", "clothed", "clothing", "comic", "cool_colors", "digital_media_(artwork)", "duo", "expressionless", "eye_contact", "eyeliner", "eyes", "eyes_closed", "female", "furniture", "green_eyes", "hair", "half-closed_eyes", "hand_on_torso", "head_on_pillow", "heterochromia", "humanoid", "letters", "lipstick", "looking_at_another", "looking_away", "lying", "lying_on_bed", "makeup", "mascara", "narrowed_eyes", "on_bed", "palette", "personal_grooming", "pink_lipstick", "playful", "purple_hair", "raised_foot", "red_lipstick", "relaxed_expression", "resting", "resting_bitch_face", "sleeping_bag", "text", "text_box", "under_blanket", "watermark"], "stage3_selected": ["3rd_party_watermark", "annoyed", "annoyed_expression", "applying_makeup", "arm_around_back", "artist_name", "atmosphere", "bed_sheet", "bedding", "bedroom", "black_lipstick", "blanket", "blonde_hair", "blue_background", "blue_eyes", "calm", "calves_up", "comic", "cool_colors", "digital_media_(artwork)", "expressionless", "eye_contact", "eyeliner", "eyes", "eyes_closed", "green_eyes", "hair", "half-closed_eyes", "hand_on_torso", "head_on_pillow", "heterochromia", "letters", "lipstick", "looking_away", "lying", "lying_on_bed", "makeup", "mascara", "palette", "pink_lipstick", "playful", "purple_hair", "raised_foot", "red_lipstick", "relaxed_expression", "resting", "resting_bitch_face", "sleeping_bag", "text", "text_box", "under_blanket"], "stage3_selected_scores": {"hair": 0.5936, "digital_media_(artwork)": 0.3347, "text": 0.5939, "blue_eyes": 0.595, "lying": 0.4445, "eyes_closed": 0.3883, "green_eyes": 0.5933, "comic": 0.3823, "blonde_hair": 0.5872, "half-closed_eyes": 0.3908, "purple_hair": 0.5591, "makeup": 0.5894, "eye_contact": 0.3673, "lipstick": 0.4782, "bedroom": 0.4873, "blue_background": 0.4036, "bedding": 0.3861, "artist_name": 0.4023, "heterochromia": 0.4, "looking_away": 0.4251, "annoyed": 0.5677, "bed_sheet": 0.3933, "mascara": 0.4405, "eyeliner": 0.4399, "lying_on_bed": 0.4059, "text_box": 0.3881, "red_lipstick": 0.463, "cool_colors": 0.3885, "playful": 0.443, "black_lipstick": 0.4382, "pink_lipstick": 0.4482, "raised_foot": 0.3045, "calves_up": 0.3068, "resting": 0.5124, "annoyed_expression": 0.7219, "head_on_pillow": 0.3867, "sleeping_bag": 0.4733, "3rd_party_watermark": 0.3932, "palette": 0.6656, "under_blanket": 0.4251, "letters": 0.3654, "hand_on_torso": 0.3883, "relaxed_expression": 0.5026, "resting_bitch_face": 0.358, "atmosphere": 0.5039, "eyes": 0.8955, "blanket": 0.4169, "calm": 0.3466, "arm_around_back": 0.3844, "expressionless": 0.4858, "applying_makeup": 0.4698}, "stage3_selected_ranks": {"hair": 8, "digital_media_(artwork)": 203, "text": 7, "blue_eyes": 6, "lying": 63, "eyes_closed": 143, "green_eyes": 9, "comic": 154, "blonde_hair": 11, "half-closed_eyes": 135, "purple_hair": 14, "makeup": 10, "eye_contact": 174, "lipstick": 46, "bedroom": 41, "blue_background": 109, "bedding": 150, "artist_name": 113, "heterochromia": 118, "looking_away": 82, "annoyed": 13, "bed_sheet": 129, "mascara": 67, "eyeliner": 68, "lying_on_bed": 106, "text_box": 145, "red_lipstick": 54, "cool_colors": 141, "playful": 65, "black_lipstick": 72, "pink_lipstick": 61, "raised_foot": 221, "calves_up": 219, "resting": 28, "annoyed_expression": 2, "head_on_pillow": 149, "sleeping_bag": 49, "3rd_party_watermark": 130, "palette": 3, "under_blanket": 83, "letters": 175, "hand_on_torso": 144, "relaxed_expression": 31, "resting_bitch_face": 178, "atmosphere": 30, "eyes": 1, "blanket": 93, "calm": 191, "arm_around_back": 151, "expressionless": 42, "applying_makeup": 52}, "stage3_selected_phrase_ranks": {"hair": 1, "digital_media_(artwork)": 10, "text": 1, "blue_eyes": 1, "lying": 1, "eyes_closed": 6, "green_eyes": 1, "comic": 10, "blonde_hair": 1, "half-closed_eyes": 9, "purple_hair": 1, "makeup": 1, "eye_contact": 10, "lipstick": 2, "bedroom": 1, "blue_background": 7, "bedding": 8, "artist_name": 5, "heterochromia": 9, "looking_away": 3, "annoyed": 2, "bed_sheet": 5, "mascara": 8, "eyeliner": 9, "lying_on_bed": 4, "text_box": 8, "red_lipstick": 5, "cool_colors": 2, "playful": 1, "black_lipstick": 10, "pink_lipstick": 7, "raised_foot": 9, "calves_up": 8, "resting": 1, "annoyed_expression": 1, "head_on_pillow": 7, "sleeping_bag": 4, "3rd_party_watermark": 3, "palette": 1, "under_blanket": 7, "letters": 9, "hand_on_torso": 7, "relaxed_expression": 6, "resting_bitch_face": 7, "atmosphere": 1, "eyes": 1, "blanket": 7, "calm": 4, "arm_around_back": 8, "expressionless": 9, "applying_makeup": 4}, "extra_evidence": {"3rd_party_watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "annoyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5677}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "applying_makeup": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4698}, "arm_around_back": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3844}, "artist_name": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4023}, "atmosphere": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5039}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "black_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4382}, "blanket": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4169}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4036}, "calm": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3466}, "calves_up": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3068}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3823}, "cool_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3885}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "expressionless": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4858}, "eye_contact": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3673}, "eyeliner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4399}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "female": {"source": "structural"}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3908}, "hand_on_torso": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3883}, "head_on_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3867}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4}, "humanoid": {"source": "structural"}, "letters": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3654}, "lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4782}, "looking_at_another": {"source": "implied"}, "looking_away": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4251}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "mascara": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4405}, "narrowed_eyes": {"source": "implied"}, "on_bed": {"source": "implied"}, "palette": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6656}, "personal_grooming": {"source": "implied"}, "pink_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4482}, "playful": {"source": "stage3", "why": "explicit", "retrieval_score": 0.443}, "raised_foot": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3045}, "red_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.463}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5026}, "resting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5124}, "resting_bitch_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.358}, "sleeping_bag": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4733}, "text_box": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3881}, "under_blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4251}, "watermark": {"source": "implied"}}, "structural": ["duo", "humanoid", "female", "clothed", "text"], "t1": 1.51, "t2": 1.68, "t3": 64.09, "t3s": 2.54, "err": null, "issues": ["Stage3 bootstrap_anchor: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"style_or_meta\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 50, \"why\": \"style_or_meta\"}, {\"i\": 51, \"why\": \"explicit\"}, {\"i\": 53, \"why\": \"explicit\"}, {\"i\": 55, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.37.why\n Field required [type=missing, input_value={'i': 55, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 bootstrap_anchor: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"style_or_meta\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"other\"}, {\"i\": 41, \"why\": \"other\"}, {\"i\": 44, \"why\": \"style_or_meta\"}, {\"i\": 45, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 51, \"why\": \"other\"}, {\"i\": 55, \"why\": \"style_or_meta\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 58, \"why\": \"other\"}, {\"i\": 60, \"why\": \"other\"}, {\"i\": 62, \"why\": \"explicit\"}, {\"i\": 65, \"why\": \"other\"}, {\"i\": 68, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.38.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.38.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 split: general=225 entity=0 copyright_filtered=2 generic_char_to_general=0 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"other\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"other\"}, {\"i\": 54, \"why\": \"explicit\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.37.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.37.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 22, "n_retrieved": 155, "n_selected": 49, "n_implied": 14, "n_structural": 4, "ret_R": 0.3182, "P": 0.2041, "R": 0.4545, "F1": 0.2817, "leaf_P": 0.0938, "leaf_R": 0.2308, "leaf_F1": 0.1333, "n_leaf_sel": 32, "n_leaf_gt": 13, "ret_P": 0.0452, "sel_given_ret": 1.4286, "over_sel": 2.23, "why": {"explicit": 30, "strong_implied": 1}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 54, "bootstrap_anchor_selected": 14, "bootstrap_anchor_kept": 5, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 4, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 114, "attempts_by_n_local": {"54": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 2, "parse_ok": 2, "parse_fail": 0, "errors": 0}, "39": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2041, "gen_R": 0.4545, "gen_F1": 0.2817, "missed": ["bass_guitar", "canid", "canine", "fur", "guitar", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "string_instrument"], "extra": ["3_claws", "4_claws", "5_fingers", "ambiguous_gender", "bangs", "bass_(disambiguation)", "blonde_hair", "bottomwear", "claws_out", "colorful", "colorful_background", "cosplay", "curved_tail", "dress", "expression_sheet", "flowing_hair", "grey_claws", "holding_hair", "long_claws", "long_hair", "long_tail", "medium_hair", "pastel_background", "playing", "playing_bass", "playing_music", "shirt", "shorts", "topless", "topwear", "torn_bottomwear", "torn_dress", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "translucent", "translucent_hair", "wavy_hair"], "ground_truth_tags": ["anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["3_claws", "4_claws", "5_fingers", "ambiguous_gender", "anthro", "bangs", "bass_(disambiguation)", "blonde_hair", "bottomwear", "claws", "claws_out", "clothed", "clothing", "colorful", "colorful_background", "cosplay", "curved_tail", "dress", "expression_sheet", "fingers", "flowing_hair", "grey_claws", "hair", "holding_hair", "long_claws", "long_hair", "long_tail", "medium_hair", "pastel_background", "playing", "playing_bass", "playing_music", "shirt", "shorts", "solo", "spade_tail", "tail", "topless", "topwear", "torn_bottomwear", "torn_clothing", "torn_dress", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "translucent", "translucent_hair", "wavy_hair"], "stage3_selected": ["3_claws", "4_claws", "5_fingers", "bangs", "bass_(disambiguation)", "blonde_hair", "claws", "claws_out", "colorful_background", "cosplay", "curved_tail", "expression_sheet", "fingers", "flowing_hair", "grey_claws", "holding_hair", "long_claws", "long_hair", "long_tail", "medium_hair", "pastel_background", "playing", "playing_bass", "playing_music", "spade_tail", "torn_clothing", "torn_dress", "torn_shirt", "torn_shorts", "translucent_hair", "wavy_hair"], "stage3_selected_scores": {"claws": 0.6292, "fingers": 0.5056, "long_hair": 0.5155, "5_fingers": 0.4857, "blonde_hair": 0.4405, "torn_clothing": 0.4526, "long_tail": 0.5009, "translucent_hair": 0.4474, "spade_tail": 0.8714, "bangs": 0.463, "grey_claws": 0.4899, "wavy_hair": 0.4761, "playing_music": 0.3471, "long_claws": 0.4927, "torn_shirt": 0.4395, "playing": 0.4732, "medium_hair": 0.4515, "claws_out": 0.4959, "3_claws": 0.5377, "expression_sheet": 0.5076, "torn_shorts": 0.4525, "flowing_hair": 0.7012, "curved_tail": 0.4714, "4_claws": 0.5825, "holding_hair": 0.4914, "colorful_background": 0.5256, "torn_dress": 0.4319, "bass_(disambiguation)": 0.5203, "playing_bass": 0.5051, "pastel_background": 0.6251, "cosplay": 0.3625}, "stage3_selected_ranks": {"claws": 5, "fingers": 50, "long_hair": 41, "5_fingers": 70, "blonde_hair": 109, "torn_clothing": 94, "long_tail": 55, "translucent_hair": 99, "spade_tail": 1, "bangs": 89, "grey_claws": 64, "wavy_hair": 77, "playing_music": 152, "long_claws": 61, "torn_shirt": 112, "playing": 79, "medium_hair": 96, "claws_out": 58, "3_claws": 23, "expression_sheet": 47, "torn_shorts": 95, "flowing_hair": 2, "curved_tail": 81, "4_claws": 12, "holding_hair": 63, "colorful_background": 32, "torn_dress": 120, "bass_(disambiguation)": 37, "playing_bass": 52, "pastel_background": 6, "cosplay": 144}, "stage3_selected_phrase_ranks": {"claws": 1, "fingers": 4, "long_hair": 2, "5_fingers": 9, "blonde_hair": 9, "torn_clothing": 3, "long_tail": 5, "translucent_hair": 10, "spade_tail": 1, "bangs": 8, "grey_claws": 10, "wavy_hair": 4, "playing_music": 9, "long_claws": 9, "torn_shirt": 8, "playing": 2, "medium_hair": 9, "claws_out": 8, "3_claws": 5, "expression_sheet": 9, "torn_shorts": 4, "flowing_hair": 1, "curved_tail": 10, "4_claws": 3, "holding_hair": 3, "colorful_background": 7, "torn_dress": 10, "bass_(disambiguation)": 1, "playing_bass": 1, "pastel_background": 1, "cosplay": 8}, "extra_evidence": {"3_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5377}, "4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5825}, "5_fingers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4857}, "ambiguous_gender": {"source": "structural"}, "bangs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.463}, "bass_(disambiguation)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5203}, "blonde_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4405}, "bottomwear": {"source": "implied"}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4959}, "colorful": {"source": "implied"}, "colorful_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5256}, "cosplay": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3625}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4714}, "dress": {"source": "implied"}, "expression_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5076}, "flowing_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7012}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4899}, "holding_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4914}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4927}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5155}, "long_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5009}, "medium_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4515}, "pastel_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6251}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4732}, "playing_bass": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5051}, "playing_music": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3471}, "shirt": {"source": "implied"}, "shorts": {"source": "implied"}, "topless": {"source": "structural"}, "topwear": {"source": "implied"}, "torn_bottomwear": {"source": "implied"}, "torn_dress": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4319}, "torn_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4395}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4525}, "torn_topwear": {"source": "implied"}, "touching_hair": {"source": "implied"}, "translucent": {"source": "implied"}, "translucent_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4474}, "wavy_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4761}}, "structural": ["solo", "anthro", "ambiguous_gender", "topless"], "t1": 1.79, "t2": 1.13, "t3": 74.53, "t3s": 1.09, "err": null, "issues": ["Stage3 split: general=159 entity=1 copyright_filtered=2 generic_char_to_general=0 unknown_type=3"]}
10
+ {"id": 2021552, "n_gt": 25, "n_retrieved": 161, "n_selected": 54, "n_implied": 18, "n_structural": 5, "ret_R": 0.64, "P": 0.4444, "R": 0.96, "F1": 0.6076, "leaf_P": 0.3548, "leaf_R": 0.7333, "leaf_F1": 0.4783, "n_leaf_sel": 31, "n_leaf_gt": 15, "ret_P": 0.0994, "sel_given_ret": 1.5, "over_sel": 2.16, "why": {"explicit": 30, "strong_implied": 2}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 59, "bootstrap_anchor_selected": 19, "bootstrap_anchor_kept": 12, "bootstrap_reranked": true, "calls_total": 4, "calls_with_selection": 4, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 1, "attempt_parse_fail": 0, "attempt_parse_ok": 4, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 115, "attempts_by_n_local": {"59": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 2, "parse_fail": 0, "errors": 1}, "41": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.2, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4444, "gen_R": 0.96, "gen_F1": 0.6076, "missed": ["looking_at_another"], "extra": ["4_claws", "5_claws", "actual_fur", "ambiguous_gender", "black_bottomwear", "black_claws", "black_clothing", "black_pants", "blue_clothing", "blue_overalls", "blue_shirt", "blue_topwear", "finger_claws", "floppy_ears", "full-length_portrait", "grey_claws", "grey_clothing", "grey_shirt", "grey_topwear", "long_ears", "looking_at_viewer", "lop_ears", "open_mouth", "overalls_only", "portrait", "three-quarter_portrait", "undershirt", "white_clothing", "white_shirt", "white_topwear"], "ground_truth_tags": ["anthro", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "fox", "fur", "grey_background", "head_markings", "lagomorph", "leporid", "looking_at_another", "mammal", "markings", "overalls", "pants", "rabbit", "shirt", "standing", "topwear"], "selected_tags": ["4_claws", "5_claws", "actual_fur", "ambiguous_gender", "anthro", "black_bottomwear", "black_claws", "black_clothing", "black_pants", "blue_clothing", "blue_overalls", "blue_shirt", "blue_topwear", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "finger_claws", "floppy_ears", "fox", "full-length_portrait", "fur", "grey_background", "grey_claws", "grey_clothing", "grey_shirt", "grey_topwear", "head_markings", "lagomorph", "leporid", "long_ears", "looking_at_viewer", "lop_ears", "mammal", "markings", "open_mouth", "overalls", "overalls_only", "pants", "portrait", "rabbit", "shirt", "standing", "three-quarter_portrait", "topwear", "undershirt", "white_clothing", "white_shirt", "white_topwear"], "stage3_selected": ["4_claws", "5_claws", "actual_fur", "black_claws", "black_pants", "blue_overalls", "blue_shirt", "claws", "crossed_arms", "facial_markings", "finger_claws", "fox", "full-length_portrait", "fur", "grey_background", "grey_claws", "grey_shirt", "head_markings", "lagomorph", "leporid", "long_ears", "lop_ears", "open_mouth", "overalls", "overalls_only", "rabbit", "simple_background", "standing", "three-quarter_portrait", "topwear", "undershirt", "white_shirt"], "stage3_selected_scores": {"fur": 0.6531, "simple_background": 0.416, "open_mouth": 0.633, "claws": 0.6303, "topwear": 0.6439, "standing": 0.6878, "lagomorph": 0.5936, "leporid": 0.5826, "rabbit": 0.6511, "grey_background": 0.6784, "long_ears": 0.4627, "full-length_portrait": 0.4928, "head_markings": 0.6327, "facial_markings": 0.6945, "three-quarter_portrait": 0.5029, "finger_claws": 0.5345, "black_claws": 0.507, "crossed_arms": 0.7285, "white_shirt": 0.8197, "overalls": 0.8776, "black_pants": 0.833, "blue_shirt": 0.7655, "grey_claws": 0.5166, "lop_ears": 0.4692, "undershirt": 0.7064, "4_claws": 0.5921, "overalls_only": 0.6711, "blue_overalls": 0.9203, "5_claws": 0.6021, "actual_fur": 0.4837, "fox": 0.6379, "grey_shirt": 0.6923}, "stage3_selected_ranks": {"fur": 41, "simple_background": 163, "open_mouth": 49, "claws": 51, "topwear": 44, "standing": 29, "lagomorph": 70, "leporid": 77, "rabbit": 42, "grey_background": 31, "long_ears": 153, "full-length_portrait": 139, "head_markings": 50, "facial_markings": 27, "three-quarter_portrait": 134, "finger_claws": 111, "black_claws": 130, "crossed_arms": 21, "white_shirt": 4, "overalls": 2, "black_pants": 3, "blue_shirt": 7, "grey_claws": 123, "lop_ears": 150, "undershirt": 26, "4_claws": 71, "overalls_only": 32, "blue_overalls": 1, "5_claws": 63, "actual_fur": 147, "fox": 47, "grey_shirt": 28}, "stage3_selected_phrase_ranks": {"fur": 1, "simple_background": 8, "open_mouth": 1, "claws": 1, "topwear": 7, "standing": 1, "lagomorph": 3, "leporid": 4, "rabbit": 1, "grey_background": 1, "long_ears": 10, "full-length_portrait": 9, "head_markings": 2, "facial_markings": 1, "three-quarter_portrait": 7, "finger_claws": 6, "black_claws": 10, "crossed_arms": 1, "white_shirt": 1, "overalls": 1, "black_pants": 1, "blue_shirt": 3, "grey_claws": 8, "lop_ears": 8, "undershirt": 3, "4_claws": 3, "overalls_only": 3, "blue_overalls": 1, "5_claws": 2, "actual_fur": 10, "fox": 1, "grey_shirt": 4}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5921}, "5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6021}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4837}, "ambiguous_gender": {"source": "structural"}, "black_bottomwear": {"source": "implied"}, "black_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.507}, "black_clothing": {"source": "implied"}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.833}, "blue_clothing": {"source": "implied"}, "blue_overalls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9203}, "blue_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7655}, "blue_topwear": {"source": "implied"}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5345}, "floppy_ears": {"source": "implied"}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4928}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5166}, "grey_clothing": {"source": "implied"}, "grey_shirt": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6923}, "grey_topwear": {"source": "implied"}, "long_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4627}, "looking_at_viewer": {"source": "structural"}, "lop_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4692}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "overalls_only": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6711}, "portrait": {"source": "implied"}, "three-quarter_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5029}, "undershirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7064}, "white_clothing": {"source": "implied"}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_topwear": {"source": "implied"}}, "structural": ["duo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 1.4, "t2": 1.38, "t3": 73.52, "t3s": 0.96, "err": null, "issues": ["Stage3 split: general=161 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"strong_implied\"}, {\"i\": 16, \"why\": \"strong_implied\"}, {\"i\": 17, \"why\": \"strong_implied\"}, {\"i\": 22, \"why\": \"strong_implied\"}, {\"i\": 23, \"why\": \"strong_implied\"}, {\"i\": 25, \"why\": \"strong_implied\"}, {\"i\": 28, \"why\": \"strong_implied\"}, {\"i\": 32, \"why\": \"strong_implied\"}, {\"i\": 34, \"why\": \"strong_implied\"}, {\"i\": 36, \"why\": \"strong_implied\"}, {\"i\": 37, \"why\": \"strong_implied\"}, {\"i\": 41}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 41}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
11
+ {"id": 2034167, "n_gt": 11, "n_retrieved": 202, "n_selected": 73, "n_implied": 22, "n_structural": 4, "ret_R": 0.6364, "P": 0.1507, "R": 1.0, "F1": 0.2619, "leaf_P": 0.1087, "leaf_R": 0.7143, "leaf_F1": 0.1887, "n_leaf_sel": 46, "n_leaf_gt": 7, "ret_P": 0.0347, "sel_given_ret": 1.5714, "over_sel": 6.64, "why": {"explicit": 29, "strong_implied": 19}, "stage3_diag": {"mode": "chunked_map_union", "chunk_strategy": "interleave", "chunk_passes": 1, "chunk_shuffle_within_call": false, "bootstrap_enabled": true, "bootstrap_anchor_pool_size": 75, "bootstrap_anchor_selected": 23, "bootstrap_anchor_kept": 12, "bootstrap_reranked": true, "calls_total": 5, "calls_with_selection": 5, "calls_exhausted_retries": 0, "attempts_total": 5, "attempt_errors": 0, "attempt_parse_fail": 0, "attempt_parse_ok": 5, "invalid_items_total": 0, "oob_indices_total": 0, "dupe_indices_total": 0, "kept_total": 142, "attempts_by_n_local": {"75": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}, "60": {"attempts": 3, "parse_ok": 3, "parse_fail": 0, "errors": 0}, "25": {"attempts": 1, "parse_ok": 1, "parse_fail": 0, "errors": 0}}, "attempt_failure_rate": 0.0, "call_exhaustion_rate": 0.0}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1507, "gen_R": 1.0, "gen_F1": 0.2619, "missed": [], "extra": ["2d_animation", "actual_fur", "amphibian", "amphibian_humanoid", "animal_humanoid", "animated", "animated_comic", "animated_png", "anthro", "big_mouth_(anatomy)", "blue_ears", "blue_fingers", "blue_inner_ear_fluff", "blue_stripes", "blue_tail", "blue_toes", "blush", "body_hair", "canid_humanoid", "canine_humanoid", "clothed", "clothing", "comic", "countershading", "curved_tail", "facial_markings", "female_humanoid", "fennec_humanoid", "fox_humanoid", "frog_humanoid", "head_markings", "humanoid", "inner_ear_fluff", "jojo_pose", "jumper", "jumping", "light_nose", "light_tail", "male", "male_humanoid", "mammal_humanoid", "markings", "midair", "nose", "open_smile", "pink_blush", "pink_body", "pink_countershading", "pink_ears", "pink_stripes", "smile", "sparkling_character", "striped_face", "stripes", "suggestive_pose", "tail", "tailed_humanoid", "tan_tail", "tanuki_humanoid", "tuft", "two_tone_tail", "walking"], "ground_truth_tags": ["blue_eyes", "blue_nose", "canid", "canine", "fur", "mammal", "open_mouth", "purple_body", "solo", "white_body", "white_fur"], "selected_tags": ["2d_animation", "actual_fur", "amphibian", "amphibian_humanoid", "animal_humanoid", "animated", "animated_comic", "animated_png", "anthro", "big_mouth_(anatomy)", "blue_ears", "blue_eyes", "blue_fingers", "blue_inner_ear_fluff", "blue_nose", "blue_stripes", "blue_tail", "blue_toes", "blush", "body_hair", "canid", "canid_humanoid", "canine", "canine_humanoid", "clothed", "clothing", "comic", "countershading", "curved_tail", "facial_markings", "female_humanoid", "fennec_humanoid", "fox_humanoid", "frog_humanoid", "fur", "head_markings", "humanoid", "inner_ear_fluff", "jojo_pose", "jumper", "jumping", "light_nose", "light_tail", "male", "male_humanoid", "mammal", "mammal_humanoid", "markings", "midair", "nose", "open_mouth", "open_smile", "pink_blush", "pink_body", "pink_countershading", "pink_ears", "pink_stripes", "purple_body", "smile", "solo", "sparkling_character", "striped_face", "stripes", "suggestive_pose", "tail", "tailed_humanoid", "tan_tail", "tanuki_humanoid", "tuft", "two_tone_tail", "walking", "white_body", "white_fur"], "stage3_selected": ["2d_animation", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "big_mouth_(anatomy)", "blue_ears", "blue_eyes", "blue_fingers", "blue_inner_ear_fluff", "blue_nose", "blue_stripes", "blue_tail", "blue_toes", "body_hair", "canine_humanoid", "curved_tail", "female_humanoid", "fennec_humanoid", "frog_humanoid", "jojo_pose", "jumper", "jumping", "light_nose", "light_tail", "male_humanoid", "midair", "nose", "open_mouth", "open_smile", "pink_blush", "pink_countershading", "pink_ears", "pink_stripes", "purple_body", "simple_background", "sparkling_character", "striped_face", "stripes", "suggestive_pose", "tail", "tailed_humanoid", "tan_tail", "tanuki_humanoid", "two_tone_tail", "walking", "white_fur"], "stage3_selected_scores": {"simple_background": 0.5779, "open_mouth": 0.5847, "tail": 0.5894, "blue_eyes": 0.5818, "white_fur": 0.577, "animal_humanoid": 0.6209, "stripes": 0.5778, "purple_body": 0.5474, "open_smile": 0.4621, "body_hair": 0.2969, "canine_humanoid": 0.9127, "blue_nose": 0.592, "blue_tail": 0.5051, "blue_stripes": 0.5366, "light_tail": 0.5359, "striped_face": 0.5337, "pink_stripes": 0.5442, "frog_humanoid": 0.5238, "curved_tail": 0.5957, "male_humanoid": 0.5448, "light_nose": 0.4525, "big_mouth_(anatomy)": 0.4509, "sparkling_character": 0.3393, "nose": 0.7033, "actual_fur": 0.4438, "animated_comic": 0.4044, "animated_png": 0.4455, "animated": 0.3689, "2d_animation": 0.3613, "two_tone_tail": 0.4799, "walking": 0.33, "tan_tail": 0.4916, "blue_ears": 0.4792, "jumping": 0.5802, "pink_ears": 0.4875, "midair": 0.3892, "pink_blush": 0.4826, "suggestive_pose": 0.4425, "tanuki_humanoid": 0.77, "tailed_humanoid": 0.55, "female_humanoid": 0.5577, "blue_inner_ear_fluff": 0.4606, "pink_countershading": 0.4929, "blue_fingers": 0.4809, "blue_toes": 0.4856, "jojo_pose": 0.43, "fennec_humanoid": 0.7855, "jumper": 0.4004}, "stage3_selected_ranks": {"simple_background": 25, "open_mouth": 21, "tail": 20, "blue_eyes": 22, "white_fur": 27, "animal_humanoid": 14, "stripes": 26, "purple_body": 39, "open_smile": 106, "body_hair": 208, "canine_humanoid": 1, "blue_nose": 19, "blue_tail": 66, "blue_stripes": 49, "light_tail": 50, "striped_face": 51, "pink_stripes": 45, "frog_humanoid": 55, "curved_tail": 16, "male_humanoid": 44, "light_nose": 117, "big_mouth_(anatomy)": 119, "sparkling_character": 188, "nose": 12, "actual_fur": 126, "animated_comic": 150, "animated_png": 122, "animated": 171, "2d_animation": 173, "two_tone_tail": 89, "walking": 196, "tan_tail": 72, "blue_ears": 90, "jumping": 23, "pink_ears": 77, "midair": 163, "pink_blush": 85, "suggestive_pose": 128, "tanuki_humanoid": 7, "tailed_humanoid": 38, "female_humanoid": 33, "blue_inner_ear_fluff": 109, "pink_countershading": 71, "blue_fingers": 88, "blue_toes": 80, "jojo_pose": 135, "fennec_humanoid": 6, "jumper": 154}, "stage3_selected_phrase_ranks": {"simple_background": 1, "open_mouth": 1, "tail": 1, "blue_eyes": 1, "white_fur": 1, "animal_humanoid": 2, "stripes": 1, "purple_body": 1, "open_smile": 2, "body_hair": 10, "canine_humanoid": 1, "blue_nose": 1, "blue_tail": 2, "blue_stripes": 3, "light_tail": 4, "striped_face": 6, "pink_stripes": 1, "frog_humanoid": 10, "curved_tail": 1, "male_humanoid": 9, "light_nose": 2, "big_mouth_(anatomy)": 7, "sparkling_character": 8, "nose": 1, "actual_fur": 3, "animated_comic": 2, "animated_png": 1, "animated": 4, "2d_animation": 5, "two_tone_tail": 9, "walking": 10, "tan_tail": 6, "blue_ears": 7, "jumping": 1, "pink_ears": 5, "midair": 3, "pink_blush": 7, "suggestive_pose": 7, "tanuki_humanoid": 7, "tailed_humanoid": 8, "female_humanoid": 6, "blue_inner_ear_fluff": 5, "pink_countershading": 9, "blue_fingers": 5, "blue_toes": 4, "jojo_pose": 7, "fennec_humanoid": 6, "jumper": 2}, "extra_evidence": {"2d_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4438}, "amphibian": {"source": "implied"}, "amphibian_humanoid": {"source": "implied"}, "animal_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6209}, "animated": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3689}, "animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4044}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4455}, "anthro": {"source": "structural"}, "big_mouth_(anatomy)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4509}, "blue_ears": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4792}, "blue_fingers": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4809}, "blue_inner_ear_fluff": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4606}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5366}, "blue_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5051}, "blue_toes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4856}, "blush": {"source": "implied"}, "body_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.2969}, "canid_humanoid": {"source": "implied"}, "canine_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9127}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "comic": {"source": "implied"}, "countershading": {"source": "implied"}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5957}, "facial_markings": {"source": "implied"}, "female_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5577}, "fennec_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7855}, "fox_humanoid": {"source": "implied"}, "frog_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5238}, "head_markings": {"source": "implied"}, "humanoid": {"source": "implied"}, "inner_ear_fluff": {"source": "implied"}, "jojo_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.43}, "jumper": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4004}, "jumping": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5802}, "light_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4525}, "light_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5359}, "male": {"source": "structural"}, "male_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5448}, "mammal_humanoid": {"source": "implied"}, "markings": {"source": "implied"}, "midair": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3892}, "nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7033}, "open_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4621}, "pink_blush": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4826}, "pink_body": {"source": "implied"}, "pink_countershading": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4929}, "pink_ears": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4875}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5442}, "smile": {"source": "implied"}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3393}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5337}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5778}, "suggestive_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4425}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5894}, "tailed_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.55}, "tan_tail": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4916}, "tanuki_humanoid": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.77}, "tuft": {"source": "implied"}, "two_tone_tail": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4799}, "walking": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.33}}, "structural": ["solo", "anthro", "male", "clothed"], "t1": 1.83, "t2": 1.72, "t3": 64.54, "t3s": 2.33, "err": null, "issues": ["Stage3 split: general=205 entity=2 copyright_filtered=2 generic_char_to_general=4 unknown_type=12"]}
data/eval_results/eval_caption_cogvlm_n10_seed42_20260220_124618.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-20T12:46:18.984193", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "per_phrase_k": 2, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 1, "min_why": "strong_implied", "expand_implications": true, "infer_structural": true, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 18}
2
+ {"id": 3285630, "n_gt": 16, "n_retrieved": 151, "n_selected": 50, "n_implied": 14, "n_structural": 5, "ret_R": 0.1875, "P": 0.2, "R": 0.625, "F1": 0.303, "leaf_P": 0.129, "leaf_R": 0.4444, "leaf_F1": 0.2, "n_leaf_sel": 31, "n_leaf_gt": 9, "ret_P": 0.0199, "sel_given_ret": 3.3333, "over_sel": 3.12, "why": {"explicit": 25, "strong_implied": 7}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2, "gen_R": 0.625, "gen_F1": 0.303, "missed": ["5_fingers", "alpha_channel", "black_hair", "brown_body", "brown_fur", "fingers"], "extra": ["beverage", "big_hands", "black_body", "black_fur", "black_hands", "black_necktie", "blowup_background", "business_suit", "container", "cup", "felis", "formal", "hair_bun", "hand_on_hand", "handpaw", "holding_beverage", "holding_container", "holding_cup", "holding_mug", "holding_object", "long_arms", "looking_at_viewer", "mug", "necktie", "orange_background", "paws", "red_hands", "shirt", "straight_arms", "suit", "teal_shirt", "teal_topwear", "topwear", "vest", "waiter", "white_arms", "white_body", "white_fur", "white_hands", "white_necktie"], "ground_truth_tags": ["5_fingers", "alpha_channel", "anthro", "black_hair", "brown_body", "brown_fur", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "beverage", "big_hands", "black_body", "black_fur", "black_hands", "black_necktie", "blowup_background", "business_suit", "clothed", "clothing", "container", "cup", "felid", "feline", "felis", "formal", "fur", "hair", "hair_bun", "hand_on_hand", "handpaw", "holding_beverage", "holding_container", "holding_cup", "holding_mug", "holding_object", "long_arms", "looking_at_viewer", "male", "mammal", "mug", "necktie", "orange_background", "paws", "red_hands", "shirt", "solo", "straight_arms", "suit", "teal_shirt", "teal_topwear", "topwear", "vest", "waiter", "white_arms", "white_body", "white_fur", "white_hands", "white_necktie"], "extra_evidence": {"beverage": {"source": "implied"}, "big_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4948}, "black_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5747}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "black_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4561}, "black_necktie": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7125}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6332}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5728}, "container": {"source": "implied"}, "cup": {"source": "implied"}, "felis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "formal": {"source": "stage3", "why": "explicit", "retrieval_score": 0.598}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6912}, "hand_on_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5228}, "handpaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5262}, "holding_beverage": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7705}, "holding_container": {"source": "implied"}, "holding_cup": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7652}, "holding_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.915}, "holding_object": {"source": "implied"}, "long_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4658}, "looking_at_viewer": {"source": "structural"}, "mug": {"source": "implied"}, "necktie": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.7297}, "orange_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6196}, "paws": {"source": "implied"}, "red_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4285}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "straight_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5259}, "suit": {"source": "implied"}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "teal_topwear": {"source": "implied"}, "topwear": {"source": "implied"}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "waiter": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5856}, "white_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4277}, "white_body": {"source": "implied"}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5953}, "white_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4517}, "white_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6419}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 2.26, "t2": 2.92, "t3": 14.4, "t3s": 1.25, "err": null, "issues": ["Stage3 split: general=154 entity=1 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
3
+ {"id": 260449, "n_gt": 17, "n_retrieved": 160, "n_selected": 42, "n_implied": 11, "n_structural": 6, "ret_R": 0.4706, "P": 0.3095, "R": 0.7647, "F1": 0.4407, "leaf_P": 0.2333, "leaf_R": 0.5833, "leaf_F1": 0.3333, "n_leaf_sel": 30, "n_leaf_gt": 12, "ret_P": 0.05, "sel_given_ret": 1.625, "over_sel": 2.47, "why": {"explicit": 12, "strong_implied": 13}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3095, "gen_R": 0.7647, "gen_F1": 0.4407, "missed": ["claws", "fur", "human", "slightly_chubby"], "extra": ["anthro", "arm_above_head", "arms_above_head", "belly_dancer_outfit", "belly_dancing", "bodily_fluids", "bottomwear", "cheeky", "crying_laughing", "dancewear", "expressions", "feral", "hand_above_head", "hands_behind_head", "holding_arms", "laugh", "loincloth", "monkey", "new_world_monkey", "raised_arm", "raised_arms", "raised_finger", "smile", "smirk", "spread_arms", "tears", "toony_expression", "trio", "wide_grin"], "ground_truth_tags": ["ape", "bear", "claws", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate", "slightly_chubby", "topless"], "selected_tags": ["anthro", "ape", "arm_above_head", "arms_above_head", "bear", "belly_dancer_outfit", "belly_dancing", "bodily_fluids", "bottomwear", "cheeky", "clothed", "clothing", "crying_laughing", "dancewear", "dancing", "expressions", "feral", "group", "hair", "hand_above_head", "hands_behind_head", "haplorhine", "holding_arms", "laugh", "loincloth", "looking_at_viewer", "male", "mammal", "monkey", "new_world_monkey", "primate", "raised_arm", "raised_arms", "raised_finger", "smile", "smirk", "spread_arms", "tears", "toony_expression", "topless", "trio", "wide_grin"], "extra_evidence": {"anthro": {"source": "structural"}, "arm_above_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3826}, "arms_above_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3935}, "belly_dancer_outfit": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.351}, "belly_dancing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.378}, "bodily_fluids": {"source": "implied"}, "bottomwear": {"source": "implied"}, "cheeky": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3862}, "crying_laughing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3837}, "dancewear": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3454}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4949}, "feral": {"source": "structural"}, "hand_above_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.374}, "hands_behind_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3616}, "holding_arms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3945}, "laugh": {"source": "implied"}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "implied"}, "new_world_monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7333}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4867}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_finger": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3777}, "smile": {"source": "implied"}, "smirk": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3536}, "spread_arms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3987}, "tears": {"source": "implied"}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4623}, "trio": {"source": "structural"}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}}, "structural": ["trio", "anthro", "feral", "male", "topless", "looking_at_viewer"], "t1": 3.26, "t2": 1.76, "t3": 11.38, "t3s": 1.04, "err": null, "issues": ["Stage3 split: general=160 entity=5 copyright_filtered=3 generic_char_to_general=1 unknown_type=3"]}
4
+ {"id": 1078019, "n_gt": 18, "n_retrieved": 133, "n_selected": 31, "n_implied": 7, "n_structural": 5, "ret_R": 0.5556, "P": 0.4516, "R": 0.7778, "F1": 0.5714, "leaf_P": 0.3182, "leaf_R": 0.5833, "leaf_F1": 0.4118, "n_leaf_sel": 22, "n_leaf_gt": 12, "ret_P": 0.0752, "sel_given_ret": 1.4, "over_sel": 1.72, "why": {"explicit": 15, "strong_implied": 6}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4516, "gen_R": 0.7778, "gen_F1": 0.5714, "missed": ["alternate_species", "male", "male/male", "open_mouth"], "extra": ["ambiguous_gender", "blush_lines", "coat", "cuddling", "embrace", "eyes", "holding_object", "holding_plushie", "inanimate_object", "looking_at_viewer", "love", "pink_blush", "red_cheeks", "relationship", "teal_clothing", "topwear", "winter_coat"], "ground_truth_tags": ["alternate_species", "anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "male", "male/male", "mammal", "open_mouth", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["ambiguous_gender", "anthro", "blue_eyes", "blush", "blush_lines", "clothed", "clothing", "coat", "cuddling", "duo", "embrace", "eyes", "holding_object", "holding_plushie", "inanimate_object", "lagomorph", "leporid", "looking_at_viewer", "love", "mammal", "pink_blush", "plushie", "rabbit", "red_cheeks", "relationship", "romantic", "romantic_couple", "teal_clothing", "teal_eyes", "topwear", "winter_coat"], "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "blush_lines": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4772}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "cuddling": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4801}, "embrace": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4349}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "holding_object": {"source": "implied"}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "inanimate_object": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4648}, "looking_at_viewer": {"source": "structural"}, "love": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.469}, "pink_blush": {"source": "stage3", "why": "explicit", "retrieval_score": 0.476}, "red_cheeks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4401}, "relationship": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6203}, "teal_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4335}, "topwear": {"source": "implied"}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4757}}, "structural": ["duo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 1.59, "t2": 1.47, "t3": 11.0, "t3s": 2.05, "err": null, "issues": ["Stage3 split: general=131 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
5
+ {"id": 1624724, "n_gt": 16, "n_retrieved": 121, "n_selected": 33, "n_implied": 2, "n_structural": 4, "ret_R": 0.1875, "P": 0.1515, "R": 0.3125, "F1": 0.2041, "leaf_P": 0.1562, "leaf_R": 0.5, "leaf_F1": 0.2381, "n_leaf_sel": 32, "n_leaf_gt": 10, "ret_P": 0.0248, "sel_given_ret": 1.6667, "over_sel": 2.06, "why": {"explicit": 29}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1515, "gen_R": 0.4167, "gen_F1": 0.2222, "missed": ["alien", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)"], "extra": ["big_eyes", "blowup_background", "blue_background", "cartoon_character", "eye_spots", "feral", "floating", "floating_hands", "floating_head", "glistening", "glistening_nose", "grinning_at_viewer", "jagged_mouth", "looking_at_viewer", "red_ears", "red_face", "small_nose", "spots", "spotted_arms", "spotted_back", "spotted_legs", "spotted_shoulders", "tan_background", "tan_chest", "tan_nose", "tan_stripes", "toony", "white_nose"], "ground_truth_tags": ["alien", "ambiguous_gender", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)", "red_nose", "smile", "solo", "tan_body"], "selected_tags": ["ambiguous_gender", "big_eyes", "blowup_background", "blue_background", "cartoon_character", "eye_spots", "feral", "floating", "floating_hands", "floating_head", "glistening", "glistening_nose", "grinning_at_viewer", "jagged_mouth", "looking_at_viewer", "red_ears", "red_face", "red_nose", "small_nose", "smile", "solo", "spots", "spotted_arms", "spotted_back", "spotted_legs", "spotted_shoulders", "tan_background", "tan_body", "tan_chest", "tan_nose", "tan_stripes", "toony", "white_nose"], "extra_evidence": {"big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6933}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5293}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5363}, "cartoon_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4997}, "eye_spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6773}, "feral": {"source": "structural"}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "floating_hands": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4326}, "floating_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4722}, "glistening": {"source": "implied"}, "glistening_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5911}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4723}, "jagged_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5895}, "looking_at_viewer": {"source": "structural"}, "red_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5731}, "red_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6394}, "small_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5298}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6222}, "spotted_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6641}, "spotted_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7}, "spotted_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6491}, "spotted_shoulders": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6126}, "tan_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5157}, "tan_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6867}, "tan_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6411}, "tan_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6487}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5156}, "white_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5939}}, "structural": ["solo", "feral", "ambiguous_gender", "looking_at_viewer"], "t1": 2.38, "t2": 1.09, "t3": 11.61, "t3s": 1.12, "err": null, "issues": ["Stage3 split: general=122 entity=0 copyright_filtered=4 generic_char_to_general=5 unknown_type=5"]}
6
+ {"id": 1325009, "n_gt": 24, "n_retrieved": 177, "n_selected": 81, "n_implied": 26, "n_structural": 5, "ret_R": 0.4167, "P": 0.2469, "R": 0.8333, "F1": 0.381, "leaf_P": 0.1064, "leaf_R": 0.3571, "leaf_F1": 0.1639, "n_leaf_sel": 47, "n_leaf_gt": 14, "ret_P": 0.0565, "sel_given_ret": 2.0, "over_sel": 3.38, "why": {"explicit": 50}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2469, "gen_R": 0.8333, "gen_F1": 0.381, "missed": ["hand_on_head", "muscular_male", "smile", "topless"], "extra": ["actual_fur", "belly", "bengal_tiger", "big_eyes", "black_chest", "blue_bottomwear", "blue_clothing", "blue_inner_ear_fluff", "blue_shorts", "brown_chest", "confident", "countershade_belly", "countershade_face", "countershade_torso", "ear_tuft", "facial_markings", "full-length_portrait", "fur_pattern", "gesture", "glistening", "glistening_eyes", "hand_on_chest", "hand_on_knee", "hand_on_leg", "hand_on_own_chest", "hand_on_own_head", "hand_on_own_knee", "hand_on_own_leg", "head_markings", "heterochromia", "inner_ear_fluff", "light_chest", "manly", "markings", "monotone_chest", "muscular_arms", "obliques", "pointing", "portrait", "posed", "raised_hand", "sound_effects", "striped_arms", "striped_body", "striped_face", "striped_fur", "striped_legs", "striped_neck", "tan_bottomwear", "tan_clothing", "tan_shorts", "white_arms", "white_body", "white_bottomwear", "white_chest", "white_clothing", "white_fur", "white_shorts", "yellow_bottomwear", "yellow_clothing", "yellow_shorts"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "looking_at_viewer", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "smile", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["actual_fur", "anthro", "belly", "bengal_tiger", "big_eyes", "black_chest", "blue_bottomwear", "blue_clothing", "blue_eyes", "blue_inner_ear_fluff", "blue_shorts", "bottomwear", "brown_chest", "chest_tuft", "clothed", "clothing", "confident", "countershade_belly", "countershade_face", "countershade_torso", "countershading", "ear_tuft", "facial_markings", "felid", "full-length_portrait", "fur", "fur_pattern", "gesture", "glistening", "glistening_eyes", "hand_on_chest", "hand_on_knee", "hand_on_leg", "hand_on_own_chest", "hand_on_own_head", "hand_on_own_knee", "hand_on_own_leg", "head_markings", "heterochromia", "inner_ear_fluff", "light_chest", "looking_at_viewer", "male", "mammal", "manly", "markings", "monotone_chest", "muscular", "muscular_anthro", "muscular_arms", "obliques", "pantherine", "pointing", "portrait", "posed", "raised_hand", "shorts", "solo", "sound_effects", "striped_arms", "striped_body", "striped_face", "striped_fur", "striped_legs", "striped_neck", "stripes", "tan_bottomwear", "tan_clothing", "tan_shorts", "tiger", "tuft", "white_arms", "white_body", "white_bottomwear", "white_chest", "white_clothing", "white_fur", "white_shorts", "yellow_bottomwear", "yellow_clothing", "yellow_shorts"], "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4579}, "belly": {"source": "implied"}, "bengal_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.438}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4728}, "black_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "blue_bottomwear": {"source": "implied"}, "blue_clothing": {"source": "implied"}, "blue_inner_ear_fluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4467}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6243}, "brown_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.722}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "countershade_belly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8323}, "countershade_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6714}, "countershade_torso": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6684}, "ear_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6019}, "facial_markings": {"source": "implied"}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4553}, "fur_pattern": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4741}, "gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "glistening": {"source": "implied"}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4944}, "hand_on_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4215}, "hand_on_knee": {"source": "implied"}, "hand_on_leg": {"source": "implied"}, "hand_on_own_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.57}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5852}, "hand_on_own_knee": {"source": "stage3", "why": "explicit", "retrieval_score": 0.601}, "hand_on_own_leg": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6095}, "head_markings": {"source": "implied"}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "inner_ear_fluff": {"source": "implied"}, "light_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8262}, "manly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6183}, "markings": {"source": "implied"}, "monotone_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8371}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7914}, "obliques": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6211}, "pointing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4531}, "portrait": {"source": "implied"}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.442}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7019}, "sound_effects": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3834}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6838}, "striped_body": {"source": "implied"}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7138}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6535}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6442}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.686}, "tan_bottomwear": {"source": "implied"}, "tan_clothing": {"source": "implied"}, "tan_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6103}, "white_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8123}, "white_body": {"source": "implied"}, "white_bottomwear": {"source": "implied"}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9241}, "white_clothing": {"source": "implied"}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4637}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6109}, "yellow_bottomwear": {"source": "implied"}, "yellow_clothing": {"source": "implied"}, "yellow_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6093}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 2.43, "t2": 1.74, "t3": 41.86, "t3s": 1.24, "err": null, "issues": ["Stage3 split: general=173 entity=2 copyright_filtered=3 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 37, \"why\": \"explicit\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 41, \"why\": \"explicit\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 49, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"explicit\"}, {\"i\": 53, \"why\": \"explicit\"}, {\"i\": 55, \"why\": \"explicit\"}, {\"i\": 57, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"other\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 36}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
7
+ {"id": 1023509, "n_gt": 21, "n_retrieved": 192, "n_selected": 70, "n_implied": 19, "n_structural": 5, "ret_R": 0.4286, "P": 0.1286, "R": 0.4286, "F1": 0.1978, "leaf_P": 0.0444, "leaf_R": 0.1818, "leaf_F1": 0.0714, "n_leaf_sel": 45, "n_leaf_gt": 11, "ret_P": 0.0469, "sel_given_ret": 1.0, "over_sel": 3.33, "why": {"explicit": 42, "strong_implied": 4}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1286, "gen_R": 0.4737, "gen_F1": 0.2022, "missed": ["armless", "boss_monster", "d6", "dialogue", "dice", "fur", "long_ears", "red_eyes", "undertale", "undertale_(series)", "white_body", "white_fur"], "extra": ["ambiguous_gender", "anthro", "attack", "bodily_fluids", "bovid_horn", "caprine_demon", "caprine_horn", "clenched_teeth", "clothed", "clothing", "comic", "crying", "dark", "darkner", "darkness", "demon", "elemental_creature", "emotional", "eublepharid", "excited", "fight", "gecko", "goat_demon", "goat_ears", "goat_horn", "group", "hands_on_ground", "horn", "human_only", "human_to_humanoid", "iguanid", "kneeling", "kneeling_on_ground", "laying_on_ground", "leaning", "leaning_forward", "leopard_gecko", "light", "lizard_tail", "lizardman", "lying_on_ground", "male_human", "medieval_fantasy", "monitor_lizard", "not_furry", "note", "on_one_leg", "one_page_comic", "pictographics", "plague_doctor", "relaxed_expression", "running_away", "scared", "species_transformation", "standing", "tears", "teeth", "torch", "transformation", "two_panel_image", "wounded"], "ground_truth_tags": ["armless", "boss_monster", "bovid", "caprine", "d6", "dialogue", "dice", "fur", "goat", "human", "lizard", "long_ears", "mammal", "red_eyes", "reptile", "scalie", "text", "undertale", "undertale_(series)", "white_body", "white_fur"], "selected_tags": ["ambiguous_gender", "anthro", "attack", "bodily_fluids", "bovid", "bovid_horn", "caprine", "caprine_demon", "caprine_horn", "clenched_teeth", "clothed", "clothing", "comic", "crying", "dark", "darkner", "darkness", "demon", "elemental_creature", "emotional", "eublepharid", "excited", "fight", "gecko", "goat", "goat_demon", "goat_ears", "goat_horn", "group", "hands_on_ground", "horn", "human", "human_only", "human_to_humanoid", "iguanid", "kneeling", "kneeling_on_ground", "laying_on_ground", "leaning", "leaning_forward", "leopard_gecko", "light", "lizard", "lizard_tail", "lizardman", "lying_on_ground", "male_human", "mammal", "medieval_fantasy", "monitor_lizard", "not_furry", "note", "on_one_leg", "one_page_comic", "pictographics", "plague_doctor", "relaxed_expression", "reptile", "running_away", "scalie", "scared", "species_transformation", "standing", "tears", "teeth", "text", "torch", "transformation", "two_panel_image", "wounded"], "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "anthro": {"source": "structural"}, "attack": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5453}, "bodily_fluids": {"source": "implied"}, "bovid_horn": {"source": "implied"}, "caprine_demon": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5623}, "caprine_horn": {"source": "implied"}, "clenched_teeth": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4782}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5599}, "crying": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5149}, "dark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5867}, "darkner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.565}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "demon": {"source": "implied"}, "elemental_creature": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4914}, "emotional": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5288}, "eublepharid": {"source": "implied"}, "excited": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5209}, "fight": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5245}, "gecko": {"source": "implied"}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5691}, "goat_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5349}, "goat_horn": {"source": "stage3", "why": "explicit", "retrieval_score": 0.535}, "group": {"source": "structural"}, "hands_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "horn": {"source": "implied"}, "human_only": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5247}, "human_to_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3295}, "iguanid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5909}, "kneeling": {"source": "implied"}, "kneeling_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.563}, "laying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7028}, "leaning": {"source": "implied"}, "leaning_forward": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5138}, "leopard_gecko": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5333}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5708}, "lizardman": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5348}, "lying_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7873}, "male_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5551}, "medieval_fantasy": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5178}, "monitor_lizard": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5652}, "not_furry": {"source": "implied"}, "note": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7374}, "on_one_leg": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5646}, "one_page_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5615}, "pictographics": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5134}, "plague_doctor": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5063}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5599}, "running_away": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5225}, "scared": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5095}, "species_transformation": {"source": "implied"}, "standing": {"source": "implied"}, "tears": {"source": "implied"}, "teeth": {"source": "implied"}, "torch": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5647}, "transformation": {"source": "implied"}, "two_panel_image": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5288}, "wounded": {"source": "stage3", "why": "explicit", "retrieval_score": 0.521}}, "structural": ["group", "anthro", "ambiguous_gender", "clothed", "text"], "t1": 2.94, "t2": 1.72, "t3": 125.58, "t3s": 4.86, "err": null, "issues": ["Stage3 split: general=183 entity=5 copyright_filtered=4 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"style_or_meta\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"style_or_meta\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"style_or_meta\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"style_or_meta\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"style_or_meta\"}, {\"i\": 56, \"why\": \"weak_implied\"}, {\"i\": 58}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.34.why\n Field required [type=missing, input_value={'i': 58}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_1: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
8
+ {"id": 335343, "n_gt": 33, "n_retrieved": 207, "n_selected": 40, "n_implied": 12, "n_structural": 5, "ret_R": 0.2727, "P": 0.175, "R": 0.2121, "F1": 0.1918, "leaf_P": 0.0741, "leaf_R": 0.1, "leaf_F1": 0.0851, "n_leaf_sel": 27, "n_leaf_gt": 20, "ret_P": 0.0435, "sel_given_ret": 0.7778, "over_sel": 1.21, "why": {"explicit": 23}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.175, "gen_R": 0.2258, "gen_F1": 0.1972, "missed": ["angry", "blonde_hair", "earth_pony", "equid", "equine", "eyes_closed", "eyeshadow", "feathers", "female", "feral", "freckles", "fur", "green_eyes", "hasbro", "horn", "horse", "mammal", "my_little_pony", "pillow", "pony", "purple_hair", "scared", "sleeping", "unicorn", "white_body", "white_fur"], "extra": ["3rd_party_watermark", "ambiguous_gender", "anime_eyes", "artist_logo", "bed_sheet", "bedding", "blanket", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "english_text", "green_ears", "half-closed_eyes", "head_on_pillow", "heterochromia", "humanoid", "letters", "lipstick", "logo", "long_hair", "lying_on_bed", "name_drop", "narrowed_eyes", "on_bed", "pink_lipstick", "purple_eyes", "resting_arm", "sitting", "sitting_on_bed", "sleepwear", "text_box", "watermark"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "duo", "earth_pony", "equid", "equine", "eyes_closed", "eyeshadow", "feathers", "female", "feral", "freckles", "fur", "furniture", "green_eyes", "hair", "hasbro", "horn", "horse", "lying", "makeup", "mammal", "my_little_pony", "pillow", "pony", "purple_hair", "scared", "sleeping", "text", "unicorn", "white_body", "white_fur"], "selected_tags": ["3rd_party_watermark", "ambiguous_gender", "anime_eyes", "artist_logo", "bed", "bed_sheet", "bedding", "blanket", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "duo", "english_text", "furniture", "green_ears", "hair", "half-closed_eyes", "head_on_pillow", "heterochromia", "humanoid", "letters", "lipstick", "logo", "long_hair", "lying", "lying_on_bed", "makeup", "name_drop", "narrowed_eyes", "on_bed", "pink_lipstick", "purple_eyes", "resting_arm", "sitting", "sitting_on_bed", "sleepwear", "text", "text_box", "watermark"], "extra_evidence": {"3rd_party_watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "ambiguous_gender": {"source": "structural"}, "anime_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4287}, "artist_logo": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3893}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4169}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "cool_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3885}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "green_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4373}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3908}, "head_on_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3867}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4}, "humanoid": {"source": "structural"}, "letters": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3654}, "lipstick": {"source": "implied"}, "logo": {"source": "implied"}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4471}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "name_drop": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3557}, "narrowed_eyes": {"source": "implied"}, "on_bed": {"source": "implied"}, "pink_lipstick": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4482}, "purple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4293}, "resting_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3734}, "sitting": {"source": "implied"}, "sitting_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "text_box": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3881}, "watermark": {"source": "implied"}}, "structural": ["duo", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 2.18, "t2": 1.9, "t3": 72.92, "t3s": 1.22, "err": null, "issues": ["Stage3 split: general=209 entity=0 copyright_filtered=1 generic_char_to_general=0 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"other\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"other\"}, {\"i\": 52, \"why\": \"other\"}, {\"i\": 53, \"why\": \"other\"}, {\"i\": 56}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.37.why\n Field required [type=missing, input_value={'i': 56}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"other\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"other\"}, {\"i\": 52, \"why\": \"other\"}, {\"i\": 53, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.37.why\n Field required [type=missing, input_value={'i': 53, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"other\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 30, \"why\": \"style_or_meta\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 39, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"other\"}, {\"i\": 49, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"other\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 56, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.37.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.37.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: gave up after 3 attempts"]}
9
+ {"id": 17482, "n_gt": 26, "n_retrieved": 173, "n_selected": 69, "n_implied": 26, "n_structural": 4, "ret_R": 0.5385, "P": 0.2754, "R": 0.7308, "F1": 0.4, "leaf_P": 0.1389, "leaf_R": 0.3125, "leaf_F1": 0.1923, "n_leaf_sel": 36, "n_leaf_gt": 16, "ret_P": 0.0809, "sel_given_ret": 1.3571, "over_sel": 2.65, "why": {"explicit": 39}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2754, "gen_R": 0.7308, "gen_F1": 0.4, "missed": ["4_fingers", "demon", "female", "fingers", "fur", "music", "slim"], "extra": ["4_claws", "accessory", "action_pose", "auburn_hair", "big_hair", "big_tail", "black_hair", "blonde_hair", "bottomwear", "canis", "claws_out", "crosslegged_pose", "cute_expression", "denim", "denim_clothing", "digital_creature", "ear_piercing", "electric_guitar", "flowing_hair", "hair_accessory", "hair_tie", "holding_guitar", "holding_hair", "holding_tail", "inspired_by_formal_art", "jeans", "leggings", "legwear", "long_tail", "male", "pants", "piercing", "playing_music", "pose", "shirt", "shorts", "toony_expression", "topless", "topwear", "torn_body", "torn_bottomwear", "torn_jeans", "torn_leggings", "torn_legwear", "torn_pants", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "wavy_hair"], "ground_truth_tags": ["4_fingers", "anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "demon", "female", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "slim", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_claws", "accessory", "action_pose", "anthro", "auburn_hair", "bass_guitar", "big_hair", "big_tail", "black_hair", "blonde_hair", "bottomwear", "canid", "canine", "canis", "claws", "claws_out", "clothed", "clothing", "crosslegged_pose", "cute_expression", "denim", "denim_clothing", "digital_creature", "ear_piercing", "electric_guitar", "flowing_hair", "guitar", "hair", "hair_accessory", "hair_tie", "holding_guitar", "holding_hair", "holding_musical_instrument", "holding_object", "holding_tail", "inspired_by_formal_art", "jeans", "leggings", "legwear", "long_tail", "male", "mammal", "musical_instrument", "pants", "piercing", "playing_music", "plucked_string_instrument", "pose", "shirt", "shorts", "solo", "spade_tail", "string_instrument", "tail", "toony_expression", "topless", "topwear", "torn_body", "torn_bottomwear", "torn_clothing", "torn_jeans", "torn_leggings", "torn_legwear", "torn_pants", "torn_shirt", "torn_shorts", "torn_topwear", "touching_hair", "wavy_hair"], "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4509}, "accessory": {"source": "implied"}, "action_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5786}, "auburn_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3745}, "big_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "big_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3803}, "black_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3857}, "blonde_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3793}, "bottomwear": {"source": "implied"}, "canis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4558}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.433}, "crosslegged_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4547}, "cute_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3951}, "denim": {"source": "implied"}, "denim_clothing": {"source": "implied"}, "digital_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3725}, "ear_piercing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3405}, "electric_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8659}, "flowing_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5653}, "hair_accessory": {"source": "implied"}, "hair_tie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3721}, "holding_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.844}, "holding_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3907}, "holding_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3825}, "inspired_by_formal_art": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3055}, "jeans": {"source": "implied"}, "leggings": {"source": "implied"}, "legwear": {"source": "implied"}, "long_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.419}, "male": {"source": "structural"}, "pants": {"source": "implied"}, "piercing": {"source": "implied"}, "playing_music": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8724}, "pose": {"source": "implied"}, "shirt": {"source": "implied"}, "shorts": {"source": "implied"}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "topless": {"source": "structural"}, "topwear": {"source": "implied"}, "torn_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3954}, "torn_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4337}, "torn_jeans": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4808}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4221}, "torn_legwear": {"source": "implied"}, "torn_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4617}, "torn_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4021}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3964}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3916}, "touching_hair": {"source": "implied"}, "wavy_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3475}}, "structural": ["solo", "anthro", "male", "topless"], "t1": 2.14, "t2": 1.86, "t3": 23.47, "t3s": 4.47, "err": null, "issues": ["Stage3 split: general=173 entity=2 copyright_filtered=4 generic_char_to_general=0 unknown_type=2"]}
10
+ {"id": 2021552, "n_gt": 40, "n_retrieved": 161, "n_selected": 48, "n_implied": 18, "n_structural": 5, "ret_R": 0.5, "P": 0.5208, "R": 0.625, "F1": 0.5682, "leaf_P": 0.3478, "leaf_R": 0.381, "leaf_F1": 0.3636, "n_leaf_sel": 23, "n_leaf_gt": 21, "ret_P": 0.1242, "sel_given_ret": 1.25, "over_sel": 1.2, "why": {"explicit": 26}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.5208, "gen_R": 0.625, "gen_F1": 0.5682, "missed": ["arctic_fox", "cheek_tuft", "facial_tuft", "fluffy", "fluffy_tail", "grey_background", "head_tuft", "looking_at_another", "looking_away", "pawpads", "tail", "toe_claws", "tuft", "white_body", "white_fur"], "extra": ["big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "clawed_fingers", "coveralls", "cross_fox", "domestic_rabbit", "dutch_rabbit", "fennec_fox", "floppy_ears", "grey_claws", "humanoid", "looking_at_viewer", "lop_ears", "open_mouth", "oryctolagus", "rabbit_ears", "red_fox", "white_clothing", "white_shirt", "white_topwear"], "ground_truth_tags": ["anthro", "arctic_fox", "bottomwear", "canid", "canine", "cheek_tuft", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "facial_tuft", "fluffy", "fluffy_tail", "fox", "fur", "grey_background", "grey_body", "grey_fur", "head_markings", "head_tuft", "lagomorph", "leporid", "looking_at_another", "looking_away", "mammal", "markings", "overalls", "pants", "pawpads", "rabbit", "shirt", "standing", "tail", "toe_claws", "topwear", "tuft", "white_body", "white_fur"], "selected_tags": ["anthro", "big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "bottomwear", "canid", "canine", "clawed_fingers", "claws", "clothed", "clothing", "coveralls", "cross_fox", "crossed_arms", "domestic_rabbit", "duo", "dutch_rabbit", "facial_markings", "fennec_fox", "floppy_ears", "fox", "fur", "grey_body", "grey_claws", "grey_fur", "head_markings", "humanoid", "lagomorph", "leporid", "looking_at_viewer", "lop_ears", "mammal", "markings", "open_mouth", "oryctolagus", "overalls", "pants", "rabbit", "rabbit_ears", "red_fox", "shirt", "standing", "topwear", "white_clothing", "white_shirt", "white_topwear"], "extra_evidence": {"big_mouth_(anatomy)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5037}, "black_bottomwear": {"source": "implied"}, "black_clothing": {"source": "implied"}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.833}, "blue_overalls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9203}, "clawed_fingers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5145}, "coveralls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5688}, "cross_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4688}, "domestic_rabbit": {"source": "implied"}, "dutch_rabbit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4737}, "fennec_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5031}, "floppy_ears": {"source": "implied"}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5166}, "humanoid": {"source": "structural"}, "looking_at_viewer": {"source": "structural"}, "lop_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4692}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "oryctolagus": {"source": "implied"}, "rabbit_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "red_fox": {"source": "implied"}, "white_clothing": {"source": "implied"}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_topwear": {"source": "implied"}}, "structural": ["duo", "anthro", "humanoid", "clothed", "looking_at_viewer"], "t1": 1.79, "t2": 1.58, "t3": 47.13, "t3s": 6.26, "err": null, "issues": ["Stage3 split: general=161 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3"]}
11
+ {"id": 2034167, "n_gt": 21, "n_retrieved": 191, "n_selected": 82, "n_implied": 28, "n_structural": 5, "ret_R": 0.381, "P": 0.1341, "R": 0.5238, "F1": 0.2136, "leaf_P": 0.0833, "leaf_R": 0.3333, "leaf_F1": 0.1333, "n_leaf_sel": 48, "n_leaf_gt": 12, "ret_P": 0.0419, "sel_given_ret": 1.375, "over_sel": 3.9, "why": {"explicit": 43, "strong_implied": 7}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1341, "gen_R": 0.5238, "gen_F1": 0.2136, "missed": ["4_toes", "canis", "feet", "female", "feral", "purple_fur", "teeth", "toes", "tongue", "wolf"], "extra": ["ambiguous_gender", "amusement_ride", "animal_humanoid", "anime_eyes", "anthro", "big_eyes", "big_tail", "blue_eyebrows", "blue_inner_ear_fluff", "blue_stripes", "blue_tail", "blue_tuft", "canid_humanoid", "canine_humanoid", "clothed", "clothing", "curved_tail", "dark_stripes", "eyebrows", "eyes", "facial_markings", "facial_stripes", "female_humanoid", "fennec_humanoid", "fluffy_fur", "fox_humanoid", "glistening", "glistening_body", "glistening_fur", "glistening_tail", "head_markings", "heterochromia", "humanoid", "inner_ear_fluff", "jumper", "jumping", "long_tail", "looking_at_viewer", "male", "mammal_humanoid", "markings", "multi_tone_fur", "multicolored_body", "multicolored_fur", "multicolored_tail", "muscular", "muscular_male", "narrow_tail", "open_smile", "pig_humanoid", "pink_stripes", "pupils", "purple_pupils", "smile", "striped_arms", "striped_back", "striped_face", "stripes", "suid", "suid_humanoid", "suina", "suina_humanoid", "tail", "tail_tuft", "tanuki_humanoid", "thin_tail", "tuft", "two_tone_body", "two_tone_fur", "wolf_humanoid", "yellow_background"], "ground_truth_tags": ["4_toes", "blue_eyes", "blue_nose", "canid", "canine", "canis", "feet", "female", "feral", "fur", "mammal", "open_mouth", "purple_body", "purple_fur", "solo", "teeth", "toes", "tongue", "white_body", "white_fur", "wolf"], "selected_tags": ["ambiguous_gender", "amusement_ride", "animal_humanoid", "anime_eyes", "anthro", "big_eyes", "big_tail", "blue_eyebrows", "blue_eyes", "blue_inner_ear_fluff", "blue_nose", "blue_stripes", "blue_tail", "blue_tuft", "canid", "canid_humanoid", "canine", "canine_humanoid", "clothed", "clothing", "curved_tail", "dark_stripes", "eyebrows", "eyes", "facial_markings", "facial_stripes", "female_humanoid", "fennec_humanoid", "fluffy_fur", "fox_humanoid", "fur", "glistening", "glistening_body", "glistening_fur", "glistening_tail", "head_markings", "heterochromia", "humanoid", "inner_ear_fluff", "jumper", "jumping", "long_tail", "looking_at_viewer", "male", "mammal", "mammal_humanoid", "markings", "multi_tone_fur", "multicolored_body", "multicolored_fur", "multicolored_tail", "muscular", "muscular_male", "narrow_tail", "open_mouth", "open_smile", "pig_humanoid", "pink_stripes", "pupils", "purple_body", "purple_pupils", "smile", "solo", "striped_arms", "striped_back", "striped_face", "stripes", "suid", "suid_humanoid", "suina", "suina_humanoid", "tail", "tail_tuft", "tanuki_humanoid", "thin_tail", "tuft", "two_tone_body", "two_tone_fur", "white_body", "white_fur", "wolf_humanoid", "yellow_background"], "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "amusement_ride": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.2638}, "animal_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6108}, "anime_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4956}, "anthro": {"source": "structural"}, "big_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4357}, "big_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5597}, "blue_eyebrows": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5104}, "blue_inner_ear_fluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5267}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6998}, "blue_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5674}, "blue_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5342}, "canid_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8514}, "canine_humanoid": {"source": "implied"}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7268}, "dark_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6047}, "eyebrows": {"source": "implied"}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7773}, "facial_markings": {"source": "implied"}, "facial_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6252}, "female_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5647}, "fennec_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7638}, "fluffy_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5831}, "fox_humanoid": {"source": "implied"}, "glistening": {"source": "implied"}, "glistening_body": {"source": "implied"}, "glistening_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5624}, "glistening_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6145}, "head_markings": {"source": "implied"}, "heterochromia": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4544}, "humanoid": {"source": "implied"}, "inner_ear_fluff": {"source": "implied"}, "jumper": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4273}, "jumping": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5557}, "long_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.636}, "looking_at_viewer": {"source": "structural"}, "male": {"source": "implied"}, "mammal_humanoid": {"source": "implied"}, "markings": {"source": "implied"}, "multi_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5458}, "multicolored_body": {"source": "implied"}, "multicolored_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5196}, "multicolored_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5462}, "muscular": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.356}, "muscular_male": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3111}, "narrow_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5697}, "open_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.502}, "pig_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5859}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7067}, "pupils": {"source": "implied"}, "purple_pupils": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5623}, "smile": {"source": "implied"}, "striped_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5998}, "striped_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6325}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6073}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6214}, "suid": {"source": "implied"}, "suid_humanoid": {"source": "implied"}, "suina": {"source": "implied"}, "suina_humanoid": {"source": "implied"}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.626}, "tail_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5506}, "tanuki_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7472}, "thin_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.587}, "tuft": {"source": "implied"}, "two_tone_body": {"source": "implied"}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5105}, "wolf_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8079}, "yellow_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5134}}, "structural": ["solo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 2.04, "t2": 2.0, "t3": 47.91, "t3s": 1.37, "err": null, "issues": ["Stage3 split: general=194 entity=2 copyright_filtered=2 generic_char_to_general=0 unknown_type=5"]}
data/eval_results/eval_caption_cogvlm_n10_seed42_20260220_220531.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-20T22:05:31.105960", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "per_phrase_k": 2, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 1, "min_why": "strong_implied", "expand_implications": true, "infer_structural": true, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 18}
2
+ {"id": 3285630, "n_gt": 16, "n_retrieved": 133, "n_selected": 32, "n_implied": 12, "n_structural": 5, "ret_R": 0.1875, "P": 0.3125, "R": 0.625, "F1": 0.4167, "leaf_P": 0.2105, "leaf_R": 0.4444, "leaf_F1": 0.2857, "n_leaf_sel": 19, "n_leaf_gt": 9, "ret_P": 0.0226, "sel_given_ret": 3.3333, "over_sel": 2.0, "why": {"explicit": 17}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3125, "gen_R": 0.625, "gen_F1": 0.4167, "missed": ["5_fingers", "alpha_channel", "black_hair", "brown_body", "brown_fur", "fingers"], "extra": ["black_body", "black_fur", "business_attire", "business_suit", "felis", "formal", "hair_bun", "heavy", "holding_mug", "holding_object", "lips", "looking_at_viewer", "mafia", "mug", "shirt", "suit", "teal_shirt", "teal_topwear", "thick_lips", "top_heavy", "topwear", "vest"], "ground_truth_tags": ["5_fingers", "alpha_channel", "anthro", "black_hair", "brown_body", "brown_fur", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "black_body", "black_fur", "business_attire", "business_suit", "clothed", "clothing", "felid", "feline", "felis", "formal", "fur", "hair", "hair_bun", "heavy", "holding_mug", "holding_object", "lips", "looking_at_viewer", "mafia", "male", "mammal", "mug", "shirt", "solo", "suit", "teal_shirt", "teal_topwear", "thick_lips", "top_heavy", "topwear", "vest"], "stage3_selected": ["black_fur", "business_attire", "business_suit", "felid", "felis", "formal", "hair_bun", "heavy", "holding_mug", "invalid_background", "mafia", "shirt", "simple_background", "teal_shirt", "thick_lips", "top_heavy", "vest"], "stage3_selected_scores": {"simple_background": 0.6952, "felid": 0.6378, "shirt": 0.7976, "felis": 0.633, "black_fur": 0.7157, "vest": 0.8387, "thick_lips": 0.4059, "hair_bun": 0.6912, "business_suit": 0.5728, "holding_mug": 0.915, "top_heavy": 0.4236, "formal": 0.598, "heavy": 0.3859, "business_attire": 0.5642, "mafia": 0.4054, "teal_shirt": 0.7466, "invalid_background": 0.6479}, "extra_evidence": {"black_body": {"source": "implied"}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "business_attire": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5642}, "business_suit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5728}, "felis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "formal": {"source": "stage3", "why": "explicit", "retrieval_score": 0.598}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6912}, "heavy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3859}, "holding_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.915}, "holding_object": {"source": "implied"}, "lips": {"source": "implied"}, "looking_at_viewer": {"source": "structural"}, "mafia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4054}, "mug": {"source": "implied"}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "suit": {"source": "implied"}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "teal_topwear": {"source": "implied"}, "thick_lips": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "top_heavy": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4236}, "topwear": {"source": "implied"}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 2.35, "t2": 3.44, "t3": 19.47, "t3s": 0.85, "err": null, "issues": ["Stage3 split: general=135 entity=1 copyright_filtered=1 generic_char_to_general=0 unknown_type=1", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"explicit\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"explicit\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 40, \"why\": \"explicit\"}, {\"i\": 42, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"explicit\"}, {\"i\": 46, \"why\": \"explicit\"}, {\"i\": 47, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"explicit\"}, {\"i\": 55, \"why\": \"explicit\"}, {\"i\": 59, \"why\": \"explicit\"}, {\"i\": 1, \"why\": \"style_or_meta\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 8, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.36.why\n Field required [type=missing, input_value={'i': 50}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
3
+ {"id": 260449, "n_gt": 17, "n_retrieved": 160, "n_selected": 40, "n_implied": 12, "n_structural": 6, "ret_R": 0.4706, "P": 0.325, "R": 0.7647, "F1": 0.4561, "leaf_P": 0.2069, "leaf_R": 0.5, "leaf_F1": 0.2927, "n_leaf_sel": 29, "n_leaf_gt": 12, "ret_P": 0.05, "sel_given_ret": 1.625, "over_sel": 2.35, "why": {"explicit": 14, "strong_implied": 10}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.325, "gen_R": 0.7647, "gen_F1": 0.4561, "missed": ["claws", "fur", "human", "slightly_chubby"], "extra": ["alpha_channel", "anthro", "arm_above_head", "arms_above_head", "balancing", "black_hair", "bodily_fluids", "bottomwear", "cheeky", "crying_laughing", "dancer_outfit", "expressions", "forearms", "humanoid", "laugh", "loincloth", "monkey", "raised_arms", "raised_finger", "raised_hand", "raised_head", "smug_expression", "spread_arms", "surprised_expression", "tears", "trio", "wide_grin"], "ground_truth_tags": ["ape", "bear", "claws", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate", "slightly_chubby", "topless"], "selected_tags": ["alpha_channel", "anthro", "ape", "arm_above_head", "arms_above_head", "balancing", "bear", "black_hair", "bodily_fluids", "bottomwear", "cheeky", "clothed", "clothing", "crying_laughing", "dancer_outfit", "dancing", "expressions", "forearms", "group", "hair", "haplorhine", "humanoid", "laugh", "loincloth", "looking_at_viewer", "male", "mammal", "monkey", "primate", "raised_arms", "raised_finger", "raised_hand", "raised_head", "smug_expression", "spread_arms", "surprised_expression", "tears", "topless", "trio", "wide_grin"], "stage3_selected": ["ape", "arm_above_head", "arms_above_head", "balancing", "bear", "black_hair", "cheeky", "crying_laughing", "dancer_outfit", "dancing", "expressions", "forearms", "loincloth", "monkey", "primate", "raised_arms", "raised_finger", "raised_hand", "raised_head", "smug_expression", "spread_arms", "surprised_expression", "transparent_background", "wide_grin"], "stage3_selected_scores": {"black_hair": 0.3428, "bear": 0.5551, "primate": 0.8894, "loincloth": 0.5634, "dancing": 0.5523, "monkey": 0.7547, "arms_above_head": 0.3935, "ape": 0.9763, "raised_arms": 0.5407, "raised_hand": 0.3752, "spread_arms": 0.3987, "raised_finger": 0.3777, "expressions": 0.4949, "wide_grin": 0.5248, "transparent_background": 0.4112, "surprised_expression": 0.3615, "forearms": 0.4005, "arm_above_head": 0.3826, "balancing": 0.4045, "crying_laughing": 0.3837, "smug_expression": 0.3908, "raised_head": 0.376, "dancer_outfit": 0.4138, "cheeky": 0.3862}, "extra_evidence": {"alpha_channel": {"source": "implied"}, "anthro": {"source": "structural"}, "arm_above_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3826}, "arms_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3935}, "balancing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4045}, "black_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3428}, "bodily_fluids": {"source": "implied"}, "bottomwear": {"source": "implied"}, "cheeky": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3862}, "crying_laughing": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3837}, "dancer_outfit": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4138}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4949}, "forearms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4005}, "humanoid": {"source": "structural"}, "laugh": {"source": "implied"}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3777}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3752}, "raised_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.376}, "smug_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3908}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "surprised_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3615}, "tears": {"source": "implied"}, "trio": {"source": "structural"}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}}, "structural": ["trio", "anthro", "humanoid", "male", "topless", "looking_at_viewer"], "t1": 1.55, "t2": 1.79, "t3": 13.49, "t3s": 2.45, "err": null, "issues": ["Stage3 split: general=160 entity=5 copyright_filtered=3 generic_char_to_general=1 unknown_type=3", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 2, \"why\": \"strong_implied\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 21, \"why\": \"strong_implied\"}, {\"i\": 31, \"why\": \"strong_implied\"}, {\"i\": 45, \"why\": \"strong_implied\"}, {\"i\": 48, \"why\": \"strong_implied\"}, {\"i\": 59, \"why\": \"strong_implied\"}, {\"i\": 60, \"why\": \"strong_implied\"}, {\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"style_or_meta\"}, {\"i\": 7, \"why\": \"weak_implied\"}, {\"i\": 15, \"why\": \"weak_implied\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 56, \"why\": \"weak_implied\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 13, \"why\": \"other\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 32, \"why\": \"other\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"other\"}, {\"i\": 41, \"why\": \"other\"}, {\"i\": 42, \"why\": \"other\"}, {\"i\": 44, \"why\": \"other\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"other\"}, {\"i\": 51}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 51}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
4
+ {"id": 1078019, "n_gt": 18, "n_retrieved": 140, "n_selected": 37, "n_implied": 7, "n_structural": 4, "ret_R": 0.5556, "P": 0.3514, "R": 0.7222, "F1": 0.4727, "leaf_P": 0.2414, "leaf_R": 0.5833, "leaf_F1": 0.3415, "n_leaf_sel": 29, "n_leaf_gt": 12, "ret_P": 0.0714, "sel_given_ret": 1.3, "over_sel": 2.06, "why": {"explicit": 17, "strong_implied": 11}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3514, "gen_R": 0.7222, "gen_F1": 0.4727, "missed": ["alternate_species", "blush", "male", "male/male", "open_mouth"], "extra": ["bedroom", "big_eyes", "coat", "confident", "cuddling", "embrace", "eyes", "facing_each_other", "female", "green_eyes", "group", "holding_object", "holding_plushie", "intimate", "love", "ownership", "passionate", "relationship", "round_eyes", "setting", "teal_clothing", "topwear", "wide_eyed", "winter_coat"], "ground_truth_tags": ["alternate_species", "anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "male", "male/male", "mammal", "open_mouth", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["anthro", "bedroom", "big_eyes", "blue_eyes", "clothed", "clothing", "coat", "confident", "cuddling", "duo", "embrace", "eyes", "facing_each_other", "female", "green_eyes", "group", "holding_object", "holding_plushie", "intimate", "lagomorph", "leporid", "love", "mammal", "ownership", "passionate", "plushie", "rabbit", "relationship", "romantic", "romantic_couple", "round_eyes", "setting", "teal_clothing", "teal_eyes", "topwear", "wide_eyed", "winter_coat"], "stage3_selected": ["anthro", "bedroom", "big_eyes", "blue_eyes", "coat", "confident", "cuddling", "duo", "embrace", "eyes", "facing_each_other", "green_eyes", "group", "holding_plushie", "intimate", "love", "ownership", "passionate", "plushie", "rabbit", "relationship", "romantic_couple", "round_eyes", "setting", "teal_clothing", "teal_eyes", "wide_eyed", "winter_coat"], "stage3_selected_scores": {"anthro": 0.4178, "duo": 0.3622, "group": 0.3368, "green_eyes": 0.4204, "rabbit": 0.5935, "bedroom": 0.4144, "romantic_couple": 0.5615, "coat": 0.6379, "plushie": 0.7451, "teal_eyes": 0.6279, "confident": 0.5156, "holding_plushie": 0.7791, "winter_coat": 0.4756, "teal_clothing": 0.4334, "relationship": 0.6201, "setting": 0.5565, "eyes": 0.8767, "blue_eyes": 0.4288, "embrace": 0.4347, "wide_eyed": 0.4611, "big_eyes": 0.4432, "cuddling": 0.4799, "love": 0.4688, "intimate": 0.4677, "passionate": 0.4291, "ownership": 0.2975, "round_eyes": 0.4886, "facing_each_other": 0.4252}, "extra_evidence": {"bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4144}, "big_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4432}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6379}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5156}, "cuddling": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4799}, "embrace": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4347}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "facing_each_other": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4252}, "female": {"source": "structural"}, "green_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4204}, "group": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3368}, "holding_object": {"source": "implied"}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "intimate": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4677}, "love": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4688}, "ownership": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.2975}, "passionate": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4291}, "relationship": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6201}, "round_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4886}, "setting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5565}, "teal_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4334}, "topwear": {"source": "implied"}, "wide_eyed": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4611}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4756}}, "structural": ["duo", "anthro", "female", "clothed"], "t1": 2.2, "t2": 1.48, "t3": 23.25, "t3s": 0.76, "err": null, "issues": ["Stage3 split: general=138 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_1: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"strong_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 7, \"why\": \"strong_implied\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"strong_implied\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"strong_implied\"}, {\"i\": 42, \"why\": \"strong_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"strong_implied\"}, {\"i\": 50, \"why\": \"weak_implied\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"strong_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 56, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
5
+ {"id": 1624724, "n_gt": 16, "n_retrieved": 124, "n_selected": 23, "n_implied": 3, "n_structural": 4, "ret_R": 0.1875, "P": 0.2174, "R": 0.3125, "F1": 0.2564, "leaf_P": 0.2381, "leaf_R": 0.5, "leaf_F1": 0.3226, "n_leaf_sel": 21, "n_leaf_gt": 10, "ret_P": 0.0242, "sel_given_ret": 1.6667, "over_sel": 1.44, "why": {"explicit": 17, "strong_implied": 1}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2174, "gen_R": 0.4167, "gen_F1": 0.2857, "missed": ["alien", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)"], "extra": ["abs", "cartoon", "feral", "floating", "front_view", "grey_nose", "grinning_at_viewer", "looking_at_viewer", "male", "midair", "muscular", "muscular_male", "round_eyes", "spots", "tan_background", "tan_face", "toony", "wide_eyed"], "ground_truth_tags": ["alien", "ambiguous_gender", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)", "red_nose", "smile", "solo", "tan_body"], "selected_tags": ["abs", "ambiguous_gender", "cartoon", "feral", "floating", "front_view", "grey_nose", "grinning_at_viewer", "looking_at_viewer", "male", "midair", "muscular", "muscular_male", "red_nose", "round_eyes", "smile", "solo", "spots", "tan_background", "tan_body", "tan_face", "toony", "wide_eyed"], "stage3_selected": ["abs", "cartoon", "floating", "front_view", "grey_nose", "grinning_at_viewer", "midair", "muscular_male", "red_nose", "round_eyes", "smile", "spots", "tan_background", "tan_body", "tan_face", "toony", "white_background", "wide_eyed"], "stage3_selected_scores": {"smile": 0.6047, "white_background": 0.6209, "abs": 0.3066, "tan_body": 0.6738, "front_view": 0.4491, "spots": 0.6289, "toony": 0.6339, "wide_eyed": 0.4708, "red_nose": 0.7437, "floating": 0.6722, "tan_background": 0.537, "grey_nose": 0.5374, "midair": 0.4363, "tan_face": 0.7174, "round_eyes": 0.8857, "grinning_at_viewer": 0.4946, "cartoon": 0.515, "muscular_male": 0.3035}, "extra_evidence": {"abs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3066}, "cartoon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.515}, "feral": {"source": "structural"}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6722}, "front_view": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4491}, "grey_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5374}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4946}, "looking_at_viewer": {"source": "structural"}, "male": {"source": "implied"}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4363}, "muscular": {"source": "implied"}, "muscular_male": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3035}, "round_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8857}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6289}, "tan_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.537}, "tan_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7174}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6339}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4708}}, "structural": ["solo", "feral", "ambiguous_gender", "looking_at_viewer"], "t1": 1.74, "t2": 1.04, "t3": 17.53, "t3s": 0.66, "err": null, "issues": ["Stage3 split: general=125 entity=0 copyright_filtered=4 generic_char_to_general=0 unknown_type=5"]}
6
+ {"id": 1325009, "n_gt": 24, "n_retrieved": 184, "n_selected": 53, "n_implied": 16, "n_structural": 5, "ret_R": 0.375, "P": 0.3396, "R": 0.75, "F1": 0.4675, "leaf_P": 0.1212, "leaf_R": 0.2857, "leaf_F1": 0.1702, "n_leaf_sel": 33, "n_leaf_gt": 14, "ret_P": 0.0489, "sel_given_ret": 2.0, "over_sel": 2.21, "why": {"explicit": 30, "strong_implied": 2}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3396, "gen_R": 0.75, "gen_F1": 0.4675, "missed": ["chest_tuft", "countershading", "muscular_anthro", "muscular_male", "smile", "topless"], "extra": ["actual_fur", "bengal_tiger", "big_biceps", "big_eyes", "big_muscles", "blue_bottomwear", "blue_clothing", "blue_inner_ear_fluff", "blue_shorts", "chest_fur", "confident", "eyes", "flexing_both_biceps", "full-length_portrait", "gesture", "glistening", "glistening_eyes", "hand_gesture", "hand_on_own_head", "hand_on_torso", "inner_ear_fluff", "muscular_arms", "muscular_thighs", "pointing", "portrait", "posed", "striped_body", "striped_fur", "suggestive_look", "suggestive_pose", "waving", "waving_at_viewer", "white_body", "white_chest", "white_fur"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "looking_at_viewer", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "smile", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["actual_fur", "anthro", "bengal_tiger", "big_biceps", "big_eyes", "big_muscles", "blue_bottomwear", "blue_clothing", "blue_eyes", "blue_inner_ear_fluff", "blue_shorts", "bottomwear", "chest_fur", "clothed", "clothing", "confident", "eyes", "felid", "flexing_both_biceps", "full-length_portrait", "fur", "gesture", "glistening", "glistening_eyes", "hand_gesture", "hand_on_head", "hand_on_own_head", "hand_on_torso", "inner_ear_fluff", "looking_at_viewer", "male", "mammal", "muscular", "muscular_arms", "muscular_thighs", "pantherine", "pointing", "portrait", "posed", "shorts", "solo", "striped_body", "striped_fur", "stripes", "suggestive_look", "suggestive_pose", "tiger", "tuft", "waving", "waving_at_viewer", "white_body", "white_chest", "white_fur"], "stage3_selected": ["actual_fur", "bengal_tiger", "big_biceps", "big_eyes", "big_muscles", "blue_eyes", "blue_inner_ear_fluff", "blue_shorts", "chest_fur", "confident", "eyes", "flexing_both_biceps", "full-length_portrait", "fur", "glistening_eyes", "hand_gesture", "hand_on_head", "hand_on_own_head", "hand_on_torso", "muscular_arms", "muscular_thighs", "pantherine", "pointing", "posed", "striped_fur", "stripes", "suggestive_look", "suggestive_pose", "tuft", "waving_at_viewer", "white_chest", "white_fur"], "stage3_selected_scores": {"fur": 0.584, "blue_eyes": 0.572, "white_fur": 0.4504, "tuft": 0.4899, "pantherine": 0.4517, "stripes": 0.4682, "full-length_portrait": 0.4406, "big_muscles": 0.5578, "striped_fur": 0.6385, "hand_on_head": 0.5934, "glistening_eyes": 0.4749, "big_eyes": 0.4603, "pointing": 0.4392, "muscular_thighs": 0.7073, "muscular_arms": 0.7949, "big_biceps": 0.6931, "blue_shorts": 0.6051, "chest_fur": 0.4606, "waving_at_viewer": 0.4038, "bengal_tiger": 0.4382, "confident": 0.4884, "white_chest": 0.92, "hand_gesture": 0.5025, "blue_inner_ear_fluff": 0.4274, "flexing_both_biceps": 0.562, "hand_on_own_head": 0.5255, "hand_on_torso": 0.556, "posed": 0.4341, "actual_fur": 0.4537, "eyes": 0.9805, "suggestive_pose": 0.6244, "suggestive_look": 0.5908}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4537}, "bengal_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4382}, "big_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6931}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4603}, "big_muscles": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5578}, "blue_bottomwear": {"source": "implied"}, "blue_clothing": {"source": "implied"}, "blue_inner_ear_fluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4274}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6051}, "chest_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4606}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4884}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9805}, "flexing_both_biceps": {"source": "stage3", "why": "explicit", "retrieval_score": 0.562}, "full-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4406}, "gesture": {"source": "implied"}, "glistening": {"source": "implied"}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4749}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5025}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5255}, "hand_on_torso": {"source": "stage3", "why": "explicit", "retrieval_score": 0.556}, "inner_ear_fluff": {"source": "implied"}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7949}, "muscular_thighs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7073}, "pointing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4392}, "portrait": {"source": "implied"}, "posed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4341}, "striped_body": {"source": "implied"}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6385}, "suggestive_look": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5908}, "suggestive_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6244}, "waving": {"source": "implied"}, "waving_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4038}, "white_body": {"source": "implied"}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.92}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4504}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 1.4, "t2": 1.46, "t3": 24.77, "t3s": 1.5, "err": null, "issues": ["Stage3 split: general=180 entity=3 copyright_filtered=2 generic_char_to_general=0 unknown_type=2"]}
7
+ {"id": 1023509, "n_gt": 21, "n_retrieved": 202, "n_selected": 42, "n_implied": 13, "n_structural": 5, "ret_R": 0.4286, "P": 0.2143, "R": 0.4286, "F1": 0.2857, "leaf_P": 0.0833, "leaf_R": 0.1818, "leaf_F1": 0.1143, "n_leaf_sel": 24, "n_leaf_gt": 11, "ret_P": 0.0446, "sel_given_ret": 1.0, "over_sel": 2.0, "why": {"explicit": 22, "strong_implied": 2}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2143, "gen_R": 0.4737, "gen_F1": 0.2951, "missed": ["armless", "boss_monster", "d6", "dialogue", "dice", "fur", "long_ears", "red_eyes", "undertale", "undertale_(series)", "white_body", "white_fur"], "extra": ["alternate_species", "ambiguous_gender", "bovid_horn", "bubble", "caprine_demon", "caprine_horn", "clothed", "clothing", "darkner", "darkness", "demon", "dust", "eublepharid", "gecko", "goat_demon", "goat_ears", "goat_horn", "group", "hands_on_ground", "horn", "humanized", "humanoid", "kneeling", "kneeling_on_ground", "larger_human", "leopard_gecko", "light", "lizard_tail", "lizardman", "male_human", "scared", "size_difference", "speech_bubble"], "ground_truth_tags": ["armless", "boss_monster", "bovid", "caprine", "d6", "dialogue", "dice", "fur", "goat", "human", "lizard", "long_ears", "mammal", "red_eyes", "reptile", "scalie", "text", "undertale", "undertale_(series)", "white_body", "white_fur"], "selected_tags": ["alternate_species", "ambiguous_gender", "bovid", "bovid_horn", "bubble", "caprine", "caprine_demon", "caprine_horn", "clothed", "clothing", "darkner", "darkness", "demon", "dust", "eublepharid", "gecko", "goat", "goat_demon", "goat_ears", "goat_horn", "group", "hands_on_ground", "horn", "human", "humanized", "humanoid", "kneeling", "kneeling_on_ground", "larger_human", "leopard_gecko", "light", "lizard", "lizard_tail", "lizardman", "male_human", "mammal", "reptile", "scalie", "scared", "size_difference", "speech_bubble", "text"], "stage3_selected": ["bubble", "caprine_demon", "darkner", "darkness", "dust", "goat", "goat_demon", "goat_ears", "goat_horn", "hands_on_ground", "human", "humanized", "kneeling_on_ground", "larger_human", "leopard_gecko", "light", "lizard", "lizard_tail", "lizardman", "male_human", "reptile", "scalie", "scared", "speech_bubble"], "stage3_selected_scores": {"scalie": 0.5669, "human": 0.6639, "reptile": 0.5533, "speech_bubble": 0.7486, "lizard": 0.8361, "goat": 0.7748, "light": 0.7781, "scared": 0.539, "bubble": 0.7514, "caprine_demon": 0.5623, "darkner": 0.565, "goat_demon": 0.5691, "lizardman": 0.5348, "darkness": 0.8346, "dust": 0.5497, "lizard_tail": 0.5708, "goat_horn": 0.535, "leopard_gecko": 0.5333, "hands_on_ground": 0.5695, "goat_ears": 0.5349, "kneeling_on_ground": 0.563, "male_human": 0.5551, "humanized": 0.401, "larger_human": 0.3995}, "extra_evidence": {"alternate_species": {"source": "implied"}, "ambiguous_gender": {"source": "structural"}, "bovid_horn": {"source": "implied"}, "bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7514}, "caprine_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5623}, "caprine_horn": {"source": "implied"}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "darkner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.565}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8346}, "demon": {"source": "implied"}, "dust": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5497}, "eublepharid": {"source": "implied"}, "gecko": {"source": "implied"}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5691}, "goat_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5349}, "goat_horn": {"source": "stage3", "why": "explicit", "retrieval_score": 0.535}, "group": {"source": "structural"}, "hands_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5695}, "horn": {"source": "implied"}, "humanized": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.401}, "humanoid": {"source": "structural"}, "kneeling": {"source": "implied"}, "kneeling_on_ground": {"source": "stage3", "why": "explicit", "retrieval_score": 0.563}, "larger_human": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3995}, "leopard_gecko": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5333}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7781}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5708}, "lizardman": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5348}, "male_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5551}, "scared": {"source": "stage3", "why": "explicit", "retrieval_score": 0.539}, "size_difference": {"source": "implied"}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7486}}, "structural": ["group", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 1.79, "t2": 1.62, "t3": 35.6, "t3s": 1.14, "err": null, "issues": ["Stage3 split: general=191 entity=5 copyright_filtered=6 generic_char_to_general=2 unknown_type=2", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"style_or_meta\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"style_or_meta\"}, {\"i\": 9, \"why\": \"explicit\"}, {\"i\": 10, \"why\": \"explicit\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"explicit\"}, {\"i\": 31, \"why\": \"explicit\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"style_or_meta\"}, {\"i\": 46, \"why\": \"style_or_meta\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"explicit\"}, {\"i\": 49, \"why\": \"explicit\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {\"i\": 58, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.35.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.35.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"style_or_meta\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 39, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 58, \"why\": \"weak_implied\"}, {\"i\": 59, \"why\": \"style_or_meta\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: attempt 3 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"style_or_meta\"}, {\"i\": 3, \"why\": \"weak_implied\"}, {\"i\": 4, \"why\": \"weak_implied\"}, {\"i\": 5, \"why\": \"style_or_meta\"}, {\"i\": 6, \"why\": \"weak_implied\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 40, \"why\": \"weak_implied\"}, {\"i\": 41, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"explicit\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"weak_implied\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"explicit\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.34.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.34.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_2: gave up after 3 attempts"]}
8
+ {"id": 335343, "n_gt": 33, "n_retrieved": 207, "n_selected": 50, "n_implied": 9, "n_structural": 5, "ret_R": 0.3333, "P": 0.24, "R": 0.3636, "F1": 0.2892, "leaf_P": 0.1842, "leaf_R": 0.35, "leaf_F1": 0.2414, "n_leaf_sel": 38, "n_leaf_gt": 20, "ret_P": 0.0531, "sel_given_ret": 1.0909, "over_sel": 1.52, "why": {"explicit": 37}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.24, "gen_R": 0.3871, "gen_F1": 0.2963, "missed": ["angry", "earth_pony", "equid", "equine", "feathers", "female", "feral", "freckles", "fur", "hasbro", "horn", "horse", "mammal", "my_little_pony", "pillow", "pony", "scared", "sleeping", "unicorn", "white_body", "white_fur"], "extra": ["3rd_party_watermark", "ambiguous_gender", "annoyed_expression", "applying_makeup", "bed_sheet", "bedding", "bedroom", "big_eyes", "blanket", "blue_eyes", "bored_expression", "brown_eyes", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "english_text", "eyeliner", "eyes", "green_ears", "half-closed_eyes", "humanoid", "long_hair", "lying_on_bed", "name_drop", "narrowed_eyes", "nightmare_fuel", "on_bed", "orange_eyes", "personal_grooming", "purple_eyes", "relaxed_expression", "resting", "sleeping_together", "sleepwear", "text_box", "watermark"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "duo", "earth_pony", "equid", "equine", "eyes_closed", "eyeshadow", "feathers", "female", "feral", "freckles", "fur", "furniture", "green_eyes", "hair", "hasbro", "horn", "horse", "lying", "makeup", "mammal", "my_little_pony", "pillow", "pony", "purple_hair", "scared", "sleeping", "text", "unicorn", "white_body", "white_fur"], "selected_tags": ["3rd_party_watermark", "ambiguous_gender", "annoyed_expression", "applying_makeup", "bed", "bed_sheet", "bedding", "bedroom", "big_eyes", "blanket", "blonde_hair", "blue_eyes", "bored_expression", "brown_eyes", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "duo", "english_text", "eyeliner", "eyes", "eyes_closed", "eyeshadow", "furniture", "green_ears", "green_eyes", "hair", "half-closed_eyes", "humanoid", "long_hair", "lying", "lying_on_bed", "makeup", "name_drop", "narrowed_eyes", "nightmare_fuel", "on_bed", "orange_eyes", "personal_grooming", "purple_eyes", "purple_hair", "relaxed_expression", "resting", "sleeping_together", "sleepwear", "text", "text_box", "watermark"], "stage3_selected": ["3rd_party_watermark", "annoyed_expression", "applying_makeup", "bed_sheet", "bedding", "bedroom", "big_eyes", "blanket", "blonde_hair", "blue_eyes", "bored_expression", "brown_eyes", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "english_text", "eyeliner", "eyes", "eyes_closed", "eyeshadow", "green_ears", "green_eyes", "half-closed_eyes", "long_hair", "lying_on_bed", "makeup", "name_drop", "nightmare_fuel", "orange_eyes", "purple_eyes", "purple_hair", "relaxed_expression", "resting", "sleeping_together", "sleepwear", "text", "text_box"], "stage3_selected_scores": {"digital_media_(artwork)": 0.3347, "text": 0.5939, "blue_eyes": 0.595, "eyes_closed": 0.3883, "green_eyes": 0.5933, "long_hair": 0.4471, "blonde_hair": 0.5872, "half-closed_eyes": 0.3908, "purple_eyes": 0.4293, "purple_hair": 0.5591, "makeup": 0.5894, "brown_eyes": 0.4044, "eyeshadow": 0.4712, "orange_eyes": 0.4306, "bedroom": 0.4873, "bedding": 0.3861, "digital_painting_(artwork)": 0.388, "big_eyes": 0.4246, "bed_sheet": 0.3933, "blanket": 0.4169, "nightmare_fuel": 0.3467, "eyeliner": 0.4399, "name_drop": 0.3542, "lying_on_bed": 0.4059, "text_box": 0.3881, "cool_colors": 0.3885, "green_ears": 0.4373, "resting": 0.5015, "annoyed_expression": 0.7219, "sleeping_together": 0.5063, "3rd_party_watermark": 0.3932, "bored_expression": 0.5259, "sleepwear": 0.444, "applying_makeup": 0.4698, "relaxed_expression": 0.5026, "eyes": 0.8955, "english_text": 0.4128}, "extra_evidence": {"3rd_party_watermark": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3932}, "ambiguous_gender": {"source": "structural"}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "applying_makeup": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4698}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4246}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4169}, "blue_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "bored_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5259}, "brown_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4044}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "cool_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3885}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "digital_painting_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "eyeliner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4399}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "green_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4373}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3908}, "humanoid": {"source": "structural"}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4471}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "name_drop": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3542}, "narrowed_eyes": {"source": "implied"}, "nightmare_fuel": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3467}, "on_bed": {"source": "implied"}, "orange_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4306}, "personal_grooming": {"source": "implied"}, "purple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4293}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5026}, "resting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5063}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "text_box": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3881}, "watermark": {"source": "implied"}}, "structural": ["duo", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 2.73, "t2": 1.71, "t3": 31.53, "t3s": 0.74, "err": null, "issues": ["Stage3 split: general=210 entity=0 copyright_filtered=0 generic_char_to_general=0 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"explicit\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"style_or_meta\"}, {\"i\": 54, \"why\": \"weak_implied\"}, {\"i\": 55, \"why\": \"weak_implied\"}, {\"i\": 56, \"why\": \"other\"}, {\"i\": 57, \"why\": \"weak_implied\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.36.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.36.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 26, "n_retrieved": 155, "n_selected": 22, "n_implied": 3, "n_structural": 4, "ret_R": 0.3077, "P": 0.4545, "R": 0.3846, "F1": 0.4167, "leaf_P": 0.2353, "leaf_R": 0.25, "leaf_F1": 0.2424, "n_leaf_sel": 17, "n_leaf_gt": 16, "ret_P": 0.0516, "sel_given_ret": 1.25, "over_sel": 0.85, "why": {"explicit": 13, "strong_implied": 2}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.4545, "gen_R": 0.3846, "gen_F1": 0.4167, "missed": ["4_fingers", "bass_guitar", "canid", "canine", "demon", "female", "fur", "guitar", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "slim", "string_instrument"], "extra": ["curled_hair", "flowing_hair", "long_claws", "long_hair", "male", "playful", "playing", "playing_bass", "playing_music", "string", "topless", "wavy_hair"], "ground_truth_tags": ["4_fingers", "anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "demon", "female", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "slim", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["anthro", "claws", "clothed", "clothing", "curled_hair", "fingers", "flowing_hair", "hair", "long_claws", "long_hair", "male", "playful", "playing", "playing_bass", "playing_music", "solo", "spade_tail", "string", "tail", "topless", "torn_clothing", "wavy_hair"], "stage3_selected": ["curled_hair", "fingers", "flowing_hair", "hair", "long_claws", "long_hair", "playful", "playing", "playing_bass", "playing_music", "spade_tail", "string", "tail", "torn_clothing", "wavy_hair"], "stage3_selected_scores": {"hair": 0.6402, "tail": 0.6463, "fingers": 0.5056, "long_hair": 0.5155, "torn_clothing": 0.4526, "spade_tail": 0.8714, "curled_hair": 0.464, "wavy_hair": 0.4761, "long_claws": 0.4927, "playing": 0.4732, "string": 0.6122, "flowing_hair": 0.7012, "playing_bass": 0.5051, "playful": 0.3751, "playing_music": 0.3471}, "extra_evidence": {"curled_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.464}, "flowing_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7012}, "long_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4927}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5155}, "male": {"source": "structural"}, "playful": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3751}, "playing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4732}, "playing_bass": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5051}, "playing_music": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3471}, "string": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6122}, "topless": {"source": "structural"}, "wavy_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4761}}, "structural": ["solo", "anthro", "male", "topless"], "t1": 1.82, "t2": 1.17, "t3": 11.51, "t3s": 1.96, "err": null, "issues": ["Stage3 split: general=159 entity=1 copyright_filtered=2 generic_char_to_general=0 unknown_type=3"]}
10
+ {"id": 2021552, "n_gt": 40, "n_retrieved": 161, "n_selected": 48, "n_implied": 16, "n_structural": 4, "ret_R": 0.5, "P": 0.5, "R": 0.6, "F1": 0.5455, "leaf_P": 0.32, "leaf_R": 0.381, "leaf_F1": 0.3478, "n_leaf_sel": 25, "n_leaf_gt": 21, "ret_P": 0.1242, "sel_given_ret": 1.2, "over_sel": 1.2, "why": {"explicit": 29}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.5, "gen_R": 0.6, "gen_F1": 0.5455, "missed": ["cheek_tuft", "facial_tuft", "fluffy", "fluffy_tail", "grey_background", "grey_body", "grey_fur", "head_tuft", "looking_at_another", "looking_away", "pawpads", "tail", "toe_claws", "tuft", "white_body", "white_fur"], "extra": ["5_claws", "big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "clawed_fingers", "coveralls", "cross_fox", "domestic_rabbit", "dutch_rabbit", "fennec_fox", "floppy_ears", "grey_claws", "looking_at_viewer", "lop_ears", "open_mouth", "oryctolagus", "rabbit_ears", "red_fox", "white_clothing", "white_shirt", "white_topwear", "wide_eyed"], "ground_truth_tags": ["anthro", "arctic_fox", "bottomwear", "canid", "canine", "cheek_tuft", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "facial_tuft", "fluffy", "fluffy_tail", "fox", "fur", "grey_background", "grey_body", "grey_fur", "head_markings", "head_tuft", "lagomorph", "leporid", "looking_at_another", "looking_away", "mammal", "markings", "overalls", "pants", "pawpads", "rabbit", "shirt", "standing", "tail", "toe_claws", "topwear", "tuft", "white_body", "white_fur"], "selected_tags": ["5_claws", "anthro", "arctic_fox", "big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "bottomwear", "canid", "canine", "clawed_fingers", "claws", "clothed", "clothing", "coveralls", "cross_fox", "crossed_arms", "domestic_rabbit", "duo", "dutch_rabbit", "facial_markings", "fennec_fox", "floppy_ears", "fox", "fur", "grey_claws", "head_markings", "lagomorph", "leporid", "looking_at_viewer", "lop_ears", "mammal", "markings", "open_mouth", "oryctolagus", "overalls", "pants", "rabbit", "rabbit_ears", "red_fox", "shirt", "standing", "topwear", "white_clothing", "white_shirt", "white_topwear", "wide_eyed"], "stage3_selected": ["5_claws", "arctic_fox", "big_mouth_(anatomy)", "black_pants", "blue_overalls", "clawed_fingers", "claws", "coveralls", "cross_fox", "crossed_arms", "dutch_rabbit", "facial_markings", "fennec_fox", "fox", "fur", "grey_claws", "lagomorph", "leporid", "lop_ears", "open_mouth", "overalls", "rabbit", "rabbit_ears", "red_fox", "shirt", "simple_background", "standing", "white_shirt", "wide_eyed"], "stage3_selected_scores": {"fur": 0.6531, "simple_background": 0.416, "open_mouth": 0.633, "claws": 0.6303, "standing": 0.6878, "fox": 0.6379, "shirt": 0.7483, "lagomorph": 0.5936, "leporid": 0.5826, "rabbit": 0.6511, "facial_markings": 0.6945, "red_fox": 0.4427, "crossed_arms": 0.7285, "wide_eyed": 0.4671, "white_shirt": 0.8197, "arctic_fox": 0.4522, "overalls": 0.8776, "black_pants": 0.833, "grey_claws": 0.5166, "clawed_fingers": 0.5145, "lop_ears": 0.4692, "rabbit_ears": 0.5992, "cross_fox": 0.4688, "coveralls": 0.5688, "big_mouth_(anatomy)": 0.5037, "dutch_rabbit": 0.4737, "blue_overalls": 0.9203, "5_claws": 0.6021, "fennec_fox": 0.5031}, "extra_evidence": {"5_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6021}, "big_mouth_(anatomy)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5037}, "black_bottomwear": {"source": "implied"}, "black_clothing": {"source": "implied"}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.833}, "blue_overalls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9203}, "clawed_fingers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5145}, "coveralls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5688}, "cross_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4688}, "domestic_rabbit": {"source": "implied"}, "dutch_rabbit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4737}, "fennec_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5031}, "floppy_ears": {"source": "implied"}, "grey_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5166}, "looking_at_viewer": {"source": "structural"}, "lop_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4692}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "oryctolagus": {"source": "implied"}, "rabbit_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "red_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4427}, "white_clothing": {"source": "implied"}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_topwear": {"source": "implied"}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4671}}, "structural": ["duo", "anthro", "clothed", "looking_at_viewer"], "t1": 2.05, "t2": 1.29, "t3": 11.0, "t3s": 1.34, "err": null, "issues": ["Stage3 split: general=161 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3"]}
11
+ {"id": 2034167, "n_gt": 21, "n_retrieved": 183, "n_selected": 59, "n_implied": 14, "n_structural": 5, "ret_R": 0.4286, "P": 0.1864, "R": 0.5238, "F1": 0.275, "leaf_P": 0.1081, "leaf_R": 0.3333, "leaf_F1": 0.1633, "n_leaf_sel": 37, "n_leaf_gt": 12, "ret_P": 0.0492, "sel_given_ret": 1.2222, "over_sel": 2.81, "why": {"explicit": 30, "strong_implied": 10}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1864, "gen_R": 0.5238, "gen_F1": 0.275, "missed": ["4_toes", "canis", "feet", "female", "feral", "purple_fur", "teeth", "toes", "tongue", "wolf"], "extra": ["2d_animation", "action_pose", "actual_fur", "ambiguous_gender", "animal_humanoid", "animated", "animated_comic", "animated_png", "anthro", "belly", "big_eyes", "blue_stripes", "blue_tail", "canid_humanoid", "canine_humanoid", "clothed", "clothing", "comic", "curved_tail", "dog_humanoid", "eyes", "fighting_pose", "glistening", "glistening_eyes", "holding_character", "humanoid", "looking_at_viewer", "mammal_humanoid", "multi_tone_fur", "multicolored_body", "multicolored_fur", "multiple_poses", "open_smile", "paw_pose", "pink_stripes", "pink_tail", "pose", "purple_belly", "smile", "stripes", "suggestive_pose", "swinging", "tail", "thick_fur", "two_tone_body", "two_tone_fur", "white_nose", "white_stripes"], "ground_truth_tags": ["4_toes", "blue_eyes", "blue_nose", "canid", "canine", "canis", "feet", "female", "feral", "fur", "mammal", "open_mouth", "purple_body", "purple_fur", "solo", "teeth", "toes", "tongue", "white_body", "white_fur", "wolf"], "selected_tags": ["2d_animation", "action_pose", "actual_fur", "ambiguous_gender", "animal_humanoid", "animated", "animated_comic", "animated_png", "anthro", "belly", "big_eyes", "blue_eyes", "blue_nose", "blue_stripes", "blue_tail", "canid", "canid_humanoid", "canine", "canine_humanoid", "clothed", "clothing", "comic", "curved_tail", "dog_humanoid", "eyes", "fighting_pose", "fur", "glistening", "glistening_eyes", "holding_character", "humanoid", "looking_at_viewer", "mammal", "mammal_humanoid", "multi_tone_fur", "multicolored_body", "multicolored_fur", "multiple_poses", "open_mouth", "open_smile", "paw_pose", "pink_stripes", "pink_tail", "pose", "purple_belly", "purple_body", "smile", "solo", "stripes", "suggestive_pose", "swinging", "tail", "thick_fur", "two_tone_body", "two_tone_fur", "white_body", "white_fur", "white_nose", "white_stripes"], "stage3_selected": ["2d_animation", "action_pose", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "big_eyes", "blue_eyes", "blue_nose", "blue_stripes", "blue_tail", "canid_humanoid", "canine_humanoid", "curved_tail", "dog_humanoid", "eyes", "fighting_pose", "glistening_eyes", "holding_character", "humanoid", "multi_tone_fur", "multicolored_fur", "multiple_poses", "open_mouth", "open_smile", "paw_pose", "pink_stripes", "pink_tail", "purple_belly", "purple_body", "stripes", "suggestive_pose", "swinging", "tail", "thick_fur", "two_tone_fur", "white_fur", "white_nose", "white_stripes"], "stage3_selected_scores": {"open_mouth": 0.5847, "tail": 0.5894, "blue_eyes": 0.5818, "white_fur": 0.577, "humanoid": 0.6713, "multicolored_fur": 0.4709, "animal_humanoid": 0.6209, "stripes": 0.5778, "purple_body": 0.5474, "two_tone_fur": 0.4614, "open_smile": 0.4621, "canid_humanoid": 0.8743, "canine_humanoid": 0.9127, "blue_nose": 0.592, "blue_tail": 0.5051, "pink_tail": 0.5165, "multiple_poses": 0.4908, "dog_humanoid": 0.8076, "blue_stripes": 0.5366, "white_stripes": 0.5319, "multi_tone_fur": 0.4733, "white_nose": 0.4856, "pink_stripes": 0.5442, "curved_tail": 0.5957, "purple_belly": 0.5117, "animated_comic": 0.4044, "eyes": 0.7508, "animated_png": 0.4455, "animated": 0.3689, "2d_animation": 0.3613, "glistening_eyes": 0.4376, "big_eyes": 0.3944, "action_pose": 0.5938, "holding_character": 0.3205, "fighting_pose": 0.4377, "paw_pose": 0.5509, "suggestive_pose": 0.4425, "swinging": 0.3355, "thick_fur": 0.4071, "actual_fur": 0.4438}, "extra_evidence": {"2d_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3613}, "action_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5938}, "actual_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4438}, "ambiguous_gender": {"source": "structural"}, "animal_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6209}, "animated": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3689}, "animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4044}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4455}, "anthro": {"source": "structural"}, "belly": {"source": "implied"}, "big_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3944}, "blue_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5366}, "blue_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5051}, "canid_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8743}, "canine_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9127}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "comic": {"source": "implied"}, "curved_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5957}, "dog_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8076}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7508}, "fighting_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4377}, "glistening": {"source": "implied"}, "glistening_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4376}, "holding_character": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3205}, "humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6713}, "looking_at_viewer": {"source": "structural"}, "mammal_humanoid": {"source": "implied"}, "multi_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4733}, "multicolored_body": {"source": "implied"}, "multicolored_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4709}, "multiple_poses": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4908}, "open_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4621}, "paw_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.5509}, "pink_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5442}, "pink_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5165}, "pose": {"source": "implied"}, "purple_belly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5117}, "smile": {"source": "implied"}, "stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5778}, "suggestive_pose": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4425}, "swinging": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3355}, "tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5894}, "thick_fur": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4071}, "two_tone_body": {"source": "implied"}, "two_tone_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4614}, "white_nose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4856}, "white_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5319}}, "structural": ["solo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 2.52, "t2": 1.47, "t3": 16.24, "t3s": 0.6, "err": null, "issues": ["Stage3 split: general=186 entity=2 copyright_filtered=2 generic_char_to_general=4 unknown_type=12"]}
data/eval_results/eval_caption_cogvlm_n10_seed42_20260221_045137.jsonl ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"_meta": true, "timestamp": "2026-02-21T04:51:37.379381", "n_samples": 10, "caption_field": "caption_cogvlm", "skip_rewrite": false, "allow_nsfw": false, "mode": "chunked_map_union", "chunk_size": 60, "per_phrase_k": 20, "temperature": 0.0, "shuffle": false, "seed": 42, "workers": 1, "min_why": "strong_implied", "expand_implications": true, "infer_structural": true, "n_errors": 0, "n_issue_samples": 10, "n_issues_total": 15}
2
+ {"id": 3285630, "n_gt": 16, "n_retrieved": 151, "n_selected": 37, "n_implied": 14, "n_structural": 5, "ret_R": 0.1875, "P": 0.2703, "R": 0.625, "F1": 0.3774, "leaf_P": 0.2105, "leaf_R": 0.4444, "leaf_F1": 0.2857, "n_leaf_sel": 19, "n_leaf_gt": 9, "ret_P": 0.0199, "sel_given_ret": 3.3333, "over_sel": 2.31, "why": {"explicit": 17, "strong_implied": 2}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2703, "gen_R": 0.625, "gen_F1": 0.3774, "missed": ["5_fingers", "alpha_channel", "black_hair", "brown_body", "brown_fur", "fingers"], "extra": ["black_body", "black_fur", "blowup_background", "dress_shirt", "felis", "formal", "hair_bun", "handpaw", "holding_mug", "holding_object", "jacket", "looking_at_viewer", "mug", "necktie", "paws", "shirt", "suit_jacket", "t-shirt", "teal_shirt", "teal_topwear", "topwear", "vest", "white_clothing", "white_necktie", "white_shirt", "white_t-shirt", "white_topwear"], "ground_truth_tags": ["5_fingers", "alpha_channel", "anthro", "black_hair", "brown_body", "brown_fur", "clothed", "clothing", "felid", "feline", "fingers", "fur", "hair", "male", "mammal", "solo"], "selected_tags": ["anthro", "black_body", "black_fur", "blowup_background", "clothed", "clothing", "dress_shirt", "felid", "feline", "felis", "formal", "fur", "hair", "hair_bun", "handpaw", "holding_mug", "holding_object", "jacket", "looking_at_viewer", "male", "mammal", "mug", "necktie", "paws", "shirt", "solo", "suit_jacket", "t-shirt", "teal_shirt", "teal_topwear", "topwear", "vest", "white_clothing", "white_necktie", "white_shirt", "white_t-shirt", "white_topwear"], "stage3_selected": ["black_fur", "blowup_background", "dress_shirt", "felid", "feline", "felis", "formal", "fur", "hair_bun", "handpaw", "holding_mug", "jacket", "shirt", "simple_background", "suit_jacket", "teal_shirt", "vest", "white_necktie", "white_t-shirt"], "stage3_selected_scores": {"simple_background": 0.6952, "felid": 0.6378, "shirt": 0.7976, "felis": 0.633, "black_fur": 0.7157, "jacket": 0.6655, "vest": 0.8387, "dress_shirt": 0.7226, "hair_bun": 0.6912, "handpaw": 0.5262, "holding_mug": 0.915, "white_t-shirt": 0.7323, "suit_jacket": 0.6849, "blowup_background": 0.6332, "formal": 0.598, "teal_shirt": 0.7466, "white_necktie": 0.6419, "fur": 0.712, "feline": 0.7023}, "stage3_selected_ranks": {"simple_background": 50, "felid": 66, "shirt": 5, "felis": 70, "black_fur": 37, "jacket": 58, "vest": 3, "dress_shirt": 34, "hair_bun": 52, "handpaw": 124, "holding_mug": 1, "white_t-shirt": 26, "suit_jacket": 54, "blowup_background": 69, "formal": 84, "teal_shirt": 19, "white_necktie": 65, "fur": 40, "feline": 45}, "extra_evidence": {"black_body": {"source": "implied"}, "black_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7157}, "blowup_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6332}, "dress_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7226}, "felis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "formal": {"source": "stage3", "why": "explicit", "retrieval_score": 0.598}, "hair_bun": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6912}, "handpaw": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5262}, "holding_mug": {"source": "stage3", "why": "explicit", "retrieval_score": 0.915}, "holding_object": {"source": "implied"}, "jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6655}, "looking_at_viewer": {"source": "structural"}, "mug": {"source": "implied"}, "necktie": {"source": "implied"}, "paws": {"source": "implied"}, "shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7976}, "suit_jacket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6849}, "t-shirt": {"source": "implied"}, "teal_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7466}, "teal_topwear": {"source": "implied"}, "topwear": {"source": "implied"}, "vest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8387}, "white_clothing": {"source": "implied"}, "white_necktie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6419}, "white_shirt": {"source": "implied"}, "white_t-shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7323}, "white_topwear": {"source": "implied"}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 2.32, "t2": 3.4, "t3": 15.47, "t3s": 1.54, "err": null, "issues": ["Stage3 split: general=154 entity=1 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
3
+ {"id": 260449, "n_gt": 17, "n_retrieved": 160, "n_selected": 49, "n_implied": 14, "n_structural": 6, "ret_R": 0.4706, "P": 0.2653, "R": 0.7647, "F1": 0.3939, "leaf_P": 0.1613, "leaf_R": 0.4167, "leaf_F1": 0.2326, "n_leaf_sel": 31, "n_leaf_gt": 12, "ret_P": 0.05, "sel_given_ret": 1.625, "over_sel": 2.88, "why": {"explicit": 20, "strong_implied": 9}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2653, "gen_R": 0.7647, "gen_F1": 0.3939, "missed": ["claws", "fur", "human", "slightly_chubby"], "extra": ["american_black_bear", "anthro", "arm_above_head", "arms_above_head", "black_bear", "bottomwear", "cheeky", "cute_expression", "expressions", "feral", "forearms", "giant_panda", "grinning_at_viewer", "kermode_bear", "loincloth", "monkey", "neutral_expression", "new_world_monkey", "one_eye_closed", "raised_arm", "raised_arms", "raised_finger", "raised_hand", "raised_head", "relaxed_expression", "smile", "smiling_at_viewer", "smirk", "smirking_at_viewer", "spread_arms", "toony_expression", "trio", "ursine", "wide_grin", "wink", "winking_at_viewer"], "ground_truth_tags": ["ape", "bear", "claws", "clothed", "clothing", "dancing", "fur", "group", "hair", "haplorhine", "human", "looking_at_viewer", "male", "mammal", "primate", "slightly_chubby", "topless"], "selected_tags": ["american_black_bear", "anthro", "ape", "arm_above_head", "arms_above_head", "bear", "black_bear", "bottomwear", "cheeky", "clothed", "clothing", "cute_expression", "dancing", "expressions", "feral", "forearms", "giant_panda", "grinning_at_viewer", "group", "hair", "haplorhine", "kermode_bear", "loincloth", "looking_at_viewer", "male", "mammal", "monkey", "neutral_expression", "new_world_monkey", "one_eye_closed", "primate", "raised_arm", "raised_arms", "raised_finger", "raised_hand", "raised_head", "relaxed_expression", "smile", "smiling_at_viewer", "smirk", "smirking_at_viewer", "spread_arms", "toony_expression", "topless", "trio", "ursine", "wide_grin", "wink", "winking_at_viewer"], "stage3_selected": ["ape", "arm_above_head", "arms_above_head", "bear", "cheeky", "cute_expression", "dancing", "expressions", "forearms", "giant_panda", "grinning_at_viewer", "hair", "kermode_bear", "loincloth", "monkey", "neutral_expression", "new_world_monkey", "primate", "raised_arm", "raised_arms", "raised_finger", "raised_hand", "raised_head", "relaxed_expression", "smirking_at_viewer", "spread_arms", "toony_expression", "wide_grin", "winking_at_viewer"], "stage3_selected_scores": {"hair": 0.5395, "bear": 0.5551, "raised_arm": 0.4867, "primate": 0.8894, "loincloth": 0.5634, "giant_panda": 0.463, "dancing": 0.5523, "monkey": 0.7547, "arms_above_head": 0.3935, "ape": 0.9763, "raised_arms": 0.5407, "raised_hand": 0.3752, "cute_expression": 0.4288, "spread_arms": 0.3987, "raised_finger": 0.3777, "expressions": 0.4949, "new_world_monkey": 0.7333, "kermode_bear": 0.4257, "toony_expression": 0.4623, "wide_grin": 0.5248, "winking_at_viewer": 0.4285, "forearms": 0.4005, "arm_above_head": 0.3826, "neutral_expression": 0.4019, "smirking_at_viewer": 0.4305, "raised_head": 0.376, "grinning_at_viewer": 0.4379, "relaxed_expression": 0.4112, "cheeky": 0.3862}, "stage3_selected_ranks": {"hair": 20, "bear": 14, "raised_arm": 28, "primate": 2, "loincloth": 12, "giant_panda": 30, "dancing": 15, "monkey": 6, "arms_above_head": 80, "ape": 1, "raised_arms": 19, "raised_hand": 99, "cute_expression": 47, "spread_arms": 77, "raised_finger": 94, "expressions": 26, "new_world_monkey": 9, "kermode_bear": 54, "toony_expression": 31, "wide_grin": 21, "winking_at_viewer": 48, "forearms": 75, "arm_above_head": 88, "neutral_expression": 74, "smirking_at_viewer": 45, "raised_head": 96, "grinning_at_viewer": 40, "relaxed_expression": 69, "cheeky": 85}, "extra_evidence": {"american_black_bear": {"source": "implied"}, "anthro": {"source": "structural"}, "arm_above_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3826}, "arms_above_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3935}, "black_bear": {"source": "implied"}, "bottomwear": {"source": "implied"}, "cheeky": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3862}, "cute_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4288}, "expressions": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4949}, "feral": {"source": "structural"}, "forearms": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4005}, "giant_panda": {"source": "stage3", "why": "explicit", "retrieval_score": 0.463}, "grinning_at_viewer": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4379}, "kermode_bear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4257}, "loincloth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5634}, "monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7547}, "neutral_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4019}, "new_world_monkey": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7333}, "one_eye_closed": {"source": "implied"}, "raised_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4867}, "raised_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5407}, "raised_finger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3777}, "raised_hand": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3752}, "raised_head": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.376}, "relaxed_expression": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4112}, "smile": {"source": "implied"}, "smiling_at_viewer": {"source": "implied"}, "smirk": {"source": "implied"}, "smirking_at_viewer": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4305}, "spread_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3987}, "toony_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4623}, "trio": {"source": "structural"}, "ursine": {"source": "implied"}, "wide_grin": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5248}, "wink": {"source": "implied"}, "winking_at_viewer": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4285}}, "structural": ["trio", "anthro", "feral", "male", "topless", "looking_at_viewer"], "t1": 1.95, "t2": 1.48, "t3": 11.48, "t3s": 0.87, "err": null, "issues": ["Stage3 split: general=160 entity=5 copyright_filtered=3 generic_char_to_general=1 unknown_type=3"]}
4
+ {"id": 1078019, "n_gt": 18, "n_retrieved": 133, "n_selected": 33, "n_implied": 10, "n_structural": 5, "ret_R": 0.5556, "P": 0.3939, "R": 0.7222, "F1": 0.5098, "leaf_P": 0.2609, "leaf_R": 0.5, "leaf_F1": 0.3429, "n_leaf_sel": 23, "n_leaf_gt": 12, "ret_P": 0.0752, "sel_given_ret": 1.3, "over_sel": 1.83, "why": {"explicit": 13, "strong_implied": 7}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.3939, "gen_R": 0.7222, "gen_F1": 0.5098, "missed": ["alternate_species", "blue_eyes", "male", "male/male", "open_mouth"], "extra": ["blush_lines", "coat", "comforting", "confident", "cuddling", "embrace", "eyes", "female", "holding_object", "holding_pillow", "holding_plushie", "intimate", "looking_at_viewer", "passionate", "pillow", "relationship", "rosy_cheeks", "teal_clothing", "topwear", "winter_coat"], "ground_truth_tags": ["alternate_species", "anthro", "blue_eyes", "blush", "clothed", "clothing", "duo", "lagomorph", "leporid", "male", "male/male", "mammal", "open_mouth", "plushie", "rabbit", "romantic", "romantic_couple", "teal_eyes"], "selected_tags": ["anthro", "blush", "blush_lines", "clothed", "clothing", "coat", "comforting", "confident", "cuddling", "duo", "embrace", "eyes", "female", "holding_object", "holding_pillow", "holding_plushie", "intimate", "lagomorph", "leporid", "looking_at_viewer", "mammal", "passionate", "pillow", "plushie", "rabbit", "relationship", "romantic", "romantic_couple", "rosy_cheeks", "teal_clothing", "teal_eyes", "topwear", "winter_coat"], "stage3_selected": ["anthro", "blush_lines", "coat", "comforting", "confident", "cuddling", "duo", "embrace", "eyes", "holding_pillow", "holding_plushie", "intimate", "passionate", "rabbit", "relationship", "romantic_couple", "rosy_cheeks", "teal_clothing", "teal_eyes", "winter_coat"], "stage3_selected_scores": {"anthro": 0.4179, "duo": 0.3624, "rabbit": 0.5936, "blush_lines": 0.4751, "coat": 0.638, "teal_eyes": 0.6281, "rosy_cheeks": 0.4718, "confident": 0.5158, "holding_plushie": 0.7791, "winter_coat": 0.4757, "teal_clothing": 0.4335, "holding_pillow": 0.4824, "eyes": 0.8767, "embrace": 0.4349, "romantic_couple": 0.5617, "cuddling": 0.4801, "intimate": 0.4399, "passionate": 0.4293, "comforting": 0.4366, "relationship": 0.6203}, "stage3_selected_ranks": {"anthro": 121, "duo": 131, "rabbit": 13, "blush_lines": 67, "coat": 7, "teal_eyes": 8, "rosy_cheeks": 68, "confident": 39, "holding_plushie": 2, "winter_coat": 66, "teal_clothing": 103, "holding_pillow": 61, "eyes": 1, "embrace": 101, "romantic_couple": 18, "cuddling": 63, "intimate": 97, "passionate": 109, "comforting": 100, "relationship": 9}, "extra_evidence": {"blush_lines": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4751}, "coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.638}, "comforting": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4366}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5158}, "cuddling": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4801}, "embrace": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4349}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8767}, "female": {"source": "structural"}, "holding_object": {"source": "implied"}, "holding_pillow": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4824}, "holding_plushie": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7791}, "intimate": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4399}, "looking_at_viewer": {"source": "structural"}, "passionate": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4293}, "pillow": {"source": "implied"}, "relationship": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.6203}, "rosy_cheeks": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4718}, "teal_clothing": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4335}, "topwear": {"source": "implied"}, "winter_coat": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4757}}, "structural": ["duo", "anthro", "female", "clothed", "looking_at_viewer"], "t1": 2.34, "t2": 1.22, "t3": 12.57, "t3s": 1.72, "err": null, "issues": ["Stage3 split: general=131 entity=2 copyright_filtered=1 generic_char_to_general=0 unknown_type=2"]}
5
+ {"id": 1624724, "n_gt": 16, "n_retrieved": 113, "n_selected": 22, "n_implied": 4, "n_structural": 5, "ret_R": 0.1875, "P": 0.1818, "R": 0.25, "F1": 0.2105, "leaf_P": 0.2105, "leaf_R": 0.4, "leaf_F1": 0.2759, "n_leaf_sel": 19, "n_leaf_gt": 10, "ret_P": 0.0265, "sel_given_ret": 1.3333, "over_sel": 1.38, "why": {"explicit": 15}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.1818, "gen_R": 0.3333, "gen_F1": 0.2353, "missed": ["alien", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)", "smile"], "extra": ["big_eyes", "carton", "cartoon", "clothed", "clothing", "feral", "floating", "grinning_at_viewer", "half-length_portrait", "portrait", "purple_mouth", "red_face", "spots", "spotted_back", "spotted_face", "text", "toony", "topless"], "ground_truth_tags": ["alien", "ambiguous_gender", "brown_eyes", "crossover", "disney", "experiment_(lilo_and_stitch)", "generation_3_pokemon", "hybrid", "lilo_and_stitch", "nintendo", "pokemon", "pokemon_(species)", "red_nose", "smile", "solo", "tan_body"], "selected_tags": ["ambiguous_gender", "big_eyes", "carton", "cartoon", "clothed", "clothing", "feral", "floating", "grinning_at_viewer", "half-length_portrait", "portrait", "purple_mouth", "red_face", "red_nose", "solo", "spots", "spotted_back", "spotted_face", "tan_body", "text", "toony", "topless"], "stage3_selected": ["big_eyes", "carton", "cartoon", "floating", "grinning_at_viewer", "half-length_portrait", "purple_mouth", "red_face", "red_nose", "spots", "spotted_back", "spotted_face", "tan_body", "toony", "white_background"], "stage3_selected_scores": {"white_background": 0.6069, "tan_body": 0.658, "spots": 0.6222, "toony": 0.5972, "half-length_portrait": 0.4869, "big_eyes": 0.6933, "red_nose": 0.7473, "floating": 0.6452, "spotted_face": 0.6973, "red_face": 0.6394, "purple_mouth": 0.4752, "carton": 0.4369, "grinning_at_viewer": 0.4723, "spotted_back": 0.7, "cartoon": 0.5002}, "stage3_selected_ranks": {"white_background": 32, "tan_body": 15, "spots": 27, "toony": 36, "half-length_portrait": 73, "big_eyes": 7, "red_nose": 3, "floating": 20, "spotted_face": 5, "red_face": 24, "purple_mouth": 77, "carton": 91, "grinning_at_viewer": 79, "spotted_back": 4, "cartoon": 70}, "extra_evidence": {"big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6933}, "carton": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4369}, "cartoon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5002}, "clothed": {"source": "implied"}, "clothing": {"source": "implied"}, "feral": {"source": "structural"}, "floating": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6452}, "grinning_at_viewer": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4723}, "half-length_portrait": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4869}, "portrait": {"source": "implied"}, "purple_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4752}, "red_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6394}, "spots": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6222}, "spotted_back": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7}, "spotted_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6973}, "text": {"source": "structural"}, "toony": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5972}, "topless": {"source": "structural"}}, "structural": ["solo", "feral", "ambiguous_gender", "topless", "text"], "t1": 1.66, "t2": 0.92, "t3": 6.72, "t3s": 0.86, "err": null, "issues": ["Stage3 split: general=114 entity=0 copyright_filtered=4 generic_char_to_general=0 unknown_type=5"]}
6
+ {"id": 1325009, "n_gt": 24, "n_retrieved": 192, "n_selected": 78, "n_implied": 24, "n_structural": 5, "ret_R": 0.375, "P": 0.2179, "R": 0.7083, "F1": 0.3333, "leaf_P": 0.0638, "leaf_R": 0.2143, "leaf_F1": 0.0984, "n_leaf_sel": 47, "n_leaf_gt": 14, "ret_P": 0.0469, "sel_given_ret": 1.8889, "over_sel": 3.25, "why": {"explicit": 50}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2179, "gen_R": 0.7083, "gen_F1": 0.3333, "missed": ["chest_tuft", "hand_on_head", "muscular", "muscular_anthro", "muscular_male", "smile", "topless"], "extra": ["actual_fur", "big_eyes", "black_chest", "blue_body", "blue_bottomwear", "blue_clothing", "blue_countershading", "blue_inner_ear_fluff", "blue_shorts", "cheek_tuft", "confident", "countershade_fur", "different_sound_effects", "facial_markings", "facial_stripes", "facial_tuft", "gesture", "grey_chest", "gripping_sound_effect", "hand_gesture", "hand_on_chest", "hand_on_knee", "hand_on_leg", "hand_on_own_head", "hand_on_own_knee", "hand_on_own_leg", "head_markings", "heterochromia", "inner_ear_fluff", "light_chest", "manly", "markings", "multicolored_bottomwear", "multicolored_clothing", "multicolored_shorts", "muscular_arms", "muscular_legs", "obliques", "on_grass", "pop_(sound_effect)", "pose", "relaxed_expression", "rubbing_sound_effect", "siberian_tiger", "sound_effects", "striped_body", "striped_face", "striped_fur", "striped_legs", "striped_neck", "tan_bottomwear", "tan_clothing", "tan_shorts", "tan_stripes", "white_arms", "white_body", "white_bottomwear", "white_chest", "white_clothing", "white_fur", "white_shorts"], "ground_truth_tags": ["anthro", "blue_eyes", "bottomwear", "chest_tuft", "clothed", "clothing", "countershading", "felid", "fur", "hand_on_head", "looking_at_viewer", "male", "mammal", "muscular", "muscular_anthro", "muscular_male", "pantherine", "shorts", "smile", "solo", "stripes", "tiger", "topless", "tuft"], "selected_tags": ["actual_fur", "anthro", "big_eyes", "black_chest", "blue_body", "blue_bottomwear", "blue_clothing", "blue_countershading", "blue_eyes", "blue_inner_ear_fluff", "blue_shorts", "bottomwear", "cheek_tuft", "clothed", "clothing", "confident", "countershade_fur", "countershading", "different_sound_effects", "facial_markings", "facial_stripes", "facial_tuft", "felid", "fur", "gesture", "grey_chest", "gripping_sound_effect", "hand_gesture", "hand_on_chest", "hand_on_knee", "hand_on_leg", "hand_on_own_head", "hand_on_own_knee", "hand_on_own_leg", "head_markings", "heterochromia", "inner_ear_fluff", "light_chest", "looking_at_viewer", "male", "mammal", "manly", "markings", "multicolored_bottomwear", "multicolored_clothing", "multicolored_shorts", "muscular_arms", "muscular_legs", "obliques", "on_grass", "pantherine", "pop_(sound_effect)", "pose", "relaxed_expression", "rubbing_sound_effect", "shorts", "siberian_tiger", "solo", "sound_effects", "striped_body", "striped_face", "striped_fur", "striped_legs", "striped_neck", "stripes", "tan_bottomwear", "tan_clothing", "tan_shorts", "tan_stripes", "tiger", "tuft", "white_arms", "white_body", "white_bottomwear", "white_chest", "white_clothing", "white_fur", "white_shorts"], "stage3_selected": ["actual_fur", "anthro", "big_eyes", "black_chest", "blue_countershading", "blue_eyes", "blue_inner_ear_fluff", "blue_shorts", "cheek_tuft", "confident", "countershade_fur", "different_sound_effects", "facial_stripes", "gesture", "grey_chest", "gripping_sound_effect", "hand_gesture", "hand_on_chest", "hand_on_own_head", "hand_on_own_knee", "heterochromia", "light_chest", "manly", "multicolored_shorts", "muscular_arms", "muscular_legs", "obliques", "on_grass", "pantherine", "pop_(sound_effect)", "pose", "relaxed_expression", "rubbing_sound_effect", "shorts", "siberian_tiger", "sound_effects", "striped_body", "striped_face", "striped_fur", "striped_legs", "striped_neck", "stripes", "tan_shorts", "tan_stripes", "tuft", "white_arms", "white_body", "white_chest", "white_fur", "white_shorts"], "stage3_selected_scores": {"anthro": 0.3973, "white_body": 0.4088, "blue_eyes": 0.6025, "white_fur": 0.4865, "tuft": 0.5284, "pantherine": 0.4785, "stripes": 0.5, "pose": 0.6545, "shorts": 0.6073, "sound_effects": 0.399, "cheek_tuft": 0.5107, "gesture": 0.6162, "striped_body": 0.6236, "striped_fur": 0.6725, "heterochromia": 0.4325, "big_eyes": 0.4844, "different_sound_effects": 0.3383, "manly": 0.6237, "hand_on_chest": 0.4457, "countershade_fur": 0.6855, "obliques": 0.6265, "muscular_arms": 0.7936, "muscular_legs": 0.7906, "blue_shorts": 0.6382, "striped_legs": 0.657, "on_grass": 0.4388, "blue_countershading": 0.6885, "confident": 0.5252, "white_arms": 0.8154, "white_chest": 0.9269, "hand_gesture": 0.5321, "striped_face": 0.7203, "hand_on_own_knee": 0.6114, "blue_inner_ear_fluff": 0.4653, "pop_(sound_effect)": 0.3394, "white_shorts": 0.6221, "striped_neck": 0.7021, "rubbing_sound_effect": 0.3533, "siberian_tiger": 0.5098, "light_chest": 0.819, "tan_shorts": 0.6233, "grey_chest": 0.7073, "hand_on_own_head": 0.597, "black_chest": 0.8215, "gripping_sound_effect": 0.3475, "facial_stripes": 0.6561, "tan_stripes": 0.7662, "multicolored_shorts": 0.6506, "relaxed_expression": 0.5095, "actual_fur": 0.4723}, "stage3_selected_ranks": {"anthro": 188, "white_body": 185, "blue_eyes": 66, "white_fur": 144, "tuft": 100, "pantherine": 148, "stripes": 131, "pose": 42, "shorts": 63, "sound_effects": 187, "cheek_tuft": 121, "gesture": 58, "striped_body": 51, "striped_fur": 36, "heterochromia": 173, "big_eyes": 145, "different_sound_effects": 196, "manly": 50, "hand_on_chest": 166, "countershade_fur": 33, "obliques": 49, "muscular_arms": 14, "muscular_legs": 15, "blue_shorts": 46, "striped_legs": 40, "on_grass": 170, "blue_countershading": 29, "confident": 103, "white_arms": 12, "white_chest": 2, "hand_gesture": 97, "striped_face": 21, "hand_on_own_knee": 60, "blue_inner_ear_fluff": 154, "pop_(sound_effect)": 195, "white_shorts": 53, "striped_neck": 26, "rubbing_sound_effect": 191, "siberian_tiger": 122, "light_chest": 11, "tan_shorts": 52, "grey_chest": 25, "hand_on_own_head": 67, "black_chest": 9, "gripping_sound_effect": 193, "facial_stripes": 41, "tan_stripes": 17, "multicolored_shorts": 44, "relaxed_expression": 123, "actual_fur": 151}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4723}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4844}, "black_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8215}, "blue_body": {"source": "implied"}, "blue_bottomwear": {"source": "implied"}, "blue_clothing": {"source": "implied"}, "blue_countershading": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6885}, "blue_inner_ear_fluff": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4653}, "blue_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6382}, "cheek_tuft": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5107}, "confident": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5252}, "countershade_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6855}, "different_sound_effects": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3383}, "facial_markings": {"source": "implied"}, "facial_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6561}, "facial_tuft": {"source": "implied"}, "gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6162}, "grey_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7073}, "gripping_sound_effect": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3475}, "hand_gesture": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5321}, "hand_on_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4457}, "hand_on_knee": {"source": "implied"}, "hand_on_leg": {"source": "implied"}, "hand_on_own_head": {"source": "stage3", "why": "explicit", "retrieval_score": 0.597}, "hand_on_own_knee": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6114}, "hand_on_own_leg": {"source": "implied"}, "head_markings": {"source": "implied"}, "heterochromia": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4325}, "inner_ear_fluff": {"source": "implied"}, "light_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.819}, "manly": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6237}, "markings": {"source": "implied"}, "multicolored_bottomwear": {"source": "implied"}, "multicolored_clothing": {"source": "implied"}, "multicolored_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6506}, "muscular_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7936}, "muscular_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7906}, "obliques": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6265}, "on_grass": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4388}, "pop_(sound_effect)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3394}, "pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6545}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5095}, "rubbing_sound_effect": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3533}, "siberian_tiger": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5098}, "sound_effects": {"source": "stage3", "why": "explicit", "retrieval_score": 0.399}, "striped_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6236}, "striped_face": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7203}, "striped_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6725}, "striped_legs": {"source": "stage3", "why": "explicit", "retrieval_score": 0.657}, "striped_neck": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7021}, "tan_bottomwear": {"source": "implied"}, "tan_clothing": {"source": "implied"}, "tan_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6233}, "tan_stripes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7662}, "white_arms": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8154}, "white_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4088}, "white_bottomwear": {"source": "implied"}, "white_chest": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9269}, "white_clothing": {"source": "implied"}, "white_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4865}, "white_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6221}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 1.2, "t2": 1.6, "t3": 25.55, "t3s": 0.73, "err": null, "issues": ["Stage3 split: general=193 entity=1 copyright_filtered=2 generic_char_to_general=0 unknown_type=3"]}
7
+ {"id": 1023509, "n_gt": 21, "n_retrieved": 187, "n_selected": 30, "n_implied": 6, "n_structural": 6, "ret_R": 0.4286, "P": 0.1333, "R": 0.1905, "F1": 0.1569, "leaf_P": 0.0455, "leaf_R": 0.0909, "leaf_F1": 0.0606, "n_leaf_sel": 22, "n_leaf_gt": 11, "ret_P": 0.0481, "sel_given_ret": 0.4444, "over_sel": 1.43, "why": {"explicit": 17, "strong_implied": 1}, "n_gt_char": 0, "n_sel_char": 2, "char_F1": 0.0, "gen_P": 0.1429, "gen_R": 0.2105, "gen_F1": 0.1702, "missed": ["armless", "boss_monster", "d6", "dialogue", "dice", "fur", "goat", "human", "lizard", "long_ears", "red_eyes", "reptile", "scalie", "undertale", "undertale_(series)", "white_body", "white_fur"], "extra": ["ambiguous_gender", "anthro", "black_speech_bubble", "bubble", "caprine_demon", "clothed", "clothing", "dark_theme", "darkness", "demon", "elemental_creature", "face_mask", "fire", "fire_creature", "frown", "goat_demon", "group", "humanoid", "light", "lizard_tail", "male_human", "sad", "scary", "speech_bubble", "unknown_character", "unnamed_character"], "ground_truth_tags": ["armless", "boss_monster", "bovid", "caprine", "d6", "dialogue", "dice", "fur", "goat", "human", "lizard", "long_ears", "mammal", "red_eyes", "reptile", "scalie", "text", "undertale", "undertale_(series)", "white_body", "white_fur"], "selected_tags": ["ambiguous_gender", "anthro", "black_speech_bubble", "bovid", "bubble", "caprine", "caprine_demon", "clothed", "clothing", "dark_theme", "darkness", "demon", "elemental_creature", "face_mask", "fire", "fire_creature", "frown", "goat_demon", "group", "humanoid", "light", "lizard_tail", "male_human", "mammal", "sad", "scary", "speech_bubble", "text", "unknown_character", "unnamed_character"], "stage3_selected": ["black_speech_bubble", "bubble", "caprine_demon", "dark_theme", "darkness", "elemental_creature", "face_mask", "fire_creature", "frown", "goat_demon", "light", "lizard_tail", "male_human", "sad", "scary", "speech_bubble", "unknown_character", "unnamed_character"], "stage3_selected_scores": {"speech_bubble": 0.7386, "light": 0.7623, "elemental_creature": 0.4919, "frown": 0.4795, "bubble": 0.7537, "sad": 0.5476, "unknown_character": 0.5187, "caprine_demon": 0.5512, "goat_demon": 0.5579, "face_mask": 0.5312, "fire_creature": 0.5179, "darkness": 0.8335, "dark_theme": 0.5815, "lizard_tail": 0.5606, "black_speech_bubble": 0.6245, "unnamed_character": 0.559, "male_human": 0.5507, "scary": 0.505}, "stage3_selected_ranks": {"speech_bubble": 8, "light": 4, "elemental_creature": 142, "frown": 158, "bubble": 6, "sad": 79, "unknown_character": 112, "caprine_demon": 70, "goat_demon": 65, "face_mask": 99, "fire_creature": 113, "darkness": 1, "dark_theme": 48, "lizard_tail": 62, "black_speech_bubble": 28, "unnamed_character": 64, "male_human": 72, "scary": 127}, "extra_evidence": {"ambiguous_gender": {"source": "structural"}, "anthro": {"source": "structural"}, "black_speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.6245}, "bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7537}, "caprine_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5512}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "dark_theme": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5815}, "darkness": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8335}, "demon": {"source": "implied"}, "elemental_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4919}, "face_mask": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5312}, "fire": {"source": "implied"}, "fire_creature": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5179}, "frown": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4795}, "goat_demon": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5579}, "group": {"source": "structural"}, "humanoid": {"source": "structural"}, "light": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7623}, "lizard_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5606}, "male_human": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5507}, "sad": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5476}, "scary": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.505}, "speech_bubble": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7386}, "unknown_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5187}, "unnamed_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.559}}, "structural": ["group", "anthro", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 2.18, "t2": 1.46, "t3": 25.68, "t3s": 2.31, "err": null, "issues": ["Stage3 split: general=180 entity=4 copyright_filtered=3 generic_char_to_general=4 unknown_type=3"]}
8
+ {"id": 335343, "n_gt": 33, "n_retrieved": 207, "n_selected": 43, "n_implied": 7, "n_structural": 5, "ret_R": 0.3333, "P": 0.2326, "R": 0.303, "F1": 0.2632, "leaf_P": 0.1515, "leaf_R": 0.25, "leaf_F1": 0.1887, "n_leaf_sel": 33, "n_leaf_gt": 20, "ret_P": 0.0531, "sel_given_ret": 0.9091, "over_sel": 1.3, "why": {"explicit": 32}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.2326, "gen_R": 0.3226, "gen_F1": 0.2703, "missed": ["angry", "earth_pony", "equid", "equine", "eyes_closed", "eyeshadow", "feathers", "female", "feral", "freckles", "fur", "hasbro", "horn", "horse", "mammal", "my_little_pony", "pillow", "pony", "scared", "sleeping", "unicorn", "white_body", "white_fur"], "extra": ["alternate_color", "ambiguous_gender", "annoyed_expression", "bed_sheet", "bedding", "bedroom", "blanket", "blue_eyes", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "ellipsis", "english_text", "eyeliner", "eyes", "half-closed_eyes", "humanoid", "letters", "long_hair", "lying_on_bed", "mascara", "narrowed_eyes", "nightmare_fuel", "on_bed", "purple_eyes", "relaxed_expression", "resting", "resting_arm", "sleeping_together", "sleepwear", "text_box"], "ground_truth_tags": ["angry", "bed", "blonde_hair", "duo", "earth_pony", "equid", "equine", "eyes_closed", "eyeshadow", "feathers", "female", "feral", "freckles", "fur", "furniture", "green_eyes", "hair", "hasbro", "horn", "horse", "lying", "makeup", "mammal", "my_little_pony", "pillow", "pony", "purple_hair", "scared", "sleeping", "text", "unicorn", "white_body", "white_fur"], "selected_tags": ["alternate_color", "ambiguous_gender", "annoyed_expression", "bed", "bed_sheet", "bedding", "bedroom", "blanket", "blonde_hair", "blue_eyes", "clothed", "clothing", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "duo", "ellipsis", "english_text", "eyeliner", "eyes", "furniture", "green_eyes", "hair", "half-closed_eyes", "humanoid", "letters", "long_hair", "lying", "lying_on_bed", "makeup", "mascara", "narrowed_eyes", "nightmare_fuel", "on_bed", "purple_eyes", "purple_hair", "relaxed_expression", "resting", "resting_arm", "sleeping_together", "sleepwear", "text", "text_box"], "stage3_selected": ["alternate_color", "annoyed_expression", "bed_sheet", "bedding", "bedroom", "blanket", "blonde_hair", "blue_eyes", "cool_colors", "digital_media_(artwork)", "digital_painting_(artwork)", "ellipsis", "english_text", "eyeliner", "eyes", "green_eyes", "half-closed_eyes", "letters", "long_hair", "lying_on_bed", "makeup", "mascara", "nightmare_fuel", "purple_eyes", "purple_hair", "relaxed_expression", "resting", "resting_arm", "sleeping_together", "sleepwear", "text", "text_box"], "stage3_selected_scores": {"digital_media_(artwork)": 0.3347, "text": 0.5939, "blue_eyes": 0.595, "green_eyes": 0.5933, "long_hair": 0.4471, "blonde_hair": 0.5872, "half-closed_eyes": 0.3908, "purple_eyes": 0.4293, "purple_hair": 0.5591, "makeup": 0.5894, "bedroom": 0.4873, "bedding": 0.3861, "digital_painting_(artwork)": 0.388, "ellipsis": 0.3904, "bed_sheet": 0.3933, "alternate_color": 0.3925, "blanket": 0.4169, "mascara": 0.4405, "nightmare_fuel": 0.3467, "eyeliner": 0.4399, "lying_on_bed": 0.4059, "text_box": 0.3881, "cool_colors": 0.3885, "resting": 0.5015, "annoyed_expression": 0.7219, "sleeping_together": 0.5063, "sleepwear": 0.444, "letters": 0.3654, "relaxed_expression": 0.5026, "resting_arm": 0.3655, "eyes": 0.8955, "english_text": 0.4128}, "stage3_selected_ranks": {"digital_media_(artwork)": 192, "text": 7, "blue_eyes": 6, "green_eyes": 9, "long_hair": 61, "blonde_hair": 11, "half-closed_eyes": 132, "purple_eyes": 76, "purple_hair": 14, "makeup": 10, "bedroom": 40, "bedding": 147, "digital_painting_(artwork)": 143, "ellipsis": 134, "bed_sheet": 127, "alternate_color": 130, "blanket": 93, "mascara": 67, "nightmare_fuel": 186, "eyeliner": 68, "lying_on_bed": 106, "text_box": 142, "cool_colors": 138, "resting": 31, "annoyed_expression": 2, "sleeping_together": 29, "sleepwear": 63, "letters": 173, "relaxed_expression": 30, "resting_arm": 172, "eyes": 1, "english_text": 99}, "extra_evidence": {"alternate_color": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3925}, "ambiguous_gender": {"source": "structural"}, "annoyed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.7219}, "bed_sheet": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3933}, "bedding": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3861}, "bedroom": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4873}, "blanket": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4169}, "blue_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.595}, "clothed": {"source": "structural"}, "clothing": {"source": "implied"}, "cool_colors": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3885}, "digital_media_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3347}, "digital_painting_(artwork)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.388}, "ellipsis": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3904}, "english_text": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4128}, "eyeliner": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4399}, "eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8955}, "half-closed_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3908}, "humanoid": {"source": "structural"}, "letters": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3654}, "long_hair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4471}, "lying_on_bed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4059}, "mascara": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4405}, "narrowed_eyes": {"source": "implied"}, "nightmare_fuel": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3467}, "on_bed": {"source": "implied"}, "purple_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4293}, "relaxed_expression": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5026}, "resting": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5015}, "resting_arm": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3655}, "sleeping_together": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5063}, "sleepwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "text_box": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3881}}, "structural": ["duo", "humanoid", "ambiguous_gender", "clothed", "text"], "t1": 1.39, "t2": 1.6, "t3": 33.02, "t3s": 0.87, "err": null, "issues": ["Stage3 split: general=210 entity=0 copyright_filtered=0 generic_char_to_general=0 unknown_type=4", "Stage3 general_chunk_0: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"weak_implied\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 35, \"why\": \"other\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"weak_implied\"}, {\"i\": 38, \"why\": \"weak_implied\"}, {\"i\": 43, \"why\": \"weak_implied\"}, {\"i\": 44, \"why\": \"weak_implied\"}, {\"i\": 45, \"why\": \"weak_implied\"}, {\"i\": 46, \"why\": \"other\"}, {\"i\": 47, \"why\": \"weak_implied\"}, {\"i\": 48, \"why\": \"weak_implied\"}, {\"i\": 49, \"why\": \"weak_implied\"}, {\"i\": 50, \"why\": \"other\"}, {\"i\": 51, \"why\": \"weak_implied\"}, {\"i\": 52, \"why\": \"weak_implied\"}, {\"i\": 53, \"why\": \"weak_implied\"}, {\"i\": 54}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 54}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE ", "Stage3 general_chunk_0: attempt 2 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"weak_implied\"}, {\"i\": 2, \"why\": \"weak_implied\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"other\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"other\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"style_or_meta\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"other\"}, {\"i\": 12, \"why\": \"explicit\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"style_or_meta\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"style_or_meta\"}, {\"i\": 18, \"why\": \"other\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"explicit\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"weak_implied\"}, {\"i\": 23, \"why\": \"explicit\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"style_or_meta\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"weak_implied\"}, {\"i\": 37, \"why\": \"other\"}, {}]}. Got: 2 validation errors for Stage3SelectionResponse\nselections.37.i\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nselections.37.why\n Field required [type=missing, input_value={}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
9
+ {"id": 17482, "n_gt": 26, "n_retrieved": 185, "n_selected": 50, "n_implied": 21, "n_structural": 5, "ret_R": 0.4615, "P": 0.34, "R": 0.6538, "F1": 0.4474, "leaf_P": 0.1923, "leaf_R": 0.3125, "leaf_F1": 0.2381, "n_leaf_sel": 26, "n_leaf_gt": 16, "ret_P": 0.0649, "sel_given_ret": 1.4167, "over_sel": 1.92, "why": {"explicit": 21, "strong_implied": 3}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.34, "gen_R": 0.6538, "gen_F1": 0.4474, "missed": ["4_fingers", "demon", "female", "fur", "hair", "holding_musical_instrument", "holding_object", "music", "slim"], "extra": ["4_claws", "acoustic_guitar", "big_tail", "bottomwear", "canis", "clawed_fingers", "claws_out", "daydream", "domestic_dog", "dream", "finger_claws", "herding_dog", "holding_guitar", "left-handed", "leggings", "legwear", "looking_at_viewer", "male", "malinois_dog", "pastoral_dog", "playing_guitar", "playing_music", "pointed_tail", "sheepdog", "shorts", "spitz", "topwear", "torn_body", "torn_bottomwear", "torn_leggings", "torn_legwear", "torn_shorts", "torn_topwear"], "ground_truth_tags": ["4_fingers", "anthro", "bass_guitar", "canid", "canine", "claws", "clothed", "clothing", "demon", "female", "fingers", "fur", "guitar", "hair", "holding_musical_instrument", "holding_object", "mammal", "music", "musical_instrument", "plucked_string_instrument", "slim", "solo", "spade_tail", "string_instrument", "tail", "torn_clothing"], "selected_tags": ["4_claws", "acoustic_guitar", "anthro", "bass_guitar", "big_tail", "bottomwear", "canid", "canine", "canis", "clawed_fingers", "claws", "claws_out", "clothed", "clothing", "daydream", "domestic_dog", "dream", "finger_claws", "fingers", "guitar", "herding_dog", "holding_guitar", "left-handed", "leggings", "legwear", "looking_at_viewer", "male", "malinois_dog", "mammal", "musical_instrument", "pastoral_dog", "playing_guitar", "playing_music", "plucked_string_instrument", "pointed_tail", "sheepdog", "shorts", "solo", "spade_tail", "spitz", "string_instrument", "tail", "topwear", "torn_body", "torn_bottomwear", "torn_clothing", "torn_leggings", "torn_legwear", "torn_shorts", "torn_topwear"], "stage3_selected": ["4_claws", "acoustic_guitar", "bass_guitar", "big_tail", "clawed_fingers", "claws_out", "daydream", "dream", "finger_claws", "fingers", "guitar", "holding_guitar", "left-handed", "malinois_dog", "musical_instrument", "playing_guitar", "pointed_tail", "spade_tail", "spitz", "torn_body", "torn_bottomwear", "torn_leggings", "torn_shorts", "torn_topwear"], "stage3_selected_scores": {"fingers": 0.4391, "spitz": 0.444, "finger_claws": 0.4391, "big_tail": 0.3836, "spade_tail": 0.6177, "musical_instrument": 0.8463, "torn_bottomwear": 0.436, "guitar": 0.9622, "torn_topwear": 0.3942, "clawed_fingers": 0.4297, "playing_guitar": 0.9316, "claws_out": 0.4376, "pointed_tail": 0.3765, "torn_shorts": 0.3993, "bass_guitar": 0.9118, "4_claws": 0.4515, "holding_guitar": 0.8442, "torn_leggings": 0.424, "acoustic_guitar": 0.8654, "torn_body": 0.397, "malinois_dog": 0.4568, "dream": 0.3663, "daydream": 0.2754, "left-handed": 0.3968}, "stage3_selected_ranks": {"fingers": 78, "spitz": 77, "finger_claws": 79, "big_tail": 145, "spade_tail": 13, "musical_instrument": 10, "torn_bottomwear": 85, "guitar": 1, "torn_topwear": 131, "clawed_fingers": 96, "playing_guitar": 2, "claws_out": 82, "pointed_tail": 151, "torn_shorts": 126, "bass_guitar": 3, "4_claws": 70, "holding_guitar": 11, "torn_leggings": 101, "acoustic_guitar": 7, "torn_body": 129, "malinois_dog": 65, "dream": 161, "daydream": 188, "left-handed": 130}, "extra_evidence": {"4_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4515}, "acoustic_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8654}, "big_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3836}, "bottomwear": {"source": "implied"}, "canis": {"source": "implied"}, "clawed_fingers": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4297}, "claws_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "daydream": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.2754}, "domestic_dog": {"source": "implied"}, "dream": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3663}, "finger_claws": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4391}, "herding_dog": {"source": "implied"}, "holding_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8442}, "left-handed": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3968}, "leggings": {"source": "implied"}, "legwear": {"source": "implied"}, "looking_at_viewer": {"source": "structural"}, "male": {"source": "structural"}, "malinois_dog": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4568}, "pastoral_dog": {"source": "implied"}, "playing_guitar": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9316}, "playing_music": {"source": "implied"}, "pointed_tail": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3765}, "sheepdog": {"source": "implied"}, "shorts": {"source": "implied"}, "spitz": {"source": "stage3", "why": "explicit", "retrieval_score": 0.444}, "topwear": {"source": "implied"}, "torn_body": {"source": "stage3", "why": "explicit", "retrieval_score": 0.397}, "torn_bottomwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.436}, "torn_leggings": {"source": "stage3", "why": "explicit", "retrieval_score": 0.424}, "torn_legwear": {"source": "implied"}, "torn_shorts": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3993}, "torn_topwear": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3942}}, "structural": ["solo", "anthro", "male", "clothed", "looking_at_viewer"], "t1": 1.16, "t2": 1.43, "t3": 22.66, "t3s": 1.13, "err": null, "issues": ["Stage3 split: general=185 entity=2 copyright_filtered=4 generic_char_to_general=0 unknown_type=2", "Stage3 general_chunk_2: candidates (local indices):\n1. official art\n2. wire\n3. left-handed\n4. dream\n5. styling hair\n6. black hair\n7. cute expression\n8. pointing\n9. demon\n10. black bracelet\n11. tail fluff\n12. torn shorts\n13. sphere creature\n14. psychedelic\n15. painting (object)\n16. nodding\n17. holding hair\n18. curled hair\n19. worried\n20. succubus\n21. ear ring\n22. tail ring\n23. torn body\n24. :o\n25. fire\n26. wanted poster\n27. flower in hair\n28. blonde hair\n29. caressing face\n30. pointed tail\n31. tied string\n32. holding tail\n33. torn topwear\n34. pun\n35. lined paper\n36. big hair\n37. auburn hair\n38. shocked expression\n39. red sclera\n40. studded bracelet\n41. big tail\n42. business attire\n43. warning sign\n44. hairclip\n45. hair tie\n46. head horn\n47. ring\n48. scaly tail\n49. burning\n50. contrapposto\n51. wavy hair\n52. hair dye\n53. demon humanoid\n54. drawstring\n55. blowing\n56. blue hair\n57. short horn\n58. ear piercing\n59. small horn\n60. side cut"]}
10
+ {"id": 2021552, "n_gt": 40, "n_retrieved": 161, "n_selected": 50, "n_implied": 16, "n_structural": 5, "ret_R": 0.5, "P": 0.46, "R": 0.575, "F1": 0.5111, "leaf_P": 0.25, "leaf_R": 0.3333, "leaf_F1": 0.2857, "n_leaf_sel": 28, "n_leaf_gt": 21, "ret_P": 0.1242, "sel_given_ret": 1.15, "over_sel": 1.25, "why": {"explicit": 30}, "n_gt_char": 0, "n_sel_char": 0, "char_F1": 1.0, "gen_P": 0.46, "gen_R": 0.575, "gen_F1": 0.5111, "missed": ["cheek_tuft", "facial_tuft", "fluffy", "fluffy_tail", "grey_background", "grey_body", "grey_fur", "head_tuft", "looking_at_another", "looking_away", "overalls", "pawpads", "tail", "toe_claws", "tuft", "white_body", "white_fur"], "extra": ["actual_fur", "ambiguous_gender", "big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "corsac_fox", "coveralls", "cross_fox", "domestic_rabbit", "dutch_rabbit", "fennec_fox", "floppy_ears", "hare", "kit_fox", "long_ears", "looking_at_viewer", "lop_ears", "open_mouth", "oryctolagus", "rabbit_ears", "red_fox", "white_clothing", "white_shirt", "white_topwear", "wide_eyed"], "ground_truth_tags": ["anthro", "arctic_fox", "bottomwear", "canid", "canine", "cheek_tuft", "claws", "clothed", "clothing", "crossed_arms", "duo", "facial_markings", "facial_tuft", "fluffy", "fluffy_tail", "fox", "fur", "grey_background", "grey_body", "grey_fur", "head_markings", "head_tuft", "lagomorph", "leporid", "looking_at_another", "looking_away", "mammal", "markings", "overalls", "pants", "pawpads", "rabbit", "shirt", "standing", "tail", "toe_claws", "topwear", "tuft", "white_body", "white_fur"], "selected_tags": ["actual_fur", "ambiguous_gender", "anthro", "arctic_fox", "big_mouth_(anatomy)", "black_bottomwear", "black_clothing", "black_pants", "blue_overalls", "bottomwear", "canid", "canine", "claws", "clothed", "clothing", "corsac_fox", "coveralls", "cross_fox", "crossed_arms", "domestic_rabbit", "duo", "dutch_rabbit", "facial_markings", "fennec_fox", "floppy_ears", "fox", "fur", "hare", "head_markings", "kit_fox", "lagomorph", "leporid", "long_ears", "looking_at_viewer", "lop_ears", "mammal", "markings", "open_mouth", "oryctolagus", "pants", "rabbit", "rabbit_ears", "red_fox", "shirt", "standing", "topwear", "white_clothing", "white_shirt", "white_topwear", "wide_eyed"], "stage3_selected": ["actual_fur", "arctic_fox", "big_mouth_(anatomy)", "black_pants", "blue_overalls", "claws", "corsac_fox", "coveralls", "cross_fox", "crossed_arms", "dutch_rabbit", "facial_markings", "fennec_fox", "fox", "fur", "hare", "kit_fox", "lagomorph", "leporid", "long_ears", "lop_ears", "open_mouth", "rabbit", "rabbit_ears", "red_fox", "shirt", "simple_background", "standing", "white_shirt", "wide_eyed"], "stage3_selected_scores": {"fur": 0.6531, "simple_background": 0.416, "open_mouth": 0.633, "claws": 0.6303, "standing": 0.6878, "fox": 0.6379, "shirt": 0.7483, "lagomorph": 0.5936, "leporid": 0.5826, "rabbit": 0.6511, "long_ears": 0.4627, "facial_markings": 0.6945, "red_fox": 0.4427, "crossed_arms": 0.7285, "wide_eyed": 0.4671, "white_shirt": 0.8197, "arctic_fox": 0.4522, "black_pants": 0.833, "hare": 0.5449, "lop_ears": 0.4692, "rabbit_ears": 0.5992, "cross_fox": 0.4688, "coveralls": 0.5688, "corsac_fox": 0.4193, "big_mouth_(anatomy)": 0.5037, "dutch_rabbit": 0.4737, "kit_fox": 0.4485, "blue_overalls": 0.9203, "actual_fur": 0.4837, "fennec_fox": 0.5031}, "stage3_selected_ranks": {"fur": 41, "simple_background": 163, "open_mouth": 49, "claws": 51, "standing": 29, "fox": 47, "shirt": 18, "lagomorph": 70, "leporid": 77, "rabbit": 42, "long_ears": 153, "facial_markings": 27, "red_fox": 159, "crossed_arms": 21, "wide_eyed": 152, "white_shirt": 4, "arctic_fox": 155, "black_pants": 3, "hare": 100, "lop_ears": 150, "rabbit_ears": 65, "cross_fox": 151, "coveralls": 83, "corsac_fox": 162, "big_mouth_(anatomy)": 132, "dutch_rabbit": 149, "kit_fox": 156, "blue_overalls": 1, "actual_fur": 147, "fennec_fox": 133}, "extra_evidence": {"actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4837}, "ambiguous_gender": {"source": "structural"}, "big_mouth_(anatomy)": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5037}, "black_bottomwear": {"source": "implied"}, "black_clothing": {"source": "implied"}, "black_pants": {"source": "stage3", "why": "explicit", "retrieval_score": 0.833}, "blue_overalls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9203}, "corsac_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4193}, "coveralls": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5688}, "cross_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4688}, "domestic_rabbit": {"source": "implied"}, "dutch_rabbit": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4737}, "fennec_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5031}, "floppy_ears": {"source": "implied"}, "hare": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5449}, "kit_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4485}, "long_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4627}, "looking_at_viewer": {"source": "structural"}, "lop_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4692}, "open_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.633}, "oryctolagus": {"source": "implied"}, "rabbit_ears": {"source": "stage3", "why": "explicit", "retrieval_score": 0.5992}, "red_fox": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4427}, "white_clothing": {"source": "implied"}, "white_shirt": {"source": "stage3", "why": "explicit", "retrieval_score": 0.8197}, "white_topwear": {"source": "implied"}, "wide_eyed": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4671}}, "structural": ["duo", "anthro", "ambiguous_gender", "clothed", "looking_at_viewer"], "t1": 1.72, "t2": 1.25, "t3": 19.89, "t3s": 1.06, "err": null, "issues": ["Stage3 split: general=161 entity=5 copyright_filtered=0 generic_char_to_general=0 unknown_type=3", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"other\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"other\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"explicit\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"other\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 10, \"why\": \"other\"}, {\"i\": 11, \"why\": \"explicit\"}, {\"i\": 12, \"why\": \"other\"}, {\"i\": 13, \"why\": \"explicit\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"other\"}, {\"i\": 16, \"why\": \"explicit\"}, {\"i\": 17, \"why\": \"other\"}, {\"i\": 18, \"why\": \"explicit\"}, {\"i\": 19, \"why\": \"other\"}, {\"i\": 20, \"why\": \"other\"}, {\"i\": 21, \"why\": \"other\"}, {\"i\": 22, \"why\": \"other\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"other\"}, {\"i\": 25, \"why\": \"explicit\"}, {\"i\": 26, \"why\": \"other\"}, {\"i\": 27, \"why\": \"other\"}, {\"i\": 28, \"why\": \"other\"}, {\"i\": 29, \"why\": \"other\"}, {\"i\": 30, \"why\": \"other\"}, {\"i\": 31, \"why\": \"other\"}, {\"i\": 32, \"why\": \"other\"}, {\"i\": 33, \"why\": \"other\"}, {\"i\": 34, \"why\": \"other\"}, {\"i\": 35, \"why\": \"explicit\"}, {\"i\": 36, \"why\": \"other\"}, {\"i\": 37, \"why\": \"other\"}, {\"i\": 38, \"why\": \"explicit\"}, {\"i\": 39}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.38.why\n Field required [type=missing, input_value={'i': 39}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}
11
+ {"id": 2034167, "n_gt": 21, "n_retrieved": 202, "n_selected": 43, "n_implied": 14, "n_structural": 1, "ret_R": 0.4286, "P": 0.2326, "R": 0.4762, "F1": 0.3125, "leaf_P": 0.0741, "leaf_R": 0.1667, "leaf_F1": 0.1026, "n_leaf_sel": 27, "n_leaf_gt": 12, "ret_P": 0.0446, "sel_given_ret": 1.1111, "over_sel": 2.05, "why": {"explicit": 27, "strong_implied": 3}, "n_gt_char": 0, "n_sel_char": 1, "char_F1": 0.0, "gen_P": 0.2381, "gen_R": 0.4762, "gen_F1": 0.3175, "missed": ["4_toes", "blue_eyes", "canis", "feet", "female", "feral", "purple_fur", "solo", "teeth", "toes", "wolf"], "extra": ["2_frame_animation", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "anime_eyes", "big_eyes", "blue_background", "canid_humanoid", "canine_humanoid", "comic", "fan_character", "fighting_pose", "glistening", "glistening_eyes", "humanoid", "jumper", "long_mouth", "looking_at_viewer", "mammal_humanoid", "midair", "mouth_full", "multicolored_body", "multicolored_fur", "open_smile", "pose", "smile", "sparkling_character", "swinging", "thick_fur", "tongue_out", "walking"], "ground_truth_tags": ["4_toes", "blue_eyes", "blue_nose", "canid", "canine", "canis", "feet", "female", "feral", "fur", "mammal", "open_mouth", "purple_body", "purple_fur", "solo", "teeth", "toes", "tongue", "white_body", "white_fur", "wolf"], "selected_tags": ["2_frame_animation", "actual_fur", "animal_humanoid", "animated", "animated_comic", "animated_png", "anime_eyes", "big_eyes", "blue_background", "blue_nose", "canid", "canid_humanoid", "canine", "canine_humanoid", "comic", "fan_character", "fighting_pose", "fur", "glistening", "glistening_eyes", "humanoid", "jumper", "long_mouth", "looking_at_viewer", "mammal", "mammal_humanoid", "midair", "mouth_full", "multicolored_body", "multicolored_fur", "open_mouth", "open_smile", "pose", "purple_body", "smile", "sparkling_character", "swinging", "thick_fur", "tongue", "tongue_out", "walking", "white_body", "white_fur"], "stage3_selected": ["2_frame_animation", "actual_fur", "animated", "animated_comic", "animated_png", "anime_eyes", "big_eyes", "blue_background", "blue_nose", "canine_humanoid", "fan_character", "fighting_pose", "fur", "glistening_eyes", "jumper", "long_mouth", "midair", "mouth_full", "multicolored_fur", "open_mouth", "open_smile", "purple_body", "sparkling_character", "swinging", "thick_fur", "tongue", "tongue_out", "walking", "white_background", "white_fur"], "stage3_selected_scores": {"fur": 0.5663, "open_mouth": 0.5847, "tongue": 0.3349, "tongue_out": 0.3395, "white_fur": 0.577, "white_background": 0.444, "multicolored_fur": 0.4709, "fan_character": 0.4108, "purple_body": 0.5474, "open_smile": 0.4621, "blue_background": 0.4316, "canine_humanoid": 0.9127, "glistening_eyes": 0.4376, "blue_nose": 0.592, "big_eyes": 0.3944, "fighting_pose": 0.4377, "midair": 0.3892, "long_mouth": 0.4451, "sparkling_character": 0.3393, "mouth_full": 0.4433, "thick_fur": 0.4071, "jumper": 0.4004, "actual_fur": 0.4438, "animated_comic": 0.4044, "animated_png": 0.4455, "2_frame_animation": 0.3858, "animated": 0.3689, "walking": 0.33, "swinging": 0.3355, "anime_eyes": 0.4788}, "stage3_selected_ranks": {"fur": 28, "open_mouth": 21, "tongue": 192, "tongue_out": 187, "white_fur": 27, "white_background": 125, "multicolored_fur": 97, "fan_character": 146, "purple_body": 39, "open_smile": 106, "blue_background": 134, "canine_humanoid": 1, "glistening_eyes": 130, "blue_nose": 19, "big_eyes": 158, "fighting_pose": 129, "midair": 163, "long_mouth": 124, "sparkling_character": 188, "mouth_full": 127, "thick_fur": 147, "jumper": 154, "actual_fur": 126, "animated_comic": 150, "animated_png": 122, "2_frame_animation": 165, "animated": 171, "walking": 196, "swinging": 191, "anime_eyes": 91}, "extra_evidence": {"2_frame_animation": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3858}, "actual_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4438}, "animal_humanoid": {"source": "implied"}, "animated": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3689}, "animated_comic": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4044}, "animated_png": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4455}, "anime_eyes": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.4788}, "big_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3944}, "blue_background": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4316}, "canid_humanoid": {"source": "implied"}, "canine_humanoid": {"source": "stage3", "why": "explicit", "retrieval_score": 0.9127}, "comic": {"source": "implied"}, "fan_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4108}, "fighting_pose": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4377}, "glistening": {"source": "implied"}, "glistening_eyes": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4376}, "humanoid": {"source": "implied"}, "jumper": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4004}, "long_mouth": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4451}, "looking_at_viewer": {"source": "structural"}, "mammal_humanoid": {"source": "implied"}, "midair": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3892}, "mouth_full": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4433}, "multicolored_body": {"source": "implied"}, "multicolored_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4709}, "open_smile": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4621}, "pose": {"source": "implied"}, "smile": {"source": "implied"}, "sparkling_character": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3393}, "swinging": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.3355}, "thick_fur": {"source": "stage3", "why": "explicit", "retrieval_score": 0.4071}, "tongue_out": {"source": "stage3", "why": "explicit", "retrieval_score": 0.3395}, "walking": {"source": "stage3", "why": "strong_implied", "retrieval_score": 0.33}}, "structural": ["looking_at_viewer"], "t1": 2.08, "t2": 1.57, "t3": 32.89, "t3s": 1.58, "err": null, "issues": ["Stage3 split: general=205 entity=2 copyright_filtered=2 generic_char_to_general=4 unknown_type=12", "Stage3 general_chunk_2: attempt 1 error: Failed to parse Stage3SelectionResponse from completion {\"selections\": [{\"i\": 1, \"why\": \"explicit\"}, {\"i\": 2, \"why\": \"explicit\"}, {\"i\": 3, \"why\": \"explicit\"}, {\"i\": 4, \"why\": \"explicit\"}, {\"i\": 5, \"why\": \"strong_implied\"}, {\"i\": 6, \"why\": \"explicit\"}, {\"i\": 7, \"why\": \"explicit\"}, {\"i\": 8, \"why\": \"explicit\"}, {\"i\": 9, \"why\": \"other\"}, {\"i\": 10, \"why\": \"weak_implied\"}, {\"i\": 11, \"why\": \"weak_implied\"}, {\"i\": 12, \"why\": \"weak_implied\"}, {\"i\": 13, \"why\": \"weak_implied\"}, {\"i\": 14, \"why\": \"explicit\"}, {\"i\": 15, \"why\": \"explicit\"}, {\"i\": 16, \"why\": \"weak_implied\"}, {\"i\": 17, \"why\": \"weak_implied\"}, {\"i\": 18, \"why\": \"weak_implied\"}, {\"i\": 19, \"why\": \"weak_implied\"}, {\"i\": 20, \"why\": \"weak_implied\"}, {\"i\": 21, \"why\": \"weak_implied\"}, {\"i\": 22, \"why\": \"explicit\"}, {\"i\": 23, \"why\": \"other\"}, {\"i\": 24, \"why\": \"weak_implied\"}, {\"i\": 25, \"why\": \"weak_implied\"}, {\"i\": 26, \"why\": \"weak_implied\"}, {\"i\": 27, \"why\": \"weak_implied\"}, {\"i\": 28, \"why\": \"explicit\"}, {\"i\": 29, \"why\": \"explicit\"}, {\"i\": 30, \"why\": \"weak_implied\"}, {\"i\": 31, \"why\": \"weak_implied\"}, {\"i\": 32, \"why\": \"weak_implied\"}, {\"i\": 33, \"why\": \"weak_implied\"}, {\"i\": 34, \"why\": \"weak_implied\"}, {\"i\": 35, \"why\": \"weak_implied\"}, {\"i\": 36, \"\": null}]}. Got: 1 validation error for Stage3SelectionResponse\nselections.35.why\n Field required [type=missing, input_value={'i': 36, '': None}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.12/v/missing\nFor troubleshooting, visit: https://docs.langchain.com/oss/python/langchain/errors/OUTPUT_PARSING_FAILURE "]}