{
"source": "https://arxiv.org/html/2608.13113v1#S4.T3",
"source_table": "Table 3, arXiv:2608.13113v1",
"qa_distribution_source": "Table 5, arXiv:2608.13113v1",
"unit": "accuracy_percent",
"direction": "higher",
"chance_percent": 25,
"notes": "Scores are transcribed as reported. Avg and Acc are the paper aggregate rows, not recalculated from rounded per-task values. Model input budgets differ. Human results average three trained annotators with unrestricted viewing.",
"models": [
{
"name": "Chat-UniVi-V1.5",
"group": "open",
"frames": "256",
"parameters": "7B"
},
{
"name": "LLaVA-NeXT-Video",
"group": "open",
"frames": "64",
"parameters": "7B"
},
{
"name": "MiniCPM-V 4.5",
"group": "open",
"frames": "256",
"parameters": "8B"
},
{
"name": "Qwen2-VL",
"group": "open",
"frames": "256",
"parameters": "7B"
},
{
"name": "Qwen2.5-VL-32B",
"group": "open",
"frames": "256",
"parameters": "32B"
},
{
"name": "Qwen3-VL-8B",
"group": "open",
"frames": "256",
"parameters": "8B"
},
{
"name": "Qwen3-VL-30B-A3B",
"group": "open",
"frames": "256",
"parameters": "30B"
},
{
"name": "ShareGPT4Video",
"group": "open",
"frames": "64",
"parameters": "8B"
},
{
"name": "ST-LLM",
"group": "open",
"frames": "256",
"parameters": null
},
{
"name": "VideoLLaMA3",
"group": "open",
"frames": "512",
"parameters": "7B"
},
{
"name": "VITA-1.5",
"group": "open",
"frames": "16",
"parameters": "7B"
},
{
"name": "Gemini 2.5 Pro",
"group": "closed",
"frames": "1fps",
"parameters": null
},
{
"name": "Human",
"group": "human",
"frames": null,
"parameters": null
}
],
"metrics": [
{
"id": "habit-inference",
"name": "Habit Inference",
"level": 1,
"values": [
58.7,
72.5,
82.6,
75.4,
81.9,
75.4,
81.9,
50.0,
65.2,
69.6,
78.3,
84.8,
97.8
],
"count": 138,
"percent": 9.6
},
{
"id": "personality-inference",
"name": "Personality Inference",
"level": 1,
"values": [
50.0,
62.5,
68.8,
56.2,
62.5,
68.8,
37.5,
50.0,
56.2,
50.0,
56.2,
81.3,
93.8
],
"count": 16,
"percent": 1.1
},
{
"id": "detail-retrieval",
"name": "Detail Retrieval",
"level": 2,
"values": [
34.1,
38.3,
66.5,
58.9,
61.7,
54.4,
56.8,
34.8,
34.8,
55.4,
51.6,
77.7,
95.1
],
"count": 287,
"percent": 19.9
},
{
"id": "spatial-relation",
"name": "Spatial Relation",
"level": 2,
"values": [
56.9,
49.0,
58.8,
54.9,
56.9,
51.0,
51.0,
51.0,
49.0,
49.0,
54.9,
86.3,
94.1
],
"count": 51,
"percent": 3.5
},
{
"id": "self-localization",
"name": "Self-localization",
"level": 2,
"values": [
29.4,
41.2,
54.1,
52.9,
49.4,
52.9,
52.9,
31.8,
35.2,
43.5,
48.2,
82.4,
97.6
],
"count": 85,
"percent": 5.9
},
{
"id": "temporal-ordering",
"name": "Temporal Ordering",
"level": 2,
"values": [
40.9,
45.7,
70.1,
59.8,
64.6,
55.9,
62.2,
36.2,
40.1,
62.2,
57.5,
75.6,
95.3
],
"count": 127,
"percent": 8.8
},
{
"id": "event-time",
"name": "Event Time",
"level": 2,
"values": [
44.4,
46.1,
57.3,
42.7,
47.9,
41.9,
50.4,
45.3,
42.7,
41.9,
49.6,
60.7,
95.7
],
"count": 117,
"percent": 8.1
},
{
"id": "object-location",
"name": "Object Location",
"level": 2,
"values": [
51.9,
50.5,
61.6,
61.6,
64.1,
55.3,
57.3,
44.7,
58.2,
54.9,
51.5,
67.0,
96.1
],
"count": 206,
"percent": 14.3
},
{
"id": "procedure-planning",
"name": "Procedure Planning",
"level": 3,
"values": [
46.0,
29.4,
68.2,
68.2,
78.6,
71.4,
70.6,
30.2,
46.0,
60.3,
66.7,
81.7,
94.4
],
"count": 126,
"percent": 8.7
},
{
"id": "event-counting",
"name": "Event Counting",
"level": 3,
"values": [
8.8,
16.8,
40.0,
36.8,
43.2,
38.4,
34.4,
16.8,
8.0,
40.8,
41.6,
52.0,
94.4
],
"count": 125,
"percent": 8.7
},
{
"id": "object-counting",
"name": "Object Counting",
"level": 3,
"values": [
33.3,
29.8,
38.6,
52.6,
50.9,
31.6,
47.4,
29.8,
31.6,
49.1,
40.4,
73.7,
94.7
],
"count": 57,
"percent": 4.0
},
{
"id": "route-reasoning",
"name": "Route Reasoning",
"level": 3,
"values": [
35.7,
23.8,
35.7,
52.4,
52.4,
28.6,
45.2,
30.9,
28.6,
45.2,
40.5,
64.3,
90.5
],
"count": 42,
"percent": 2.9
},
{
"id": "cross-view-spatial-reasoning",
"name": "Cross-view Spatial Reasoning",
"level": 3,
"values": [
23.3,
30.0,
43.3,
40.0,
50.0,
46.7,
46.7,
26.7,
20.0,
40.0,
33.3,
60.0,
93.3
],
"count": 30,
"percent": 2.1
},
{
"id": "direction-judgement",
"name": "Direction Judgement",
"level": 3,
"values": [
38.9,
33.3,
38.9,
50.0,
47.2,
47.2,
47.2,
41.7,
27.8,
41.7,
47.2,
58.3,
86.1
],
"count": 36,
"percent": 2.5
},
{
"id": "avg",
"name": "Avg",
"level": null,
"values": [
39.5,
40.6,
56.0,
54.5,
58.0,
51.4,
53.0,
37.1,
38.8,
50.3,
51.3,
71.8,
94.2
]
},
{
"id": "acc",
"name": "Acc",
"level": null,
"values": [
39.9,
41.7,
60.6,
57.0,
60.8,
53.7,
56.7,
36.9,
40.8,
53.1,
53.6,
72.6,
95.1
]
}
]
}