memray hschechter commited on
Commit
578a5b2
·
1 Parent(s): de9d6ee

Add EmbeddingGemma-2 evaluation scores (#192)

Browse files

- Add EmbeddingGemma-2 evaluation scores (de8758748f918bede1aea6dcadd147f16a2d3edb)


Co-authored-by: Henrique Schechter Vera <hschechter@users.noreply.huggingface.co>

Files changed (1) hide show
  1. scores/EmbeddingGemma-2.json +764 -0
scores/EmbeddingGemma-2.json ADDED
@@ -0,0 +1,764 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "model_name": "EmbeddingGemma-2",
4
+ "model_size": "0.438",
5
+ "embedding_dimension": 768,
6
+ "max_length_tokens": 8192,
7
+ "model_release_date": "2026-10-06",
8
+ "data_source": "Self-Reported",
9
+ "url": "https://huggingface.co/google/embeddinggemma-2",
10
+ "report_generated_date": "2026-10-04T14:52:34.155145"
11
+ },
12
+ "metrics": {
13
+ "image": {
14
+ "A-OKVQA": {
15
+ "hit@1": 0.352,
16
+ "hit@5": 0.66,
17
+ "hit@10": 0.751,
18
+ "mrr": 0.4894555584036258,
19
+ "mrr@10": 0.47852063492063496,
20
+ "ndcg@5": 0.5145081612467047,
21
+ "ndcg@10": 0.5440319846297221,
22
+ "ndcg_linear@5": 0.5145081612467047
23
+ },
24
+ "CIRR": {
25
+ "hit@1": 0.401,
26
+ "hit@5": 0.869,
27
+ "hit@10": 0.935,
28
+ "mrr": 0.605247176320826,
29
+ "mrr@10": 0.6018944444444445,
30
+ "ndcg@5": 0.6628500173492062,
31
+ "ndcg@10": 0.6841589706600825,
32
+ "ndcg_linear@5": 0.6628500173492062
33
+ },
34
+ "ChartQA": {
35
+ "hit@1": 0.189,
36
+ "hit@5": 0.367,
37
+ "hit@10": 0.443,
38
+ "mrr": 0.27691337724965515,
39
+ "mrr@10": 0.26569087301587296,
40
+ "ndcg@5": 0.28342017469359204,
41
+ "ndcg@10": 0.3079004559601625,
42
+ "ndcg_linear@5": 0.28342017469359204
43
+ },
44
+ "Country211": {
45
+ "hit@1": 0.185,
46
+ "hit@5": 0.424,
47
+ "hit@10": 0.546,
48
+ "mrr": 0.3032786992575371,
49
+ "mrr@10": 0.28764206349206345,
50
+ "ndcg@5": 0.3094687204942733,
51
+ "ndcg@10": 0.3486695627979483,
52
+ "ndcg_linear@5": 0.3094687204942733
53
+ },
54
+ "DocVQA": {
55
+ "hit@1": 0.515,
56
+ "hit@5": 0.753,
57
+ "hit@10": 0.82,
58
+ "mrr": 0.6277390552993807,
59
+ "mrr@10": 0.6209503968253969,
60
+ "ndcg@5": 0.6474117169925452,
61
+ "ndcg@10": 0.6692835999019502,
62
+ "ndcg_linear@5": 0.6474117169925452
63
+ },
64
+ "EDIS": {
65
+ "hit@1": 0.807,
66
+ "hit@5": 0.941,
67
+ "hit@10": 0.967,
68
+ "mrr": 0.866164484957964,
69
+ "mrr@10": 0.8646158730158731,
70
+ "ndcg@5": 0.881249708371607,
71
+ "ndcg@10": 0.8896769400885012,
72
+ "ndcg_linear@5": 0.881249708371607
73
+ },
74
+ "FashionIQ": {
75
+ "hit@1": 0.279,
76
+ "hit@5": 0.534,
77
+ "hit@10": 0.646,
78
+ "mrr": 0.4028261132566756,
79
+ "mrr@10": 0.3903662698412699,
80
+ "ndcg@5": 0.41489550397903147,
81
+ "ndcg@10": 0.45123833145154596,
82
+ "ndcg_linear@5": 0.41489550397903147
83
+ },
84
+ "GQA": {
85
+ "hit@1": 0.614,
86
+ "hit@5": 0.877,
87
+ "hit@10": 0.92,
88
+ "mrr": 0.7260412976292057,
89
+ "mrr@10": 0.7222789682539683,
90
+ "ndcg@5": 0.7565880675190193,
91
+ "ndcg@10": 0.7707873943768512,
92
+ "ndcg_linear@5": 0.7565880675190193
93
+ },
94
+ "HatefulMemes": {
95
+ "hit@1": 0.514,
96
+ "hit@5": 1.0,
97
+ "hit@10": 1.0,
98
+ "mrr": 0.757,
99
+ "mrr@10": 0.757,
100
+ "ndcg@5": 0.8206318602357284,
101
+ "ndcg@10": 0.8206318602357284,
102
+ "ndcg_linear@5": 0.8206318602357284
103
+ },
104
+ "ImageNet-1K": {
105
+ "hit@1": 0.65,
106
+ "hit@5": 0.892,
107
+ "hit@10": 0.942,
108
+ "mrr": 0.7557240503639441,
109
+ "mrr@10": 0.7528730158730159,
110
+ "ndcg@5": 0.783126653845566,
111
+ "ndcg@10": 0.7990633137778012,
112
+ "ndcg_linear@5": 0.783126653845566
113
+ },
114
+ "ImageNet-A": {
115
+ "hit@1": 0.398,
116
+ "hit@5": 0.75,
117
+ "hit@10": 0.848,
118
+ "mrr": 0.5549240120846187,
119
+ "mrr@10": 0.5492305555555554,
120
+ "ndcg@5": 0.5896776537947478,
121
+ "ndcg@10": 0.6216746819404655,
122
+ "ndcg_linear@5": 0.5896776537947478
123
+ },
124
+ "ImageNet-R": {
125
+ "hit@1": 0.874,
126
+ "hit@5": 0.986,
127
+ "hit@10": 0.991,
128
+ "mrr": 0.9273334365825745,
129
+ "mrr@10": 0.9268944444444444,
130
+ "ndcg@5": 0.9416473792378206,
131
+ "ndcg@10": 0.9432917533612719,
132
+ "ndcg_linear@5": 0.9416473792378206
133
+ },
134
+ "InfographicsVQA": {
135
+ "hit@1": 0.227,
136
+ "hit@5": 0.444,
137
+ "hit@10": 0.546,
138
+ "mrr": 0.3375985800036458,
139
+ "mrr@10": 0.3251273809523809,
140
+ "ndcg@5": 0.3447410127414168,
141
+ "ndcg@10": 0.3776705805668205,
142
+ "ndcg_linear@5": 0.3447410127414168
143
+ },
144
+ "MSCOCO": {
145
+ "hit@1": 0.632,
146
+ "hit@5": 0.789,
147
+ "hit@10": 0.841,
148
+ "mrr": 0.7106500350556304,
149
+ "mrr@10": 0.7051980158730159,
150
+ "ndcg@5": 0.7213013289764899,
151
+ "ndcg@10": 0.7381038770666598,
152
+ "ndcg_linear@5": 0.7213013289764899
153
+ },
154
+ "MSCOCO_i2t": {
155
+ "hit@1": 0.728,
156
+ "hit@5": 0.944,
157
+ "hit@10": 0.975,
158
+ "mrr": 0.8208246652796981,
159
+ "mrr@10": 0.8195198412698412,
160
+ "ndcg@5": 0.8475519354491046,
161
+ "ndcg@10": 0.8579237536533877,
162
+ "ndcg_linear@5": 0.8475519354491046
163
+ },
164
+ "MSCOCO_t2i": {
165
+ "hit@1": 0.777,
166
+ "hit@5": 0.954,
167
+ "hit@10": 0.983,
168
+ "mrr": 0.8535446015001864,
169
+ "mrr@10": 0.8526019841269841,
170
+ "ndcg@5": 0.8751242264966991,
171
+ "ndcg@10": 0.8847345150174667,
172
+ "ndcg_linear@5": 0.8751242264966991
173
+ },
174
+ "N24News": {
175
+ "hit@1": 0.381,
176
+ "hit@5": 0.747,
177
+ "hit@10": 0.878,
178
+ "mrr": 0.5400244176201522,
179
+ "mrr@10": 0.5320234126984128,
180
+ "ndcg@5": 0.5723438839411544,
181
+ "ndcg@10": 0.614741597903652,
182
+ "ndcg_linear@5": 0.5723438839411544
183
+ },
184
+ "NIGHTS": {
185
+ "hit@1": 0.667,
186
+ "hit@5": 0.966,
187
+ "hit@10": 0.987,
188
+ "mrr": 0.7979682043213556,
189
+ "mrr@10": 0.7971630952380951,
190
+ "ndcg@5": 0.8378597947351424,
191
+ "ndcg@10": 0.8448937493832229,
192
+ "ndcg_linear@5": 0.8378597947351424
193
+ },
194
+ "OK-VQA": {
195
+ "hit@1": 0.434,
196
+ "hit@5": 0.745,
197
+ "hit@10": 0.839,
198
+ "mrr": 0.5725276338714869,
199
+ "mrr@10": 0.5661801587301587,
200
+ "ndcg@5": 0.6013839453492872,
201
+ "ndcg@10": 0.6318522749614458,
202
+ "ndcg_linear@5": 0.6013839453492872
203
+ },
204
+ "OVEN": {
205
+ "hit@1": 0.68,
206
+ "hit@5": 0.876,
207
+ "hit@10": 0.932,
208
+ "mrr": 0.7677856604037729,
209
+ "mrr@10": 0.7653499999999999,
210
+ "ndcg@5": 0.7875850993463452,
211
+ "ndcg@10": 0.8057185110580729,
212
+ "ndcg_linear@5": 0.7875850993463452
213
+ },
214
+ "ObjectNet": {
215
+ "hit@1": 0.655,
216
+ "hit@5": 0.883,
217
+ "hit@10": 0.919,
218
+ "mrr": 0.7553486186418554,
219
+ "mrr@10": 0.751804761904762,
220
+ "ndcg@5": 0.7813281723434164,
221
+ "ndcg@10": 0.7928849860617837,
222
+ "ndcg_linear@5": 0.7813281723434164
223
+ },
224
+ "Place365": {
225
+ "hit@1": 0.353,
226
+ "hit@5": 0.661,
227
+ "hit@10": 0.756,
228
+ "mrr": 0.49608768789253993,
229
+ "mrr@10": 0.486584126984127,
230
+ "ndcg@5": 0.5207587965443671,
231
+ "ndcg@10": 0.5515767036658638,
232
+ "ndcg_linear@5": 0.5207587965443671
233
+ },
234
+ "RefCOCO-Matching": {
235
+ "hit@1": 0.724,
236
+ "hit@5": 1.0,
237
+ "hit@10": 1.0,
238
+ "mrr": 0.8616666666666667,
239
+ "mrr@10": 0.8616666666666667,
240
+ "ndcg@5": 0.8978747524785794,
241
+ "ndcg@10": 0.8978747524785794,
242
+ "ndcg_linear@5": 0.8978747524785794
243
+ },
244
+ "RefCOCO": {
245
+ "hit@1": 0.829,
246
+ "hit@5": 0.978,
247
+ "hit@10": 0.99,
248
+ "mrr": 0.8933526303506099,
249
+ "mrr@10": 0.8927507936507937,
250
+ "ndcg@5": 0.9132100671970795,
251
+ "ndcg@10": 0.9170226657951486,
252
+ "ndcg_linear@5": 0.9132100671970795
253
+ },
254
+ "SUN397": {
255
+ "hit@1": 0.675,
256
+ "hit@5": 0.921,
257
+ "hit@10": 0.957,
258
+ "mrr": 0.782764703092061,
259
+ "mrr@10": 0.7804293650793651,
260
+ "ndcg@5": 0.8120372524667199,
261
+ "ndcg@10": 0.8241385482330595,
262
+ "ndcg_linear@5": 0.8120372524667199
263
+ },
264
+ "ScienceQA": {
265
+ "hit@1": 0.269,
266
+ "hit@5": 0.575,
267
+ "hit@10": 0.726,
268
+ "mrr": 0.4177625089784668,
269
+ "mrr@10": 0.40633055555555553,
270
+ "ndcg@5": 0.4332867477915349,
271
+ "ndcg@10": 0.482230130084527,
272
+ "ndcg_linear@5": 0.4332867477915349
273
+ },
274
+ "TextVQA": {
275
+ "hit@1": 0.741,
276
+ "hit@5": 0.878,
277
+ "hit@10": 0.905,
278
+ "mrr": 0.8039221659014103,
279
+ "mrr@10": 0.8015027777777778,
280
+ "ndcg@5": 0.8180099440570526,
281
+ "ndcg@10": 0.8269374006132875,
282
+ "ndcg_linear@5": 0.8180099440570526
283
+ },
284
+ "VOC2007": {
285
+ "hit@1": 0.752,
286
+ "hit@5": 0.972,
287
+ "hit@10": 0.995,
288
+ "mrr": 0.8446821011290051,
289
+ "mrr@10": 0.8443119047619048,
290
+ "ndcg@5": 0.8740300239112544,
291
+ "ndcg@10": 0.8817728667368578,
292
+ "ndcg_linear@5": 0.8740300239112544
293
+ },
294
+ "VisDial": {
295
+ "hit@1": 0.765,
296
+ "hit@5": 0.935,
297
+ "hit@10": 0.967,
298
+ "mrr": 0.8391409064908703,
299
+ "mrr@10": 0.8374265873015873,
300
+ "ndcg@5": 0.8585624372054541,
301
+ "ndcg@10": 0.869265119031295,
302
+ "ndcg_linear@5": 0.8585624372054541
303
+ },
304
+ "Visual7W-Pointing": {
305
+ "hit@1": 0.91,
306
+ "hit@5": 0.976,
307
+ "hit@10": 0.986,
308
+ "mrr": 0.9407746051847553,
309
+ "mrr@10": 0.9399952380952382,
310
+ "ndcg@5": 0.948084915799739,
311
+ "ndcg@10": 0.9514297391280604,
312
+ "ndcg_linear@5": 0.948084915799739
313
+ },
314
+ "Visual7W": {
315
+ "hit@1": 0.385,
316
+ "hit@5": 0.725,
317
+ "hit@10": 0.845,
318
+ "mrr": 0.5386436764530227,
319
+ "mrr@10": 0.5315170634920635,
320
+ "ndcg@5": 0.5673500308490059,
321
+ "ndcg@10": 0.6068576519809303,
322
+ "ndcg_linear@5": 0.5673500308490059
323
+ },
324
+ "VisualNews_i2t": {
325
+ "hit@1": 0.6,
326
+ "hit@5": 0.783,
327
+ "hit@10": 0.832,
328
+ "mrr": 0.6816213303872509,
329
+ "mrr@10": 0.6759515873015873,
330
+ "ndcg@5": 0.6977034049147782,
331
+ "ndcg@10": 0.7136780216357829,
332
+ "ndcg_linear@5": 0.6977034049147782
333
+ },
334
+ "VisualNews_t2i": {
335
+ "hit@1": 0.585,
336
+ "hit@5": 0.733,
337
+ "hit@10": 0.791,
338
+ "mrr": 0.6560841292177779,
339
+ "mrr@10": 0.6499996031746031,
340
+ "ndcg@5": 0.6650404720683368,
341
+ "ndcg@10": 0.6837591584473537,
342
+ "ndcg_linear@5": 0.6650404720683368
343
+ },
344
+ "VizWiz": {
345
+ "hit@1": 0.464,
346
+ "hit@5": 0.683,
347
+ "hit@10": 0.724,
348
+ "mrr": 0.5611011544058288,
349
+ "mrr@10": 0.5566777777777777,
350
+ "ndcg@5": 0.5842509226671125,
351
+ "ndcg@10": 0.5976887198414288,
352
+ "ndcg_linear@5": 0.5842509226671125
353
+ },
354
+ "WebQA": {
355
+ "hit@1": 0.897,
356
+ "hit@5": 0.986,
357
+ "hit@10": 0.994,
358
+ "mrr": 0.93943769171692,
359
+ "mrr@10": 0.9392892857142857,
360
+ "ndcg@5": 0.9504748949850282,
361
+ "ndcg@10": 0.9531694409026967,
362
+ "ndcg_linear@5": 0.9504748949850282
363
+ },
364
+ "Wiki-SS-NQ": {
365
+ "hit@1": 0.714,
366
+ "hit@5": 0.896,
367
+ "hit@10": 0.933,
368
+ "mrr": 0.7969800563599774,
369
+ "mrr@10": 0.7946861111111111,
370
+ "ndcg@5": 0.8163114205315097,
371
+ "ndcg@10": 0.8287051220783258,
372
+ "ndcg_linear@5": 0.8163114205315097
373
+ }
374
+ },
375
+ "video": {
376
+ "ActivityNetQA": {
377
+ "hit@1": 0.521,
378
+ "hit@5": 1.0,
379
+ "mrr": 0.7605,
380
+ "ndcg@5": 0.8232153519607281,
381
+ "ndcg@10": 0.8232153519607281,
382
+ "ndcg_linear@5": 0.8232153519607281
383
+ },
384
+ "Breakfast": {
385
+ "hit@1": 0.3556581986143187,
386
+ "hit@5": 0.859122401847575,
387
+ "mrr": 0.5699127534000513,
388
+ "ndcg@5": 0.6272061693767046,
389
+ "ndcg@10": 0.6742020333066189,
390
+ "ndcg_linear@5": 0.6272061693767046
391
+ },
392
+ "Charades-STA": {
393
+ "hit@1": 0.2558459422283356,
394
+ "hit@5": 0.6519944979367263,
395
+ "mrr": 0.43762963690749107,
396
+ "ndcg@5": 0.4574943728260989,
397
+ "ndcg@10": 0.5675804035220362,
398
+ "ndcg_linear@5": 0.4574943728260989
399
+ },
400
+ "DiDeMo": {
401
+ "hit@1": 0.5249003984063745,
402
+ "hit@5": 0.7569721115537849,
403
+ "mrr": 0.6314462953529382,
404
+ "ndcg@5": 0.651308776587172,
405
+ "ndcg@10": 0.6735482789104176,
406
+ "ndcg_linear@5": 0.651308776587172
407
+ },
408
+ "EgoSchema": {
409
+ "hit@1": 0.566,
410
+ "hit@5": 1.0,
411
+ "mrr": 0.7311333333333333,
412
+ "ndcg@5": 0.7984151736319034,
413
+ "ndcg@10": 0.7984151736319034,
414
+ "ndcg_linear@5": 0.7984151736319034
415
+ },
416
+ "HMDB51": {
417
+ "hit@1": 0.553,
418
+ "hit@5": 0.807,
419
+ "mrr": 0.6691967387306713,
420
+ "ndcg@5": 0.691570441813318,
421
+ "ndcg@10": 0.7159941722136407,
422
+ "ndcg_linear@5": 0.691570441813318
423
+ },
424
+ "K700": {
425
+ "hit@1": 0.547,
426
+ "hit@5": 0.801,
427
+ "mrr": 0.6614229807817694,
428
+ "ndcg@5": 0.685966662372165,
429
+ "ndcg@10": 0.705365275925303,
430
+ "ndcg_linear@5": 0.685966662372165
431
+ },
432
+ "MSR-VTT": {
433
+ "hit@1": 0.509,
434
+ "hit@5": 0.736,
435
+ "mrr": 0.6135421236674015,
436
+ "ndcg@5": 0.6313083725269414,
437
+ "ndcg@10": 0.6570741924231366,
438
+ "ndcg_linear@5": 0.6313083725269414
439
+ },
440
+ "MSVD": {
441
+ "hit@1": 0.6746268656716418,
442
+ "hit@5": 0.9,
443
+ "mrr": 0.7753262426312134,
444
+ "ndcg@5": 0.7988314316025045,
445
+ "ndcg@10": 0.817775007508517,
446
+ "ndcg_linear@5": 0.7988314316025045
447
+ },
448
+ "MVBench": {
449
+ "hit@1": 0.438,
450
+ "hit@5": 1.0,
451
+ "mrr": 0.6615541666666666,
452
+ "ndcg@5": 0.7469911287021453,
453
+ "ndcg@10": 0.7469911287021453,
454
+ "ndcg_linear@5": 0.7469911287021453
455
+ },
456
+ "MomentSeeker": {
457
+ "hit@1": 0.4787765293383271,
458
+ "hit@5": 0.9051186017478152,
459
+ "mrr": 0.6551114182668489,
460
+ "ndcg@5": 0.7080558629435917,
461
+ "ndcg@10": 0.7389227282890873,
462
+ "ndcg_linear@5": 0.7080558629435917
463
+ },
464
+ "NExTQA": {
465
+ "hit@1": 0.5274404483886035,
466
+ "hit@5": 1.0,
467
+ "mrr": 0.7048906274326638,
468
+ "ndcg@5": 0.7786147159635359,
469
+ "ndcg@10": 0.7786147159635359,
470
+ "ndcg_linear@5": 0.7786147159635359
471
+ },
472
+ "QVHighlight": {
473
+ "hit@1": 0.7820867959372114,
474
+ "hit@5": 0.96398891966759,
475
+ "mrr": 0.8643011036362837,
476
+ "ndcg@5": 0.8859457139278087,
477
+ "ndcg@10": 0.8977481508549356,
478
+ "ndcg_linear@5": 0.8859457139278087
479
+ },
480
+ "SmthSmthV2": {
481
+ "hit@1": 0.598,
482
+ "hit@5": 0.833,
483
+ "mrr": 0.7048300295408914,
484
+ "ndcg@5": 0.7247244109117655,
485
+ "ndcg@10": 0.7543879226981658,
486
+ "ndcg_linear@5": 0.7247244109117655
487
+ },
488
+ "UCF101": {
489
+ "hit@1": 0.759,
490
+ "hit@5": 0.936,
491
+ "mrr": 0.8370420752469588,
492
+ "ndcg@5": 0.8576768765002054,
493
+ "ndcg@10": 0.8647014016643914,
494
+ "ndcg_linear@5": 0.8576768765002054
495
+ },
496
+ "VATEX": {
497
+ "hit@1": 0.4805716837874051,
498
+ "hit@5": 0.7778025904421617,
499
+ "mrr": 0.6104341878050024,
500
+ "ndcg@5": 0.6389041825618447,
501
+ "ndcg@10": 0.6680834179015435,
502
+ "ndcg_linear@5": 0.6389041825618447
503
+ },
504
+ "Video-MME": {
505
+ "hit@1": 0.3788888888888889,
506
+ "hit@5": 1.0,
507
+ "mrr": 0.6118827160493827,
508
+ "ndcg@5": 0.7091309958379056,
509
+ "ndcg@10": 0.7091309958379056,
510
+ "ndcg_linear@5": 0.7091309958379056
511
+ },
512
+ "YouCook2": {
513
+ "hit@1": 0.17080843032400125,
514
+ "hit@5": 0.37653350110097517,
515
+ "mrr": 0.2714840957655189,
516
+ "ndcg@5": 0.27757283647971825,
517
+ "ndcg@10": 0.3095328244106524,
518
+ "ndcg_linear@5": 0.27757283647971825
519
+ }
520
+ },
521
+ "visdoc": {
522
+ "MMLongBench-doc": {
523
+ "ndcg@5": 0.450758814236983,
524
+ "ndcg@10": 0.4241051274696162,
525
+ "hit@1": 0.6264916467780429,
526
+ "hit@5": 0.7434367541766109,
527
+ "mrr": 0.6846131830285731,
528
+ "num_queries": 838,
529
+ "num_corpus": 6492,
530
+ "ndcg_linear@5": 0.450758814236983
531
+ },
532
+ "MMLongBench-page-fixed": {
533
+ "ndcg@5": 0.4634150424163121,
534
+ "ndcg@10": 0.4860724008747348,
535
+ "hit@1": 0.39618138424821003,
536
+ "hit@5": 0.6085918854415274,
537
+ "mrr": 0.4928192419881348,
538
+ "num_queries": 838,
539
+ "num_corpus": 6492,
540
+ "ndcg_linear@5": 0.4634150424163121
541
+ },
542
+ "ViDoSeek-page-fixed": {
543
+ "ndcg@5": 0.8229202309191093,
544
+ "ndcg@10": 0.8371070850074852,
545
+ "hit@1": 0.6882661996497373,
546
+ "hit@5": 0.9316987740805605,
547
+ "mrr": 0.7932916167999776,
548
+ "num_queries": 1142,
549
+ "num_corpus": 5349,
550
+ "ndcg_linear@5": 0.8229202309191093
551
+ },
552
+ "ViDoSeek-doc": {
553
+ "ndcg@5": 0.7949515684801567,
554
+ "ndcg@10": 0.7561461985950078,
555
+ "hit@1": 0.9859894921190894,
556
+ "hit@5": 0.9964973730297724,
557
+ "mrr": 0.9900663391894617,
558
+ "num_queries": 1142,
559
+ "num_corpus": 5349,
560
+ "ndcg_linear@5": 0.7949515684801567
561
+ },
562
+ "VisRAG_ArxivQA": {
563
+ "ndcg@5": 0.6814418846609271,
564
+ "ndcg@10": 0.6973338955764089,
565
+ "hit@1": 0.6066176470588235,
566
+ "hit@5": 0.7438725490196079,
567
+ "mrr": 0.6718777424078283,
568
+ "num_queries": 816,
569
+ "num_corpus": 8066,
570
+ "ndcg_linear@5": 0.6814418846609271
571
+ },
572
+ "VisRAG_ChartQA": {
573
+ "ndcg@5": 0.7232574696726612,
574
+ "ndcg@10": 0.739573013339356,
575
+ "hit@1": 0.6031746031746031,
576
+ "hit@5": 0.8095238095238095,
577
+ "mrr": 0.7055757452072949,
578
+ "num_queries": 63,
579
+ "num_corpus": 500,
580
+ "ndcg_linear@5": 0.7232574696726612
581
+ },
582
+ "VisRAG_InfoVQA": {
583
+ "ndcg@5": 0.8325872362815218,
584
+ "ndcg@10": 0.8438526239963108,
585
+ "hit@1": 0.7381615598885793,
586
+ "hit@5": 0.9066852367688022,
587
+ "mrr": 0.8144302013277015,
588
+ "num_queries": 718,
589
+ "num_corpus": 459,
590
+ "ndcg_linear@5": 0.8325872362815218
591
+ },
592
+ "VisRAG_MP-DocVQA": {
593
+ "ndcg@5": 0.6766858922125144,
594
+ "ndcg@10": 0.6918402397804315,
595
+ "hit@1": 0.5617597292724196,
596
+ "hit@5": 0.7749576988155669,
597
+ "mrr": 0.6559263988613074,
598
+ "num_queries": 591,
599
+ "num_corpus": 741,
600
+ "ndcg_linear@5": 0.6766858922125144
601
+ },
602
+ "VisRAG_PlotQA": {
603
+ "ndcg@5": 0.5551899440867997,
604
+ "ndcg@10": 0.5837983923332378,
605
+ "hit@1": 0.41367323290845887,
606
+ "hit@5": 0.6790266512166859,
607
+ "mrr": 0.5345889606958261,
608
+ "num_queries": 863,
609
+ "num_corpus": 9593,
610
+ "ndcg_linear@5": 0.5551899440867997
611
+ },
612
+ "VisRAG_SlideVQA": {
613
+ "ndcg@5": 0.8908274602204853,
614
+ "ndcg@10": 0.8956744857199754,
615
+ "hit@1": 0.8399280575539568,
616
+ "hit@5": 0.9622302158273381,
617
+ "mrr": 0.8946820302162474,
618
+ "num_queries": 556,
619
+ "num_corpus": 1284,
620
+ "ndcg_linear@5": 0.8908274602204853
621
+ },
622
+ "ViDoRe_arxivqa": {
623
+ "ndcg@5": 0.7056352690768556,
624
+ "ndcg@10": 0.7196746763356245,
625
+ "hit@1": 0.632,
626
+ "hit@5": 0.768,
627
+ "mrr": 0.6964459758559202,
628
+ "num_queries": 500,
629
+ "num_corpus": 500,
630
+ "ndcg_linear@5": 0.7056352690768556
631
+ },
632
+ "ViDoRe_biomedical_lectures_v2_multilingual": {
633
+ "ndcg@5": 0.4699178696485379,
634
+ "ndcg@10": 0.4967771084596303,
635
+ "hit@1": 0.415625,
636
+ "hit@5": 0.7203125,
637
+ "mrr": 0.5485551214298156,
638
+ "num_queries": 640,
639
+ "num_corpus": 1016,
640
+ "ndcg_linear@5": 0.4699178696485379
641
+ },
642
+ "ViDoRe_docvqa": {
643
+ "ndcg@5": 0.3659572005602061,
644
+ "ndcg@10": 0.3873734103499988,
645
+ "hit@1": 0.2771618625277162,
646
+ "hit@5": 0.4523281596452328,
647
+ "mrr": 0.3609032947623907,
648
+ "num_queries": 451,
649
+ "num_corpus": 500,
650
+ "ndcg_linear@5": 0.3659572005602061
651
+ },
652
+ "ViDoRe_economics_reports_v2_multilingual": {
653
+ "ndcg@5": 0.5381426949845595,
654
+ "ndcg@10": 0.5253333849395884,
655
+ "hit@1": 0.6379310344827587,
656
+ "hit@5": 0.8793103448275862,
657
+ "mrr": 0.7452132546468191,
658
+ "num_queries": 232,
659
+ "num_corpus": 452,
660
+ "ndcg_linear@5": 0.5381426949845595
661
+ },
662
+ "ViDoRe_esg_reports_human_labeled_v2": {
663
+ "ndcg@5": 0.7822251570313639,
664
+ "ndcg@10": 0.8491451223959837,
665
+ "hit@1": 0.5384615384615384,
666
+ "hit@5": 0.6538461538461539,
667
+ "mrr": 0.5841922540573646,
668
+ "num_queries": 104,
669
+ "num_corpus": 3076,
670
+ "ndcg_linear@5": 0.7822251570313639
671
+ },
672
+ "ViDoRe_esg_reports_v2_multilingual": {
673
+ "ndcg@5": 0.5425137076002731,
674
+ "ndcg@10": 0.5786672525435137,
675
+ "hit@1": 0.5175438596491229,
676
+ "hit@5": 0.8333333333333334,
677
+ "mrr": 0.6559308879529052,
678
+ "num_queries": 228,
679
+ "num_corpus": 1538,
680
+ "ndcg_linear@5": 0.5425137076002731
681
+ },
682
+ "ViDoRe_infovqa": {
683
+ "ndcg@5": 0.7879220399881894,
684
+ "ndcg@10": 0.7993470701962244,
685
+ "hit@1": 0.6983805668016194,
686
+ "hit@5": 0.8603238866396761,
687
+ "mrr": 0.7719324086653583,
688
+ "num_queries": 494,
689
+ "num_corpus": 500,
690
+ "ndcg_linear@5": 0.7879220399881894
691
+ },
692
+ "ViDoRe_shiftproject": {
693
+ "ndcg@5": 0.6354026172209697,
694
+ "ndcg@10": 0.6701952698683278,
695
+ "hit@1": 0.48,
696
+ "hit@5": 0.79,
697
+ "mrr": 0.6023397892956717,
698
+ "num_queries": 100,
699
+ "num_corpus": 999,
700
+ "ndcg_linear@5": 0.6354026172209697
701
+ },
702
+ "ViDoRe_syntheticDocQA_artificial_intelligence": {
703
+ "ndcg@5": 0.8177553921539996,
704
+ "ndcg@10": 0.8309248329271962,
705
+ "hit@1": 0.71,
706
+ "hit@5": 0.9,
707
+ "mrr": 0.7971291554357591,
708
+ "num_queries": 100,
709
+ "num_corpus": 968,
710
+ "ndcg_linear@5": 0.8177553921539996
711
+ },
712
+ "ViDoRe_syntheticDocQA_energy": {
713
+ "ndcg@5": 0.7867065065186136,
714
+ "ndcg@10": 0.8031592266528728,
715
+ "hit@1": 0.69,
716
+ "hit@5": 0.86,
717
+ "mrr": 0.7727233547701909,
718
+ "num_queries": 100,
719
+ "num_corpus": 975,
720
+ "ndcg_linear@5": 0.7867065065186136
721
+ },
722
+ "ViDoRe_syntheticDocQA_government_reports": {
723
+ "ndcg@5": 0.818685816716124,
724
+ "ndcg@10": 0.8278610653972609,
725
+ "hit@1": 0.67,
726
+ "hit@5": 0.93,
727
+ "mrr": 0.786040158227229,
728
+ "num_queries": 100,
729
+ "num_corpus": 972,
730
+ "ndcg_linear@5": 0.818685816716124
731
+ },
732
+ "ViDoRe_syntheticDocQA_healthcare_industry": {
733
+ "ndcg@5": 0.8760132720993477,
734
+ "ndcg@10": 0.8893023645660051,
735
+ "hit@1": 0.79,
736
+ "hit@5": 0.94,
737
+ "mrr": 0.8612970085470084,
738
+ "num_queries": 100,
739
+ "num_corpus": 963,
740
+ "ndcg_linear@5": 0.8760132720993477
741
+ },
742
+ "ViDoRe_tabfquad": {
743
+ "ndcg@5": 0.9161691407732645,
744
+ "ndcg@10": 0.9173596169637408,
745
+ "hit@1": 0.85,
746
+ "hit@5": 0.9678571428571429,
747
+ "mrr": 0.9008970564440351,
748
+ "num_queries": 280,
749
+ "num_corpus": 70,
750
+ "ndcg_linear@5": 0.9161691407732645
751
+ },
752
+ "ViDoRe_tatdqa": {
753
+ "ndcg@5": 0.3474833939701265,
754
+ "ndcg@10": 0.38026550267203135,
755
+ "hit@1": 0.24301336573511542,
756
+ "hit@5": 0.44532199270959905,
757
+ "mrr": 0.34276758470384544,
758
+ "num_queries": 1646,
759
+ "num_corpus": 271,
760
+ "ndcg_linear@5": 0.3474833939701265
761
+ }
762
+ }
763
+ }
764
+ }