Skip to content

Commit b2f1bac

Browse files
zhongzhouTan-coderzhongzhouTan-coder
committed
[feature] add refcoco/+/g base64 support
1 parent 8aaa9fa commit b2f1bac

8 files changed

Lines changed: 282 additions & 91 deletions

File tree

‎ais_bench/benchmark/configs/datasets/refcoco/refcoco_gen.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,21 +33,21 @@
3333
)
3434

3535
_splits = [
36-
('RefCOCO_val', 'val'),
37-
('RefCOCO_test', 'test'),
38-
('RefCOCO_testA', 'testA'),
39-
('RefCOCO_testB', 'testB'),
36+
'val',
37+
'test',
38+
'testA',
39+
'testB',
4040
]
4141

4242
refcoco_datasets = [
4343
dict(
44-
abbr=abbr,
44+
abbr='RefCOCO_' + split,
4545
type=RefCOCODataset,
4646
path='ais_bench/datasets/RefCOCO/data',
4747
split=split,
4848
reader_cfg=refcoco_reader_cfg,
4949
infer_cfg=refcoco_infer_cfg,
5050
eval_cfg=refcoco_eval_cfg,
5151
)
52-
for abbr, split in _splits
52+
for split in _splits
5353
]

‎ais_bench/benchmark/configs/datasets/refcoco/refcoco_gen_base64.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
2+
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
3+
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
4+
from ais_bench.benchmark.datasets import RefCOCODataset
5+
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
6+
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator
7+
8+
9+
refcoco_reader_cfg = dict(
10+
input_columns=['question', 'image'],
11+
output_column='answer'
12+
)
13+
14+
refcoco_infer_cfg = dict(
15+
prompt_template=dict(
16+
type=MMPromptTemplate,
17+
template=dict(
18+
round=[
19+
dict(role='HUMAN', prompt_mm={
20+
'text': {'type': 'text', 'text': '{question}'},
21+
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
22+
})
23+
]
24+
)
25+
),
26+
retriever=dict(type=ZeroRetriever),
27+
inferencer=dict(type=GenInferencer),
28+
)
29+
30+
refcoco_eval_cfg = dict(
31+
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
32+
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
33+
)
34+
35+
_splits = [
36+
'val',
37+
'test',
38+
'testA',
39+
'testB',
40+
]
41+
42+
refcoco_datasets = [
43+
dict(
44+
abbr='RefCOCO_base64_' + split,
45+
type=RefCOCODataset,
46+
path='ais_bench/datasets/RefCOCO/data',
47+
split=split,
48+
image_type=IMAGE_BASE64_TYPE,
49+
reader_cfg=refcoco_reader_cfg,
50+
infer_cfg=refcoco_infer_cfg,
51+
eval_cfg=refcoco_eval_cfg,
52+
)
53+
for split in _splits
54+
]

‎ais_bench/benchmark/configs/datasets/refcoco_plus/refcoco_plus_gen.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88

99
refcoco_plus_reader_cfg = dict(
10-
input_columns=['content'],
10+
input_columns=['question', 'image'],
1111
output_column='answer'
1212
)
1313

@@ -33,20 +33,20 @@
3333
)
3434

3535
_splits = [
36-
('RefCOCOPlus_val', 'val'),
37-
('RefCOCOPlus_testA', 'testA'),
38-
('RefCOCOPlus_testB', 'testB'),
36+
'val',
37+
'testA',
38+
'testB',
3939
]
4040

4141
refcoco_plus_datasets = [
4242
dict(
43-
abbr=abbr,
43+
abbr='RefCOCOPlus_' + split,
4444
type=RefCOCOPlusDataset,
4545
path='ais_bench/datasets/RefCOCOplus/data',
4646
split=split,
4747
reader_cfg=refcoco_plus_reader_cfg,
4848
infer_cfg=refcoco_plus_infer_cfg,
4949
eval_cfg=refcoco_plus_eval_cfg,
5050
)
51-
for abbr, split in _splits
51+
for split in _splits
5252
]

‎ais_bench/benchmark/configs/datasets/refcoco_plus/refcoco_plus_gen_base64.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
2+
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
3+
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
4+
from ais_bench.benchmark.datasets import RefCOCOPlusDataset
5+
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
6+
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator
7+
8+
9+
refcoco_plus_reader_cfg = dict(
10+
input_columns=['question', 'image'],
11+
output_column='answer'
12+
)
13+
14+
refcoco_plus_infer_cfg = dict(
15+
prompt_template=dict(
16+
type=MMPromptTemplate,
17+
template=dict(
18+
round=[
19+
dict(role='HUMAN', prompt_mm={
20+
'text': {'type': 'text', 'text': '{question}'},
21+
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
22+
})
23+
]
24+
)
25+
),
26+
retriever=dict(type=ZeroRetriever),
27+
inferencer=dict(type=GenInferencer),
28+
)
29+
30+
refcoco_plus_eval_cfg = dict(
31+
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
32+
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
33+
)
34+
35+
_splits = [
36+
'val',
37+
'testA',
38+
'testB',
39+
]
40+
41+
refcoco_plus_datasets = [
42+
dict(
43+
abbr='RefCOCOPlus_base64_' + split,
44+
type=RefCOCOPlusDataset,
45+
path='ais_bench/datasets/RefCOCOplus/data',
46+
split=split,
47+
image_type=IMAGE_BASE64_TYPE,
48+
reader_cfg=refcoco_plus_reader_cfg,
49+
infer_cfg=refcoco_plus_infer_cfg,
50+
eval_cfg=refcoco_plus_eval_cfg,
51+
)
52+
for split in _splits
53+
]

‎ais_bench/benchmark/configs/datasets/refcocog/refcocog_gen.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88

99
refcocog_reader_cfg = dict(
10-
input_columns=['content'],
10+
input_columns=['question', 'image'],
1111
output_column='answer'
1212
)
1313

@@ -33,19 +33,19 @@
3333
)
3434

3535
_splits = [
36-
('RefCOCOg_val', 'val'),
37-
('RefCOCOg_test', 'test'),
36+
'val',
37+
'test',
3838
]
3939

4040
refcocog_datasets = [
4141
dict(
42-
abbr=abbr,
42+
abbr='RefCOCOg_' + split,
4343
type=RefCOCOgDataset,
4444
path='ais_bench/datasets/RefCOCOg/data',
4545
split=split,
4646
reader_cfg=refcocog_reader_cfg,
4747
infer_cfg=refcocog_infer_cfg,
4848
eval_cfg=refcocog_eval_cfg,
4949
)
50-
for abbr, split in _splits
50+
for split in _splits
5151
]

‎ais_bench/benchmark/configs/datasets/refcocog/refcocog_gen_base64.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
2+
from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
3+
from ais_bench.benchmark.openicl.icl_prompt_template import MMPromptTemplate
4+
from ais_bench.benchmark.datasets import RefCOCOgDataset
5+
from ais_bench.benchmark.datasets.refcoco import IMAGE_BASE64_TYPE, refcoco_bbox_postprocess
6+
from ais_bench.benchmark.openicl.icl_evaluator import BBoxIoUEvaluator
7+
8+
9+
refcocog_reader_cfg = dict(
10+
input_columns=['question', 'image'],
11+
output_column='answer'
12+
)
13+
14+
refcocog_infer_cfg = dict(
15+
prompt_template=dict(
16+
type=MMPromptTemplate,
17+
template=dict(
18+
round=[
19+
dict(role='HUMAN', prompt_mm={
20+
'text': {'type': 'text', 'text': '{question}'},
21+
'image': {'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,{image}'}},
22+
})
23+
]
24+
)
25+
),
26+
retriever=dict(type=ZeroRetriever),
27+
inferencer=dict(type=GenInferencer),
28+
)
29+
30+
refcocog_eval_cfg = dict(
31+
evaluator=dict(type=BBoxIoUEvaluator, iou_threshold=0.5, coord_scale=1000.0),
32+
pred_postprocessor=dict(type=refcoco_bbox_postprocess),
33+
)
34+
35+
_splits = [
36+
'val',
37+
'test',
38+
]
39+
40+
refcocog_datasets = [
41+
dict(
42+
abbr='RefCOCOg_base64_' + split,
43+
type=RefCOCOgDataset,
44+
path='ais_bench/datasets/RefCOCOg/data',
45+
split=split,
46+
image_type=IMAGE_BASE64_TYPE,
47+
reader_cfg=refcocog_reader_cfg,
48+
infer_cfg=refcocog_infer_cfg,
49+
eval_cfg=refcocog_eval_cfg,
50+
)
51+
for split in _splits
52+
]

‎ais_bench/benchmark/datasets/refcoco/__init__.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1-
from ais_bench.benchmark.datasets.refcoco.refcoco import RefCOCODataset, refcoco_bbox_postprocess # noqa: F401
1+
from ais_bench.benchmark.datasets.refcoco.refcoco import ( # noqa: F401
2+
IMAGE_BASE64_TYPE,
3+
IMAGE_PATH_TYPE,
4+
TEMP_IMAGE_STORE_DIR,
5+
RefCOCODataset,
6+
refcoco_bbox_postprocess,
7+
)
28
from ais_bench.benchmark.datasets.refcoco.refcoco_g import RefCOCOgDataset # noqa: F401
39
from ais_bench.benchmark.datasets.refcoco.refcoco_plus import RefCOCOPlusDataset # noqa: F401

0 commit comments

Comments
 (0)