tasks/<task_name>/
├── <task_name>.py              # Task config (one per split/variant) [REQUIRED]
├── process.py                  # Data processor                      [REQUIRED for HuggingFace data]
├── evaluate.py                 # Custom eval function                [OPTIONAL]
├── post_prompt.py              # Shared prompt config via _base_     [OPTIONAL]
└── <name>_evaluator.py         # Custom evaluator class              [OPTIONAL]

from datasets import load_dataset
ds = load_dataset("org/DatasetName", split="test")
print(ds.column_names, len(ds))
for k, v in ds[0].items():
    if k != "image":
        print(f"  {k}: {repr(v)[:200]}")

import os
import os.path as osp
import json
from datasets import load_dataset
def process(cfg):
    dataset = load_dataset(cfg.dataset_path, split=cfg.split)
    output_dir = osp.join(cfg.processed_dataset_path, cfg.split)
    img_dir = osp.join(output_dir, "img")
    os.makedirs(img_dir, exist_ok=True)
    content = []
    for item in dataset:
        entry = {
            "question_id": item["id"],
            "question": item["question"],
            "question_type": "multiple-choice",
            "answer": item["answer"],
            "options": item["options"],
            "img_path": osp.join("img", f"{item['id']}.png"),
        }
        item["image"].save(osp.join(output_dir, entry["img_path"]))
        content.append(entry)
    with open(osp.join(output_dir, "data.json"), "w") as f:
        json.dump(content, f, indent=2, ensure_ascii=False)

entry["img_path"] = []
for i in range(num_images):
    path = f"img/{qid}_{i + 1}.jpg"
    entry["img_path"].append(path)
    item[f"image_{i + 1}"].save(osp.join(output_dir, path))
img_prefix = "".join(f"<image {i + 1}> " for i in range(len(entry["img_path"])))
entry["question"] = img_prefix + entry["question"]

dataset = dict(
    type="VqaBaseDataset",
    data_root="/path/to/data",
    anno_file="data.json",       # or a list of json files
    name="my_task",
)

config = dict(
    dataset_path="org/DatasetName",
    split="test",
    processed_dataset_path="DatasetName",
    processor="process.py",
)
dataset = dict(
    type="VqaBaseDataset",
    config=config,
    prompt_template=dict(type="PromptTemplate"),
    name="my_task_test",
)
evaluator = dict(type="BaseEvaluator")

register_evaluator = {"my_evaluator.py": "MyEvaluator"}
dataset = dict(...)
evaluator = dict(type="MyEvaluator", custom_param="value")

# post_prompt.py
def post_prompt_func(annotation: dict, **kwargs):
    return "Answer with the option letter."
dataset = dict(
    type="VqaBaseDataset",
    prompt_template=dict(type="PromptTemplate", post_prompt=post_prompt_func),
)

# my_task_test.py
_base_ = "post_prompt.py"
config = dict(...)
dataset = dict(type="VqaBaseDataset", config=config, name="my_task_test")
evaluator = dict(...)

evaluator = dict(type="BaseEvaluator", detailed_keys=["sub_task"])

# evaluate.py
def get_result(annotations: dict, predictions: list) -> dict:
    """
    annotations: dict keyed by question_id (string)
    predictions: list of dicts with "question_id", "answer", etc.
    Returns: dict of metric_name -> value
    """
    correct = 0
    for pred in predictions:
        qid = str(pred["question_id"])
        gt = annotations[qid]
        if pred["answer"].strip() == gt["answer"].strip():
            correct += 1
    return {"accuracy": round(correct / len(predictions) * 100, 2)}

from flagevalmm.evaluator import BaseEvaluator
from flagevalmm.registry import EVALUATORS
@EVALUATORS.register_module()
class MyEvaluator(BaseEvaluator):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
    def cal_accuracy(self, annotations, predictions, *args, **kwargs):
        results = {}
        # ... compute metrics from annotations and predictions ...
        return results

def post_prompt_func(annotation: dict, **kwargs):
    qtype = annotation.get("question_type", "")
    if qtype == "multiple-choice":
        return "Answer with the option letter only."
    else:
        return "Provide a brief answer."
dataset = dict(
    type="VqaBaseDataset",
    prompt_template=dict(type="PromptTemplate", post_prompt=post_prompt_func),
)

flagevalmm --tasks tasks/<task_name>/<task_name>.py \
           --cfg <model_config>.json \
           --try-run --debug

flagevalmm --tasks tasks/<task_name>/<task_name>.py \
           --cfg <model_config>.json

flagevalmm --tasks tasks/<task_name>/split_a.py tasks/<task_name>/split_b.py \
           --cfg <model_config>.json

{
    "model_name": "openai/gpt-5-mini",
    "api_key": "<your-api-key>",
    "url": "https://openrouter.ai/api/v1/chat/completions",
    "use_cache": false,
    "num_workers": 3,
    "max_image_size": 4718592,
    "max_tokens": 28000,
    "max_long_side": 1000,
    "output_dir": "results/openai/gpt-5-mini"
}

flagevalmm --tasks tasks/<task_name>/<task_name>.py \
           --without-infer \
           --output-dir ./previous_results

Is data on HuggingFace?
├── Yes → config dict + process.py
└── No  → data_root + anno_file (skip process.py)
 
Question format?
├── Multiple-choice → BaseEvaluator (automatic)
├── Open-ended      → ExtractEvaluator or OpenEvaluator (LLM judge)
├── Custom metric   → eval_func="evaluate.py" (simple) or custom class (complex)
└── Standard NLP    → CocoEvaluator, MmmuEvaluator, etc.
 
Multiple splits or variants?
├── Yes → _base_ = "post_prompt.py" for shared config
└── No  → inline prompt_template in task config
 
Modality?
├── Single image → VqaBaseDataset, img_path as string
├── Multi-image  → VqaBaseDataset, img_path as list, add <image N> tokens
├── Video        → VideoDataset, video_path field
└── Text-only    → VqaBaseDataset, no img_path