Detectron2: Results do not correspond to current coco set

[skye95git]

If you do not know the root cause of the problem, and wish someone to help you, please
post according to this template:

Instructions To Reproduce the Issue:

Check https://stackoverflow.com/help/minimal-reproducible-example for how to ask good questions.
Simplify the steps to reproduce the issue using suggestions from the above link, and provide them below:

1. full code you wrote or full changes you made (git diff)

# import some common detectron2 utilities
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog

import os
import numpy as np
import json
import cv2
import random
from detectron2.utils.visualizer import Visualizer
from detectron2.structures import BoxMode

def get_vehicle_dicts(img_dir):
    img_name_list = [file_name for file_name in os.listdir(img_dir) \
                     if file_name.lower().endswith('jpg')]
    # print(img_name_list)

    dataset_dicts = []
    i = 0
    for file_name in img_name_list:
        i = i+1
        # print(file_name[0:-4])
        img_full_name = os.path.join(img_dir, file_name)
        # print(img_full_name)
        label_full_name = os.path.join(img_dir, file_name.replace(".jpg",'.json'))
        # print(label_full_name)

        record = {}
        height, width = cv2.imread(img_full_name).shape[:2]
        record["file_name"] = img_full_name
        # record["image_id"] = file_name[0:-4]
        record["image_id"] = i
        record["height"] = height
        record["width"] = width

        with open(label_full_name) as f:
            imgs_anns = json.load(f)
        # print(imgs_anns)
        # print(imgs_anns["shapes"][0]["points"][0])

        objs = []
        for region in imgs_anns["shapes"]:
            pointsList = np.array(region["points"])
            px = pointsList[:,0]
            # print(px)
            py = pointsList[:,1]
            # print(py)
            poly = [(x + 0.5, y + 0.5) for x, y in zip(px, py)]
            poly = [p for x in poly for p in x]
            obj = {
                "bbox": [np.min(px), np.min(py), np.max(px), np.max(py)],
                "bbox_mode": BoxMode.XYXY_ABS,
                "segmentation": [poly],
                "category_id": 0,
            }
            objs.append(obj)
        record["annotations"] = objs
        dataset_dicts.append(record)
    # print(dataset_dicts)
    return dataset_dicts


from detectron2.data import DatasetCatalog, MetadataCatalog

for d in ["train", "val"]:
    DatasetCatalog.register("vehicle_" + d, lambda d=d: get_vehicle_dicts("vehicle/" + d))
    MetadataCatalog.get("vehicle_" + d).set(thing_classes=["vehicle"])
    MetadataCatalog.get("vehicle_" + d).evaluator_type = "coco"
vehicle_metadata = MetadataCatalog.get("vehicle_train")

dataset_dicts = get_vehicle_dicts("vehicle/train")
for d in random.sample(dataset_dicts, 3):
    print(d["file_name"])
    img = cv2.imread(d["file_name"])
    visualizer = Visualizer(img[:, :, ::-1], metadata=vehicle_metadata, scale=0.5)
    out = visualizer.draw_dataset_dict(d)
    print(os.path.join('vehicle/label', d["file_name"][14:]))
    label_img = out.get_image()[:, :, ::-1]
    cv2.imwrite(os.path.join('vehicle/label', d["file_name"][14:]), label_img)

from detectron2.engine import DefaultTrainer
from detectron2.config import get_cfg

cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("vehicle_train",)
cfg.DATASETS.TEST = ()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")  # Let training initialize from model zoo
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = 0.00025  # pick a good LR
cfg.SOLVER.MAX_ITER = 300    # 300 iterations seems good enough for this toy dataset; you may need to train longer for a practical dataset
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   # faster, and good enough for this toy dataset (default: 512)
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  # only has one class (ballon)

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set the testing threshold for this model
cfg.DATASETS.TEST = ("vehicle_val", )
predictor = DefaultPredictor(cfg)

cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set the testing threshold for this model
cfg.DATASETS.TEST = ("vehicle_val", )
predictor = DefaultPredictor(cfg)

from detectron2.utils.visualizer import ColorMode
dataset_dicts = get_vehicle_dicts("vehicle/val")
for d in random.sample(dataset_dicts, 3):
    im = cv2.imread(d["file_name"])
    outputs = predictor(im)
    v = Visualizer(im[:, :, ::-1],
                   metadata=vehicle_metadata,
                   scale=0.8,
                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels
    )
    v = v.draw_instance_predictions(outputs["instances"].to("cpu"))
    res_img = v.get_image()[:, :, ::-1]
    cv2.imwrite(os.path.join('vehicle/res', d["file_name"][14:]), res_img)
    # cv2_imshow(v.get_image()[:, :, ::-1])

from detectron2.evaluation import COCOEvaluator, inference_on_dataset
from detectron2.data import build_detection_test_loader
evaluator = COCOEvaluator("vehicle_val", cfg, False, output_dir="./output/")
val_loader = build_detection_test_loader(cfg, "vehicle_val")
inference_on_dataset(trainer.model, val_loader, evaluator)

1. what exact command you run:
   python custom_dataset.py(Thecustom_dataset.py file contents are shown above)
2. __full logs__ you observed:

[06/19 09:55:19 d2.evaluation.coco_evaluation]: Evaluating predictions ...
Loading and preparing results...
Traceback (most recent call last):
  File "custom_dataset.py", line 148, in <module>
    inference_on_dataset(trainer.model, val_loader, evaluator)
  File "/public/home/linjiayi/PCL/detectron2/detectron2/evaluation/evaluator.py", line 176, in inference_on_dataset
    results = evaluator.evaluate()
  File "/public/home/linjiayi/PCL/detectron2/detectron2/evaluation/coco_evaluation.py", line 143, in evaluate
    self._eval_predictions(set(self._tasks), predictions)
  File "/public/home/linjiayi/PCL/detectron2/detectron2/evaluation/coco_evaluation.py", line 186, in _eval_predictions
    if len(coco_results) > 0
  File "/public/home/linjiayi/PCL/detectron2/detectron2/evaluation/coco_evaluation.py", line 493, in _evaluate_predictions_on_coco
    coco_dt = coco_gt.loadRes(coco_results)
  File "/public/home/linjiayi/.local/lib/python3.6/site-packages/pycocotools-2.0-py3.6-linux-x86_64.egg/pycocotools/coco.py", line 325, in loadRes
    'Results do not correspond to current coco set'
AssertionError: Results do not correspond to current coco set

Expected behavior:

I want to get the AP evaluated on my dataset.

If there are no obvious error in "what you observed" provided above,
please tell us the expected behavior.

If you expect the model to converge / work better, note that we do not give suggestions
on how to train a new model.
Only in one of the two conditions we will help with it:
(1) You're unable to reproduce the results in detectron2 model zoo.
(2) It indicates a detectron2 bug.

Environment:

Provide your environment information using the following command:

wget -nc -q https://github.com/facebookresearch/detectron2/raw/master/detectron2/utils/collect_env.py && python collect_env.py

If your issue looks like an installation issue / environment issue,
please first try to solve it with the instructions in
https://detectron2.readthedocs.io/tutorials/install.html#common-installation-issues

[ppwwyyxx]

This error can happen if get_vehicle_dicts does not return the same data each time it's called with the same argument, e.g. if it uses different image_id.
However, since no runnable code is provided, we're not able to reproduce it and see if it is the case.

[skye95git]

This error can happen if get_vehicle_dicts does not return the same data each time it's called with the same argument, e.g. if it uses different image_id.
However, since no runnable code is provided, we're not able to reproduce it and see if it is the case.

Thanks，but I don't understand what it means to return the same data. Does it mean that the same picture returns the same information every time? For example, the same image is going to labeled image_id:99 every time.

[skye95git]

This error can happen if get_vehicle_dicts does not return the same data each time it's called with the same argument, e.g. if it uses different image_id.
However, since no runnable code is provided, we're not able to reproduce it and see if it is the case.

Thank you, I have solved this problem. All I need to do is delete the previous files in the output folder.

[ppwwyyxx]

Sounds reasonable. I think you might have seen a warning that says "Using previously cached COCO format annotations at '/path/to/file'. You need to clear the cache file if your dataset has been modified."