Detectron2: Visualize DensePose results

[ItamarSafriel]

Hello,
I wrote this code to use detectron's DensePose on a single image:

img = plt.imread("./image.jpg")
plt.imshow(img)

cfg = get_cfg()
add_densepose_config(cfg)

cfg.merge_from_file("../detectron2_repo/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x.yaml")
cfg.MODEL.DEVICE = "cpu"
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 

cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/143908701/model_final_dd99d2.pkl"
predictor = DefaultPredictor(cfg)
outputs = predictor(img)

How do I Visualize the results I get?
I found this notebook and this notebook but couldn't figure how to make it work (they use a .png file while I get an array of predictions).
Can some one please refer me to a notebook/tutorial (or just provide an explanation it here) that explains how to use the apply net tool (that is mentioned on the detectron2 github) but not through the terminal?

[ppwwyyxx]

Visualization can be done using apply_net.py and you can see how it is implemented to write your own https://github.com/facebookresearch/detectron2/blob/master/projects/DensePose/apply_net.py#L174

[italojs]

import cv2
import numpy as np

from typing import ClassVar, Dict

from detectron2.config import get_cfg
from detectron2.structures.instances import Instances
from detectron2.engine.defaults import DefaultPredictor

from densepose import add_densepose_config
from densepose.vis.base import CompoundVisualizer
from densepose.vis.bounding_box import ScoredBoundingBoxVisualizer
from densepose.vis.extractor import CompoundExtractor, create_extractor

from densepose.vis.densepose import (
    DensePoseResultsContourVisualizer,
    DensePoseResultsFineSegmentationVisualizer,
    DensePoseResultsUVisualizer,
    DensePoseResultsVVisualizer,
)

cfg = get_cfg()
add_densepose_config(cfg)

cfg.merge_from_file("configs/densepose_rcnn_R_50_FPN_s1x.yaml")
cfg.MODEL.DEVICE = "cuda"
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 

cfg.MODEL.WEIGHTS = "densepose_rcnn_R_50_FPN_s1x.pkl"
predictor = DefaultPredictor(cfg)

VISUALIZERS: ClassVar[Dict[str, object]] = {
    "dp_contour": DensePoseResultsContourVisualizer,
    "dp_segm": DensePoseResultsFineSegmentationVisualizer,
    "dp_u": DensePoseResultsUVisualizer,
    "dp_v": DensePoseResultsVVisualizer,
    "bbox": ScoredBoundingBoxVisualizer,
}

vis_specs = ['dp_contour', 'bbox']
visualizers = []
extractors = []
for vis_spec in vis_specs:
    vis = VISUALIZERS[vis_spec]()
    visualizers.append(vis)
    extractor = create_extractor(vis)
    extractors.append(extractor)
visualizer = CompoundVisualizer(visualizers)
extractor = CompoundExtractor(extractors)

context = {
    "extractor": extractor,
    "visualizer": visualizer
}

visualizer = context["visualizer"]
extractor = context["extractor"]

captura = cv2.VideoCapture(0)

def predict(img):
    outputs = predictor(img)['instances']
    image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    image = np.tile(image[:, :, np.newaxis], [1, 1, 3])
    data = extractor(outputs)
    image_vis = visualizer.visualize(image, data)
    return image_vis

while(1):
    ret, frame = captura.read()
    cv2.imshow("Video", predict(frame))

    k = cv2.waitKey(30) & 0xff
    if k == 27:
        break

captura.release()
cv2.destroyAllWindows()

[ItamarSafriel]

import cv2
import numpy as np

from typing import ClassVar, Dict

from detectron2.config import get_cfg
from detectron2.structures.instances import Instances
from detectron2.engine.defaults import DefaultPredictor

from densepose import add_densepose_config
from densepose.vis.base import CompoundVisualizer
from densepose.vis.bounding_box import ScoredBoundingBoxVisualizer
from densepose.vis.extractor import CompoundExtractor, create_extractor

from densepose.vis.densepose import (
    DensePoseResultsContourVisualizer,
    DensePoseResultsFineSegmentationVisualizer,
    DensePoseResultsUVisualizer,
    DensePoseResultsVVisualizer,
)

cfg = get_cfg()
add_densepose_config(cfg)

cfg.merge_from_file("configs/densepose_rcnn_R_50_FPN_s1x.yaml")
cfg.MODEL.DEVICE = "cuda"
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 

cfg.MODEL.WEIGHTS = "densepose_rcnn_R_50_FPN_s1x.pkl"
predictor = DefaultPredictor(cfg)

VISUALIZERS: ClassVar[Dict[str, object]] = {
    "dp_contour": DensePoseResultsContourVisualizer,
    "dp_segm": DensePoseResultsFineSegmentationVisualizer,
    "dp_u": DensePoseResultsUVisualizer,
    "dp_v": DensePoseResultsVVisualizer,
    "bbox": ScoredBoundingBoxVisualizer,
}

vis_specs = ['dp_contour', 'bbox']
visualizers = []
extractors = []
for vis_spec in vis_specs:
    vis = VISUALIZERS[vis_spec]()
    visualizers.append(vis)
    extractor = create_extractor(vis)
    extractors.append(extractor)
visualizer = CompoundVisualizer(visualizers)
extractor = CompoundExtractor(extractors)

context = {
    "extractor": extractor,
    "visualizer": visualizer
}

visualizer = context["visualizer"]
extractor = context["extractor"]

captura = cv2.VideoCapture(0)

def predict(img):
    outputs = predictor(img)['instances']
    image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    image = np.tile(image[:, :, np.newaxis], [1, 1, 3])
    data = extractor(outputs)
    image_vis = visualizer.visualize(image, data)
    return image_vis

while(1):
    ret, frame = captura.read()
    cv2.imshow("Video", predict(frame))

    k = cv2.waitKey(30) & 0xff
    if k == 27:
        break

captura.release()
cv2.destroyAllWindows()

Thank you very much! I would never be able to figure this out on my own! (maybe a sample code could be added to the densepose repo?)

btw, after I get all the data from the densepose model, can I get an approximation of the person 'width'? that is, the estimated distance between the front and the back of the head? (or any pair of points in 3D space?)

[italojs]

you're welcome :)
approximation of the person width is another topic, can you create another issue, please?
if the Visualize DensePose results issue has been solved, please, close this :)