Darknet: Inputs from load_image(python) or array_to_image(arr) cause big detection difference under python?

[AlexLuya]

Demo code like this:

def detect(net, meta, im, thresh=.5, hier_thresh=.5, nms=.45):
    boxes = make_boxes(net)
    probs = make_probs(net)
    num =   num_boxes(net)
    network_detect(net, im, thresh, hier_thresh, nms, boxes, probs)
    res = []
    for j in range(num):
        for i in range(meta.classes):
            if probs[j][i] > 0:
                res.append((meta.names[i], probs[j][i], (boxes[j].x, boxes[j].y, boxes[j].w, boxes[j].h)))
    res = sorted(res, key=lambda x: -x[1])
    free_image(im)
    free_ptrs(cast(probs, POINTER(c_void_p)), num)
    return res

def array_to_image(arr):
    arr = arr.transpose(2,0,1)
    c = arr.shape[0]
    h = arr.shape[1]
    w = arr.shape[2]
    arr = (arr/255.0).flatten()
    data = c_array(c_float, arr)
    return IMAGE(w,h,c,data)

def detectAndShow(image):
    r = detect(net, meta, image)
    print "Object count:",len(r)
      #show result for debuging
    frame = cv2.imread("data/aoyang.png")
    for p in r:
        width =  int(p[2][2])
        height = int(p[2][3])
        left = int(p[2][0] - (width / 2))#center_x = p[2][0]
        top = int(p[2][1] - (height / 2))-10#center_y = p[2][1] 

        cv2.rectangle(frame,(left,top),(left+width,top+height),(255, 0, 0),2)
    cv2.imshow("Result", frame)
    cv2.waitKey(0)

net = load_net("cfg/yolo.cfg", "models/yolo-from-darknet.weights", 0)
meta = load_meta("cfg/coco.data")   
if __name__ == "__main__":
    #net = load_net("cfg/densenet201.cfg", "/home/pjreddie/trained/densenet201.weights", 0)
    #im = load_image("data/wolf.jpg", 0, 0)
    #meta = load_meta("cfg/imagenet1k.data")
    #r = classify(net, meta, im)
    #print r[:10]
    detectAndShow(load_image("data/aoyang.png", 0, 0))

    detectAndShow(array_to_image(cv2.imread("data/aoyang.png")))

detectAndShow(load_image("data/aoyang.png", 0, 0)) result

detectAndShow(array_to_image(cv2.imread("data/aoyang.png"))) result

[sayanmutd]

@AlexLuya I have used your code, but i'm getting segmentation fault. Could me explain what does load_image takes as input?

[raubtaube]

I have the same issue and suspect, that the image conversion is somehow affecting the image. The image is either loaded using

load_image(image, 0, 0)

or for numpy arrays using

def numpy2darknet_img(image): # v1
    image = (image / 255.).astype(np.float32)
    # image = image[..., ::-1]

    assert (image.shape[2] == 3)

    # convert rgb image(h, w, c) to bgr (c, h, w)
    # image = np.flip(image, 2)
    data = np.swapaxes(image, 2, 1)
    data = np.swapaxes(data, 1, 0)
    print("type: ", data.shape, data.dtype)

    data = np.ascontiguousarray(data)

    im = IMAGE(w=image.shape[1], h=image.shape[0], c=3, data=data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)))
    return im

or

def nparray_to_image(img): # v2
    data = img.ctypes.data_as(POINTER(c_ubyte))
    image = ndarray_image(data, img.ctypes.shape, img.ctypes.strides)
    return image

I get the following differences in scores and also detections using yolov3.weights and cfg:

type:  (3, 448, 352) float32
type:  (448, 352, 3) uint8
scream.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file |   labels v1 |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | bed           |      0.251978 |         nan |         nan | suitcase    |    0.164378 |
|  1 | suitcase      |      0.204248 |         nan |         nan | nan         |  nan        |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 500, 500) float32
type:  (500, 500, 3) uint8
giraffe.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file |   labels v1 |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | giraffe       |      0.997737 |         nan |         nan | giraffe     |    0.998335 |
|  1 | zebra         |      0.943354 |         nan |         nan | zebra       |    0.921862 |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 424, 640) float32
type:  (424, 640, 3) uint8
person.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file |   labels v1 |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | person        |      0.999897 |         nan |         nan | person      |    0.99973  |
|  1 | horse         |      0.99682  |         nan |         nan | horse       |    0.99826  |
|  2 | dog           |      0.992973 |         nan |         nan | dog         |    0.997563 |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 512, 773) float32
type:  (512, 773, 3) uint8
horses.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file | labels v1   |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | horse         |      0.975982 | horse       |    0.975019 | horse       |    0.965628 |
|  1 | horse         |      0.971411 | horse       |    0.971272 | horse       |    0.939207 |
|  2 | horse         |      0.910332 | horse       |    0.910194 | horse       |    0.885436 |
|  3 | horse         |      0.890003 | horse       |    0.890166 | horse       |    0.84182  |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 512, 773) float32
type:  (512, 773, 3) uint8
eagle.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file | labels v1   |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | bird          |      0.996067 | bird        |    0.993382 | bird        |    0.984983 |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 900, 1352) float32
type:  (900, 1352, 3) uint8
kite.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file | labels v1   |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | person        |      0.98291  | person      |    0.98291  | person      |    0.985033 |
|  1 | person        |      0.978767 | person      |    0.97877  | person      |    0.979192 |
|  2 | kite          |      0.978025 | kite        |    0.978025 | kite        |    0.978203 |
|  3 | person        |      0.967153 | person      |    0.96715  | person      |    0.974742 |
|  4 | person        |      0.941577 | person      |    0.94158  | person      |    0.929231 |
|  5 | person        |      0.938144 | person      |    0.938148 | kite        |    0.900459 |
|  6 | person        |      0.931154 | person      |    0.93115  | person      |    0.893875 |
|  7 | kite          |      0.906267 | kite        |    0.906279 | person      |    0.885582 |
|  8 | kite          |      0.849734 | kite        |    0.849733 | kite        |    0.812317 |
|  9 | kite          |      0.811424 | kite        |    0.811424 | kite        |    0.767514 |
| 10 | kite          |      0.786251 | kite        |    0.786256 | kite        |    0.707421 |
| 11 | person        |      0.708312 | person      |    0.70832  | person      |    0.705917 |
| 12 | kite          |      0.702775 | kite        |    0.702784 | kite        |    0.435649 |
| 13 | kite          |      0.53671  | kite        |    0.536706 | person      |    0.390132 |
| 14 | person        |      0.528164 | person      |    0.528169 | kite        |    0.336128 |
| 15 | person        |      0.273023 | person      |    0.273021 | person      |    0.335488 |
| 16 | surfboard     |      0.265535 | surfboard   |    0.265538 | surfboard   |    0.31242  |
| 17 | person        |      0.201481 | person      |    0.201483 | person      |    0.217044 |
| 18 | person        |      0.187219 | person      |    0.187221 | surfboard   |    0.198175 |
| 19 | surfboard     |      0.172651 | surfboard   |    0.172658 | person      |    0.173972 |
| 20 | person        |      0.156615 | person      |    0.156621 | surfboard   |    0.168909 |
| 21 | surfboard     |      0.153914 | surfboard   |    0.153912 | person      |    0.163669 |
| 22 | surfboard     |      0.14266  | surfboard   |    0.142645 | bird        |    0.162107 |
| 23 | person        |      0.129112 | person      |    0.129123 | surfboard   |    0.132225 |
| 24 | nan           |    nan        | nan         |  nan        | surfboard   |    0.131453 |
| 25 | nan           |    nan        | nan         |  nan        | bird        |    0.12721  |
| 26 | nan           |    nan        | nan         |  nan        | person      |    0.101212 |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

type:  (3, 576, 768) float32
type:  (576, 768, 3) uint8
dog.jpg
+----+---------------+---------------+-------------+-------------+-------------+-------------+
|    | labels file   |   scores file | labels v1   |   scores v1 | labels v2   |   scores v2 |
|----+---------------+---------------+-------------+-------------+-------------+-------------|
|  0 | bicycle       |      0.994193 | chair       |           1 | bicycle     |    0.9982   |
|  1 | dog           |      0.99046  | backpack    |           1 | dog         |    0.995851 |
|  2 | truck         |      0.911003 | sports ball |           1 | truck       |    0.929284 |
|  3 | bicycle       |      0.205394 | backpack    |           1 | car         |    0.111475 |
|  4 | car           |      0.12637  | sports ball |           1 | nan         |  nan        |
|  5 | motorbike     |      0.111666 | backpack    |           1 | nan         |  nan        |
|  6 | nan           |    nan        | sports ball |           1 | nan         |  nan        |
|  7 | nan           |    nan        | chair       |           1 | nan         |  nan        |
|  8 | nan           |    nan        | chair       |           1 | nan         |  nan        |
|  9 | nan           |    nan        | chair       |           1 | nan         |  nan        |
+----+---------------+---------------+-------------+-------------+-------------+-------------+

[raubtaube]

It was indeed due to how the image arrays are passed to darknet. V2 to works now, and give the exact same results as if I would use load_image from darknet. V2 expects the images to be in BGR, so convert appropriately if you do not use opencv to read in images.

[dineshresearch]

I am was also facing the same issue. By passing the numpy array using the above mentioned code my detection rate and accuracy are getting affected (decreased). Please kindly help me out

Update: @raubtaube Thanks a lot by changing the colour space using the img =cv2.cvtColor(img, cv2.COLOR_RGB2BGR) I was able to solve the issue and now it works like charm. Best Solution and saved lot of time and effort