Tfjs: Node - Shape does not match typed-array in bindData()

Created on 11 Feb 2019 · 6Comments · Source: tensorflow/tfjs

TensorFlow.js version

TFJS: 0.15.1
TFJS-Node: 0.3.0

Browser version

Node version: 11.6.0
NPM version: 6.7.0

Describe the problem or feature request

I was trying to access the webcam and use the posenet model on it, but when I estimate a single pose, I got this error:

(node:12844) UnhandledPromiseRejectionWarning: Error: Shape does not match typed-array in bindData() (num_elements=648, array_length=0)
    at NodeJSKernelBackend.getInputTensorIds (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-node\dist\nodejs_kernel_backend.js:146:38)
    at NodeJSKernelBackend.executeSingleOutput (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-node\dist\nodejs_kernel_backend.js:186:73)
    at NodeJSKernelBackend.conv2d (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-node\dist\nodejs_kernel_backend.js:650:21)
    at environment_1.ENV.engine.runKernel.x (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\ops\conv.js:85:86)
    at G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\engine.js:129:26
    at Engine.scopedRun (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\engine.js:101:23)
    at Engine.runKernel (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\engine.js:127:14)
    at conv2d_ (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\ops\conv.js:85:40)
    at Object.conv2d (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\ops\operation.js:23:29)
    at Tensor.conv2d (G:\NodeJS\node-posenet\node_modules\@tensorflow\tfjs-core\dist\tensor.js:808:26)

Code to reproduce the bug / link to feature request

const tf = require('@tensorflow/tfjs');

tf.disableDeprecationWarnings();

require('@tensorflow/tfjs-node');
const NodeWebcam = require('node-webcam');
const Posenet = require('@tensorflow-models/posenet');
global.XMLHttpRequest = require('xmlhttprequest').XMLHttpRequest;
const requestAnimationFrame = require('raf');
const { createCanvas, Image } = require('canvas');

const posenetConfiguration = {
  imageScaleFactor: 0.6,
  multiplier: 0.75,
  outputStride: 16,
  reversed: false,
};

const canvas = createCanvas(1920, 1080);
const ctx = canvas.getContext('2d');

let webcam = null;
let net = null;

const initWebcam = () => {
  return new Promise((resolve) => {
    webcam = NodeWebcam.create({
      callbackReturn: 'base64',
    });
    console.log('Webcam resolved');
    resolve();
  });
};

const initModel = () => {
  return new Promise((resolve, reject) => {
    Posenet.load(posenetConfiguration.multiplier)
      .then((posenet) => {
        net = posenet;
        console.log('Posenet resolved');
        resolve();
      })
      .catch((err) => {
        reject(err);
      });
  });
};

const getHand = (data) => {
  return new Promise((resolve, reject) => {
    net
      .estimateSinglePose(data, posenetConfiguration.imageScaleFactor, posenetConfiguration.reversed, posenetConfiguration.outputStride)
      .then((pose) => {
        const handKeyPoints = pose.keypoints.filter((item) => {
          return item.part === 'rightWrist' || item.part === 'leftWrist';
        });

        handKeyPoints.sort((a, b) => {
          return a.score > b.score ? 1 : -1;
        });

        resolve(this.getPartLocation(handKeyPoints[0]));
      })
      .catch((err) => {
        reject(err);
      });
  });
};

const loadImage = (base64) => {
  return new Promise((resolve) => {
    const image = new Image();
    image.onload = () => {
      ctx.drawImage(image, 0, 0, 1920, 1080);
      resolve(image);
    };

    image.src = base64;
  });
};

const render = () => {
  webcam.capture('webcam', (err, data) => {
    if (err) {
      console.error(err);
      return false;
    }

    loadImage(data).then((image) => {
      // const tfImage = tf.browser.fromPixels(canvas);

      getHand(canvas).then((hand) => {
        console.log(hand.position);
        requestAnimationFrame(render());
      });
    });
  });
};

initWebcam()
  .then(() => initModel())
  .then(() => {
    render();
  })
  .catch((err) => {
    console.error(err);
  });

Source