Tf-pose-estimation: Inference output on Java

[jax79sg]

Hi, i am writing some codes to port tf-pose-estimation to Android phones. TensorFlow Mobile Java for a start before moving on to Snapdragon Neural Processing Engine SDK running on DSP/GPU. Would like to check if anyone can guide on;

1. How to setup the output variable in Java (Last 2 lines below failed)
2. How to convert decode the output into the joint structures in Java

    //bitmap is android.graphics.Bitmap taken off the camera
    intValues = new int[bitmap.getWidth() * bitmap.getHeight()];  
    bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
    floatValues = new float[bitmap.getWidth() * bitmap.getWidth() * 3];
    for (int i = 0; i < intValues.length; ++i) 
    {
      floatValues[i * 3 + 0] = ((intValues[i] >> 16) & 0xFF) / 255.0f;
      floatValues[i * 3 + 1] = ((intValues[i] >> 8) & 0xFF) / 255.0f;
      floatValues[i * 3 + 2] = (intValues[i] & 0xFF) / 255.0f;
    }

    // Copy the input data into TensorFlow.
    inferenceInterface.feed("image", floatValues,1,bitmap.getWidth(), bitmap.getHeight(),3);

    // Run the inference call.
    inferenceInterface.run("Openpose/concat_stage7", logStats);

    float[] outputs0 = new float[19];
    inferenceInterface.fetch("Openpose/concat_stage7", outputs0); //Execution of this line fails with bufferoverflow

[ouceduxzk]

I will write some code for explaining and maybe java codes to make it running on android

[zyxcambridge]

private static final float MINIMUM_CONFIDENCE_TF_OD_API = 0.6f;
private static final String MODEL_FILE = "file:///android_asset/graph_opt.pb";

// private static final String MODEL_FILE = "file:///android_asset/fcn_310_2.h5.pb";
private static final String LABEL_FILE =
"file:///android_asset/retrained_graph_phone.txt";
private static final int TF_OD_API_INPUT_SIZE = 64;
private static final int TF_OD_API_INPUT_SIZE_width = 368;
private static final int TF_OD_API_INPUT_SIZE_height = 368;

private static final int IMAGE_MEAN = 128;
private static final float IMAGE_STD = 128.0f;
private static final String INPUT_NAME = "image";
private static final String OUTPUT_NAME = "Openpose/concat_stage7";


    c.intValues = new int[inputSize_W * inputSize_H];
    c.floatValues = new float[inputSize_W * inputSize_H * 3];
    c.outputs = new float[46*46*57];



    Trace.beginSection("preprocessBitmap");
    // Preprocess the image data from 0-255 int to normalized float based
    // on the provided parameters.
    bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
    for (int i = 0; i < intValues.length; ++i) {
        final int val = intValues[i];
        //以下为改纯RGB输入
        floatValues[i * 3 + 2] = ((val >> 16) & 0xFF); //R
        floatValues[i * 3 + 1] = ((val >> 8) & 0xFF);  //G
        floatValues[i * 3 + 0] = (val & 0xFF);        //B
    }



    // Copy the output Tensor back into the output array.
    Trace.beginSection("fetch");
    inferenceInterface.fetch(outputName, outputs);

    Log.e("outputs inputSize_W", String.valueOf(inputSize_W));
    Log.e("outputs inputSize_H", String.valueOf((inputSize_H)));
    Log.e("outputs outputs", String.valueOf((outputs)));
    Log.e("tensorflow intent time",System.currentTimeMillis() - outTime +"   ");

// try {
// BufferedWriter bw = new BufferedWriter( new OutputStreamWriter(new FileOutputStream(Environment.getExternalStorageDirectory()+"/outputs.txt")));
// bw.write(Arrays.toString(outputs),0,Arrays.toString(outputs).length());
// bw.flush(); bw.close();
// }
// catch (FileNotFoundException e) {
// e.printStackTrace();
// }
// catch (IOException e) {
// e.printStackTrace();
// }

    long zhuanhuan = System.currentTimeMillis();
    int[] b = new int[46 * 46 * 57];
    float[] floats = null;
    Log.e(" 长度", outputs.length + "");
    float[][] floats5 = new float[(outputs.length / 57) * 19][19];
    for (int i = 0, sum = -1; i < outputs.length; i++) {
        if (0 <= i % 57 && i % 57 < 19) {
            if (0 == i % 57)
                sum++;
            if (outputs[i] > 0.1)
                floats5[sum][i % 19] = outputs[i];

        }
    }
    //单纯去掉第19个数组
    float[][] floats2 = new float[(outputs.length / 57) * 19][18];
    for (int i = 0; i < floats5.length; i++) {
        System.arraycopy(floats5[i], 0, floats2[i], 0, 18);
    }
    float max = 0;
    float[] floats3 = new float[(outputs.length / 57) * 19];
    for (int i = 0; i < floats2.length; i++) {
        for (int j = 0; j < floats2[i].length; j++) {
            if (floats2[i][j] > max)
                max = floats2[i][j];
        }
        floats3[i] = max;
        max = 0;
    }

    float[] floats4 = new float[((outputs.length / 57) * 19) * 18];
    Log.e("abcde", floats2.length + " ");
    for (int i = 0, sum = 0; i < floats2.length; i++) {
        for (int j = 0; j < floats2[i].length; j++) {

            floats4[sum] = floats2[i][j];
            sum++;
        }
    }



    if (!ssss)
        try {
            ssss = true;

// OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(Environment.getExternalStorageDirectory() + "/5566.txt"));
FileWriter fw = new FileWriter(Environment.getExternalStorageDirectory() + "/5566.txt");
for (int i = 0; i < outputs.length; i++) {
fw.write((outputs[i]) + ",");
}
fw.flush();
Log.e("成功", "成功");
fw.close();
} catch (IOException e) {
e.printStackTrace();
}

    for (int j = 0; j < floats3.length; j++) {

        int c = (int) -(floats3[j] * 255);
        //4通道输出  分别是 ARGB  其中 A通道C<<24是为了让有颜色的地方透明度低些 让没有颜色的地方透明度高一些
        b[j] = c;

    }


    Log.e("zhuanhuan",System.currentTimeMillis() - zhuanhuan+" ");
    final Bitmap bitmap1 = Bitmap.createBitmap(b, 46, 46, Bitmap.Config.ARGB_8888);

    if (iv12 != null) {
        iv12.post(new Runnable() {
            @Override
            public void run() {
                iv12.setImageBitmap(bitmap1);
            }
        });
    }

    int res_len = outputs.length;
    Log.i(TAG, "inferenceInterface.fetch(outputName, outputs); " + " " + outputs[res_len - 4] + " " + outputs[res_len - 3] + " " + outputs[res_len - 2] + " " + outputs[res_len - 1]);

    Trace.endSection();

[GajjarMihir]

@zyxcambridge I got the output of 'Openpose/concat_stage7' node in to an array. Can you please explain the post processing steps that you performed on the output. And how to extract the coordinates from this array? Your help will be highly appreciated.

It would be great if anyone can help me.

[liljom]

I got it running in a desktop Java app. I'll post repo as soon as I cleaned it up.
If it's very urgent here is the source I used (an Android demo):
https://github.com/yuxitong/TensorFlowDemo

and the pose estimation part is:
Camera2BasicFragment3.java
TensorFlowImageClassifier3.java

I fixed it not being able to handle other then 1:1 ratio.

Update:
here is the cleaned up version: https://github.com/liljom/openpose-tf-mobilenet-java