Mediapipe: How to show the point position of fingers?

[crabbit0629]

I'm using the handtrackinggpu for detecting hands , but I want to know what is the each keypoint position of fingers? Any help greatly appreciated !!

[lisbravo]

[crabbit0629]

@lisbravo How do you do that ? can it show coordinates?

[lisbravo]

`
::mediapipe::Status LandmarksToRenderDataCalculator::Process(
CalculatorContext* cc) {
auto render_data = absl::make_unique();
bool visualize_depth = options_.visualize_landmark_depth();
float z_min = 0.f;
float z_max = 0.f;

if (cc->Inputs().HasTag(kLandmarksTag)) {
const auto& landmarks =
cc->Inputs().Tag(kLandmarksTag).Get>();
RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
<< "Number of entries in landmark connections must be a multiple of 2";
if (visualize_depth) {
GetMinMaxZ(landmarks, &z_min, &z_max);
}
// Only change rendering if there are actually z values other than 0.
visualize_depth &= ((z_max - z_min) > 1e-3);
for (const auto& landmark : landmarks) {
auto* landmark_data_render =
AddPointRenderData(options_, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(false);
landmark_data->set_x(landmark.x());
landmark_data->set_y(landmark.y());
}
if (visualize_depth) {
AddConnectionsWithDepth(landmarks, /normalized=/false, z_min, z_max,
render_data.get());
} else {
AddConnections(landmarks, /normalized=/false, render_data.get());
}
}

if (cc->Inputs().HasTag(kNormLandmarksTag)) {
const auto& landmarks = cc->Inputs()
.Tag(kNormLandmarksTag)
.Get>();
RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
<< "Number of entries in landmark connections must be a multiple of 2";
if (visualize_depth) {
GetMinMaxZ(landmarks, &z_min, &z_max);
}
// Only change rendering if there are actually z values other than 0.
visualize_depth &= ((z_max - z_min) > 1e-3);
for (const auto& landmark : landmarks) {
auto* landmark_data_render =
AddPointRenderData(options_, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(true);
landmark_data->set_x(landmark.x());
landmark_data->set_y(landmark.y());

  const unsigned char lmIndex = &landmark - &landmarks[0];
  auto* landmarks_text_render = render_data->add_render_annotations();
  auto* landmark_text =
  landmarks_text_render->mutable_text();
  std::string dispText = "LM:";
  dispText.append(std::to_string(lmIndex));
  landmark_text->set_normalized(true);
  landmark_text->set_display_text(dispText); 
  landmark_text->set_font_height(0.03);
  landmark_text->set_baseline(landmark.y());
  landmark_text->set_left(landmark.x() + 0.02);
  landmarks_text_render->mutable_color()->set_r(0);
  landmarks_text_render->mutable_color()->set_g(0);
  landmarks_text_render->mutable_color()-> set_b(255);
  landmarks_text_render->set_scene_tag(kLandmarkLabel);

  auto* landmarks_col_text_render = render_data->add_render_annotations();
  landmarks_col_text_render->mutable_color()->set_r(0);
  landmarks_col_text_render->mutable_color()->set_g(0);
  landmarks_col_text_render->mutable_color()-> set_b(255);
  landmarks_col_text_render->set_scene_tag(kLandmarkLabel);
  auto* landmark_col_text =
  landmarks_col_text_render->mutable_text();
  std::string dispColText = "LM:";
  dispColText.append(std::to_string(lmIndex));

  if((lmIndex>5) && (lmIndex<8)){
   /* dispColText.append(" X:");
    dispColText.append(std::to_string(landmark.x()));
    dispColText.append(" Y:");
    dispColText.append(std::to_string(landmark.y()));*/


    dispColText.append(" Ø:");   
    dispColText.append(std::to_string(angleBetweenLines(landmark.x(), landmark.y(),
                                          landmarks[lmIndex-1].x(),landmarks[lmIndex-1].y(),
                                          landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y())));  //float x0, float y0, float x1, float y1, float x2, float y2
    std::cout << dispColText << "\t"; 
    if(lmIndex==7) std::cout << "\n";      
  }

  if(lmIndex==5){
    /*dispColText.append(" X:");
    dispColText.append(std::to_string(landmark.x()));
    dispColText.append(" Y:");
    dispColText.append(std::to_string(landmark.y()));*/

    dispColText.append(" Ø:");   
    dispColText.append(std::to_string(angleBetweenLines(landmark.x(), landmark.y(),
                                          landmarks[0].x(),landmarks[0].y(),
                                          landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y())));  //float x0, float y0, float x1, float y1, float x2, float y2

    std::cout << dispColText << "\t" ;       
  }

  landmark_col_text->set_display_text(dispColText); 
  landmark_col_text->set_font_height(10); 
  landmark_col_text->set_normalized(false); 
  landmark_col_text->set_left(5); 
  landmark_col_text->set_baseline(10 + (lmIndex * 12));


  /*if((&landmark - &landmarks[0])==0){
    std::cout  << "X:" << std::to_string(landmark.x()) << " - " << landmark_data->x() << "\n";
  }*/



}
if (visualize_depth) {
  AddConnectionsWithDepth(landmarks, /*normalized=*/true, z_min, z_max,
                          render_data.get());
} else {
  AddConnections(landmarks, /*normalized=*/true, render_data.get());
}

}
`

[crabbit0629]

@lisbravo Sorry ~ which file is this ? Then is it able to show the point coordinates?

[lisbravo]

use the source, luke

[crabbit0629]

@lisbravo Is in this path ? mediapipe/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

[dipampatel18]

@lisbravo I cloned your repository and when I try to build the hand tracking demo for desktop using-

bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1

It gives me an error
BUILD file not found in any of the following directories.

This error is because there are no folders for hand_tracking in your branch. Can you help me this please?

The hello_world works fine and the object_detection demo for desktop also builds successfully.

[lisbravo]

@crabbit0629

@lisbravo Is in this path ? mediapipe/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

Yes, that's the file you need to modify with the code I provide earlier

@dipampatel18

@lisbravo I cloned your repository and when I try to build the hand tracking demo for desktop using-

bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1

It gives me an error
BUILD file not found in any of the following directories.

This error is because there are no folders for hand_tracking in your branch. Can you help me this please?

The hello_world works fine and the object_detection demo for desktop also builds successfully.

I'm still working heavily and my repo may not be up to date but I'm committing now. Beware that is a work in progress and you may need to modify the graph since I'm adding new calculators that may not compile fine just yet

[mgyong]

@lisbravo Thanks for sharing the code. on a different note, curious about your use case on MediaPipe. Appreciate any feedback about your use of MediaPipe. If you could share with me your feedback on MediaPipe that is appreciated to mgyong at google dot com

[lisbravo]

@mgyong 4 sure!

[dipampatel18]

@crabbit0629

@lisbravo Is in this path ? mediapipe/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

Yes, that's the file you need to modify with the code I provide earlier

@dipampatel18

@lisbravo I cloned your repository and when I try to build the hand tracking demo for desktop using-
bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
It gives me an error
BUILD file not found in any of the following directories.
This error is because there are no folders for hand_tracking in your branch. Can you help me this please?
The hello_world works fine and the object_detection demo for desktop also builds successfully.

I'm still working heavily and my repo may not be up to date but I'm committing now. Beware that is a work in progress and you may need to modify the graph since I'm adding new calculators that may not compile fine just yet

@lisbravo Excellent!! Yeah I am seeing the updates. And as you mentioned, the compile didn't go well
C++ compilation of rule '//mediapipe/calculators/video:opencv_video_imshow_calculator'

Thanks for the contribution though. I am looking forward to fix this.

[lisbravo]

@dipampatel18

@lisbravo Excellent!! Yeah I am seeing the updates. And as you mentioned, the compile didn't go well
C++ compilation of rule '//mediapipe/calculators/video:opencv_video_imshow_calculator'

Thanks for the contribution though. I am looking forward to fix this.

That file should not give you much problem compiling, actually its a pretty simple mod of the Encoder Calculator, can you post the compiler errors?

[dipampatel18]

@dipampatel18

@lisbravo Excellent!! Yeah I am seeing the updates. And as you mentioned, the compile didn't go well
C++ compilation of rule '//mediapipe/calculators/video:opencv_video_imshow_calculator'
Thanks for the contribution though. I am looking forward to fix this.

That file should not give you much problem compiling, actually its a pretty simple mod of the Encoder Calculator, can you post the compiler errors?

@lisbravo When I build the hand_detection or any other modules, I get the following error-

INFO: Analyzed target //mediapipe/examples/desktop/hand_detection:hand_detection_tflite (4 packages loaded, 639 targets configured).
INFO: Found 1 target...
ERROR: /home/dipam/mediapipe/mediapipe/calculators/video/BUILD:116:1: C++ compilation of rule '//mediapipe/calculators/video:opencv_video_imshow_calculator' failed (Exit 1) gcc failed: error executing command /usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -g0 -O2 '-D_FORTIFY_SOURCE=1' -DNDEBUG -ffunction-sections ... (remaining 72 argument(s) skipped)

Use --sandbox_debug to see verbose messages from the sandbox mediapipe/calculators/video/opencv_video_imshow_calculator.cc: In member function 'mediapipe::Status mediapipe::OpenCvVideoImShowCalculator::SetUpVideoWriter()': mediapipe/calculators/video/opencv_video_imshow_calculator.cc:163:35: error: 'WINDOW_GUI_EXPANDED' is not a member of 'cv' cv::namedWindow("MediaPipe",cv::WINDOW_GUI_EXPANDED);
^~~~~~~
Target //mediapipe/examples/desktop/hand_detection:hand_detection_tflite failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 10.977s, Critical Path: 9.95s
INFO: 7 processes: 7 linux-sandbox.
FAILED: Build did NOT complete successfully

[averafter]

@lisbravo do you mind to explain why
const unsigned char lmIndex = &landmark - &landmarks[0];
can get the index?

[lisbravo]

@averafter check my entire project https://www.deuxexsilicon.com/handcommander/

[averafter]

@lisbravo , thanks!! It definitely helps!!

Besides that, I encountered following compilation issue, do you have any idea?

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:336:59: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
const unsigned char lmIndex = &landmark - &landmarks[0];

[lisbravo]

@averafter will look into that error later
also check my project announcement at https://github.com/google/mediapipe/issues/594

[ShiramineRisa]

@lisbravo
I'm a new mediapipe user, I attempted to use your code to print the information by android studio, but it ended up failing with:

_ERROR: /home/shirakami/mediapipe/mediapipe/calculators/util/BUILD:599:11: C++ compilation of rule '//mediapipe/calculators/util:landmarks_to_render_data_calculator' failed (Exit 1): clang failed: error executing command external/androidndk/ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/clang -gcc-toolchain external/androidndk/ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64 -target ... (remaining 71 argument(s) skipped)

Use --sandbox_debug to see verbose messages from the sandbox clang failed: error executing command external/androidndk/ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/clang -gcc-toolchain external/androidndk/ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64 -target ... (remaining 71 argument(s) skipped)

Use --sandbox_debug to see verbose messages from the sandbox
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:335:59: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
const unsigned char lmIndex = &landmark - &landmarks[0];
~~~^~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:369:74: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex-1].x(),landmarks[lmIndex-1].y(),
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:369:99: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex-1].x(),landmarks[lmIndex-1].y(),
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:370:74: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y()))); //float x0, float y0, float x1, float y1, float x2, float y2
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:370:99: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y()))); //float x0, float y0, float x1, float y1, float x2, float y2
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:383:74: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[0].x(),landmarks[0].y(),
~~~^~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:383:91: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[0].x(),landmarks[0].y(),
~~~^~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:384:74: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y()))); //float x0, float y0, float x1, float y1, float x2, float y2
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:384:99: error: type 'const mediapipe::NormalizedLandmarkList' does not provide a subscript operator
landmarks[lmIndex+1].x(),landmarks[lmIndex+1].y()))); //float x0, float y0, float x1, float y1, float x2, float y2
~~~^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:404:11: error: no matching function for call to 'AddConnectionsWithDepth'
AddConnectionsWithDepth(landmarks, /normalized=/true, z_min, z_max,
^~~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:99:6: note: candidate function template not viable: requires 11 arguments, but 5 were provided
void AddConnectionsWithDepth(const LandmarkListType& landmarks,
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:407:11: error: no matching function for call to 'AddConnections'
AddConnections(landmarks, /normalized=/true, render_data.get());
^~~~
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:143:6: note: candidate function template not viable: requires 10 arguments, but 3 were provided
void AddConnections(const LandmarkListType& landmarks,
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:414:3: error: unknown type name 'cc'
cc->Outputs()
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:414:5: error: cannot use arrow operator on a type
cc->Outputs()
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:417:3: error: expected unqualified-id
return ::mediapipe::OkStatus();
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:420:21: error: unknown type name 'LandmarksToRenderDataCalculator'; did you mean 'mediapipe::LandmarksToRenderDataCalculator'?
REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
^~~~~~~~~
mediapipe::LandmarksToRenderDataCalculator
./mediapipe/framework/calculator_registry.h:25:57: note: expanded from macro 'REGISTER_CALCULATOR'
absl::make_unique); \
^
./mediapipe/framework/deps/registration.h:388:41: note: expanded from macro 'REGISTER_FACTORY_FUNCTION_QUALIFIED'
RegistryType::Register(#name, __VA_ARGS__))
^
./mediapipe/calculators/util/landmarks_to_render_data_calculator.h:48:7: note: 'mediapipe::LandmarksToRenderDataCalculator' declared here
class LandmarksToRenderDataCalculator : public CalculatorBase {
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:420:21: error: use of undeclared identifier 'LandmarksToRenderDataCalculator'
REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:420:1: error: expected a type
REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
^
./mediapipe/framework/calculator_registry.h:26:3: note: expanded from macro 'REGISTER_CALCULATOR'
REGISTER_FACTORY_FUNCTION_QUALIFIED( \
^
./mediapipe/framework/deps/registration.h:388:53: note: expanded from macro 'REGISTER_FACTORY_FUNCTION_QUALIFIED'
RegistryType::Register(#name, __VA_ARGS__))
^
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc:421:2: error: expected ';' after top level declarator
} // namespace mediapipe
^
;

(using Android Studio 4.1, ubuntu 20.04)
(I'm not a English user, sorry for my poor grammar.)