Blenderproc: Dark scenes in shapenet_with_scenenet
Hi,
After the recent fix which introduced modifications to lighting (SurfaceLighting etc.) many of the scenes became dark in the shapenet_with_scenenet example:
Here is the config file I used:
# Args: <obj_file> <texture_file> <path_to_shape-net-core> <output_dir>
{
"version": 3,
"setup": {
"blender_install_path": "/home_local/<env:USER>/blender/",
"pip": [
"h5py", "pandas"
]
},
"modules": [
{
"module": "main.Initializer",
"config":{
"global": {
# "output_dir": "<args:3>"
"output_dir": "examples/shapenet_with_scenenet/<args:2>"
}
}
},
{
"module": "loader.SceneNetLoader",
"config": {
# after downloading the scenenet dataset it should be located inside of resources/scenenet/SceneNetData/
# "file_path": "<args:0>",
"file_path": "resources/scenenet/SceneNetData/<args:0>",
# "texture_folder": "<args:1>",
"texture_folder": "resources/texture_library",
"add_properties": {
"cp_physics": False
}
}
},
{
"module": "object.FloorExtractor",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_category_id": 1
}
}, # this one you need to now beforehand
"compare_angle_degrees" : 7.5,
"compare_height": 0.15,
"name_for_split_obj": "floor", # this is the new name of the object
"add_properties": {
"cp_category_id": 2
},
"should_skip_if_object_is_already_there": True
}
},
{
"module": "object.FloorExtractor",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_category_id": 1
}
},
"should_skip_if_object_is_already_there": True,
"up_vector_upwards": False, # the polygons are now facing downwards: [0, 0, -1]
"compare_angle_degrees": 7.5,
"compare_height": 0.15,
"add_properties": {
# "cp_category_id": 22
"cp_category_id": 2
},
"name_for_split_obj": "ceiling"
}
},
{
"module": "lighting.SurfaceLighting",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"name": ".*[l|L]amp.*"
}
},
"emission_strength": 15.0,
# "emission_strength": 2.0,
"keep_using_base_color": True
}
},
{
"module": "lighting.SurfaceLighting",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"name": ".*[c|C]eiling.*"
}
},
"emission_strength": 2.0,
# "keep_using_base_color": True
}
},
{
"module": "loader.ShapeNetLoader",
"config": {
# "data_path": "<args:2>",
"data_path": "/data/guyga/datasets/ShapeNetCore.v2",
# "used_synset_id": "02801938",
"used_synset_id": "<args:1>",
"add_properties": {
"cp_shape_net_object": True,
# set the custom property physics to True
"cp_physics": True
}
}
},
{
"module": "manipulators.EntityManipulator",
"config": {
# get all shape net objects, as we have only loaded one this returns only one entity
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
},
# Sets the location of this entity above a bed
"location": {
"provider": "sampler.UpperRegionSampler",
"min_height": 0.3,
"use_ray_trace_check": True,
"to_sample_on": {
"provider": "getter.Entity",
"conditions": {
# "cp_category_id": 4, # 4 is the category of the bed
"cp_category_id": 2, # Floor
"type": "MESH"
}
}
},
# by adding a modifier we avoid that the objects falls through other objects during the physics simulation
"cf_add_modifier": {
"name": "Solidify",
"thickness": 0.0025
}
}
},
{
"module": "object.PhysicsPositioning",
"config": {
"solver_iters": 30,
"substeps_per_frame": 40,
"min_simulation_time": 0.5,
"max_simulation_time": 4,
"check_object_interval": 0.25,
"mass_scaling": True,
"mass_factor": 2000,
"collision_margin": 0.00001,
"collision_shape": "MESH"
# "collision_shape": "CONVEX_HULL"
}
},
{
"module": "camera.CameraSampler",
"config": {
"cam_poses": [
{
# "number_of_samples": 5,
"number_of_samples": 4,
# "number_of_samples": 3,
"proximity_checks": {
"min": 0.5
},
"check_if_objects_visible": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
},
"location": {
"provider":"sampler.PartSphere",
"center": {
"provider": "getter.POI",
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
}
},
"distance_above_center": 0.5,
"radius": 2,
"mode": "SURFACE"
},
"rotation": {
"format": "look_at",
"value": {
"provider": "getter.POI",
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
}
}
}
}
]
}
},
# {
# "module": "camera.CameraSampler",
# "config": {
# "cam_poses": [
# {
# "min_interest_score": 0.4,
# # "number_of_samples": 5,
# "number_of_samples": 3,
# "location": {
# "provider":"sampler.PartSphere",
# "center": {
# "provider": "getter.POI",
# "selector": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_shape_net_object": True,
# "type": "MESH"
# }
# }
# },
# "distance_above_center": 0.5,
# "radius": 2,
# "mode": "SURFACE"
# },
# "rotation": {
# "format": "look_at",
# "value": {
# "provider": "getter.POI",
# "selector": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_shape_net_object": True,
# "type": "MESH"
# }
# }
# }
# }
# }
# ]
# }
# },
{
"module": "renderer.RgbRenderer",
"config": {
"output_is_temp": False,
"output_key": "colors",
"samples": 150,
# "render_distance": True,
# "render_normals": True,
"image_type": "JPEG",
}
},
{
"module": "renderer.SegMapRenderer",
"config": {
"map_by": ["instance", "class"]
}
},
{
"module": "writer.CocoAnnotationsWriter",
"config": {
# "supercategory": "chairs_and_tables",
# "append_to_existing_output": True
}
}
# {
# "module": "writer.Hdf5Writer",
# "config": {
# "postprocessing_modules": {
# "distance": [
# {
# "module": "postprocessing.TrimRedundantChannels",
# }
# ]
# },
# "append_to_existing_output": True
# }
# }
]
}
ggaziv
All 20 comments
Hi @ggaziv,
unfortunately I cannot really reproduce your issue. I now went through several scenes, but I could not detect the effect you described.
Could you please list some of the scene names which you found to be darker after the last update?
cornerfarmer
on 8 Jan 2021
And maybe also set a certain seed so that we can be sure that we have the exact same result as you have.
You can do that, by setting an environment variable too:
Just do:
export BLENDER_PROC_RANDOM_SEED=15
themasterlink
on 8 Jan 2021
Hi, I think this is somewhat related to camera sampling with that sphere.
When I use camera sampling like in SceneNet alone it works fine (in the shapenet_with_scenenet example), e.g.,:
but then with the sphere/ball sampling it does weird stuff in some images (who knows where it actually samples):
Hope this helps.
Guy
ggaziv
on 11 Jan 2021
But both of them are well illuminated. Except this one image of that vase and I suspect the object just fell out of the room.
The sampling might be weird, but that is what you picked in your config?
Could you please provide a scene example in which it fails constantly, so that we can have a look at that.
themasterlink
on 11 Jan 2021
@ggaziv Any news on this?
themasterlink
on 23 Jan 2021
@themasterlink I think indeed the problem is related to the object falling out of the scene. Don't know why this happens but it still does.
Unfortunately I don't currently have concrete seeds to suggest beyond that config file and the examples.
ggaziv
on 24 Jan 2021
@ggaziv Hi,
It is strange since judging by this
{
"module": "object.PhysicsPositioning",
"config": {
"solver_iters": 30,
"substeps_per_frame": 40,
"min_simulation_time": 0.5,
"max_simulation_time": 4,
"check_object_interval": 0.25,
"mass_scaling": True,
"mass_factor": 2000,
"collision_margin": 0.00001,
"collision_shape": "MESH"
# "collision_shape": "CONVEX_HULL"
}
},
I would expect objects have proper collisions and stay inside. Did you try increasing solver_iters and substeps_per_frame?
ideas-man
on 24 Jan 2021
Did not try to play with those parameters. Here is another example of bottles outside the background scene (hence dark).
BTW, aren't you trying to drop the object at the center of the scene? What if the scene is not convex?
ggaziv
on 24 Jan 2021
Hey @ggaziv,
thanks for the clarification. If use_ray_trace_check is enabled (as it is in your config), it is always assured that the object is sampled straight above the floor, even if it is not convex.
The problem here is caused by the scene net models.
Here is the 1Bedroom/bedroom_68.obj scene with the floor outlined in orange:
As you can see the floor extends beyond the actual room. So even if objects are sampled above the floor they can fall into darkness.
One way of solving this would be to manually clean all scenes, but that is probably quite tedious. Alternatively one might write a check to make sure the sampled object position is inside the walls, but that is not so easy to check.
cornerfarmer
on 25 Jan 2021
Interesting @cornerfarmer !
Is there a way to access the full floor outline and use it to create a heuristically scaled version of it for sampling?
ggaziv
on 25 Jan 2021
With the following code you can iterate over the vertex coordinates of the floor, if that is what you want:
import bpy
obj = bpy.context.scene.objects["Floor"]
for vert in obj.data.vertices:
print(obj.matrix_world @ vert.co)
As the original question, was answered and the problem identified. I am going to close this issue and will mark it as no fix. As we do not fix errors in existing datasets ("the floor is bigger than the actual room").
If you feel up to fixing this dynamically, we would be delighted for a PR.
themasterlink
on 28 Jan 2021
Not sure the problem was identified, since I tried removing the random drop of ShapeNet object in the scene such that it is dropped at the center. Nevertheless I still got those dark images in some of the scenes.
I ended up following a heuristics approach to eliminate such images in post-processing by measuring the STD across RGB channels. In those dark/gray images that value is very close to zero in most of the pixels in the image.
ggaziv
on 1 Feb 2021
Hey @ggaziv,
I now looked through some scenes and it seems that sometimes the ceiling is not extracted and is therefore not set as a light source. I am going to look into it more closely and make sure it works for all scenes. To be sure I cover your issue, can you post some of the scene names where you experience these issues?
cornerfarmer
on 1 Feb 2021
1Living-room_living_room_11
1Living-room_living_room_11
1Living-room_living_room_11
1Living-room_living_room_11
1Living-room_living_room_33
1Living-room_living_room_33
1Living-room_living_room_33
1Living-room_living_room_33
1Living-room_lr_5_blender_mat_and_name
1Living-room_lr_5_blender_mat_and_name
1Living-room_lr_5_blender_mat_and_name
1Living-room_lr_5_blender_mat_and_name
1Living-room_lr_kt7_blender_scene
1Living-room_lr_kt7_blender_scene
1Living-room_lr_kt7_blender_scene
1Living-room_lr_kt7_blender_scene
1Office_1_hereisfree_labels
1Office_1_hereisfree_labels
1Office_1_hereisfree_labels
1Office_1_hereisfree_labels
1Office_2_crazy3dfree_labels
1Office_2_crazy3dfree_labels
1Office_2_crazy3dfree_labels
1Office_2_crazy3dfree_labels
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_4_3dmodel777
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_labels
1Office_23_3dmodelfree
1Office_23_3dmodelfree
1Office_23_3dmodelfree
1Office_23_3dmodelfree
1Office_66office_scene
1Office_66office_scene
1Office_66office_scene
1Office_66office_scene
ggaziv
on 1 Feb 2021
@cornerfarmer Any progress on that front?
Need more examples? This phenomenon is incredibly frequent.
ggaziv
on 8 Feb 2021
Hey @ggaziv,
yes, we gonna very soon (probably tomorrow or the day after) release a new version which fixes bad lighting inside the rooms.
However, most of the images you posted were taken outside the room, where of course the lighting is bad. As explained above the scenenet dataset is not very clean in the sense that the floor extends outside the room. Therefore, just sampling above the floor, does not necessary always create locations inside the room.
Improving the sampling is not trivial, as there is no other easy way to check whether one is inside the room. One heuristic check could be for example to send rays in all directions and verify that they hit an object. However this will not always work, as there are holes in the walls.
I am going to think about this a bit more tomorrow, maybe I can come up with a better idea here.
cornerfarmer
on 8 Feb 2021
Hey @ggaziv,
Version 1.9.0 should fix most of these issues, can you tell us if that is the case? It does not fix the problem, that the floors are bigger than the houses itself.
Best,
Max
themasterlink
on 11 Feb 2021
I assume this is resolved now, if not please add more details and reopen the issue.
themasterlink
on 16 Feb 2021
Hi @themasterlink, I am afraid the problem persists:
Here are names of scenes with problems:
1Bathroom_107_labels
1Bathroom_1_labels
1Bathroom_28_labels
1Bathroom_29_labels
1Bathroom_2_labels
1Bathroom_2_labels_test
1Bathroom_34_labels
1Bathroom_3_labels
1Bathroom_4_labels
1Bathroom_5_labels
1Bathroom_69_labels
1Bathroom_6_labels
1Bedroom_3_labels
1Bedroom_77_labels
1Bedroom_bedroom27
1Bedroom_bedroom_1
1Bedroom_bedroom_68
1Bedroom_bedroom_wenfagx
1Bedroom_dah07243_labels
1Bedroom_juseragn9154_labels
1Bedroom_xih07063_labels
1Kitchen_1-14_labels
1Kitchen_102
1Kitchen_13_labels
1Kitchen_1_labels
1Kitchen_2
1Kitchen_35_labels
1Kitchen_37
1Kitchen_9
1Kitchen_kitchen_106_blender_name_and_mat
1Kitchen_kitchen_16_blender_name_and_mat
1Kitchen_kitchen_76_blender_name_and_mat
1Living-room_cnh_blender_name_and_mat
1Living-room_living_room_11
1Living-room_living_room_33
1Living-room_livingroom_74
1Living-room_lr_5_blender_mat_and_name
1Living-room_lr_66_blender_name_and_mat
1Living-room_lr_kt7_blender_scene
1Living-room_pg_blender_name_and_mat
1Living-room_room_89_blender_no_paintings
1Living-room_yoa_blender_name_mat
1Office_14_3dmodelfree
1Office_1_3dmodel777_labels
1Office_1_hereisfree_labels
1Office_23_3dmodelfree
1Office_2_crazy3dfree_labels
1Office_2_hereisfree_labels
1Office_3_3dmodel777
1Office_4_3dmodel777
1Office_4_hereisfree_labels
1Office_5_hereisfree_labels
1Office_66office_scene
1Office_7_crazy3dfree_labels
1Office_7_crazy3dfree_old_labels
1Office_8_crazy3dfree_labels
1Office_8_crazy3dfree_old_labels
And here is my config file:
# Args: <obj_file> <texture_file> <path_to_shape-net-core> <output_dir>
{
"version": 3,
"setup": {
"blender_install_path": "/home_local/<env:USER>/blender/",
"pip": [
"h5py", "pandas"
]
},
"modules": [
{
"module": "main.Initializer",
"config":{
"global": {
# "output_dir": "<args:3>"
"output_dir": "examples/shapenet_with_scenenet/<args:2>"
}
}
},
{
"module": "loader.SceneNetLoader",
"config": {
# after downloading the scenenet dataset it should be located inside of resources/scenenet/SceneNetData/
# "file_path": "<args:0>",
"file_path": "resources/scenenet/SceneNetData/<args:0>",
# "texture_folder": "<args:1>",
"texture_folder": "resources/texture_library",
"add_properties": {
"cp_physics": False
}
}
},
{
"module": "object.FloorExtractor",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_category_id": 1
}
}, # this one you need to now beforehand
"compare_angle_degrees" : 7.5,
"compare_height": 0.15,
"name_for_split_obj": "floor", # this is the new name of the object
"add_properties": {
"cp_category_id": 2
},
"should_skip_if_object_is_already_there": True
}
},
{
"module": "object.FloorExtractor",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_category_id": 1
}
},
"should_skip_if_object_is_already_there": True,
"up_vector_upwards": False, # the polygons are now facing downwards: [0, 0, -1]
"compare_angle_degrees": 7.5,
"compare_height": 0.15,
"add_properties": {
"cp_category_id": 22
# "cp_category_id": 2
},
"name_for_split_obj": "ceiling"
}
},
{
"module": "lighting.SurfaceLighting",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"name": ".*[l|L]amp.*"
}
},
"emission_strength": 15.0,
# "emission_strength": 2.0,
"keep_using_base_color": True
}
},
{
"module": "lighting.SurfaceLighting",
"config": {
"selector": {
"provider": "getter.Entity",
"conditions": {
"name": ".*[c|C]eiling.*"
}
},
"emission_strength": 2.0,
"keep_using_base_color": True
}
},
{
"module": "loader.ShapeNetLoader",
"config": {
# "data_path": "<args:2>",
"data_path": "/data/guyga/datasets/ShapeNetCore.v2",
# "num_of_objs_to_sample": 2,
# "scale_factor_minmax": [1., 1.],
# "scale_factor_minmax": [.3, .9],
# "scale_factor_minmax": [.2, .5],
"scale_factor_minmax": [.4, .8],
# "used_synset_id": "02801938",
"used_synset_id": "<args:1>",
"add_properties": {
"cp_shape_net_object": True,
# set the custom property physics to True
"cp_physics": True
}
}
},
{
"module": "manipulators.EntityManipulator",
"config": {
# get all shape net objects, as we have only loaded one this returns only one entity
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
},
# Sets the location of this entity above a bed
"location": {
"provider": "sampler.UpperRegionSampler",
"min_height": 0.3,
"use_ray_trace_check": True,
"to_sample_on": {
"provider": "getter.Entity",
"conditions": {
# "cp_category_id": 4, # 4 is the category of the bed
"cp_category_id": 2, # Floor
"type": "MESH"
}
}
},
# by adding a modifier we avoid that the objects falls through other objects during the physics simulation
"cf_add_modifier": {
"name": "Solidify",
"thickness": 0.0025
}
}
},
{
"module": "object.PhysicsPositioning",
"config": {
"solver_iters": 30,
"substeps_per_frame": 40,
"min_simulation_time": 0.5,
"max_simulation_time": 4,
"check_object_interval": 0.25,
"mass_scaling": True,
"mass_factor": 2000,
"collision_margin": 0.00001,
"collision_shape": "MESH"
# "collision_shape": "CONVEX_HULL"
}
},
# { # Sampling like when there is scene only
# "module": "camera.CameraSampler",
# "config": {
# "min_var_diff_translation": 5.0,
# "cam_poses": [{
# # "number_of_samples": 5,
# "number_of_samples": 4,
# "proximity_checks": {
# "min": 1.0
# },
# # "min_interest_score": 0.1,
# "min_interest_score": 0.4,
# "location": {
# "provider": "sampler.UpperRegionSampler",
# "min_height": 1.5,
# "max_height": 1.8,
# "to_sample_on": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_category_id": 2
# }
# }
# },
# "rotation": {
# "value": {
# "provider":"sampler.Uniform3d",
# "max":[1.2217, 0, 6.283185307],
# "min":[1.2217, 0, 0]
# }
# },
# "check_if_pose_above_object_list": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_category_id": 2,
# "type": "MESH"
# }
# }
# }]
# }
# },
{ # Sampling with sphere/ball
"module": "camera.CameraSampler",
"config": {
"cam_poses": [
{
# "number_of_samples": 5,
"number_of_samples": 4,
# "number_of_samples": 3,
"proximity_checks": {
"min": 0.5
},
"check_if_objects_visible": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
},
"location": {
"provider":"sampler.PartSphere",
"center": {
"provider": "getter.POI",
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
}
},
# "distance_above_center": 0.5,
"distance_above_center": 1,
# "radius": 2,
# "radius": 4,
"radius": 6,
# "mode": "SURFACE"
"mode": "INTERIOR"
},
"rotation": {
"format": "look_at",
"value": {
"provider": "getter.POI",
"selector": {
"provider": "getter.Entity",
"conditions": {
"cp_shape_net_object": True,
"type": "MESH"
}
},
"noise_amp": 1,
},
# "inplane_rot": { # randomly inplane rotate the cameras in an interval of +/- 45 degree.
# "provider": "sampler.Value",
# "type": "float",
# "min": -0.7854,
# "max": 0.7854
# }
}
}
]
}
},
# {
# "module": "camera.CameraSampler",
# "config": {
# "cam_poses": [
# {
# "min_interest_score": 0.4,
# # "number_of_samples": 5,
# "number_of_samples": 3,
# "location": {
# "provider":"sampler.PartSphere",
# "center": {
# "provider": "getter.POI",
# "selector": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_shape_net_object": True,
# "type": "MESH"
# }
# }
# },
# "distance_above_center": 0.5,
# "radius": 2,
# "mode": "SURFACE"
# },
# "rotation": {
# "format": "look_at",
# "value": {
# "provider": "getter.POI",
# "selector": {
# "provider": "getter.Entity",
# "conditions": {
# "cp_shape_net_object": True,
# "type": "MESH"
# }
# }
# }
# }
# }
# ]
# }
# },
{
"module": "renderer.RgbRenderer",
"config": {
"output_is_temp": False,
"output_key": "colors",
"samples": 150,
# "render_distance": True,
# "render_normals": True,
"image_type": "JPEG",
}
},
{
"module": "renderer.SegMapRenderer",
"config": {
"map_by": ["instance", "class"]
}
},
{
"module": "writer.CocoAnnotationsWriter",
"config": {
# "supercategory": "chairs_and_tables",
# "append_to_existing_output": True
}
}
# {
# "module": "writer.Hdf5Writer",
# "config": {
# "postprocessing_modules": {
# "distance": [
# {
# "module": "postprocessing.TrimRedundantChannels",
# }
# ]
# },
# "append_to_existing_output": True
# }
# }
]
}
Related issues
luigifaticoso
路
4Comments
b-yogesh
路
5Comments
mikkeljakobsen
路
6Comments
elientumba2019
路
4Comments
chrisxu1995
路
5Comments