pythonpoint-cloudsopen3d

Open a new window when clicking on a mesh in Open3D

Below is an example in Open3D, in which I created a bunny mesh, as well as a circle mesh. Now I want to create a new window and show an image when I click on the circle mesh. Does anyone know how to do that ? Thank you very much.

import numpy as np
import open3d as o3d

def main():
    mesh = o3d.io.read_triangle_mesh(o3d.data.BunnyMesh().path)
    pcd1 = mesh.sample_points_poisson_disk(10000)

    visualizer = o3d.visualization.Visualizer()
    visualizer.create_window()

    visualizer.add_geometry(pcd1)

    # mesh = convert_image_to_mesh(img)
    # mesh.scale(scale=0.1, center=pcd1.get_center())
    # visualizer.add_geometry(mesh)

    # Show xyz axes
    rendering_options = visualizer.get_render_option()
    rendering_options.show_coordinate_frame = True
    rendering_options.background_color = np.asarray([0.5, 0.5, 0.5])

    mesh = o3d.geometry.TriangleMesh.create_sphere(radius=0.1)
    mesh.scale(scale=0.3, center=mesh.get_center())

    visualizer.add_geometry(mesh)

    visualizer.run()

if __name__ == "__main__":
    main()

Example

Solution

  • I have found the answer, it is based on an example in: https://github.com/isl-org/Open3D/blob/main/examples/python/visualization/mouse_and_point_coord.py. It turns out that I had to use the open3d.visualization.gui class instead of using the Visualizer class, which is admittedly confusing.

    Basically, I used a o3d.t.geometry.RaycastingScene instance, which has some functions to check distance from the mesh to a query point. However, the mesh needs to be an instance of the o3d.t.geometry.TriangleMesh class, and when I tried o3d.geometry.TriangleMesh it did not work, which is very confusing. Apparently it's some sort of new API for GPU acceleration, as explained here: https://github.com/isl-org/Open3D/discussions/6130

    Here is the code which I have modified based on that link:

    import numpy as np
import open3d as o3d
import open3d.core as o3c
import open3d.visualization.gui as gui
import open3d.visualization.rendering as rendering

import cv2
from pdb import set_trace

class ExampleApp:

    def __init__(self, cloud):
        # We will create a SceneWidget that fills the entire window, and then
        # a label in the lower left on top of the SceneWidget to display the
        # coordinate.
        app = gui.Application.instance
        self.window = app.create_window("Open3D - GetCoord Example", 1024, 768)
        # Since we want the label on top of the scene, we cannot use a layout,
        # so we need to manually layout the window's children.
        self.window.set_on_layout(self._on_layout)
        self.widget3d = gui.SceneWidget()
        self.window.add_child(self.widget3d)
        self.info = gui.Label("")
        self.info.visible = False
        self.window.add_child(self.info)

        self.widget3d.scene = rendering.Open3DScene(self.window.renderer)

        mat = rendering.MaterialRecord()
        mat.shader = "defaultUnlit"
        # Point size is in native pixels, but "pixel" means different things to
        # different platforms (macOS, in particular), so multiply by Window scale
        # factor.
        mat.point_size = 3 * self.window.scaling
        # set_trace()
        self.widget3d.scene.add_geometry("Point Cloud", cloud, mat)

        #mesh = o3d.io.read_triangle_mesh(o3d.data.BunnyMesh().path)
        mesh = o3d.t.geometry.TriangleMesh.create_sphere(radius=0.1)
        mesh.scale(scale=0.3, center=mesh.get_center())
        # mesh.translate(np.array([1000, 1000, 1000,]) + np.array([0.0, 3.0, 0.0]), relative=False) # Move a little bit up 

        self.widget3d.scene.add_geometry("Point Cloud 2", mesh, mat)
        # set_trace()
        self.mesh = mesh

        bounds = self.widget3d.scene.bounding_box
        center = bounds.get_center()
        self.widget3d.setup_camera(60, bounds, center)
        #self.widget3d.look_at(center, center - [0, 0, 3], [0, -1, 0])

        self.widget3d.set_on_mouse(self._on_mouse_widget3d)


        scene = o3d.t.geometry.RaycastingScene()
        _ = scene.add_triangles(mesh)  # we do not need the geometry ID for mesh
        self.scene = scene


    def _on_layout(self, layout_context):
        r = self.window.content_rect
        self.widget3d.frame = r
        pref = self.info.calc_preferred_size(layout_context,
                                             gui.Widget.Constraints())
        self.info.frame = gui.Rect(r.x,
                                   r.get_bottom() - pref.height, pref.width,
                                   pref.height)

    def _on_mouse_widget3d(self, event):
        # We could override BUTTON_DOWN without a modifier, but that would
        # interfere with manipulating the scene.
        if event.type == gui.MouseEvent.Type.BUTTON_DOWN:

            def depth_callback(depth_image):
                # Coordinates are expressed in absolute coordinates of the
                # window, but to dereference the image correctly we need them
                # relative to the origin of the widget. Note that even if the
                # scene widget is the only thing in the window, if a menubar
                # exists it also takes up space in the window (except on macOS).
                x = event.x - self.widget3d.frame.x
                y = event.y - self.widget3d.frame.y
                # Note that np.asarray() reverses the axes.
                depth = np.asarray(depth_image)[y, x]

                if depth == 1.0:  # clicked on nothing (i.e. the far plane)
                    text = ""
                else:
                    world = self.widget3d.scene.camera.unproject(
                        x, y, depth, self.widget3d.frame.width,
                        self.widget3d.frame.height)
                    query_point = o3d.core.Tensor([[world[0], world[1], world[2]]], dtype=o3d.core.Dtype.Float32)
                    signed_distance = self.scene.compute_signed_distance(query_point)
                    if signed_distance < 0.05:
                        text = "({:.3f}, {:.3f}, {:.3f}). In mesh!".format(
                            world[0], world[1], world[2])
                        newwindow = gui.Application.instance.create_window("Image", 1024, 1024)
                        image_widget = o3d.visualization.gui.ImageWidget("lena.png")
                        newwindow.add_child(image_widget)
                    else:    
                        text = "({:.3f}, {:.3f}, {:.3f})".format(
                            world[0], world[1], world[2])

                # This is not called on the main thread, so we need to
                # post to the main thread to safely access UI items.
                def update_label():
                    self.info.text = text
                    self.info.visible = (text != "")
                    # We are sizing the info label to be exactly the right size,
                    # so since the text likely changed width, we need to
                    # re-layout to set the new frame.
                    self.window.set_needs_layout()

                gui.Application.instance.post_to_main_thread(
                    self.window, update_label)

            self.widget3d.scene.scene.render_to_depth_image(depth_callback)
            return gui.Widget.EventCallbackResult.HANDLED
        return gui.Widget.EventCallbackResult.IGNORED


def main():
    app = gui.Application.instance
    app.initialize()

    # This example will also work with a triangle mesh, or any 3D object.
    # If you use a triangle mesh you will probably want to set the material
    # shader to "defaultLit" instead of "defaultUnlit".
    mesh = o3d.io.read_triangle_mesh(o3d.data.BunnyMesh().path)
    pcd = mesh.sample_points_poisson_disk(10000)

    ex = ExampleApp(pcd)
    app.run()


if __name__ == "__main__":
    main()