javaandroidgoogle-project-tangopoint-cloudsandroid-studio-3.0

How to set a point on an object via point-cloud/depth perception?


Problem: My app cannot set a point on/near a point of the latest point-cloud data (on an object) through the click of a button.

Click this link to see the android app I have developed so far; I developed my app from the tango point-to-point measurement sample app

This app's features...

As a head-start clue, my error originates from the getDepthAtCenter method (at the bottom of the second method below), since "No Depth Point." is posted on Android Studio's LogCat whilst the app is running.

Here is the setPoint method (upon clicking the green button)....

public void setPoint(View button) {
        MotionEvent buttonClick;
        final Button msrTarget = (Button) findViewById(R.id.target);

        button.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View view) {
                //(x,y,...) coordinates for the the middle of the target (uncertain if this is right)
                float x = msrTarget.getX() / msrTarget.getWidth();
                float y = msrTarget.getY() / msrTarget.getHeight();

                // z, you need to deduce how to calculate the distance from the tablet to the object under measurement
                // First,
                try {
                    // Place point near the clicked point using the latest point cloud data.
                    // Synchronize against concurrent access to the RGB timestamp in the OpenGL thread
                    // and a possible service disconnection due to an onPause event.
                    MeasuredPoint newMeasuredPoint;
                    synchronized (this) {
                        //z depth
                        newMeasuredPoint = getDepthAtCenter(x, y);

                    }
                    if (newMeasuredPoint != null) {
                        // Update a line endpoint to the new location.
                        // This update is made thread-safe by the renderer.
                        updateLine(newMeasuredPoint);
                        Log.w(TAG, "Point was Updated.");
                    } else {
                        Log.w(TAG, "Point was Null.");
                    }

                } catch (TangoException t) {
                    Toast.makeText(getApplicationContext(),
                            R.string.failed_measurement,
                            Toast.LENGTH_SHORT).show();
                    Log.e(TAG, getString(R.string.failed_measurement), t);
                } catch (SecurityException t) {
                    Toast.makeText(getApplicationContext(),
                            R.string.failed_permissions,
                            Toast.LENGTH_SHORT).show();
                    Log.e(TAG, getString(R.string.failed_permissions), t);
                }
            }
        });
    }

Here is the getDepthAtCenter method....

// Using the Tango Support Library with point-cloud data to calculate the depth
    // of the point. It returns Vector3 in OpenGL world space.
    private MeasuredPoint getDepthAtCenter(float x, float y) {
        TangoPointCloudData pointCloud = pointCloudManager.getLatestPointCloud();

        // There is no point cloud
        if (pointCloud == null) {
            Log.w(TAG, "No Point Cloud.");
            return null;
        }

        double rgbTimestamp;
        TangoImageBuffer imageBuffer = mCurrentImageBuffer;
        if (bilateralCheckbox.isChecked()) {
            rgbTimestamp = imageBuffer.timestamp; // CPU.
        } else {
            rgbTimestamp = mRgbTimestampGlThread; // GPU.
        }

        TangoPoseData depthlTcolorPose = TangoSupport.getPoseAtTime(
                rgbTimestamp,
                TangoPoseData.COORDINATE_FRAME_CAMERA_DEPTH,
                TangoPoseData.COORDINATE_FRAME_CAMERA_COLOR,
                TangoSupport.ENGINE_TANGO,
                TangoSupport.ENGINE_TANGO,
                TangoSupport.ROTATION_IGNORED);
        if (depthlTcolorPose.statusCode != TangoPoseData.POSE_VALID) {
            Log.w(TAG, "Could not get color camera transform at time " + rgbTimestamp);
            return null;
        }

        float[] depthPoint;
        //if the bilateral checkbox is checked, get the depth at point bilateral
        if (bilateralCheckbox.isChecked()) {
            depthPoint = TangoDepthInterpolation.getDepthAtPointBilateral(
                    pointCloud,
                    new double[]{0.0, 0.0, 0.0},
                    new double[]{0.0, 0.0, 0.0, 1.0},
                    imageBuffer,
                    x, y,
                    displayRotation,
                    depthlTcolorPose.translation,
                    depthlTcolorPose.rotation);
        } else {
            //Otherwise, get the nearest neighbour as the point-cloud point
            depthPoint = TangoDepthInterpolation.getDepthAtPointNearestNeighbor(
                    pointCloud,
                    new double[]{0.0, 0.0, 0.0},
                    new double[]{0.0, 0.0, 0.0, 1.0},
                    x, y,
                    displayRotation,
                    depthlTcolorPose.translation,
                    depthlTcolorPose.rotation);
        }

        // There is no depth point
        if (depthPoint == null) {
            Log.w(TAG, "No Depth Point.");
            return null;
        }

        return new MeasuredPoint(rgbTimestamp, depthPoint);
    }

This is the TangoDepthInterpolation class...

package com.google.tango.depthinterpolation;

import com.google.atap.tangoservice.Tango;
import com.google.atap.tangoservice.TangoInvalidException;
import com.google.atap.tangoservice.TangoPointCloudData;
import com.google.atap.tangoservice.TangoPoseData;
import com.google.atap.tangoservice.experimental.TangoImageBuffer;
import com.google.tango.depthinterpolation.TangoDepthInterpolationJNIInterface.ByteDepthBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;

public class TangoDepthInterpolation {
    private static final String TAG = TangoDepthInterpolation.class.getSimpleName();

    public TangoDepthInterpolation() {
    }

    public static float[] getDepthAtPointNearestNeighbor(TangoPointCloudData pointCloud, double[] pointCloudTranslation, double[] pointCloudOrientation, float u, float v, int displayRotation, double[] colorCameraTranslation, double[] colorCameraOrientation) throws TangoInvalidException {
        float[] colorCameraPoint = new float[3];
        int result = TangoDepthInterpolationJNIInterface.getDepthAtPointNearestNeighbor(pointCloud, pointCloudTranslation, pointCloudOrientation, u, v, displayRotation, colorCameraTranslation, colorCameraOrientation, colorCameraPoint);
        if(result == -1) {
            return null;
        } else {
            if(result != 0) {
                Tango.throwTangoExceptionIfNeeded(result);
            }

            return colorCameraPoint;
        }
    }

    public static float[] getDepthAtPointBilateral(TangoPointCloudData pointCloud, double[] pointCloudTranslation, double[] pointCloudOrientation, TangoImageBuffer imageBuffer, float u, float v, int displayRotation, double[] colorCameraTranslation, double[] colorCameraOrientation) throws TangoInvalidException {
        float[] colorCameraPoint = new float[3];
        int result = TangoDepthInterpolationJNIInterface.getDepthAtPointBilateral(pointCloud, pointCloudTranslation, pointCloudOrientation, imageBuffer, u, v, displayRotation, colorCameraTranslation, colorCameraOrientation, colorCameraPoint);
        if(result == -1) {
            return null;
        } else {
            if(result != 0) {
                Tango.throwTangoExceptionIfNeeded(result);
            }

            return colorCameraPoint;
        }
    }
}

Solution

  • I solved my own issue by entering this depth perception configuration in my main class.

    try {
        mConfig = new TangoConfig();
        mConfig = mTango.getConfig(TangoConfig.CONFIG_TYPE_CURRENT);
        mConfig.putBoolean(TangoConfig.KEY_BOOLEAN_DEPTH, true);
    } catch (TangoErrorException e) {
        // handle exception
    }