projection of GT on to the image. too many negatives.

[mdfaheem786]

Hi, great dataset for experimenting deep learning network for autonomous vehicles.

but while using the sdk for projection of GT on to images, got many negatives, is there a way to handle these negatives.

attached is one sample of projection ( 057 folder, 60.jpg - front camera)

thanking you in advances.

[mdfaheem786]

Here is my code, please kindly let me know where is the issue.

import pandaset
from pandaset import geometry
import numpy as np
import cv2
from typing import List, Optional, Tuple
import copy

import glob
import json
import os
from typing import overload, List, TypeVar, Dict
import pandas as pd
import random
from PIL import Image
from python.pandaset.sensors import Intrinsics

random.seed(9876543210)

def dump_3d_bbox_on_image(
coords, img, fpath,
color: Optional[Tuple[int, int, int]] = None,
thickness: float = 4,
) -> None:

canvas = img.copy()
canvas = cv2.cvtColor(canvas, cv2.COLOR_RGB2BGR)
coords = coords.astype(np.int32)
for index in range(coords.shape[0]):
    projected_points2d = coords[index]
    bbox = projected_points2d.tolist()
    cv2.line(canvas, bbox[0], bbox[1], color, thickness)
    cv2.line(canvas, bbox[0], bbox[4], color, thickness)
    cv2.line(canvas, bbox[0], bbox[3], color, thickness)
    cv2.line(canvas, bbox[1], bbox[2], color, thickness)
    cv2.line(canvas, bbox[1], bbox[5], color, thickness)
    cv2.line(canvas, bbox[2], bbox[3], color, thickness)
    cv2.line(canvas, bbox[2], bbox[6], color, thickness)
    cv2.line(canvas, bbox[3], bbox[7], color, thickness)
    cv2.line(canvas, bbox[4], bbox[7], color, thickness)
    cv2.line(canvas, bbox[4], bbox[5], color, thickness)
    cv2.line(canvas, bbox[5], bbox[6], color, thickness)
    cv2.line(canvas, bbox[6], bbox[7], color, thickness)
canvas = canvas.astype(np.uint8)
canvas = cv2.cvtColor(canvas, cv2.COLOR_BGR2RGB)

cv2.imwrite(fpath, canvas)
return

class CameraData:
size: None
def init(self, sze) -> None:
self.size = sze
return

if name == "main":

datasetPath = "/home/pandaset/"
out = "out"
numFolders = 4
numImgs = 5
allSeq = sorted(glob.glob(f'{datasetPath}/*/'))
seqrandidx = random.sample(range(0,len(allSeq)), numFolders)
# seq = "001"
front_camera_dir = "camera/front_camera/"
front_camera_dir = "camera/back_camera/"
front_camera_dir = "camera/front_right_camera/"
cuboids_dir = "annotations/cuboids/"
data_file_extension = "pkl.gz"

for idx in seqrandidx:
    seq = allSeq[idx]
    # seq = seq[:-4] + "057/"
    
    cuboidsFullPath = os.path.join(seq, cuboids_dir)
    frontFullPath = os.path.join(seq, front_camera_dir)
    lstfiles = sorted(glob.glob(f'{cuboidsFullPath}/*.{data_file_extension}'))

    imgrandidx = random.sample(range(0,len(lstfiles)), numImgs)
    for itemidx in imgrandidx:
        cubpath = lstfiles[itemidx]
        syncidx = cubpath.split("/")[-1]
        syncidx = syncidx[:-7]
        pklgzName = syncidx + ".jpg"
        imgpath = os.path.join(frontFullPath, pklgzName)
        ori_image = Image.open(imgpath)
        img = np.array(ori_image)
        cuboids0 = pd.read_pickle(cubpath)

        # remove nan values in the camera_used
        ori_cuboids0 = copy.deepcopy(cuboids0)
        cuboids0 = cuboids0[cuboids0["camera_used"] == -1]

        posepath = os.path.join(frontFullPath, "poses.json")
        intrinsicspath = os.path.join(frontFullPath, "intrinsics.json")
        with open(posepath, 'r') as f:
            pose = json.load(f)
        with open(intrinsicspath, 'r') as f:
            file_data = json.load(f)
            intrinsics = Intrinsics(
                fx=file_data['fx'],
                fy=file_data['fy'],
                cx=file_data['cx'],
                cy=file_data['cy'])
            
        box = [            
            cuboids0[  "position.x"], cuboids0[  "position.y"], cuboids0[  "position.z"],
            cuboids0["dimensions.x"], cuboids0["dimensions.y"], cuboids0["dimensions.z"],
            cuboids0["yaw"]
            ]
                
        intsyncidx = int(syncidx)

        lstProj2d = list()
        for index, row in cuboids0.iterrows():
                box = [
                    row[  "position.x"], row[  "position.y"], row[  "position.z"],
                    row["dimensions.x"], row["dimensions.y"], row["dimensions.z"],
                    row["yaw"]
                ]
                corners = geometry.center_box_to_corners(box)
            
                camera_data = CameraData((1920, 1080))

                projected_points2d, camera_points_3d, inner_indices = \
                    geometry.projection(
                        lidar_points=corners,                
                        # camera_data=None,
                        camera_data=camera_data,
                        camera_pose=pose[intsyncidx],
                        camera_intrinsics=intrinsics,
                        filter_outliers=True)  # False)  # True)  # False)
                projected_points2d = projected_points2d.tolist()
                if len(projected_points2d) == 8:
                    lstProj2d.append(projected_points2d)

        lstProj2d = np.asarray(lstProj2d)
        outname = out + "/"
        temp = imgpath[len(datasetPath):]
        temp = temp.split("/")
        for item in temp:
            outname += "_" + item
        print(f"outname - {outname}")
        dump_3d_bbox_on_image(coords=lstProj2d, img=img, fpath=outname, color=(0,0,255))

exit(0)

@pfmark - please kindly help me...

[william-ljz]

Please tell me, has it been resolved?

[Alexmsit]

I think the wrong boxes in the plot are the boxes which are located behind the camera. You need to make shure that only boxes in front of the camera are plotted, e.g. with:

if center_cam[2] < 0.0: continue