parameters: (nusc, pointsensor_token: str, camera_token: str)
return_statement: return coloring, mask
docstring: Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image
      plane, then retrieve the colors of the closest image pixels.
      :param pointsensor_token: Lidar/radar sample_data token.
      :param camera_token: Camera sample data token.
      :return (coloring <np.float: 3, n>, mask <np.bool: m>). Returns the colors for n points that reproject into the
          image out of m total points. The mask indicates which points are selected.
docstring_summary: Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image
      plane, then retrieve the colors of the closest image pixels.
      :param pointsensor_token: Lidar/radar sample_data token.
      :param camera_token: Camera sample data token.
      :return (coloring <np.float: 3, n>, mask <np.bool: m>). Returns the colors for n points that reproject into the
          image out of m total points. The mask indicates which points are selected.
docstring_tokens: Given a point sensor ( lidar / radar ) token and camera sample_data token load point - cloud and map it to the image plane then retrieve the colors of the closest image pixels . : param pointsensor_token : Lidar / radar sample_data token . : param camera_token : Camera sample data token . : return ( coloring <np . float : 3 n > mask <np . bool : m > ) . Returns the colors for n points that reproject into the image out of m total points . The mask indicates which points are selected .
function: def pointcloud_color_from_image(nusc, pointsensor_token: str, camera_token: str) -> Tuple[np.array, np.array]:
      """
      Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image
      plane, then retrieve the colors of the closest image pixels.
      :param pointsensor_token: Lidar/radar sample_data token.
      :param camera_token: Camera sample data token.
      :return (coloring <np.float: 3, n>, mask <np.bool: m>). Returns the colors for n points that reproject into the
          image out of m total points. The mask indicates which points are selected.
      """
  
      cam = nusc.get('sample_data', camera_token)
      pointsensor = nusc.get('sample_data', pointsensor_token)
      print(nusc.dataroot)
      print(pointsensor['filename'])
      pc = PointCloud.from_file(osp.join(nusc.dataroot, pointsensor['filename']))
      im = Image.open(osp.join(nusc.dataroot, cam['filename']))
  
      # Points live in the point sensor frame. So they need to be transformed via global to the image plane.
      # First step: transform the point-cloud to the ego vehicle frame for the timestamp of the sweep.
      cs_record = nusc.get('calibrated_sensor', pointsensor['calibrated_sensor_token'])
      pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix)
      pc.translate(np.array(cs_record['translation']))
  
      # Second step: transform to the global frame.
      poserecord = nusc.get('ego_pose', pointsensor['ego_pose_token'])
      pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix)
      pc.translate(np.array(poserecord['translation']))
  
      # Third step: transform into the ego vehicle frame for the timestamp of the image.
      poserecord = nusc.get('ego_pose', cam['ego_pose_token'])
      pc.translate(-np.array(poserecord['translation']))
      pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix.T)
  
      # Fourth step: transform into the camera.
      cs_record = nusc.get('calibrated_sensor', cam['calibrated_sensor_token'])
      pc.translate(-np.array(cs_record['translation']))
      pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix.T)
  
      # Fifth step: actually take a "picture" of the point cloud.
      # Grab the depths (camera frame z axis points away from the camera).
      depths = pc.points[2, :]
  
      # Take the actual picture (matrix multiplication with camera-matrix + renormalization).
      points = view_points(pc.points[:3, :], np.array(cs_record['camera_intrinsic']), normalize=True)
  
      # Remove points that are either outside or behind the camera. Leave a margin of 1 pixel for aesthetic reasons.
      mask = np.ones(depths.shape[0], dtype=bool)
      mask = np.logical_and(mask, depths > 0)
      mask = np.logical_and(mask, points[0, :] > 1)
      mask = np.logical_and(mask, points[0, :] < im.size[0] - 1)
      mask = np.logical_and(mask, points[1, :] > 1)
      mask = np.logical_and(mask, points[1, :] < im.size[1] - 1)
      points = points[:, mask]
  
      # Pick the colors of the points
      im_data = np.array(im)
      coloring = np.zeros(points.shape)
      for i, p in enumerate(points.transpose()):
          point = p[:2].round().astype(np.int32)
          coloring[:, i] = im_data[point[1], point[0], :]
  
      return coloring, mask
function_tokens: def pointcloud_color_from_image ( nusc , pointsensor_token : str , camera_token : str ) -> Tuple [ np . array , np . array ] : cam = nusc . get ( 'sample_data' , camera_token ) pointsensor = nusc . get ( 'sample_data' , pointsensor_token ) print ( nusc . dataroot ) print ( pointsensor [ 'filename' ] ) pc = PointCloud . from_file ( osp . join ( nusc . dataroot , pointsensor [ 'filename' ] ) ) im = Image . open ( osp . join ( nusc . dataroot , cam [ 'filename' ] ) ) # Points live in the point sensor frame. So they need to be transformed via global to the image plane. # First step: transform the point-cloud to the ego vehicle frame for the timestamp of the sweep. cs_record = nusc . get ( 'calibrated_sensor' , pointsensor [ 'calibrated_sensor_token' ] ) pc . rotate ( Quaternion ( cs_record [ 'rotation' ] ) . rotation_matrix ) pc . translate ( np . array ( cs_record [ 'translation' ] ) ) # Second step: transform to the global frame. poserecord = nusc . get ( 'ego_pose' , pointsensor [ 'ego_pose_token' ] ) pc . rotate ( Quaternion ( poserecord [ 'rotation' ] ) . rotation_matrix ) pc . translate ( np . array ( poserecord [ 'translation' ] ) ) # Third step: transform into the ego vehicle frame for the timestamp of the image. poserecord = nusc . get ( 'ego_pose' , cam [ 'ego_pose_token' ] ) pc . translate ( - np . array ( poserecord [ 'translation' ] ) ) pc . rotate ( Quaternion ( poserecord [ 'rotation' ] ) . rotation_matrix . T ) # Fourth step: transform into the camera. cs_record = nusc . get ( 'calibrated_sensor' , cam [ 'calibrated_sensor_token' ] ) pc . translate ( - np . array ( cs_record [ 'translation' ] ) ) pc . rotate ( Quaternion ( cs_record [ 'rotation' ] ) . rotation_matrix . T ) # Fifth step: actually take a "picture" of the point cloud. # Grab the depths (camera frame z axis points away from the camera). depths = pc . points [ 2 , : ] # Take the actual picture (matrix multiplication with camera-matrix + renormalization). points = view_points ( pc . points [ : 3 , : ] , np . array ( cs_record [ 'camera_intrinsic' ] ) , normalize = True ) # Remove points that are either outside or behind the camera. Leave a margin of 1 pixel for aesthetic reasons. mask = np . ones ( depths . shape [ 0 ] , dtype = bool ) mask = np . logical_and ( mask , depths > 0 ) mask = np . logical_and ( mask , points [ 0 , : ] > 1 ) mask = np . logical_and ( mask , points [ 0 , : ] < im . size [ 0 ] - 1 ) mask = np . logical_and ( mask , points [ 1 , : ] > 1 ) mask = np . logical_and ( mask , points [ 1 , : ] < im . size [ 1 ] - 1 ) points = points [ : , mask ] # Pick the colors of the points im_data = np . array ( im ) coloring = np . zeros ( points . shape ) for i , p in enumerate ( points . transpose ( ) ) : point = p [ : 2 ] . round ( ) . astype ( np . int32 ) coloring [ : , i ] = im_data [ point [ 1 ] , point [ 0 ] , : ] return coloring , mask