(self)
return loader, batch_sample
[summary]
        build training graph
        
        Returns:
            data loader and batch sample for train() to initialize
            undefined placeholders
[summary]
        build training graph
        
        Returns:
            data loader and batch sample for train() to initialize
            undefined placeholders
[ summary ] build training graph Returns : data loader and batch sample for train () to initialize undefined placeholders
def build_train_graph(self):
        '''[summary]
        build training graph
        
        Returns:
            data loader and batch sample for train() to initialize
            undefined placeholders
        '''

        opt = self.opt
        is_read_pose = opt.with_pose or opt.pose_weight > 0
        loader = DataLoader(opt.dataset_dir,
                            opt.batch_size,
                            opt.img_height,
                            opt.img_width,
                            opt.num_source,
                            opt.num_scales,
                            is_read_pose,
                            opt.match_num)
        with tf.name_scope("data_loading"):
            batch_sample = loader.load_train_batch()
            # give additional batch_size info since the input is undetermined placeholder
            inputs = batch_sample.get_next()
            tgt_image = inputs[0]
            src_image_stack = inputs[1]
            intrinsics = inputs[2]
            #[bs, 128, 416, 3]
            tgt_image.set_shape([opt.batch_size, opt.img_height, opt.img_width, 3])
            # [bs, 128, 416, 6]
            src_image_stack.set_shape([opt.batch_size, opt.img_height, opt.img_width, 3*opt.num_source])
            # [bs, 4, 3, 3]
            intrinsics.set_shape([opt.batch_size, opt.num_scales, 3, 3])
            if is_read_pose:
                poses = inputs[3]
                poses.set_shape([opt.batch_size, 3, 6])
            if opt.match_num > 0:
                matches = inputs[3]
                matches.set_shape([opt.batch_size, opt.num_source, opt.match_num, 4])
            tgt_image = self.preprocess_image(tgt_image)
            src_image_stack = self.preprocess_image(src_image_stack)

        with tf.name_scope("depth_prediction"):
            pred_disp, _ = disp_net_res50(tgt_image, is_training=True)
            if opt.with_pose:   # cannot normalize pose here due to given scale
                pred_depth = [1. / d for d in pred_disp]
            else:
                pred_depth = [1. / self.spatial_normalize(d) for d in pred_disp]

        with tf.name_scope("pose_and_explainability_prediction"):
            pred_poses, _ = pose_net(tgt_image, src_image_stack, is_training=True)

        with tf.name_scope("compute_loss"):
            pixel_loss = 0
            smooth_loss = 0
            pose_loss = 0
            ssim_loss = 0
            match_loss = 0
            tgt_image_all = []
            src_image_stack_all = []
            mask_stack_all = []
            proj_image_stack_all = []
            proj_error_stack_all = []
            for s in range(opt.num_scales):
                # Scale the source and target images for computing loss at the according scale.
                curr_tgt_image = tf.image.resize_area(tgt_image, 
                    [int(opt.img_height/(2**s)), int(opt.img_width/(2**s))])                
                curr_src_image_stack = tf.image.resize_area(src_image_stack, 
                    [int(opt.img_height/(2**s)), int(opt.img_width/(2**s))])

                if opt.smooth_weight > 0:
                    smooth_loss += opt.smooth_weight/(2**s) * \
                        self.compute_smooth_loss(pred_disp[s], curr_tgt_image)

                for i in range(opt.num_source):
                    # Inverse warp the source image to the target image frame
                    if is_read_pose:
                        relative_pose = get_relative_pose(poses[:,0,:], poses[:,i+1,:])
                        relative_rot = tf.slice(relative_pose, [0, 0, 0], [-1, 3, 3])
                        relative_rot_vec = mat2euler(relative_rot)
                        relative_trans_vec = tf.slice(relative_pose, [0, 0, 3], [-1, 3, 1])
                        relative_pose_vec = tf.squeeze(tf.concat([relative_rot_vec, relative_trans_vec], axis=1))
                    
                    if opt.with_pose:
                        warp_pose = relative_pose
                        pose_is_vec = False
                    else:
                        warp_pose = pred_poses[:,i,:]
                        pose_is_vec = True
                    
                    curr_proj_image, mask = projective_inverse_warp(
                        curr_src_image_stack[:,:,:,3*i:3*(i+1)], 
                        tf.squeeze(pred_depth[s], axis=3), 
                        warp_pose, intrinsics[:,s,:,:], is_vec=pose_is_vec)
                    curr_proj_error = tf.abs(curr_proj_image - curr_tgt_image)
                    curr_proj_error = tf.multiply(curr_proj_error, mask)

                    # below-threshold mask
                    perct_thresh = tf.contrib.distributions.percentile(curr_proj_error, q=99, axis=[1,2])
                    perct_thresh = tf.expand_dims(tf.expand_dims(perct_thresh, 1), 1)
                    curr_proj_error = tf.clip_by_value(curr_proj_error, 0, perct_thresh)
                    above_perct_thresh_region = tf.reduce_max(tf.cast(tf.equal(curr_proj_error, perct_thresh), 'float32'), axis=3)
                    above_perct_thresh_region = tf.greater_equal(above_perct_thresh_region, 1.0)
                    suppresion_mask = tf.expand_dims(1.0 - tf.cast(above_perct_thresh_region, 'float32'), axis=3)
                    curr_proj_error = tf.multiply(curr_proj_error, suppresion_mask)
                    mask = tf.multiply(mask, suppresion_mask)
                    pixel_loss += tf.reduce_mean(curr_proj_error) 

                    # SSIM loss
                    if opt.ssim_weight > 0:
                        ssim_mask = slim.avg_pool2d(mask, 3, 1, 'VALID')
                        ssim_loss += tf.reduce_mean(
                            ssim_mask * self.compute_ssim_loss(curr_proj_image, curr_tgt_image))

                    # Relative pose error
                    if opt.pose_weight > 0 and s == 0:  # only do it for highest resolution
                        pose_loss += tf.reduce_mean(self.compute_pose_loss(relative_pose_vec, pred_poses[:, i, :]))
                    
                    # Matches loss (fundamental matrix)
                    if opt.match_num > 0 and s == 0:  # only do it for highest resolution
                        match_loss += self.compute_match_loss(matches[:, i, :, :], tf.squeeze(
                            pred_depth[s], axis=3), pred_poses[:, i, :], intrinsics[:, s, :, :])

                    # Prepare images for tensorboard summaries
                    if i == 0:
                        proj_image_stack = curr_proj_image
                        mask_stack = mask
                        proj_error_stack = curr_proj_error
                    else:
                        proj_image_stack = tf.concat([proj_image_stack, curr_proj_image], axis=3)
                        mask_stack = tf.concat([mask_stack, mask], axis=3)
                        proj_error_stack = tf.concat([proj_error_stack, curr_proj_error], axis=3)
                tgt_image_all.append(curr_tgt_image)
                src_image_stack_all.append(curr_src_image_stack)
                proj_image_stack_all.append(proj_image_stack)
                mask_stack_all.append(mask_stack)
                proj_error_stack_all.append(proj_error_stack)
            total_loss = opt.ssim_weight * ssim_loss + \
                (1 - opt.ssim_weight) * pixel_loss + \
                smooth_loss + opt.pose_weight * pose_loss + opt.match_weight * match_loss

        with tf.name_scope("train_op"):
            train_vars = [var for var in tf.trainable_variables()]
            optim = tf.train.AdamOptimizer(opt.learning_rate, opt.beta1)
            # self.grads_and_vars = optim.compute_gradients(total_loss, 
            #                                               var_list=train_vars)
            # self.train_op = optim.apply_gradients(self.grads_and_vars)
            self.train_op = slim.learning.create_train_op(total_loss, optim)
            self.global_step = tf.Variable(0, name='global_step', trainable=False)
            self.incr_global_step = tf.assign(self.global_step, self.global_step+1)

        # Collect tensors that are useful later (e.g. tf summary)
        self.pred_depth = pred_depth
        self.pred_poses = pred_poses
        self.steps_per_epoch = loader.steps_per_epoch
        self.total_loss = total_loss
        self.pixel_loss = pixel_loss
        self.pose_loss = pose_loss
        self.smooth_loss = smooth_loss
        self.ssim_loss = ssim_loss
        self.match_loss = match_loss
        self.tgt_image_all = tgt_image_all
        self.src_image_stack_all = src_image_stack_all
        self.proj_image_stack_all = proj_image_stack_all
        self.mask_stack_all = mask_stack_all
        self.proj_error_stack_all = proj_error_stack_all
        return loader, batch_sample
def build_train_graph ( self ) : opt = self . opt is_read_pose = opt . with_pose or opt . pose_weight > 0 loader = DataLoader ( opt . dataset_dir , opt . batch_size , opt . img_height , opt . img_width , opt . num_source , opt . num_scales , is_read_pose , opt . match_num ) with tf . name_scope ( "data_loading" ) : batch_sample = loader . load_train_batch ( ) # give additional batch_size info since the input is undetermined placeholder inputs = batch_sample . get_next ( ) tgt_image = inputs [ 0 ] src_image_stack = inputs [ 1 ] intrinsics = inputs [ 2 ] #[bs, 128, 416, 3] tgt_image . set_shape ( [ opt . batch_size , opt . img_height , opt . img_width , 3 ] ) # [bs, 128, 416, 6] src_image_stack . set_shape ( [ opt . batch_size , opt . img_height , opt . img_width , 3 * opt . num_source ] ) # [bs, 4, 3, 3] intrinsics . set_shape ( [ opt . batch_size , opt . num_scales , 3 , 3 ] ) if is_read_pose : poses = inputs [ 3 ] poses . set_shape ( [ opt . batch_size , 3 , 6 ] ) if opt . match_num > 0 : matches = inputs [ 3 ] matches . set_shape ( [ opt . batch_size , opt . num_source , opt . match_num , 4 ] ) tgt_image = self . preprocess_image ( tgt_image ) src_image_stack = self . preprocess_image ( src_image_stack ) with tf . name_scope ( "depth_prediction" ) : pred_disp , _ = disp_net_res50 ( tgt_image , is_training = True ) if opt . with_pose : # cannot normalize pose here due to given scale pred_depth = [ 1. / d for d in pred_disp ] else : pred_depth = [ 1. / self . spatial_normalize ( d ) for d in pred_disp ] with tf . name_scope ( "pose_and_explainability_prediction" ) : pred_poses , _ = pose_net ( tgt_image , src_image_stack , is_training = True ) with tf . name_scope ( "compute_loss" ) : pixel_loss = 0 smooth_loss = 0 pose_loss = 0 ssim_loss = 0 match_loss = 0 tgt_image_all = [ ] src_image_stack_all = [ ] mask_stack_all = [ ] proj_image_stack_all = [ ] proj_error_stack_all = [ ] for s in range ( opt . num_scales ) : # Scale the source and target images for computing loss at the according scale. curr_tgt_image = tf . image . resize_area ( tgt_image , [ int ( opt . img_height / ( 2 ** s ) ) , int ( opt . img_width / ( 2 ** s ) ) ] ) curr_src_image_stack = tf . image . resize_area ( src_image_stack , [ int ( opt . img_height / ( 2 ** s ) ) , int ( opt . img_width / ( 2 ** s ) ) ] ) if opt . smooth_weight > 0 : smooth_loss += opt . smooth_weight / ( 2 ** s ) * self . compute_smooth_loss ( pred_disp [ s ] , curr_tgt_image ) for i in range ( opt . num_source ) : # Inverse warp the source image to the target image frame if is_read_pose : relative_pose = get_relative_pose ( poses [ : , 0 , : ] , poses [ : , i + 1 , : ] ) relative_rot = tf . slice ( relative_pose , [ 0 , 0 , 0 ] , [ - 1 , 3 , 3 ] ) relative_rot_vec = mat2euler ( relative_rot ) relative_trans_vec = tf . slice ( relative_pose , [ 0 , 0 , 3 ] , [ - 1 , 3 , 1 ] ) relative_pose_vec = tf . squeeze ( tf . concat ( [ relative_rot_vec , relative_trans_vec ] , axis = 1 ) ) if opt . with_pose : warp_pose = relative_pose pose_is_vec = False else : warp_pose = pred_poses [ : , i , : ] pose_is_vec = True curr_proj_image , mask = projective_inverse_warp ( curr_src_image_stack [ : , : , : , 3 * i : 3 * ( i + 1 ) ] , tf . squeeze ( pred_depth [ s ] , axis = 3 ) , warp_pose , intrinsics [ : , s , : , : ] , is_vec = pose_is_vec ) curr_proj_error = tf . abs ( curr_proj_image - curr_tgt_image ) curr_proj_error = tf . multiply ( curr_proj_error , mask ) # below-threshold mask perct_thresh = tf . contrib . distributions . percentile ( curr_proj_error , q = 99 , axis = [ 1 , 2 ] ) perct_thresh = tf . expand_dims ( tf . expand_dims ( perct_thresh , 1 ) , 1 ) curr_proj_error = tf . clip_by_value ( curr_proj_error , 0 , perct_thresh ) above_perct_thresh_region = tf . reduce_max ( tf . cast ( tf . equal ( curr_proj_error , perct_thresh ) , 'float32' ) , axis = 3 ) above_perct_thresh_region = tf . greater_equal ( above_perct_thresh_region , 1.0 ) suppresion_mask = tf . expand_dims ( 1.0 - tf . cast ( above_perct_thresh_region , 'float32' ) , axis = 3 ) curr_proj_error = tf . multiply ( curr_proj_error , suppresion_mask ) mask = tf . multiply ( mask , suppresion_mask ) pixel_loss += tf . reduce_mean ( curr_proj_error ) # SSIM loss if opt . ssim_weight > 0 : ssim_mask = slim . avg_pool2d ( mask , 3 , 1 , 'VALID' ) ssim_loss += tf . reduce_mean ( ssim_mask * self . compute_ssim_loss ( curr_proj_image , curr_tgt_image ) ) # Relative pose error if opt . pose_weight > 0 and s == 0 : # only do it for highest resolution pose_loss += tf . reduce_mean ( self . compute_pose_loss ( relative_pose_vec , pred_poses [ : , i , : ] ) ) # Matches loss (fundamental matrix) if opt . match_num > 0 and s == 0 : # only do it for highest resolution match_loss += self . compute_match_loss ( matches [ : , i , : , : ] , tf . squeeze ( pred_depth [ s ] , axis = 3 ) , pred_poses [ : , i , : ] , intrinsics [ : , s , : , : ] ) # Prepare images for tensorboard summaries if i == 0 : proj_image_stack = curr_proj_image mask_stack = mask proj_error_stack = curr_proj_error else : proj_image_stack = tf . concat ( [ proj_image_stack , curr_proj_image ] , axis = 3 ) mask_stack = tf . concat ( [ mask_stack , mask ] , axis = 3 ) proj_error_stack = tf . concat ( [ proj_error_stack , curr_proj_error ] , axis = 3 ) tgt_image_all . append ( curr_tgt_image ) src_image_stack_all . append ( curr_src_image_stack ) proj_image_stack_all . append ( proj_image_stack ) mask_stack_all . append ( mask_stack ) proj_error_stack_all . append ( proj_error_stack ) total_loss = opt . ssim_weight * ssim_loss + ( 1 - opt . ssim_weight ) * pixel_loss + smooth_loss + opt . pose_weight * pose_loss + opt . match_weight * match_loss with tf . name_scope ( "train_op" ) : train_vars = [ var for var in tf . trainable_variables ( ) ] optim = tf . train . AdamOptimizer ( opt . learning_rate , opt . beta1 ) # self.grads_and_vars = optim.compute_gradients(total_loss,  #                                               var_list=train_vars) # self.train_op = optim.apply_gradients(self.grads_and_vars) self . train_op = slim . learning . create_train_op ( total_loss , optim ) self . global_step = tf . Variable ( 0 , name = 'global_step' , trainable = False ) self . incr_global_step = tf . assign ( self . global_step , self . global_step + 1 ) # Collect tensors that are useful later (e.g. tf summary) self . pred_depth = pred_depth self . pred_poses = pred_poses self . steps_per_epoch = loader . steps_per_epoch self . total_loss = total_loss self . pixel_loss = pixel_loss self . pose_loss = pose_loss self . smooth_loss = smooth_loss self . ssim_loss = ssim_loss self . match_loss = match_loss self . tgt_image_all = tgt_image_all self . src_image_stack_all = src_image_stack_all self . proj_image_stack_all = proj_image_stack_all self . mask_stack_all = mask_stack_all self . proj_error_stack_all = proj_error_stack_all return loader , batch_sample