(config_path,
          model_dir,
          refine_weight = 5,
          result_path=None,
          create_folder=False,
          display_step=50,
          summary_step=5,
          pretrained_path=None,
          pretrained_include=None,
          pretrained_exclude=None,
          freeze_include=None,
          freeze_exclude=None,
          multi_gpu=False,
          measure_time=False,
          resume=False)
train a VoxelNet model specified by a config file.
train a VoxelNet model specified by a config file.
train a VoxelNet model specified by a config file .
def train(config_path,
          model_dir,
          refine_weight = 5,
          result_path=None,
          create_folder=False,
          display_step=50,
          summary_step=5,
          pretrained_path=None,
          pretrained_include=None,
          pretrained_exclude=None,
          freeze_include=None,
          freeze_exclude=None,
          multi_gpu=False,
          measure_time=False,
          resume=False):
    """train a VoxelNet model specified by a config file.
    """
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    
    model_dir = str(Path(model_dir).resolve())
    if create_folder:
        if Path(model_dir).exists():
            model_dir = torchplus.train.create_folder(model_dir)
    model_dir = Path(model_dir)
    if not resume and model_dir.exists():
        raise ValueError("model dir exists and you don't specify resume.")
    model_dir.mkdir(parents=True, exist_ok=True)
    if result_path is None:
        result_path = model_dir / 'results'
    config_file_bkp = "pipeline.config"
    if isinstance(config_path, str):
        # directly provide a config object. this usually used
        # when you want to train with several different parameters in
        # one script.
        config = pipeline_pb2.TrainEvalPipelineConfig()
        with open(config_path, "r") as f:
            proto_str = f.read()
            text_format.Merge(proto_str, config)
    else:
        config = config_path
        proto_str = text_format.MessageToString(config, indent=2)
    with (model_dir / config_file_bkp).open("w") as f:
        f.write(proto_str)

    input_cfg = config.train_input_reader
    eval_input_cfg = config.eval_input_reader
    model_cfg = config.model.second
    train_cfg = config.train_config

    net = build_network(model_cfg, measure_time).to(device)
    # if train_cfg.enable_mixed_precision:
    #     net.half()
    #     net.metrics_to_float()
    #     net.convert_norm_to_float(net)
    target_assigner = net.target_assigner
    voxel_generator = net.voxel_generator
    print("num parameters:", len(list(net.parameters())))
    torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
    if pretrained_path is not None:
        model_dict = net.state_dict()
        pretrained_dict = torch.load(pretrained_path)
        pretrained_dict = filter_param_dict(pretrained_dict, pretrained_include, pretrained_exclude)
        new_pretrained_dict = {}
        for k, v in pretrained_dict.items():
            if k in model_dict and v.shape == model_dict[k].shape:
                new_pretrained_dict[k] = v        
        print("Load pretrained parameters:")
        for k, v in new_pretrained_dict.items():
            print(k, v.shape)
        model_dict.update(new_pretrained_dict) 
        net.load_state_dict(model_dict)
        freeze_params_v2(dict(net.named_parameters()), freeze_include, freeze_exclude)
        net.clear_global_step()
        net.clear_metrics()
    if multi_gpu:
        net_parallel = torch.nn.DataParallel(net)
    else:
        net_parallel = net
    optimizer_cfg = train_cfg.optimizer
    loss_scale = train_cfg.loss_scale_factor
    fastai_optimizer = optimizer_builder.build(
        optimizer_cfg,
        net,
        mixed=False,
        loss_scale=loss_scale)
    if loss_scale < 0:
        loss_scale = "dynamic"
    if train_cfg.enable_mixed_precision:
        max_num_voxels = input_cfg.preprocess.max_number_of_voxels * input_cfg.batch_size
        assert max_num_voxels < 65535, "spconv fp16 training only support this"
        from apex import amp
        net, amp_optimizer = amp.initialize(net, fastai_optimizer,
                                        opt_level="O2",
                                        keep_batchnorm_fp32=True,
                                        loss_scale=loss_scale
                                        )
        net.metrics_to_float()
    else:
        amp_optimizer = fastai_optimizer
    torchplus.train.try_restore_latest_checkpoints(model_dir,
                                                   [fastai_optimizer])
    lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, amp_optimizer,
                                              train_cfg.steps)
    if train_cfg.enable_mixed_precision:
        float_dtype = torch.float16
    else:
        float_dtype = torch.float32

    if multi_gpu:
        num_gpu = torch.cuda.device_count()
        print(f"MULTI-GPU: use {num_gpu} gpu")
        collate_fn = merge_second_batch_multigpu
    else:
        collate_fn = merge_second_batch
        num_gpu = 1

    ######################
    # PREPARE INPUT
    ######################
    dataset = input_reader_builder.build(
        input_cfg,
        model_cfg,
        training=True,
        voxel_generator=voxel_generator,
        target_assigner=target_assigner,
        multi_gpu=multi_gpu)
    eval_dataset = input_reader_builder.build(
        eval_input_cfg,
        model_cfg,
        training=False,
        voxel_generator=voxel_generator,
        target_assigner=target_assigner)

    dataloader = torch.utils.data.DataLoader(
        dataset,
        batch_size=input_cfg.batch_size * num_gpu,
        shuffle=True,
        num_workers=input_cfg.preprocess.num_workers * num_gpu,
        pin_memory=False,
        collate_fn=collate_fn,
        worker_init_fn=_worker_init_fn,
        drop_last=not multi_gpu)
    eval_dataloader = torch.utils.data.DataLoader(
        eval_dataset,
        batch_size=eval_input_cfg.batch_size, # only support multi-gpu train
        shuffle=False,
        num_workers=eval_input_cfg.preprocess.num_workers,
        pin_memory=False,
        collate_fn=merge_second_batch)

    ######################
    # TRAINING
    ######################
    model_logging = SimpleModelLog(model_dir)
    model_logging.open()
    model_logging.log_text(proto_str + "\n", 0, tag="config")
    start_step = net.get_global_step()
    total_step = train_cfg.steps
    t = time.time()
    steps_per_eval = train_cfg.steps_per_eval
    clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch

    amp_optimizer.zero_grad()
    step_times = []
    step = start_step
    try:
        while True:
            if clear_metrics_every_epoch:
                net.clear_metrics()
            for example in dataloader:
                lr_scheduler.step(net.get_global_step())
                time_metrics = example["metrics"]
                example.pop("metrics")
                example_torch = example_convert_to_torch(example, float_dtype)

                batch_size = example["anchors"].shape[0]

                ret_dict = net_parallel(example_torch, refine_weight)
                cls_preds = ret_dict["cls_preds"]
                loss = ret_dict["loss"].mean()
                cls_loss_reduced = ret_dict["cls_loss_reduced"].mean()
                loc_loss_reduced = ret_dict["loc_loss_reduced"].mean()
                cls_pos_loss = ret_dict["cls_pos_loss"].mean()
                cls_neg_loss = ret_dict["cls_neg_loss"].mean()
                loc_loss = ret_dict["loc_loss"]
                cls_loss = ret_dict["cls_loss"]
                
                cared = ret_dict["cared"]
                labels = example_torch["labels"]
                if train_cfg.enable_mixed_precision:
                    with amp.scale_loss(loss, amp_optimizer) as scaled_loss:
                        scaled_loss.backward()
                else:
                    loss.backward()
                torch.nn.utils.clip_grad_norm_(net.parameters(), 10.0)
                amp_optimizer.step()
                amp_optimizer.zero_grad()
                net.update_global_step()
                net_metrics = net.update_metrics(cls_loss_reduced,
                                                 loc_loss_reduced, cls_preds,
                                                 labels, cared)

                step_time = (time.time() - t)
                step_times.append(step_time)
                t = time.time()
                metrics = {}
                num_pos = int((labels > 0)[0].float().sum().cpu().numpy())
                num_neg = int((labels == 0)[0].float().sum().cpu().numpy())
                if 'anchors_mask' not in example_torch:
                    num_anchors = example_torch['anchors'].shape[1]
                else:
                    num_anchors = int(example_torch['anchors_mask'][0].sum())
                global_step = net.get_global_step()

                if global_step % display_step == 0:
                    if measure_time:
                        for name, val in net.get_avg_time_dict().items():
                            print(f"avg {name} time = {val * 1000:.3f} ms")

                    loc_loss_elem = [
                        float(loc_loss[:, :, i].sum().detach().cpu().numpy() /
                              batch_size) for i in range(loc_loss.shape[-1])
                    ]
                    metrics["runtime"] = {
                        "step": global_step,
                        "steptime": np.mean(step_times),
                    }
                    metrics["runtime"].update(time_metrics[0])
                    step_times = []
                    metrics.update(net_metrics)
                    metrics["loss"]["loc_elem"] = loc_loss_elem
                    metrics["loss"]["cls_pos_rt"] = float(
                        cls_pos_loss.detach().cpu().numpy())
                    metrics["loss"]["cls_neg_rt"] = float(
                        cls_neg_loss.detach().cpu().numpy())

                    ########################################
                    if model_cfg.rpn.module_class_name == "PSA" or model_cfg.rpn.module_class_name == "RefineDet":
                        coarse_loss = ret_dict["coarse_loss"]
                        refine_loss = ret_dict["refine_loss"]
                        metrics["coarse_loss"] = float(coarse_loss.detach().cpu().numpy())
                        metrics["refine_loss"] = float(refine_loss.detach().cpu().numpy())
                    ########################################

                    if model_cfg.use_direction_classifier:
                        dir_loss_reduced = ret_dict["dir_loss_reduced"].mean()
                        metrics["loss"]["dir_rt"] = float(
                            dir_loss_reduced.detach().cpu().numpy())

                    metrics["misc"] = {
                        "num_vox": int(example_torch["voxels"].shape[0]),
                        "num_pos": int(num_pos),
                        "num_neg": int(num_neg),
                        "num_anchors": int(num_anchors),
                        "lr": float(amp_optimizer.lr),
                        "mem_usage": psutil.virtual_memory().percent,
                    }
                    model_logging.log_metrics(metrics, global_step)

                if global_step % steps_per_eval == 0:
                    torchplus.train.save_models(model_dir, [net, amp_optimizer],
                                                net.get_global_step())
                    net.eval()
                    result_path_step = result_path / f"step_{net.get_global_step()}"
                    result_path_step.mkdir(parents=True, exist_ok=True)
                    model_logging.log_text("#################################",
                                        global_step)
                    model_logging.log_text("# EVAL", global_step)
                    model_logging.log_text("#################################",
                                        global_step)
                    model_logging.log_text("Generate output labels...", global_step)
                    t = time.time()
                    detections = []
                    detections_coarse = []
                    detections_refine = []
                    prog_bar = ProgressBar()
                    net.clear_timer()
                    prog_bar.start((len(eval_dataset) + eval_input_cfg.batch_size - 1)
                                // eval_input_cfg.batch_size)
                    for example in iter(eval_dataloader):
                        example = example_convert_to_torch(example, float_dtype)
                        if model_cfg.rpn.module_class_name == "PSA" or model_cfg.rpn.module_class_name == "RefineDet":
                            coarse, refine = net(example)
                            detections_coarse += coarse
                            detections_refine += refine
                            detections = detections_coarse
                        else:
                            detections += net(example)
                        prog_bar.print_bar()

                    sec_per_ex = len(eval_dataset) / (time.time() - t)
                    model_logging.log_text(
                        f'generate label finished({sec_per_ex:.2f}/s). start eval:',
                        global_step)
                    result_dict = eval_dataset.dataset.evaluation(
                        detections, str(result_path_step))
                    for k, v in result_dict["results"].items():
                        model_logging.log_text("Evaluation {}".format(k), global_step)
                        model_logging.log_text(v, global_step)
                    model_logging.log_metrics(result_dict["detail"], global_step)
                    with open(result_path_step / "result.pkl", 'wb') as f:
                        pickle.dump(detections, f)

                    if model_cfg.rpn.module_class_name == "PSA" or model_cfg.rpn.module_class_name == "RefineDet":
                        print("\n")
                        print("After Refine:")
                        with open(result_path_step / "result_refine.pkl", 'wb') as f:
                            pickle.dump(detections_refine, f)
                        result_dict_refine = eval_dataset.dataset.evaluation(
                            detections_refine, str(result_path_step))
                        for k, v in result_dict_refine["results"].items():
                            model_logging.log_text("Evaluation {}".format(k), global_step)
                            model_logging.log_text(v, global_step)
                        model_logging.log_metrics(result_dict_refine["detail"], global_step)

                    net.train()
                step += 1
                if step >= total_step:
                    break
            if step >= total_step:
                break
    except Exception as e:
        print(json.dumps(example["metadata"], indent=2))
        model_logging.log_text(str(e), step)
        model_logging.log_text(json.dumps(example["metadata"], indent=2), step)
        torchplus.train.save_models(model_dir, [net, amp_optimizer],
                                    step)
        raise e
    finally:
        model_logging.close()
    torchplus.train.save_models(model_dir, [net, amp_optimizer],
                                net.get_global_step())
def train ( config_path , model_dir , refine_weight = 5 , result_path = None , create_folder = False , display_step = 50 , summary_step = 5 , pretrained_path = None , pretrained_include = None , pretrained_exclude = None , freeze_include = None , freeze_exclude = None , multi_gpu = False , measure_time = False , resume = False ) : device = torch . device ( "cuda" if torch . cuda . is_available ( ) else "cpu" ) model_dir = str ( Path ( model_dir ) . resolve ( ) ) if create_folder : if Path ( model_dir ) . exists ( ) : model_dir = torchplus . train . create_folder ( model_dir ) model_dir = Path ( model_dir ) if not resume and model_dir . exists ( ) : raise ValueError ( "model dir exists and you don't specify resume." ) model_dir . mkdir ( parents = True , exist_ok = True ) if result_path is None : result_path = model_dir / 'results' config_file_bkp = "pipeline.config" if isinstance ( config_path , str ) : # directly provide a config object. this usually used # when you want to train with several different parameters in # one script. config = pipeline_pb2 . TrainEvalPipelineConfig ( ) with open ( config_path , "r" ) as f : proto_str = f . read ( ) text_format . Merge ( proto_str , config ) else : config = config_path proto_str = text_format . MessageToString ( config , indent = 2 ) with ( model_dir / config_file_bkp ) . open ( "w" ) as f : f . write ( proto_str ) input_cfg = config . train_input_reader eval_input_cfg = config . eval_input_reader model_cfg = config . model . second train_cfg = config . train_config net = build_network ( model_cfg , measure_time ) . to ( device ) # if train_cfg.enable_mixed_precision: #     net.half() #     net.metrics_to_float() #     net.convert_norm_to_float(net) target_assigner = net . target_assigner voxel_generator = net . voxel_generator print ( "num parameters:" , len ( list ( net . parameters ( ) ) ) ) torchplus . train . try_restore_latest_checkpoints ( model_dir , [ net ] ) if pretrained_path is not None : model_dict = net . state_dict ( ) pretrained_dict = torch . load ( pretrained_path ) pretrained_dict = filter_param_dict ( pretrained_dict , pretrained_include , pretrained_exclude ) new_pretrained_dict = { } for k , v in pretrained_dict . items ( ) : if k in model_dict and v . shape == model_dict [ k ] . shape : new_pretrained_dict [ k ] = v print ( "Load pretrained parameters:" ) for k , v in new_pretrained_dict . items ( ) : print ( k , v . shape ) model_dict . update ( new_pretrained_dict ) net . load_state_dict ( model_dict ) freeze_params_v2 ( dict ( net . named_parameters ( ) ) , freeze_include , freeze_exclude ) net . clear_global_step ( ) net . clear_metrics ( ) if multi_gpu : net_parallel = torch . nn . DataParallel ( net ) else : net_parallel = net optimizer_cfg = train_cfg . optimizer loss_scale = train_cfg . loss_scale_factor fastai_optimizer = optimizer_builder . build ( optimizer_cfg , net , mixed = False , loss_scale = loss_scale ) if loss_scale < 0 : loss_scale = "dynamic" if train_cfg . enable_mixed_precision : max_num_voxels = input_cfg . preprocess . max_number_of_voxels * input_cfg . batch_size assert max_num_voxels < 65535 , "spconv fp16 training only support this" from apex import amp net , amp_optimizer = amp . initialize ( net , fastai_optimizer , opt_level = "O2" , keep_batchnorm_fp32 = True , loss_scale = loss_scale ) net . metrics_to_float ( ) else : amp_optimizer = fastai_optimizer torchplus . train . try_restore_latest_checkpoints ( model_dir , [ fastai_optimizer ] ) lr_scheduler = lr_scheduler_builder . build ( optimizer_cfg , amp_optimizer , train_cfg . steps ) if train_cfg . enable_mixed_precision : float_dtype = torch . float16 else : float_dtype = torch . float32 if multi_gpu : num_gpu = torch . cuda . device_count ( ) print ( f"MULTI-GPU: use {num_gpu} gpu" ) collate_fn = merge_second_batch_multigpu else : collate_fn = merge_second_batch num_gpu = 1 ###################### # PREPARE INPUT ###################### dataset = input_reader_builder . build ( input_cfg , model_cfg , training = True , voxel_generator = voxel_generator , target_assigner = target_assigner , multi_gpu = multi_gpu ) eval_dataset = input_reader_builder . build ( eval_input_cfg , model_cfg , training = False , voxel_generator = voxel_generator , target_assigner = target_assigner ) dataloader = torch . utils . data . DataLoader ( dataset , batch_size = input_cfg . batch_size * num_gpu , shuffle = True , num_workers = input_cfg . preprocess . num_workers * num_gpu , pin_memory = False , collate_fn = collate_fn , worker_init_fn = _worker_init_fn , drop_last = not multi_gpu ) eval_dataloader = torch . utils . data . DataLoader ( eval_dataset , batch_size = eval_input_cfg . batch_size , # only support multi-gpu train shuffle = False , num_workers = eval_input_cfg . preprocess . num_workers , pin_memory = False , collate_fn = merge_second_batch ) ###################### # TRAINING ###################### model_logging = SimpleModelLog ( model_dir ) model_logging . open ( ) model_logging . log_text ( proto_str + "\n" , 0 , tag = "config" ) start_step = net . get_global_step ( ) total_step = train_cfg . steps t = time . time ( ) steps_per_eval = train_cfg . steps_per_eval clear_metrics_every_epoch = train_cfg . clear_metrics_every_epoch amp_optimizer . zero_grad ( ) step_times = [ ] step = start_step try : while True : if clear_metrics_every_epoch : net . clear_metrics ( ) for example in dataloader : lr_scheduler . step ( net . get_global_step ( ) ) time_metrics = example [ "metrics" ] example . pop ( "metrics" ) example_torch = example_convert_to_torch ( example , float_dtype ) batch_size = example [ "anchors" ] . shape [ 0 ] ret_dict = net_parallel ( example_torch , refine_weight ) cls_preds = ret_dict [ "cls_preds" ] loss = ret_dict [ "loss" ] . mean ( ) cls_loss_reduced = ret_dict [ "cls_loss_reduced" ] . mean ( ) loc_loss_reduced = ret_dict [ "loc_loss_reduced" ] . mean ( ) cls_pos_loss = ret_dict [ "cls_pos_loss" ] . mean ( ) cls_neg_loss = ret_dict [ "cls_neg_loss" ] . mean ( ) loc_loss = ret_dict [ "loc_loss" ] cls_loss = ret_dict [ "cls_loss" ] cared = ret_dict [ "cared" ] labels = example_torch [ "labels" ] if train_cfg . enable_mixed_precision : with amp . scale_loss ( loss , amp_optimizer ) as scaled_loss : scaled_loss . backward ( ) else : loss . backward ( ) torch . nn . utils . clip_grad_norm_ ( net . parameters ( ) , 10.0 ) amp_optimizer . step ( ) amp_optimizer . zero_grad ( ) net . update_global_step ( ) net_metrics = net . update_metrics ( cls_loss_reduced , loc_loss_reduced , cls_preds , labels , cared ) step_time = ( time . time ( ) - t ) step_times . append ( step_time ) t = time . time ( ) metrics = { } num_pos = int ( ( labels > 0 ) [ 0 ] . float ( ) . sum ( ) . cpu ( ) . numpy ( ) ) num_neg = int ( ( labels == 0 ) [ 0 ] . float ( ) . sum ( ) . cpu ( ) . numpy ( ) ) if 'anchors_mask' not in example_torch : num_anchors = example_torch [ 'anchors' ] . shape [ 1 ] else : num_anchors = int ( example_torch [ 'anchors_mask' ] [ 0 ] . sum ( ) ) global_step = net . get_global_step ( ) if global_step % display_step == 0 : if measure_time : for name , val in net . get_avg_time_dict ( ) . items ( ) : print ( f"avg {name} time = {val * 1000:.3f} ms" ) loc_loss_elem = [ float ( loc_loss [ : , : , i ] . sum ( ) . detach ( ) . cpu ( ) . numpy ( ) / batch_size ) for i in range ( loc_loss . shape [ - 1 ] ) ] metrics [ "runtime" ] = { "step" : global_step , "steptime" : np . mean ( step_times ) , } metrics [ "runtime" ] . update ( time_metrics [ 0 ] ) step_times = [ ] metrics . update ( net_metrics ) metrics [ "loss" ] [ "loc_elem" ] = loc_loss_elem metrics [ "loss" ] [ "cls_pos_rt" ] = float ( cls_pos_loss . detach ( ) . cpu ( ) . numpy ( ) ) metrics [ "loss" ] [ "cls_neg_rt" ] = float ( cls_neg_loss . detach ( ) . cpu ( ) . numpy ( ) ) ######################################## if model_cfg . rpn . module_class_name == "PSA" or model_cfg . rpn . module_class_name == "RefineDet" : coarse_loss = ret_dict [ "coarse_loss" ] refine_loss = ret_dict [ "refine_loss" ] metrics [ "coarse_loss" ] = float ( coarse_loss . detach ( ) . cpu ( ) . numpy ( ) ) metrics [ "refine_loss" ] = float ( refine_loss . detach ( ) . cpu ( ) . numpy ( ) ) ######################################## if model_cfg . use_direction_classifier : dir_loss_reduced = ret_dict [ "dir_loss_reduced" ] . mean ( ) metrics [ "loss" ] [ "dir_rt" ] = float ( dir_loss_reduced . detach ( ) . cpu ( ) . numpy ( ) ) metrics [ "misc" ] = { "num_vox" : int ( example_torch [ "voxels" ] . shape [ 0 ] ) , "num_pos" : int ( num_pos ) , "num_neg" : int ( num_neg ) , "num_anchors" : int ( num_anchors ) , "lr" : float ( amp_optimizer . lr ) , "mem_usage" : psutil . virtual_memory ( ) . percent , } model_logging . log_metrics ( metrics , global_step ) if global_step % steps_per_eval == 0 : torchplus . train . save_models ( model_dir , [ net , amp_optimizer ] , net . get_global_step ( ) ) net . eval ( ) result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step . mkdir ( parents = True , exist_ok = True ) model_logging . log_text ( "#################################" , global_step ) model_logging . log_text ( "# EVAL" , global_step ) model_logging . log_text ( "#################################" , global_step ) model_logging . log_text ( "Generate output labels..." , global_step ) t = time . time ( ) detections = [ ] detections_coarse = [ ] detections_refine = [ ] prog_bar = ProgressBar ( ) net . clear_timer ( ) prog_bar . start ( ( len ( eval_dataset ) + eval_input_cfg . batch_size - 1 ) // eval_input_cfg . batch_size ) for example in iter ( eval_dataloader ) : example = example_convert_to_torch ( example , float_dtype ) if model_cfg . rpn . module_class_name == "PSA" or model_cfg . rpn . module_class_name == "RefineDet" : coarse , refine = net ( example ) detections_coarse += coarse detections_refine += refine detections = detections_coarse else : detections += net ( example ) prog_bar . print_bar ( ) sec_per_ex = len ( eval_dataset ) / ( time . time ( ) - t ) model_logging . log_text ( f'generate label finished({sec_per_ex:.2f}/s). start eval:' , global_step ) result_dict = eval_dataset . dataset . evaluation ( detections , str ( result_path_step ) ) for k , v in result_dict [ "results" ] . items ( ) : model_logging . log_text ( "Evaluation {}" . format ( k ) , global_step ) model_logging . log_text ( v , global_step ) model_logging . log_metrics ( result_dict [ "detail" ] , global_step ) with open ( result_path_step / "result.pkl" , 'wb' ) as f : pickle . dump ( detections , f ) if model_cfg . rpn . module_class_name == "PSA" or model_cfg . rpn . module_class_name == "RefineDet" : print ( "\n" ) print ( "After Refine:" ) with open ( result_path_step / "result_refine.pkl" , 'wb' ) as f : pickle . dump ( detections_refine , f ) result_dict_refine = eval_dataset . dataset . evaluation ( detections_refine , str ( result_path_step ) ) for k , v in result_dict_refine [ "results" ] . items ( ) : model_logging . log_text ( "Evaluation {}" . format ( k ) , global_step ) model_logging . log_text ( v , global_step ) model_logging . log_metrics ( result_dict_refine [ "detail" ] , global_step ) net . train ( ) step += 1 if step >= total_step : break if step >= total_step : break except Exception as e : print ( json . dumps ( example [ "metadata" ] , indent = 2 ) ) model_logging . log_text ( str ( e ) , step ) model_logging . log_text ( json . dumps ( example [ "metadata" ] , indent = 2 ) , step ) torchplus . train . save_models ( model_dir , [ net , amp_optimizer ] , step ) raise e finally : model_logging . close ( ) torchplus . train . save_models ( model_dir , [ net , amp_optimizer ] , net . get_global_step ( ) )