(self)
Runs the training pipeline with all given parameters in Trainer.
Runs the training pipeline with all given parameters in Trainer.
Runs the training pipeline with all given parameters in Trainer .
def train(self):
        """
        Runs the training pipeline with all given parameters in Trainer.
        """
        # Restore models
        global_step = self._restore_models_and_step()
        print("INFO: Starting training from global step {}...".format(
            global_step))

        try:
            start_time = time.time()

            # Iterate through data
            iter_dataloader = iter(self.dataloader)
            while global_step < self.num_steps:
                log_data = metric_log.MetricLog()  # log data for tensorboard

                # -------------------------
                #   One Training Step
                # -------------------------
                # Update n_dis times for D
                for i in range(self.n_dis):
                    iter_dataloader, real_batch = self._fetch_data(
                        iter_dataloader=iter_dataloader)

                    # -----------------------
                    #   Update G Network
                    # -----------------------
                    # Update G, but only once.
                    if i == 0:
                        errG = self.netG.train_step(
                            real_batch,
                            netD=self.netD,
                            optG=self.optG)
                        log_data.add_metric("errG", errG.item(), group="loss")

                    # ------------------------
                    #   Update D Network
                    # -----------------------
                    errD, D_x, D_Gz = self.netD.train_step(real_batch,
                                                           netG=self.netG,
                                                           optD=self.optD)
                    log_data.add_metric("errD", errD.item(), group="loss")
                    log_data.add_metric("D_x", D_x.item(), group="prob")
                    log_data.add_metric("D_Gz", D_Gz.item(), group="prob")

                # --------------------------------
                #   Update Training Variables
                # -------------------------------
                global_step += 1

                log_data = self.scheduler.step(log_data=log_data,
                                               global_step=global_step)

                # -------------------------
                #   Logging and Metrics
                # -------------------------
                if global_step % self.log_steps == 0:
                    self.logger.write_summaries(log_data=log_data,
                                                global_step=global_step)

                if global_step % self.print_steps == 0:
                    curr_time = time.time()
                    self.logger.print_log(global_step=global_step,
                                          log_data=log_data,
                                          time_taken=(curr_time - start_time) /
                                          self.print_steps)
                    start_time = curr_time

                if global_step % self.vis_steps == 0:
                    self.logger.vis_images(netG=self.netG,
                                           global_step=global_step)

                if global_step % self.save_steps == 0:
                    print("INFO: Saving checkpoints...")
                    self.netG.save_checkpoint(directory=self.netG_ckpt_dir,
                                              global_step=global_step,
                                              optimizer=self.optG)

                    self.netD.save_checkpoint(directory=self.netD_ckpt_dir,
                                              global_step=global_step,
                                              optimizer=self.optD)

            # Save models at the very end of training
            if self.save_when_end:
                print("INFO: Saving final checkpoints...")
                self.netG.save_checkpoint(directory=self.netG_ckpt_dir,
                                          global_step=global_step,
                                          optimizer=self.optG)

                self.netD.save_checkpoint(directory=self.netD_ckpt_dir,
                                          global_step=global_step,
                                          optimizer=self.optD)

        except KeyboardInterrupt:
            print("INFO: Saving checkpoints from keyboard interrupt...")
            self.netG.save_checkpoint(directory=self.netG_ckpt_dir,
                                      global_step=global_step,
                                      optimizer=self.optG)

            self.netD.save_checkpoint(directory=self.netD_ckpt_dir,
                                      global_step=global_step,
                                      optimizer=self.optD)

        finally:
            self.logger.close_writers()

        print("INFO: Training Ended.")
def train ( self ) : # Restore models global_step = self . _restore_models_and_step ( ) print ( "INFO: Starting training from global step {}..." . format ( global_step ) ) try : start_time = time . time ( ) # Iterate through data iter_dataloader = iter ( self . dataloader ) while global_step < self . num_steps : log_data = metric_log . MetricLog ( ) # log data for tensorboard # ------------------------- #   One Training Step # ------------------------- # Update n_dis times for D for i in range ( self . n_dis ) : iter_dataloader , real_batch = self . _fetch_data ( iter_dataloader = iter_dataloader ) # ----------------------- #   Update G Network # ----------------------- # Update G, but only once. if i == 0 : errG = self . netG . train_step ( real_batch , netD = self . netD , optG = self . optG ) log_data . add_metric ( "errG" , errG . item ( ) , group = "loss" ) # ------------------------ #   Update D Network # ----------------------- errD , D_x , D_Gz = self . netD . train_step ( real_batch , netG = self . netG , optD = self . optD ) log_data . add_metric ( "errD" , errD . item ( ) , group = "loss" ) log_data . add_metric ( "D_x" , D_x . item ( ) , group = "prob" ) log_data . add_metric ( "D_Gz" , D_Gz . item ( ) , group = "prob" ) # -------------------------------- #   Update Training Variables # ------------------------------- global_step += 1 log_data = self . scheduler . step ( log_data = log_data , global_step = global_step ) # ------------------------- #   Logging and Metrics # ------------------------- if global_step % self . log_steps == 0 : self . logger . write_summaries ( log_data = log_data , global_step = global_step ) if global_step % self . print_steps == 0 : curr_time = time . time ( ) self . logger . print_log ( global_step = global_step , log_data = log_data , time_taken = ( curr_time - start_time ) / self . print_steps ) start_time = curr_time if global_step % self . vis_steps == 0 : self . logger . vis_images ( netG = self . netG , global_step = global_step ) if global_step % self . save_steps == 0 : print ( "INFO: Saving checkpoints..." ) self . netG . save_checkpoint ( directory = self . netG_ckpt_dir , global_step = global_step , optimizer = self . optG ) self . netD . save_checkpoint ( directory = self . netD_ckpt_dir , global_step = global_step , optimizer = self . optD ) # Save models at the very end of training if self . save_when_end : print ( "INFO: Saving final checkpoints..." ) self . netG . save_checkpoint ( directory = self . netG_ckpt_dir , global_step = global_step , optimizer = self . optG ) self . netD . save_checkpoint ( directory = self . netD_ckpt_dir , global_step = global_step , optimizer = self . optD ) except KeyboardInterrupt : print ( "INFO: Saving checkpoints from keyboard interrupt..." ) self . netG . save_checkpoint ( directory = self . netG_ckpt_dir , global_step = global_step , optimizer = self . optG ) self . netD . save_checkpoint ( directory = self . netD_ckpt_dir , global_step = global_step , optimizer = self . optD ) finally : self . logger . close_writers ( ) print ( "INFO: Training Ended." )