(self, w, pi, delta, trainloader, testloader, valloader,
                 loss_fn)
return best_model, statistics
Main L-C compression method.
    
        :param w: Input model.
        :type w: torch.nn.Module
        :param pi: Compression function.
        :param delta: Decompression function.
        :param trainloader: Training dataloader.
        :param testloader: Test dataloader.
        :param valloader: Validation dataloader.
        :param loss_fn: Loss criterion.
Main L-C compression method.
    
        :param w: Input model.
        :type w: torch.nn.Module
        :param pi: Compression function.
        :param delta: Decompression function.
        :param trainloader: Training dataloader.
        :param testloader: Test dataloader.
        :param valloader: Validation dataloader.
        :param loss_fn: Loss criterion.
Main L - C compression method . : param w : Input model . : type w : torch . nn . Module : param pi : Compression function . : param delta : Decompression function . : param trainloader : Training dataloader . : param testloader : Test dataloader . : param valloader : Validation dataloader . : param loss_fn : Loss criterion .
def compress(self, w, pi, delta, trainloader, testloader, valloader,
                 loss_fn):
        """
        Main L-C compression method.
    
        :param w: Input model.
        :type w: torch.nn.Module
        :param pi: Compression function.
        :param delta: Decompression function.
        :param trainloader: Training dataloader.
        :param testloader: Test dataloader.
        :param valloader: Validation dataloader.
        :param loss_fn: Loss criterion.
        """
        statistics = {}
        # Save engine configuration
        statistics.update(self._engine_config)

        _model_stat_fn = self.debugging_flags['custom_model_statistics']\
                   if 'custom_model_statistics' in self.debugging_flags\
                   else util.empty_stat_fn
        _disable_train_stats = self.debugging_flags['disable_train_stats']\
                     if 'disable_train_stats' in self.debugging_flags\
                     else False
        timer_lc = EventTimer()

        if self.use_cuda: cudnn.benchmark = True
        logger.debug("[Condensa] cuDNN VERSION: {}".format(cudnn.version()))

        validate = (valloader is not None)
        test = (testloader is not None)

        # Copy model to GPU0 memory
        if self.use_cuda: w = w.cuda(0)

        # Mark all compressible modules in w
        with record_mode():
            pi(w)

        with torch.no_grad():
            theta = deepcopy(w)
        self.zero_(theta)

        with torch.no_grad():
            lm = deepcopy(w)
        self.zero_(lm)

        with torch.no_grad():
            best_model = deepcopy(w)

        # Enable data-parallelism in  L step
        if self.use_cuda and self.distributed:
            ngpus = torch.cuda.device_count()
            logger.info('[Condensa] {} GPUs enabled for L-step'.format(ngpus))
            w = torch.nn.DataParallel(w)

        mu = 0.
        learning_rate = self.lr

        optimizer = self.l_optimizer(w,
                                     lr=learning_rate,
                                     **self.l_optimizer_params)
        optimizer.reset_state()

        if not _disable_train_stats:
            w_train_loss, w_train_stats = _model_stat_fn(w, loss_fn, trainloader)
            logger.info('[Condensa] w TRAIN\tloss={:.5f}, {}'
                .format(w_train_loss,
                ', '.join(['{}:{}'.format(k, v) for k,v in w_train_stats.items()])))
        if validate:
            w_val_loss, w_val_stats = _model_stat_fn(w, loss_fn, valloader)
            logger.info('[Condensa] w VAL\tloss={:.5f}, {}'
                .format(w_val_loss,
                ', '.join(['{}:{}'.format(k, v) for k,v in w_val_stats.items()])))
        if test:
            w_test_loss, w_test_stats = _model_stat_fn(w, loss_fn, testloader)
            logger.info('[Condensa] w TEST\tloss={:.5f}, {}'
                .format(w_test_loss,
                ', '.join(['{}:{}'.format(k, v) for k,v in w_test_stats.items()])))

        best_loss = sys.float_info.max
        train_losses = []
        if validate: val_losses = []
        if test: test_losses = []
        outer_lr_scheduler = None
        if self.lr_decay is not None:
            outer_lr_scheduler = ExpDecayedLR(self.lr, self.lr_decay)
        elif self.lr_schedule is not None:
            outer_lr_scheduler = DecayedLR(self.lr, self.lr_schedule,
                                           self.lr_multiplier)
        for j in range(0, self.steps):
            n_sgd_iter = (self.mb_iterations_first_l
                          if j == 1 else self.mb_iterations_per_l)

            # Set up outer learning rate
            learning_rate = self.lr
            if outer_lr_scheduler is not None:
                learning_rate = outer_lr_scheduler.learning_rate

            logger.info(
                '[Condensa] LC Iteration {}:\tmu={:.5f}, lr={:.5f}'.format(
                    j, mu, learning_rate))

            inner_lr_scheduler = None
            if self.lr_end is not None:
                inner_lr_scheduler = IntervalLR(learning_rate, self.lr_end,
                                                n_sgd_iter)

            # L step
            # Switch to training mode
            i = 0
            w.train()
            iterator = iter(trainloader)
            if logger.isEnabledFor(logging.INFO) and j>0:
                pbar = tqdm(total=n_sgd_iter, ascii=True)
            while True:
                if j == 0:
                    logger.info('[Condensa] Skipping first L-step')
                    break
                if j == 1 and i >= self.mb_iterations_first_l:
                    break
                if j > 1 and i >= self.mb_iterations_per_l:
                    break

                try:
                    inputs, targets = next(iterator)
                except StopIteration:
                    iterator = iter(trainloader)
                    inputs, targets = next(iterator)

                if self.use_cuda:
                    if not inputs.is_cuda: inputs = inputs.cuda()
                    if not targets.is_cuda:
                        targets = targets.cuda(non_blocking=True)
                outputs = w(inputs)
                loss = loss_fn(outputs, targets)

                optimizer.zero_grad()

                loss.backward()
                optimizer.step(learning_rate, mu, theta, lm)

                if inner_lr_scheduler is not None:
                    inner_lr_scheduler.step()
                    learning_rate = inner_lr_scheduler.learning_rate

                if logger.isEnabledFor(logging.INFO):
                    pbar.update()
                i += 1

            if logger.isEnabledFor(logging.INFO) and j>0:
                pbar.close()
            logger.info('')

            if self.use_cuda: torch.cuda.synchronize()

            w.eval()
            # C step and theta update
            try:
                theta.load_state_dict(w.module.state_dict())
            except AttributeError:
                theta.load_state_dict(w.state_dict())
            if mu > 0:
                try:
                    wmodules = w.module.modules()
                except AttributeError:
                    wmodules = w.modules()
                with record_mode():
                    pi(theta)
                with torch.no_grad():
                    for w_m, theta_m, lm_m in zip(wmodules, theta.modules(),
                                                  lm.modules()):
                        if hasattr(theta_m, 'condense'):
                            for pname in theta_m.condense:
                                getattr(theta_m, pname).data = (
                                    getattr(w_m, pname).detach() -
                                    getattr(lm_m, pname).data / mu)

            pi(theta)

            if not _disable_train_stats:
                nested_train_loss, nested_train_stats = _model_stat_fn(theta, loss_fn, trainloader)
                train_losses.append(nested_train_loss)
                logger.info(
                    '[Condensa] Nested (theta) TRAIN\tloss={:.5f}, {}'
                    .format(nested_train_loss,
                    ', '.join(['{}:{}'.format(k, v) for k,v in nested_train_stats.items()])))
            if validate:
                nested_val_loss, nested_val_stats = _model_stat_fn(theta, loss_fn, valloader)
                val_losses.append(nested_val_loss)
                logger.info(
                    '[Condensa] Nested (theta) VAL\tloss={:.5f}, {}'
                    .format(nested_val_loss,
                    ', '.join(['{}:{}'.format(k, v) for k,v in nested_val_stats.items()])))
            if test:
                nested_test_loss, nested_test_stats = _model_stat_fn(theta, loss_fn, testloader)
                test_losses.append(nested_test_loss)
                logger.info(
                    '[Condensa] Nested (theta) TEST\tloss={:.5f}, {}'
                    .format(nested_test_loss,
                    ', '.join(['{}:{}'.format(k, v) for k,v in nested_test_stats.items()])))

            if validate:
                if nested_val_loss < best_loss:
                    logger.info('[Condensa] Saving model based on VAL')
                    best_loss = nested_val_loss
                    # Deep-copy required here to preserve dtypes
                    best_model = deepcopy(theta)
            elif test:
                if nested_test_loss < best_loss:
                    logger.info('[Condensa] Saving model based on TEST')
                    best_loss = nested_test_loss
                    # Deep-copy required here to preserve dtypes
                    best_model = deepcopy(theta)
            else:
                logger.info('[Condensa] Saving model based on most recent')
                best_model = deepcopy(theta)

            # theta <- delta(theta)
            delta(theta)

            # LM update
            if mu > 0:
                try:
                    wmodules = w.module.modules()
                except AttributeError:
                    wmodules = w.modules()
                for w_m, theta_m, lm_m in zip(wmodules, theta.modules(),
                                              lm.modules()):
                    if hasattr(theta_m, 'condense'):
                        for pname in theta_m.condense:
                            getattr(
                                lm_m,
                                pname).data = (getattr(lm_m, pname).data - mu *
                                               (getattr(w_m, pname).detach() -
                                                getattr(theta_m, pname).data))

            optimizer.reset_state()
            # Update mu
            mu = self._update_mu(mu, self.mu_init, self.mu_multiplier,
                                 self.mu_cap)
            # Update LR schedule
            if outer_lr_scheduler is not None: outer_lr_scheduler.step()

        statistics['elapsed_lc'] = timer_lc.elapsed_seconds
        statistics['train_losses'] = train_losses
        if test: statistics['test_losses'] = test_losses
        if validate: statistics['val_losses'] = val_losses
        return best_model, statistics
def compress ( self , w , pi , delta , trainloader , testloader , valloader , loss_fn ) : statistics = { } # Save engine configuration statistics . update ( self . _engine_config ) _model_stat_fn = self . debugging_flags [ 'custom_model_statistics' ] if 'custom_model_statistics' in self . debugging_flags else util . empty_stat_fn _disable_train_stats = self . debugging_flags [ 'disable_train_stats' ] if 'disable_train_stats' in self . debugging_flags else False timer_lc = EventTimer ( ) if self . use_cuda : cudnn . benchmark = True logger . debug ( "[Condensa] cuDNN VERSION: {}" . format ( cudnn . version ( ) ) ) validate = ( valloader is not None ) test = ( testloader is not None ) # Copy model to GPU0 memory if self . use_cuda : w = w . cuda ( 0 ) # Mark all compressible modules in w with record_mode ( ) : pi ( w ) with torch . no_grad ( ) : theta = deepcopy ( w ) self . zero_ ( theta ) with torch . no_grad ( ) : lm = deepcopy ( w ) self . zero_ ( lm ) with torch . no_grad ( ) : best_model = deepcopy ( w ) # Enable data-parallelism in  L step if self . use_cuda and self . distributed : ngpus = torch . cuda . device_count ( ) logger . info ( '[Condensa] {} GPUs enabled for L-step' . format ( ngpus ) ) w = torch . nn . DataParallel ( w ) mu = 0. learning_rate = self . lr optimizer = self . l_optimizer ( w , lr = learning_rate , * * self . l_optimizer_params ) optimizer . reset_state ( ) if not _disable_train_stats : w_train_loss , w_train_stats = _model_stat_fn ( w , loss_fn , trainloader ) logger . info ( '[Condensa] w TRAIN\tloss={:.5f}, {}' . format ( w_train_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in w_train_stats . items ( ) ] ) ) ) if validate : w_val_loss , w_val_stats = _model_stat_fn ( w , loss_fn , valloader ) logger . info ( '[Condensa] w VAL\tloss={:.5f}, {}' . format ( w_val_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in w_val_stats . items ( ) ] ) ) ) if test : w_test_loss , w_test_stats = _model_stat_fn ( w , loss_fn , testloader ) logger . info ( '[Condensa] w TEST\tloss={:.5f}, {}' . format ( w_test_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in w_test_stats . items ( ) ] ) ) ) best_loss = sys . float_info . max train_losses = [ ] if validate : val_losses = [ ] if test : test_losses = [ ] outer_lr_scheduler = None if self . lr_decay is not None : outer_lr_scheduler = ExpDecayedLR ( self . lr , self . lr_decay ) elif self . lr_schedule is not None : outer_lr_scheduler = DecayedLR ( self . lr , self . lr_schedule , self . lr_multiplier ) for j in range ( 0 , self . steps ) : n_sgd_iter = ( self . mb_iterations_first_l if j == 1 else self . mb_iterations_per_l ) # Set up outer learning rate learning_rate = self . lr if outer_lr_scheduler is not None : learning_rate = outer_lr_scheduler . learning_rate logger . info ( '[Condensa] LC Iteration {}:\tmu={:.5f}, lr={:.5f}' . format ( j , mu , learning_rate ) ) inner_lr_scheduler = None if self . lr_end is not None : inner_lr_scheduler = IntervalLR ( learning_rate , self . lr_end , n_sgd_iter ) # L step # Switch to training mode i = 0 w . train ( ) iterator = iter ( trainloader ) if logger . isEnabledFor ( logging . INFO ) and j > 0 : pbar = tqdm ( total = n_sgd_iter , ascii = True ) while True : if j == 0 : logger . info ( '[Condensa] Skipping first L-step' ) break if j == 1 and i >= self . mb_iterations_first_l : break if j > 1 and i >= self . mb_iterations_per_l : break try : inputs , targets = next ( iterator ) except StopIteration : iterator = iter ( trainloader ) inputs , targets = next ( iterator ) if self . use_cuda : if not inputs . is_cuda : inputs = inputs . cuda ( ) if not targets . is_cuda : targets = targets . cuda ( non_blocking = True ) outputs = w ( inputs ) loss = loss_fn ( outputs , targets ) optimizer . zero_grad ( ) loss . backward ( ) optimizer . step ( learning_rate , mu , theta , lm ) if inner_lr_scheduler is not None : inner_lr_scheduler . step ( ) learning_rate = inner_lr_scheduler . learning_rate if logger . isEnabledFor ( logging . INFO ) : pbar . update ( ) i += 1 if logger . isEnabledFor ( logging . INFO ) and j > 0 : pbar . close ( ) logger . info ( '' ) if self . use_cuda : torch . cuda . synchronize ( ) w . eval ( ) # C step and theta update try : theta . load_state_dict ( w . module . state_dict ( ) ) except AttributeError : theta . load_state_dict ( w . state_dict ( ) ) if mu > 0 : try : wmodules = w . module . modules ( ) except AttributeError : wmodules = w . modules ( ) with record_mode ( ) : pi ( theta ) with torch . no_grad ( ) : for w_m , theta_m , lm_m in zip ( wmodules , theta . modules ( ) , lm . modules ( ) ) : if hasattr ( theta_m , 'condense' ) : for pname in theta_m . condense : getattr ( theta_m , pname ) . data = ( getattr ( w_m , pname ) . detach ( ) - getattr ( lm_m , pname ) . data / mu ) pi ( theta ) if not _disable_train_stats : nested_train_loss , nested_train_stats = _model_stat_fn ( theta , loss_fn , trainloader ) train_losses . append ( nested_train_loss ) logger . info ( '[Condensa] Nested (theta) TRAIN\tloss={:.5f}, {}' . format ( nested_train_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in nested_train_stats . items ( ) ] ) ) ) if validate : nested_val_loss , nested_val_stats = _model_stat_fn ( theta , loss_fn , valloader ) val_losses . append ( nested_val_loss ) logger . info ( '[Condensa] Nested (theta) VAL\tloss={:.5f}, {}' . format ( nested_val_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in nested_val_stats . items ( ) ] ) ) ) if test : nested_test_loss , nested_test_stats = _model_stat_fn ( theta , loss_fn , testloader ) test_losses . append ( nested_test_loss ) logger . info ( '[Condensa] Nested (theta) TEST\tloss={:.5f}, {}' . format ( nested_test_loss , ', ' . join ( [ '{}:{}' . format ( k , v ) for k , v in nested_test_stats . items ( ) ] ) ) ) if validate : if nested_val_loss < best_loss : logger . info ( '[Condensa] Saving model based on VAL' ) best_loss = nested_val_loss # Deep-copy required here to preserve dtypes best_model = deepcopy ( theta ) elif test : if nested_test_loss < best_loss : logger . info ( '[Condensa] Saving model based on TEST' ) best_loss = nested_test_loss # Deep-copy required here to preserve dtypes best_model = deepcopy ( theta ) else : logger . info ( '[Condensa] Saving model based on most recent' ) best_model = deepcopy ( theta ) # theta <- delta(theta) delta ( theta ) # LM update if mu > 0 : try : wmodules = w . module . modules ( ) except AttributeError : wmodules = w . modules ( ) for w_m , theta_m , lm_m in zip ( wmodules , theta . modules ( ) , lm . modules ( ) ) : if hasattr ( theta_m , 'condense' ) : for pname in theta_m . condense : getattr ( lm_m , pname ) . data = ( getattr ( lm_m , pname ) . data - mu * ( getattr ( w_m , pname ) . detach ( ) - getattr ( theta_m , pname ) . data ) ) optimizer . reset_state ( ) # Update mu mu = self . _update_mu ( mu , self . mu_init , self . mu_multiplier , self . mu_cap ) # Update LR schedule if outer_lr_scheduler is not None : outer_lr_scheduler . step ( ) statistics [ 'elapsed_lc' ] = timer_lc . elapsed_seconds statistics [ 'train_losses' ] = train_losses if test : statistics [ 'test_losses' ] = test_losses if validate : statistics [ 'val_losses' ] = val_losses return best_model , statistics