code: func (s *shards) sendSamplesWithBackoff(ctx context.Context, samples []prompb.TimeSeries) error {
  	backoff := s.qm.cfg.MinBackoff
  	req, highest, err := buildWriteRequest(samples)
  	if err != nil {
  		// Failing to build the write request is non-recoverable, since it will
  		// only error if marshaling the proto to bytes fails.
  		return err
  	}
  
  	for {
  		select {
  		case <-ctx.Done():
  			return ctx.Err()
  		default:
  		}
  		begin := time.Now()
  		err := s.qm.client.Store(ctx, req)
  
  		s.qm.sentBatchDuration.Observe(time.Since(begin).Seconds())
  
  		if err == nil {
  			s.qm.succeededSamplesTotal.Add(float64(len(samples)))
  			s.qm.highestSentTimestampMetric.Set(float64(highest / 1000))
  			return nil
  		}
  
  		if _, ok := err.(recoverableError); !ok {
  			return err
  		}
  		s.qm.retriedSamplesTotal.Add(float64(len(samples)))
  		level.Debug(s.qm.logger).Log("msg", "failed to send batch, retrying", "err", err)
  
  		time.Sleep(time.Duration(backoff))
  		backoff = backoff * 2
  		if backoff > s.qm.cfg.MaxBackoff {
  			backoff = s.qm.cfg.MaxBackoff
  		}
  	}
  }
docs: // sendSamples to the remote storage with backoff for recoverable errors.