docs: def token_perplexity_micro(eval_data, predictions, scores, learner='ignored'):
      '''
      Return the micro-averaged per-token perplexity `exp(-score / num_tokens)`
      computed over the entire corpus, as a length-1 list of floats.
      The log scores in `scores` should be base e (`exp`, `log`).
  
      >>> refs = [Instance(None, ''),
      ...         Instance(None, ''),
      ...         Instance(None, '2')]
      >>> scores = [np.log(1.0), np.log(0.25), np.log(1 / 64.)]
      >>> perplexity = token_perplexity_micro(refs, None, scores)
      >>> [round(p) for p in perplexity]
      ... # sequence perplexities: [1, 4, 64]
      ... # per-token perplexities: [1, 4, 8]
      ... # micro-average: gmean([1, 4, 8, 8])
      [4.0]
      '''
      lens = np.array([len(_maybe_tokenize(inst.output)) + 1 for inst in eval_data])
      return [np.exp(np.average(-np.array(scores) / lens, weights=lens))]