class Tokenizer:
    pass


class Token:
    def __init__(self, text, offset, data=None):
        self.offset = offset
        self.text = text
        self.end = offset + len(text)
        self.data = data if data else {}

    def set(self, prop, info):
        self.data[prop] = info

    def get(self, prop, default=None):
        return self.data.get(prop, default)

import functools


class Tokenizer:
    pass


@functools.total_ordering
class Token:
    def __init__(self, text, offset, data=None):
        self.offset = offset
        self.text = text
        self.end = offset + len(text)
        self.data = data if data else {}

    def set(self, prop, info):
        self.data[prop] = info

    def get(self, prop, default=None):
        return self.data.get(prop, default)

    def __eq__(self, other):
        if not isinstance(other, Token):
            return NotImplemented
        return self.text == other.text

    def __lt__(self, other):
        if not isinstance(other, Token):
            return NotImplemented
        return self.text < other.text

Fix to make sanitize_examples() be able to sort tokens
Fix to make sanitize_examples() be able to sort tokens

+ import functools
+ 
+ 
  class Tokenizer:
      pass
  
  
+ @functools.total_ordering
  class Token:
      def __init__(self, text, offset, data=None):
          self.offset = offset
          self.text = text
          self.end = offset + len(text)
          self.data = data if data else {}
  
      def set(self, prop, info):
          self.data[prop] = info
  
      def get(self, prop, default=None):
          return self.data.get(prop, default)
  
+     def __eq__(self, other):
+         if not isinstance(other, Token):
+             return NotImplemented
+         return self.text == other.text
+ 
+     def __lt__(self, other):
+         if not isinstance(other, Token):
+             return NotImplemented
+         return self.text < other.text
+ 
Fix to make sanitize_examples() be able to sort tokens
## Code Before:
class Tokenizer:
    pass


class Token:
    def __init__(self, text, offset, data=None):
        self.offset = offset
        self.text = text
        self.end = offset + len(text)
        self.data = data if data else {}

    def set(self, prop, info):
        self.data[prop] = info

    def get(self, prop, default=None):
        return self.data.get(prop, default)

## Instruction:
Fix to make sanitize_examples() be able to sort tokens
## Code After:
import functools


class Tokenizer:
    pass


@functools.total_ordering
class Token:
    def __init__(self, text, offset, data=None):
        self.offset = offset
        self.text = text
        self.end = offset + len(text)
        self.data = data if data else {}

    def set(self, prop, info):
        self.data[prop] = info

    def get(self, prop, default=None):
        return self.data.get(prop, default)

    def __eq__(self, other):
        if not isinstance(other, Token):
            return NotImplemented
        return self.text == other.text

    def __lt__(self, other):
        if not isinstance(other, Token):
            return NotImplemented
        return self.text < other.text
