
import re

class MegaHALTokenizer:
    def split(self, phrase):
        if len(phrase) == 0:
            return []

        # add ending punctuation if it is missing
        if phrase[-1] not in ".!?":
            phrase = phrase + "."

        # megahal traditionally considers [a-z0-9] as word characters.
        # Let's see what happens if we add [_']
        words = re.findall("([\w']+|[^\w']+)", phrase.upper())
        return words


import re

class MegaHALTokenizer:
    def split(self, phrase):
        if len(phrase) == 0:
            return []

        # add ending punctuation if it is missing
        if phrase[-1] not in ".!?":
            phrase = phrase + "."

        # megahal traditionally considers [a-z0-9] as word characters.
        # Let's see what happens if we add [_']
        words = re.findall("([\w']+|[^\w']+)", phrase.upper(), re.UNICODE)
        return words

Use the re.UNICODE flag (i.e., Python character tables) in findall()
Use the re.UNICODE flag (i.e., Python character tables) in findall()

  
  import re
  
  class MegaHALTokenizer:
      def split(self, phrase):
          if len(phrase) == 0:
              return []
  
          # add ending punctuation if it is missing
          if phrase[-1] not in ".!?":
              phrase = phrase + "."
  
          # megahal traditionally considers [a-z0-9] as word characters.
          # Let's see what happens if we add [_']
-         words = re.findall("([\w']+|[^\w']+)", phrase.upper())
+         words = re.findall("([\w']+|[^\w']+)", phrase.upper(), re.UNICODE)
          return words
  
Use the re.UNICODE flag (i.e., Python character tables) in findall()
## Code Before:

import re

class MegaHALTokenizer:
    def split(self, phrase):
        if len(phrase) == 0:
            return []

        # add ending punctuation if it is missing
        if phrase[-1] not in ".!?":
            phrase = phrase + "."

        # megahal traditionally considers [a-z0-9] as word characters.
        # Let's see what happens if we add [_']
        words = re.findall("([\w']+|[^\w']+)", phrase.upper())
        return words

## Instruction:
Use the re.UNICODE flag (i.e., Python character tables) in findall()
## Code After:

import re

class MegaHALTokenizer:
    def split(self, phrase):
        if len(phrase) == 0:
            return []

        # add ending punctuation if it is missing
        if phrase[-1] not in ".!?":
            phrase = phrase + "."

        # megahal traditionally considers [a-z0-9] as word characters.
        # Let's see what happens if we add [_']
        words = re.findall("([\w']+|[^\w']+)", phrase.upper(), re.UNICODE)
        return words
