from nltk.sentiment.util import mark_negation
from nltk.util import trigrams
import re
import validators
from .happy_tokenizer import Tokenizer


class SentimentTokenizer(object):
    def __init__(self):
        self.tknzr = Tokenizer()

    @staticmethod
    def reduce_lengthening(text):
        """
        Replace repeated character sequences of length 3 or greater with sequences
        of length 3.
        """
        pattern = re.compile(r"(.)\1{2,}")
        return pattern.sub(r"\1\1\1", text)

    @staticmethod
    def replace_username(token):
        return '@__user__' if token.startswith('@') else token

    @staticmethod
    def replace_link(token):
        return '__url__' if validators.url(token) else token

    def __call__(self, t):
        t = self.reduce_lengthening(t)
        tokens = t.split(' ')

        cleaned_tokens = []
        for token in tokens:
            token = self.replace_username(token)
            token = self.replace_link(token)
            cleaned_tokens.append(token)

        rebuild_str = ' '.join(cleaned_tokens)

        negated_tokens = mark_negation(list(self.tknzr.tokenize(rebuild_str)))
        list_of_trigrams = list(trigrams(negated_tokens))
        return list([' '.join(s) for s in list_of_trigrams])


from nltk.sentiment.util import mark_negation
from nltk.util import trigrams
import re
import validators
from .happy_tokenizer import Tokenizer


class SentimentTokenizer(object):
    def __init__(self):
        self.tknzr = Tokenizer()

    @staticmethod
    def reduce_lengthening(text):
        """
        Replace repeated character sequences of length 3 or greater with sequences
        of length 3.
        """
        pattern = re.compile(r"(.)\1{2,}")
        return pattern.sub(r"\1\1\1", text)

    @staticmethod
    def replace_username(token):
        return '@__user__' if token.startswith('@') else token

    @staticmethod
    def replace_link(token):
        return '__url__' if validators.url(token) else token

    def __call__(self, t):
        t = self.reduce_lengthening(t)
        tokens = t.split(' ')

        cleaned_tokens = []
        for token in tokens:
            token = self.replace_username(token)
            token = self.replace_link(token)
            cleaned_tokens.append(token)

        rebuild_str = ' '.join(cleaned_tokens)

        negated_tokens = mark_negation(list(self.tknzr.tokenize(rebuild_str)))
        list_of_trigrams = list([' '.join(s) for s in trigrams(negated_tokens)])
        return list_of_trigrams


Use map to loop instead of mapping a list
Use map to loop instead of mapping a list

  from nltk.sentiment.util import mark_negation
  from nltk.util import trigrams
  import re
  import validators
  from .happy_tokenizer import Tokenizer
  
  
  class SentimentTokenizer(object):
      def __init__(self):
          self.tknzr = Tokenizer()
  
      @staticmethod
      def reduce_lengthening(text):
          """
          Replace repeated character sequences of length 3 or greater with sequences
          of length 3.
          """
          pattern = re.compile(r"(.)\1{2,}")
          return pattern.sub(r"\1\1\1", text)
  
      @staticmethod
      def replace_username(token):
          return '@__user__' if token.startswith('@') else token
  
      @staticmethod
      def replace_link(token):
          return '__url__' if validators.url(token) else token
  
      def __call__(self, t):
          t = self.reduce_lengthening(t)
          tokens = t.split(' ')
  
          cleaned_tokens = []
          for token in tokens:
              token = self.replace_username(token)
              token = self.replace_link(token)
              cleaned_tokens.append(token)
  
          rebuild_str = ' '.join(cleaned_tokens)
  
          negated_tokens = mark_negation(list(self.tknzr.tokenize(rebuild_str)))
-         list_of_trigrams = list(trigrams(negated_tokens))
+         list_of_trigrams = list([' '.join(s) for s in trigrams(negated_tokens)])
-         return list([' '.join(s) for s in list_of_trigrams])
+         return list_of_trigrams
  
  
Use map to loop instead of mapping a list
## Code Before:
from nltk.sentiment.util import mark_negation
from nltk.util import trigrams
import re
import validators
from .happy_tokenizer import Tokenizer


class SentimentTokenizer(object):
    def __init__(self):
        self.tknzr = Tokenizer()

    @staticmethod
    def reduce_lengthening(text):
        """
        Replace repeated character sequences of length 3 or greater with sequences
        of length 3.
        """
        pattern = re.compile(r"(.)\1{2,}")
        return pattern.sub(r"\1\1\1", text)

    @staticmethod
    def replace_username(token):
        return '@__user__' if token.startswith('@') else token

    @staticmethod
    def replace_link(token):
        return '__url__' if validators.url(token) else token

    def __call__(self, t):
        t = self.reduce_lengthening(t)
        tokens = t.split(' ')

        cleaned_tokens = []
        for token in tokens:
            token = self.replace_username(token)
            token = self.replace_link(token)
            cleaned_tokens.append(token)

        rebuild_str = ' '.join(cleaned_tokens)

        negated_tokens = mark_negation(list(self.tknzr.tokenize(rebuild_str)))
        list_of_trigrams = list(trigrams(negated_tokens))
        return list([' '.join(s) for s in list_of_trigrams])


## Instruction:
Use map to loop instead of mapping a list
## Code After:
from nltk.sentiment.util import mark_negation
from nltk.util import trigrams
import re
import validators
from .happy_tokenizer import Tokenizer


class SentimentTokenizer(object):
    def __init__(self):
        self.tknzr = Tokenizer()

    @staticmethod
    def reduce_lengthening(text):
        """
        Replace repeated character sequences of length 3 or greater with sequences
        of length 3.
        """
        pattern = re.compile(r"(.)\1{2,}")
        return pattern.sub(r"\1\1\1", text)

    @staticmethod
    def replace_username(token):
        return '@__user__' if token.startswith('@') else token

    @staticmethod
    def replace_link(token):
        return '__url__' if validators.url(token) else token

    def __call__(self, t):
        t = self.reduce_lengthening(t)
        tokens = t.split(' ')

        cleaned_tokens = []
        for token in tokens:
            token = self.replace_username(token)
            token = self.replace_link(token)
            cleaned_tokens.append(token)

        rebuild_str = ' '.join(cleaned_tokens)

        negated_tokens = mark_negation(list(self.tknzr.tokenize(rebuild_str)))
        list_of_trigrams = list([' '.join(s) for s in trigrams(negated_tokens)])
        return list_of_trigrams

