(self)
return tokens
@return: An array of unique words (tokens) with the bottom THRESHOLD_COUNT least
                 frequent words converted to '*UNK*'
def get_vocab(self):
        """
        @return: An array of unique words (tokens) with the bottom THRESHOLD_COUNT least
                 frequent words converted to '*UNK*'
        """
        # Load the lyric data if it hasn't been loaded already
        if len(self.lyrics) == 0:
            self.load_lyrics()

        # Collapses the 2D array to a 1D array of words
        all_words = reduce(lambda a,b: a + b, self.lyrics)

        # TODO: Find out why this UNK code causes differences between Linux and OS X vocabularies
        # # convert THRESHOLD_COUNT frequent words to '*UNK*'
        # THRESHOLD_COUNT = 10
        # least_referenced = Counter(all_words).most_common()[:-(THRESHOLD_COUNT + 1):-1]
        # least_referenced = [tup[0] for tup in least_referenced] # grab word from (word, count) tuple
        # print least_referenced
        #
        # self.lyrics = [map(lambda word: c.UNK if word in least_referenced else word, song)
        #                for song in self.lyrics]
        # # reset all_words to include UNKs
        # all_words = reduce(lambda a, b: a + b, self.lyrics)

        # get a sorted list of unique word tokens
        tokens = sorted(list(set(all_words)))

        # creates a map from word to index
        self.vocab_lookup = dict((word, i) for i, word in enumerate(tokens))
        # Converts words in self.lyrics to the appropriate indices.
        self.lyric_indices = [map(lambda word: self.vocab_lookup[word], song)
                              for song in self.lyrics]

        print len(tokens)

        return tokens
def get_vocab ( self ) : # Load the lyric data if it hasn't been loaded already if len ( self . lyrics ) == 0 : self . load_lyrics ( ) # Collapses the 2D array to a 1D array of words all_words = reduce ( lambda a , b : a + b , self . lyrics ) # TODO: Find out why this UNK code causes differences between Linux and OS X vocabularies # # convert THRESHOLD_COUNT frequent words to '*UNK*' # THRESHOLD_COUNT = 10 # least_referenced = Counter(all_words).most_common()[:-(THRESHOLD_COUNT + 1):-1] # least_referenced = [tup[0] for tup in least_referenced] # grab word from (word, count) tuple # print least_referenced # # self.lyrics = [map(lambda word: c.UNK if word in least_referenced else word, song) #                for song in self.lyrics] # # reset all_words to include UNKs # all_words = reduce(lambda a, b: a + b, self.lyrics) # get a sorted list of unique word tokens tokens = sorted ( list ( set ( all_words ) ) ) # creates a map from word to index self . vocab_lookup = dict ( ( word , i ) for i , word in enumerate ( tokens ) ) # Converts words in self.lyrics to the appropriate indices. self . lyric_indices = [ map ( lambda word : self . vocab_lookup [ word ] , song ) for song in self . lyrics ] print len ( tokens ) return tokens