import numpy as np
import unittest

from torchtext import vocab
from collections import Counter


class TestVocab(unittest.TestCase):
    def test_vocab(self):
        c = Counter(['hello', 'world'])
        v = vocab.Vocab(c, vectors='glove.test_twitter.27B.200d')

        self.assertEqual(v.itos, ['<unk>', '<pad>', 'hello', 'world'])
        vectors = v.vectors.numpy()

        # The first 5 entries in each vector.
        expected_glove_twitter = {
            'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
            'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
        }

        for word in ['hello', 'world']:
            self.assertTrue(
                np.allclose(
                    vectors[v.stoi[word], :5], expected_glove_twitter[word]
                )
            )

        self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))


if __name__ == '__main__':
    unittest.main()

from __future__ import unicode_literals
from collections import Counter
import unittest

import numpy as np
from torchtext import vocab


class TestVocab(unittest.TestCase):
    def test_vocab(self):
        c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5, 'freq_too_low': 2})
        v = vocab.Vocab(c, min_freq=3, specials=['<pad>', '<bos>'],
                        vectors='glove.test_twitter.27B.200d')

        self.assertEqual(v.itos, ['<unk>', '<pad>', '<bos>',
                                  'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world'])
        vectors = v.vectors.numpy()

        # The first 5 entries in each vector.
        expected_glove_twitter = {
            'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
            'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
        }

        for word in ['hello', 'world']:
            self.assertTrue(
                np.allclose(
                    vectors[v.stoi[word], :5], expected_glove_twitter[word]
                )
            )

        self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))


if __name__ == '__main__':
    unittest.main()

Test vocab min_freq and specials vocab args, as well as unicode input
Test vocab min_freq and specials vocab args, as well as unicode input

- import numpy as np
+ from __future__ import unicode_literals
+ from collections import Counter
  import unittest
  
+ import numpy as np
  from torchtext import vocab
- from collections import Counter
  
  
  class TestVocab(unittest.TestCase):
      def test_vocab(self):
-         c = Counter(['hello', 'world'])
-         v = vocab.Vocab(c, vectors='glove.test_twitter.27B.200d')
+         c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5, 'freq_too_low': 2})
+         v = vocab.Vocab(c, min_freq=3, specials=['<pad>', '<bos>'],
+                         vectors='glove.test_twitter.27B.200d')
  
-         self.assertEqual(v.itos, ['<unk>', '<pad>', 'hello', 'world'])
+         self.assertEqual(v.itos, ['<unk>', '<pad>', '<bos>',
+                                   'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world'])
          vectors = v.vectors.numpy()
  
          # The first 5 entries in each vector.
          expected_glove_twitter = {
              'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
              'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
          }
  
          for word in ['hello', 'world']:
              self.assertTrue(
                  np.allclose(
                      vectors[v.stoi[word], :5], expected_glove_twitter[word]
                  )
              )
  
          self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))
  
  
  if __name__ == '__main__':
      unittest.main()
  
Test vocab min_freq and specials vocab args, as well as unicode input
## Code Before:
import numpy as np
import unittest

from torchtext import vocab
from collections import Counter


class TestVocab(unittest.TestCase):
    def test_vocab(self):
        c = Counter(['hello', 'world'])
        v = vocab.Vocab(c, vectors='glove.test_twitter.27B.200d')

        self.assertEqual(v.itos, ['<unk>', '<pad>', 'hello', 'world'])
        vectors = v.vectors.numpy()

        # The first 5 entries in each vector.
        expected_glove_twitter = {
            'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
            'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
        }

        for word in ['hello', 'world']:
            self.assertTrue(
                np.allclose(
                    vectors[v.stoi[word], :5], expected_glove_twitter[word]
                )
            )

        self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))


if __name__ == '__main__':
    unittest.main()

## Instruction:
Test vocab min_freq and specials vocab args, as well as unicode input
## Code After:
from __future__ import unicode_literals
from collections import Counter
import unittest

import numpy as np
from torchtext import vocab


class TestVocab(unittest.TestCase):
    def test_vocab(self):
        c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5, 'freq_too_low': 2})
        v = vocab.Vocab(c, min_freq=3, specials=['<pad>', '<bos>'],
                        vectors='glove.test_twitter.27B.200d')

        self.assertEqual(v.itos, ['<unk>', '<pad>', '<bos>',
                                  'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world'])
        vectors = v.vectors.numpy()

        # The first 5 entries in each vector.
        expected_glove_twitter = {
            'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
            'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
        }

        for word in ['hello', 'world']:
            self.assertTrue(
                np.allclose(
                    vectors[v.stoi[word], :5], expected_glove_twitter[word]
                )
            )

        self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))


if __name__ == '__main__':
    unittest.main()
