from __future__ import unicode_literals

from ..util import make_tempdir

import pytest


@pytest.mark.parametrize('text', ["I can't do this"])
def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
    tokenizer_b = en_tokenizer.to_bytes()
    new_tokenizer = en_tokenizer.from_bytes(tokenizer_b)
    assert new_tokenizer.to_bytes() == tokenizer_b
    doc1 = en_tokenizer(text)
    doc2 = new_tokenizer(text)
    assert [token.text for token in doc1] == [token.text for token in doc2]


def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
    tokenizer = en_tokenizer
    with make_tempdir() as d:
        file_path = d / 'tokenizer'
        tokenizer.to_disk(file_path)
        tokenizer_d = en_tokenizer.from_disk(file_path)
        assert tokenizer.to_bytes() == tokenizer_d.to_bytes()

from __future__ import unicode_literals

from ...util import get_lang_class
from ..util import make_tempdir, assert_packed_msg_equal

import pytest


def load_tokenizer(b):
    tok = get_lang_class('en').Defaults.create_tokenizer()
    tok.from_bytes(b)
    return tok


@pytest.mark.parametrize('text', ["I💜you", "they’re", "“hello”"])
def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
    tokenizer = en_tokenizer
    new_tokenizer = load_tokenizer(tokenizer.to_bytes())
    assert_packed_msg_equal(new_tokenizer.to_bytes(), tokenizer.to_bytes())
    # assert new_tokenizer.to_bytes() == tokenizer.to_bytes()
    doc1 = tokenizer(text)
    doc2 = new_tokenizer(text)
    assert [token.text for token in doc1] == [token.text for token in doc2]


def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
    tokenizer = en_tokenizer
    with make_tempdir() as d:
        file_path = d / 'tokenizer'
        tokenizer.to_disk(file_path)
        tokenizer_d = en_tokenizer.from_disk(file_path)
        assert tokenizer.to_bytes() == tokenizer_d.to_bytes()

Update serialization tests for tokenizer
Update serialization tests for tokenizer

  from __future__ import unicode_literals
  
- from ..util import make_tempdir
+ from ...util import get_lang_class
+ from ..util import make_tempdir, assert_packed_msg_equal
  
  import pytest
  
  
- @pytest.mark.parametrize('text', ["I can't do this"])
+ def load_tokenizer(b):
+     tok = get_lang_class('en').Defaults.create_tokenizer()
+     tok.from_bytes(b)
+     return tok
+ 
+ 
+ @pytest.mark.parametrize('text', ["I💜you", "they’re", "“hello”"])
  def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
-     tokenizer_b = en_tokenizer.to_bytes()
+     tokenizer = en_tokenizer
-     new_tokenizer = en_tokenizer.from_bytes(tokenizer_b)
+     new_tokenizer = load_tokenizer(tokenizer.to_bytes())
+     assert_packed_msg_equal(new_tokenizer.to_bytes(), tokenizer.to_bytes())
-     assert new_tokenizer.to_bytes() == tokenizer_b
+     # assert new_tokenizer.to_bytes() == tokenizer.to_bytes()
-     doc1 = en_tokenizer(text)
+     doc1 = tokenizer(text)
      doc2 = new_tokenizer(text)
      assert [token.text for token in doc1] == [token.text for token in doc2]
  
  
  def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
      tokenizer = en_tokenizer
      with make_tempdir() as d:
          file_path = d / 'tokenizer'
          tokenizer.to_disk(file_path)
          tokenizer_d = en_tokenizer.from_disk(file_path)
          assert tokenizer.to_bytes() == tokenizer_d.to_bytes()
  
Update serialization tests for tokenizer
## Code Before:
from __future__ import unicode_literals

from ..util import make_tempdir

import pytest


@pytest.mark.parametrize('text', ["I can't do this"])
def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
    tokenizer_b = en_tokenizer.to_bytes()
    new_tokenizer = en_tokenizer.from_bytes(tokenizer_b)
    assert new_tokenizer.to_bytes() == tokenizer_b
    doc1 = en_tokenizer(text)
    doc2 = new_tokenizer(text)
    assert [token.text for token in doc1] == [token.text for token in doc2]


def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
    tokenizer = en_tokenizer
    with make_tempdir() as d:
        file_path = d / 'tokenizer'
        tokenizer.to_disk(file_path)
        tokenizer_d = en_tokenizer.from_disk(file_path)
        assert tokenizer.to_bytes() == tokenizer_d.to_bytes()

## Instruction:
Update serialization tests for tokenizer
## Code After:
from __future__ import unicode_literals

from ...util import get_lang_class
from ..util import make_tempdir, assert_packed_msg_equal

import pytest


def load_tokenizer(b):
    tok = get_lang_class('en').Defaults.create_tokenizer()
    tok.from_bytes(b)
    return tok


@pytest.mark.parametrize('text', ["I💜you", "they’re", "“hello”"])
def test_serialize_tokenizer_roundtrip_bytes(en_tokenizer, text):
    tokenizer = en_tokenizer
    new_tokenizer = load_tokenizer(tokenizer.to_bytes())
    assert_packed_msg_equal(new_tokenizer.to_bytes(), tokenizer.to_bytes())
    # assert new_tokenizer.to_bytes() == tokenizer.to_bytes()
    doc1 = tokenizer(text)
    doc2 = new_tokenizer(text)
    assert [token.text for token in doc1] == [token.text for token in doc2]


def test_serialize_tokenizer_roundtrip_disk(en_tokenizer):
    tokenizer = en_tokenizer
    with make_tempdir() as d:
        file_path = d / 'tokenizer'
        tokenizer.to_disk(file_path)
        tokenizer_d = en_tokenizer.from_disk(file_path)
        assert tokenizer.to_bytes() == tokenizer_d.to_bytes()
