from ..language import Language
from ..tokenizer import Tokenizer
from ..tagger import Tagger


class CharacterTokenizer(Tokenizer):
    def __call__(self, text):
        return self.tokens_from_list(list(text))


class Chinese(Language):
    lang = u'zh'

    def __call__(self, text):
        doc = self.tokenizer.tokens_from_list(list(text))
        self.tagger(doc)
        self.merge_characters(doc)
        return doc

    def merge_characters(self, doc):
        start = 0
        chunks = []
        for token in doc:
            if token.tag_ != 'CHAR':
                chunk = doc[start : token.i + 1]
                chunks.append(chunk)
                start = token.i + 1
        text = doc.text
        for chunk in chunks:
            chunk.merge(chunk[-1].tag_, chunk.text, u'')

import jieba

from ..language import Language
from ..tokens import Doc


class Chinese(Language):
    lang = u'zh'

    def make_doc(self, text):
        words = list(jieba.cut(text, cut_all=True))
        return Doc(self.vocab, words=words, spaces=[False]*len(words))

Add draft Jieba tokenizer for Chinese
Add draft Jieba tokenizer for Chinese

+ import jieba
+ 
  from ..language import Language
+ from ..tokens import Doc
- from ..tokenizer import Tokenizer
- from ..tagger import Tagger
- 
- 
- class CharacterTokenizer(Tokenizer):
-     def __call__(self, text):
-         return self.tokens_from_list(list(text))
  
  
  class Chinese(Language):
      lang = u'zh'
  
-     def __call__(self, text):
+     def make_doc(self, text):
+         words = list(jieba.cut(text, cut_all=True))
+         return Doc(self.vocab, words=words, spaces=[False]*len(words))
-         doc = self.tokenizer.tokens_from_list(list(text))
-         self.tagger(doc)
-         self.merge_characters(doc)
-         return doc
  
-     def merge_characters(self, doc):
-         start = 0
-         chunks = []
-         for token in doc:
-             if token.tag_ != 'CHAR':
-                 chunk = doc[start : token.i + 1]
-                 chunks.append(chunk)
-                 start = token.i + 1
-         text = doc.text
-         for chunk in chunks:
-             chunk.merge(chunk[-1].tag_, chunk.text, u'')
- 
Add draft Jieba tokenizer for Chinese
## Code Before:
from ..language import Language
from ..tokenizer import Tokenizer
from ..tagger import Tagger


class CharacterTokenizer(Tokenizer):
    def __call__(self, text):
        return self.tokens_from_list(list(text))


class Chinese(Language):
    lang = u'zh'

    def __call__(self, text):
        doc = self.tokenizer.tokens_from_list(list(text))
        self.tagger(doc)
        self.merge_characters(doc)
        return doc

    def merge_characters(self, doc):
        start = 0
        chunks = []
        for token in doc:
            if token.tag_ != 'CHAR':
                chunk = doc[start : token.i + 1]
                chunks.append(chunk)
                start = token.i + 1
        text = doc.text
        for chunk in chunks:
            chunk.merge(chunk[-1].tag_, chunk.text, u'')

## Instruction:
Add draft Jieba tokenizer for Chinese
## Code After:
import jieba

from ..language import Language
from ..tokens import Doc


class Chinese(Language):
    lang = u'zh'

    def make_doc(self, text):
        words = list(jieba.cut(text, cut_all=True))
        return Doc(self.vocab, words=words, spaces=[False]*len(words))
