code: def call_moses_tokenizer(workspace_dir: str,
                           input_fname: str,
                           output_fname: str,
                           lang_code: str,
                           num_threads: int = 4):
      
      tokenizer_fname = os.path.join(workspace_dir,
                                     DIR_THIRD_PARTY,
                                     MOSES_DEST,
                                     "scripts",
                                     "tokenizer",
                                     "tokenizer.perl")
      with bin_open(input_fname) as inp, gzip.open(output_fname, "wb") as out, open(os.devnull, "wb") as devnull:
          tokenizer = subprocess.Popen(["perl", tokenizer_fname, "-l", lang_code, "-threads", str(num_threads)],
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=devnull)
          tokenizer_thread = threading.Thread(target=copy_out, args=(tokenizer.stdout, out))
          tokenizer_thread.start()
          for line in inp:
              tokenizer.stdin.write(line)
          tokenizer.stdin.close()
          tokenizer_thread.join()
          tokenizer.wait()
docs: Call Moses tokenizer.
  
      :param workspace_dir: Workspace third-party directory where Moses
                              tokenizer is checked out.
      :param input_fname: Path of raw input file, plain text or gzipped.
      :param output_fname: Path of tokenized output file, gzipped.
      :param lang_code: Language code for rules and non-breaking prefixes.
      :param num_threads: Number of threads to use.