code: public static int generateDataset(String file, String test, String standard){
          int textCharCount=0;
          try(BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf-8"));
              BufferedWriter testWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(test),"utf-8"));
              BufferedWriter standardWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(standard),"utf-8"))){
              String line;
              int duplicateCount=0;
              Set<String> set = new HashSet<>();
              while( (line = reader.readLine()) != null ){
                  //不把空格当做标点符号
                  List<String> list = Punctuation.seg(line, false, ' ');
                  for(String item : list){
                      item = item.trim();
                      //忽略空行和长度为一的行
                      if("".equals(item)
                              || item.length()==1){
                          continue;
                      }
                      //忽略重复的内容
                      if(set.contains(item)){
                          duplicateCount++;
                          continue;
                      }
                      set.add(item);
                      String testItem = item.replaceAll(" ", "");
                      textCharCount += testItem.length();
                      testWriter.write(testItem+"\n");
                      standardWriter.write(item+"\n");
                  }
              }
              LOGGER.info("重复行数为："+duplicateCount);
          } catch (IOException ex) {
              LOGGER.error("生成测试数据集和标准数据集失败：", ex);
          }
          return textCharCount;
      }
docs: 生成测试数据集和标准数据集
  @param file 已分词文本，词之间空格分隔
  @param test 生成测试数据集文件路径
  @param standard 生成标准数据集文件路径
  @return 测试数据集字符数