code: public static String dehyphenate(LineNumberReader reader, String docId) {
  
          String line;
          StringBuilder sb = new StringBuilder();
  
          try {
              while ((line = reader.readLine()) != null) {
                  String lineOut = line;
                  Matcher matcher = patternHyphen.matcher(line);
  
                  // Allows to append consecutive lines with hyphens
                  while (matcher.find() && ((line = reader.readLine()) != null)) {
                      String behind[] = lineOut.split(WORD_SEPARATOR);
                      String forward[] = line.trim().split(WORD_SEPARATOR);
  
                      if (behind.length == 0) {
                          lineOut = lineOut + "\n" + line;
                          continue;
                      }
  
                      String w1 = behind[(behind.length) - 1];
                      String w2 = forward[0];
                      if (shouldDehyphenate(w1, w2)) {
                          // Remove hyphen from first line and concatenates second
                          lineOut = lineOut.substring(0, lineOut.length() - 1)
                                  .trim() + line.trim();
                      } else {
                          // just concanates the two lines
                          lineOut = lineOut + line;
                      }
                      matcher = patternHyphen.matcher(lineOut);
                  }
  
                  sb.append(lineOut.trim());
                  sb.append("\n");
              }
          } catch (Throwable t) {
              //TODO happens sometimes, e.g. docId 1279669
              LOG.warn("failed to dehyphenate, docId " + docId, print(t)); 
          }
          return sb.toString();
      }
docs: Removes ligatures, multiple spaces and hypens from a text file