code: static private void checkDuplication(FileSystem fs, Path file, Path sorted,
      Configuration conf) throws IOException {
      SequenceFile.Reader in = null;
      try {
        SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
          new Text.Comparator(), Text.class, Text.class, conf);
        sorter.sort(file, sorted);
        in = new SequenceFile.Reader(fs, sorted, conf);
  
        Text prevdst = null, curdst = new Text();
        Text prevsrc = null, cursrc = new Text(); 
        for(; in.next(curdst, cursrc); ) {
          if (prevdst != null && curdst.equals(prevdst)) {
            throw new DuplicationException(
              "Invalid input, there are duplicated files in the sources: "
              + prevsrc + ", " + cursrc);
          }
          prevdst = curdst;
          curdst = new Text();
          prevsrc = cursrc;
          cursrc = new Text();
        }
      }
      finally {
        checkAndClose(in);
      }
    }
docs: Check whether the file list have duplication.