code: public static void main(String [] args){
          if (args.length != 2){
              displayHelp();
              return;
          }
          
          //Read the input data
          try{
              String option = args[0];
              if (option.equalsIgnoreCase("-inputfile")){
                  BufferedReader in = new BufferedReader(
                          new InputStreamReader(new FileInputStream(args[1]), "UTF-8"));
                  BufferedWriter out = new BufferedWriter(
                          new OutputStreamWriter(new FileOutputStream(args[1] + ".tkn") , "UTF-8"));
  
                  String line = "";
                  while ((line = in.readLine()) != null){                
                      out.write(PennTokenizer.tokenize(line));
                      out.write("\n");
                  }
                  
                  in.close();
                  out.close();
              }
              
              else if (option.equalsIgnoreCase("-inputdir")){
                  System.out.println("Tokenize input");
                  //segment only files ends with .sent
                  File inputDir = new File(args[1]);
                  File [] childrent = inputDir.listFiles(new FilenameFilter() {
                      public boolean accept(File dir, String name) {
                          return name.endsWith(".sent");
                      }
                  });
                  
                  for (int i = 0; i < childrent.length; ++i){                    
                      BufferedReader in = new BufferedReader(
                              new InputStreamReader(new FileInputStream(childrent[i]), "UTF-8"));
                      BufferedWriter out = new BufferedWriter(
                              new OutputStreamWriter(new FileOutputStream(childrent[i] + ".tkn") , "UTF-8"));
  
                      String line = "";
                      while ((line = in.readLine()) != null){                
                          out.write(PennTokenizer.tokenize(line));
                          out.write("\n");
                      }
  
                      in.close();
                      out.close();    
                  }
              }           
          } catch (Exception e){
              System.out.println("Error:" + e.getMessage());
          }
      }
docs: The main method.
  
  @param args the arguments