// Path: src/main/java/com/davidehrmann/classifier4J/IClassifier.java
// public interface IClassifier<I> {
//     /**
//      * Default value to use if the implementation cannot work out how
//      * well a string matches.
//      */
//     public static double NEUTRAL_PROBABILITY = 0.5d;
// 
//     /**
//      * The minimum likelihood that a string matches
//      */
//     public static double LOWER_BOUND = 0.01d;
// 
//     /**
//      * The maximum likelihood that a string matches
//      */
//     public static double UPPER_BOUND = 0.99d;
// 
//     /**
//      * Default cutoff value used by default implementation of 
//      * isMatch. Any match probability greater than or equal to this
//      * value will be classified as a match. 
//      * 
//      * The value is 0.9d
//      * 
//      */
//     public static double DEFAULT_CUTOFF = 0.9d;
// 
//     /**
//      * 
//      * Sets the cutoff below which the input is not considered a match
//      * 
//      * @param cutoff the level below which isMatch will return false. Should be between 0 and 1.
//      */
//     public void setMatchCutoff(double cutoff);
// 
//     /**
//      *
//      * Function to determine the probability string matches a criteria.
//      *   
//      * @param input the string to classify
//      * @return the likelihood that this string is a match for this net.sf.classifier4j. 1 means 100% likely.
//          *
//          * @throws ClassifierException If a non-recoverable problem occurs
//      */
//     public double classify(I input) throws ClassifierException;
// 
//     /**
//      * 
//      * Function to determine if a string matches a criteria.
//      * 
//      * @param input the string to classify
//      * @return true if the input string has a probability >= the cutoff probability of 
//      * matching
//          *
//          * @throws ClassifierException If a non-recoverable problem occurs
//      */
//     public boolean isMatch(I input) throws ClassifierException;
// 
//     /**
//      * Convenience method which takes a match probability
//      * (calculated by {@link IClassifier#classify(java.lang.String)})
//      * and checks if it would be classified as a match or not
//      * 
//      * @param matchProbability 
//      * @return true if match, false otherwise
//      */
//     public boolean isMatch(double matchProbability);
// }
// 
// Path: src/main/java/com/davidehrmann/classifier4J/util/ToStringBuilder.java
// public class ToStringBuilder implements Serializable {
//     StringBuffer output = new StringBuffer("");
// 
//     /**
//      * @param classifier
//      */
//     public ToStringBuilder(Object o) {
//         if (o == null) {
//             throw new IllegalArgumentException("Object cannot be null");
//         }
//         output = new StringBuffer(o.getClass().getName() + " " );
//     }
// 
// 
//     public ToStringBuilder append(String name, Object o) {
//         if (o == null) {
//             output.append(name + ": null");    
//         } else {
//             output.append(name + ": " + o.toString());
//         }
//         return this;     
//     }
//     
//     public ToStringBuilder append(String name, double num) {
//         output.append(name + ": " + num);
//         return this;
//     }
//     
//     public String toString() {
//         return output.toString();        
//     }
//     
//     
// }


import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.davidehrmann.classifier4j.IClassifier;
import com.davidehrmann.classifier4j.util.ToStringBuilder;

        } else if (tokenizerConfig == BREAK_ON_WHITESPACE) {
            regexp = "\\s+";
        } else {
            throw new IllegalStateException("Illegal tokenizer configuration. customTokenizerRegExp = null & tokenizerConfig = " + tokenizerConfig);
        }

        ArrayList<String> words = new ArrayList<String>(input.length() / 4);
        
        int lastEnd = 0;
        Matcher matcher = Pattern.compile(regexp).matcher(input);
        
        while (matcher.find()) {
        	int start = matcher.start();
        	
        	if (start != lastEnd) {
        		words.add(input.substring(lastEnd, start));
        	}
        	
        	lastEnd = matcher.end();
        }
        
        if (lastEnd < input.length()) {
        	words.add(input.substring(lastEnd));
        }
        
        return words.toArray(new String[words.size()]);
    }

    public String toString() {


// Path: src/main/java/com/davidehrmann/classifier4J/IClassifier.java
// public interface IClassifier<I> {
//     /**
//      * Default value to use if the implementation cannot work out how
//      * well a string matches.
//      */
//     public static double NEUTRAL_PROBABILITY = 0.5d;
// 
//     /**
//      * The minimum likelihood that a string matches
//      */
//     public static double LOWER_BOUND = 0.01d;
// 
//     /**
//      * The maximum likelihood that a string matches
//      */
//     public static double UPPER_BOUND = 0.99d;
// 
//     /**
//      * Default cutoff value used by default implementation of 
//      * isMatch. Any match probability greater than or equal to this
//      * value will be classified as a match. 
//      * 
//      * The value is 0.9d
//      * 
//      */
//     public static double DEFAULT_CUTOFF = 0.9d;
// 
//     /**
//      * 
//      * Sets the cutoff below which the input is not considered a match
//      * 
//      * @param cutoff the level below which isMatch will return false. Should be between 0 and 1.
//      */
//     public void setMatchCutoff(double cutoff);
// 
//     /**
//      *
//      * Function to determine the probability string matches a criteria.
//      *   
//      * @param input the string to classify
//      * @return the likelihood that this string is a match for this net.sf.classifier4j. 1 means 100% likely.
//          *
//          * @throws ClassifierException If a non-recoverable problem occurs
//      */
//     public double classify(I input) throws ClassifierException;
// 
//     /**
//      * 
//      * Function to determine if a string matches a criteria.
//      * 
//      * @param input the string to classify
//      * @return true if the input string has a probability >= the cutoff probability of 
//      * matching
//          *
//          * @throws ClassifierException If a non-recoverable problem occurs
//      */
//     public boolean isMatch(I input) throws ClassifierException;
// 
//     /**
//      * Convenience method which takes a match probability
//      * (calculated by {@link IClassifier#classify(java.lang.String)})
//      * and checks if it would be classified as a match or not
//      * 
//      * @param matchProbability 
//      * @return true if match, false otherwise
//      */
//     public boolean isMatch(double matchProbability);
// }
// 
// Path: src/main/java/com/davidehrmann/classifier4J/util/ToStringBuilder.java
// public class ToStringBuilder implements Serializable {
//     StringBuffer output = new StringBuffer("");
// 
//     /**
//      * @param classifier
//      */
//     public ToStringBuilder(Object o) {
//         if (o == null) {
//             throw new IllegalArgumentException("Object cannot be null");
//         }
//         output = new StringBuffer(o.getClass().getName() + " " );
//     }
// 
// 
//     public ToStringBuilder append(String name, Object o) {
//         if (o == null) {
//             output.append(name + ": null");    
//         } else {
//             output.append(name + ": " + o.toString());
//         }
//         return this;     
//     }
//     
//     public ToStringBuilder append(String name, double num) {
//         output.append(name + ": " + num);
//         return this;
//     }
//     
//     public String toString() {
//         return output.toString();        
//     }
//     
//     
// }


// Path: src/main/java/com/davidehrmann/classifier4J/tokenizer/SimpleStringTokenizer.java
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.davidehrmann.classifier4j.IClassifier;
import com.davidehrmann.classifier4j.util.ToStringBuilder;

        } else if (tokenizerConfig == BREAK_ON_WHITESPACE) {
            regexp = "\\s+";
        } else {
            throw new IllegalStateException("Illegal tokenizer configuration. customTokenizerRegExp = null & tokenizerConfig = " + tokenizerConfig);
        }

        ArrayList<String> words = new ArrayList<String>(input.length() / 4);
        
        int lastEnd = 0;
        Matcher matcher = Pattern.compile(regexp).matcher(input);
        
        while (matcher.find()) {
        	int start = matcher.start();
        	
        	if (start != lastEnd) {
        		words.add(input.substring(lastEnd, start));
        	}
        	
        	lastEnd = matcher.end();
        }
        
        if (lastEnd < input.length()) {
        	words.add(input.substring(lastEnd));
        }
        
        return words.toArray(new String[words.size()]);
    }

    public String toString() {


        ToStringBuilder toStringBuilder = new ToStringBuilder(this);
