// Path: src/main/java/codemining/languagetools/ITokenizer.java
// public static class FullToken implements Serializable {
// 
// 	private static final long serialVersionUID = -49456240173307314L;
// 
// 	public static final Function<FullToken, String> TOKEN_NAME_CONVERTER = new Function<FullToken, String>() {
// 		@Override
// 		public String apply(final FullToken input) {
// 			return input.token;
// 		}
// 	};
// 
// 	public final String token;
// 
// 	public final String tokenType;
// 
// 	public FullToken(final FullToken other) {
// 		token = other.token;
// 		tokenType = other.tokenType;
// 	}
// 
// 	public FullToken(final String tokName, final String tokType) {
// 		token = tokName;
// 		tokenType = tokType;
// 	}
// 
// 	@Override
// 	public boolean equals(final Object obj) {
// 		if (!(obj instanceof FullToken)) {
// 			return false;
// 		}
// 		final FullToken other = (FullToken) obj;
// 		return other.token.equals(token)
// 				&& other.tokenType.equals(tokenType);
// 	}
// 
// 	@Override
// 	public int hashCode() {
// 		return Objects.hashCode(token, tokenType);
// 	}
// 
// 	@Override
// 	public String toString() {
// 		return token + " (" + tokenType + ")";
// 	}
// 
// }


import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkPositionIndex;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import codemining.languagetools.ITokenizer.FullToken;
import codemining.util.SettingsLoader;
/**
 * 
 */
package codemining.languagetools;





/**
 * Utility function relevant to tokenization.
 * 
 * @author Miltos Allamanis <m.allamanis@ed.ac.uk>
 * 
 */
public class TokenizerUtils {

	public static final int TAB_INDENT_SIZE = (int) SettingsLoader
			.getNumericSetting("tabSize", 4);

	/**
	 * Return the column of the given position.
	 * 
	 * @param code
	 * @param position
	 * @return
	 */
	public static int getColumnOfPosition(final String code, final int position) {
		checkPositionIndex(position, code.length());
		int newLinePosition = code.substring(0, position).lastIndexOf("\n");
		if (newLinePosition == -1) {
			newLinePosition = 0; // Start of file.
		}
		final int tabCount = StringUtils.countMatches(
				code.substring(newLinePosition, position), "\t");
		return position - newLinePosition + (TAB_INDENT_SIZE - 1) * tabCount;
	}

	/**
	 * Crudely join tokens together.
	 * 
	 * @param tokens
	 * @param sb
	 * @return
	 */
	public final static StringBuffer joinFullTokens(
// Path: src/main/java/codemining/languagetools/ITokenizer.java
// public static class FullToken implements Serializable {
// 
// 	private static final long serialVersionUID = -49456240173307314L;
// 
// 	public static final Function<FullToken, String> TOKEN_NAME_CONVERTER = new Function<FullToken, String>() {
// 		@Override
// 		public String apply(final FullToken input) {
// 			return input.token;
// 		}
// 	};
// 
// 	public final String token;
// 
// 	public final String tokenType;
// 
// 	public FullToken(final FullToken other) {
// 		token = other.token;
// 		tokenType = other.tokenType;
// 	}
// 
// 	public FullToken(final String tokName, final String tokType) {
// 		token = tokName;
// 		tokenType = tokType;
// 	}
// 
// 	@Override
// 	public boolean equals(final Object obj) {
// 		if (!(obj instanceof FullToken)) {
// 			return false;
// 		}
// 		final FullToken other = (FullToken) obj;
// 		return other.token.equals(token)
// 				&& other.tokenType.equals(tokenType);
// 	}
// 
// 	@Override
// 	public int hashCode() {
// 		return Objects.hashCode(token, tokenType);
// 	}
// 
// 	@Override
// 	public String toString() {
// 		return token + " (" + tokenType + ")";
// 	}
// 
// }


// Path: src/main/java/codemining/languagetools/TokenizerUtils.java
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkPositionIndex;
import java.lang.reflect.InvocationTargetException;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import codemining.languagetools.ITokenizer.FullToken;
import codemining.util.SettingsLoader;

/**
 * 
 */
package codemining.languagetools;





/**
 * Utility function relevant to tokenization.
 * 
 * @author Miltos Allamanis <m.allamanis@ed.ac.uk>
 * 
 */
public class TokenizerUtils {

	public static final int TAB_INDENT_SIZE = (int) SettingsLoader
			.getNumericSetting("tabSize", 4);

	/**
	 * Return the column of the given position.
	 * 
	 * @param code
	 * @param position
	 * @return
	 */
	public static int getColumnOfPosition(final String code, final int position) {
		checkPositionIndex(position, code.length());
		int newLinePosition = code.substring(0, position).lastIndexOf("\n");
		if (newLinePosition == -1) {
			newLinePosition = 0; // Start of file.
		}
		final int tabCount = StringUtils.countMatches(
				code.substring(newLinePosition, position), "\t");
		return position - newLinePosition + (TAB_INDENT_SIZE - 1) * tabCount;
	}

	/**
	 * Crudely join tokens together.
	 * 
	 * @param tokens
	 * @param sb
	 * @return
	 */
	public final static StringBuffer joinFullTokens(
			final List<FullToken> tokens, final StringBuffer sb) {