/FileExtraction/Java_unseen/slavpetrov_berkeleyparser/src/edu/berkeley/nlp/tokenizer/PTBLineLexer.java
/**
 * 
 */
package edu.berkeley.nlp.tokenizer;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import edu.berkeley.nlp.util.IOUtils;
import edu.berkeley.nlp.util.Iterators;
import edu.berkeley.nlp.util.StringUtils;

/**
 * Similar to PTBLexer. However, instead of reading from a Reader this class is
 * given a line and returns a list of tokenized Strings.
 * 
 * @author petrov
 * 
 */
public class PTBLineLexer extends PTBLexer implements LineTokenizer {

	public PTBLineLexer() {
		super((java.io.Reader) null);
	}

	public List<String> tokenize(String line) {
		PTBTokenizer toker = new PTBTokenizer(new StringReader(line), true);
		List<?> elems = toker.tokenize();
		List<String> toks = new ArrayList<String>();
		for (Object o : elems) {
			toks.add(o.toString());
		}
		return toks;
	}

	public List<String> tokenizeLine(String line) throws IOException {
		LinkedList<String> tokenized = new LinkedList<String>();
		int nEl = line.length();
		char[] array = line.toCharArray();
		yy_buffer = line.toCharArray();// new char[nEl+1];
		// for(int i=0;i<nEl;i++) yy_buffer[i] = array[i];
		// yy_buffer[nEl] = (char)YYEOF;
		yy_startRead = 0;
		yy_endRead = yy_buffer.length;
		yy_atBOL = true;
		yy_atEOF = false;
		yy_currentPos = yy_markedPos = yy_pushbackPos = 0;
		yyline = yychar = yycolumn = 0;
		yy_lexical_state = YYINITIAL;
		while (yy_markedPos < yy_endRead)
			tokenized.add(next());
		return tokenized;
	}

	private boolean yy_refill() throws java.io.IOException {
		return true;
	}

	public static void main(String[] argv) {
		PTBLineLexer tokenizer = new PTBLineLexer();
		try {
			for (String line : Iterators.able(IOUtils.lineIterator(argv[0]))) {
				final List<String> tokenizeLine = tokenizer.tokenizeLine(line);
				if (tokenizeLine.get(tokenizeLine.size() - 1) == null)
					tokenizeLine.remove(tokenizeLine.size() - 1);
				System.out.println(StringUtils.join(tokenizeLine));
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			throw new RuntimeException(e);

		}
	}

}
