// Path: src/main/java/vn/vitk/util/Converter.java
// public class Converter implements Serializable {
// 	private static final long serialVersionUID = -8916193514910249287L;
// 	
// 	private Map<String, Pattern> patterns = new HashMap<String, Pattern>();
// 	
// 	public Converter(String regexpFileName) {
// 		BufferedReader br = null;
// 		try {
// 			br = new BufferedReader(new InputStreamReader(new FileInputStream(regexpFileName), "UTF-8"));
// 			String line = null;
// 			while ((line = br.readLine()) != null) {
// 				String[] parts = line.trim().split("\\s+");
// 				if (parts.length == 2) {
// 					patterns.put(parts[0].trim(), Pattern.compile(parts[1].trim()));
// 				} else {
// 					System.err.println("Bad format regexp file: " + line);
// 				}
// 			}
// 			br.close();
// 		} catch (IOException e) {
// 			e.printStackTrace();
// 		} finally {
// 			try {
// 				br.close();
// 			} catch (IOException e) {
// 				e.printStackTrace();
// 			}
// 		}
// 		
// 	}
// 	
// 	public String convert(String token) {
// 		for (String type : patterns.keySet()) {
// 			Pattern pattern = patterns.get(type);
// 			String t = token.replace('_', ' ');
// 			if (pattern.matcher(t).matches()) {
// 				if (type.contains("entity"))
// 					return "entity";
// 				else if (type.contains("name"))
// 					return "name";
// 				else if (type.contains("allcap"))
// 					return "allcap";
// 				else if (type.contains("email"))
// 					return "email";
// 				else if (type.contains("date"))
// 					return "date";
// 				else if (type.contains("hour"))
// 					return "hour";
// 				else if (type.contains("number"))
// 					return "number";
// 				else if (type.contains("fraction"))
// 					return "fraction";
// 				else if (type.contains("punctuation"))
// 					return "punct";
// 			}
// 		}
// 		return "normal";
// 	}
// 	
// 	public static void main(String[] args) {
// 		Converter transformer = new Converter("dat/tok/regexp.txt");
// 		String[] tokens = {"H5N1", "H5N1;Np", "2006/11/12", "22/07/2012;D", "khanhlh@mail.com", "12:05", "299.2", "33/3", "Jean-Claude Juncker", "Lê Hồng Quang", "không có gì"};
// 		for (String token : tokens)
// 			System.out.println(token + "=>" + transformer.convert(token));
// 	}
// }


import java.io.Serializable;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.sql.Row;
import scala.Tuple2;
import vn.vitk.util.Converter;
package vn.vitk.tag;




/**
 * @author Phuong LE-HONG
 *         <p>
 *         May 11, 2016, 5:57:52 PM
 *         <p>
 *         A context extractor extracts all labeled contexts from a training dataset
 *         containing labeled sequences.
 *         
 */
public class ContextExtractor implements Serializable {

	private static final long serialVersionUID = 6481557600701316137L;
	
	private MarkovOrder markovOrder = MarkovOrder.FIRST;
	
// Path: src/main/java/vn/vitk/util/Converter.java
// public class Converter implements Serializable {
// 	private static final long serialVersionUID = -8916193514910249287L;
// 	
// 	private Map<String, Pattern> patterns = new HashMap<String, Pattern>();
// 	
// 	public Converter(String regexpFileName) {
// 		BufferedReader br = null;
// 		try {
// 			br = new BufferedReader(new InputStreamReader(new FileInputStream(regexpFileName), "UTF-8"));
// 			String line = null;
// 			while ((line = br.readLine()) != null) {
// 				String[] parts = line.trim().split("\\s+");
// 				if (parts.length == 2) {
// 					patterns.put(parts[0].trim(), Pattern.compile(parts[1].trim()));
// 				} else {
// 					System.err.println("Bad format regexp file: " + line);
// 				}
// 			}
// 			br.close();
// 		} catch (IOException e) {
// 			e.printStackTrace();
// 		} finally {
// 			try {
// 				br.close();
// 			} catch (IOException e) {
// 				e.printStackTrace();
// 			}
// 		}
// 		
// 	}
// 	
// 	public String convert(String token) {
// 		for (String type : patterns.keySet()) {
// 			Pattern pattern = patterns.get(type);
// 			String t = token.replace('_', ' ');
// 			if (pattern.matcher(t).matches()) {
// 				if (type.contains("entity"))
// 					return "entity";
// 				else if (type.contains("name"))
// 					return "name";
// 				else if (type.contains("allcap"))
// 					return "allcap";
// 				else if (type.contains("email"))
// 					return "email";
// 				else if (type.contains("date"))
// 					return "date";
// 				else if (type.contains("hour"))
// 					return "hour";
// 				else if (type.contains("number"))
// 					return "number";
// 				else if (type.contains("fraction"))
// 					return "fraction";
// 				else if (type.contains("punctuation"))
// 					return "punct";
// 			}
// 		}
// 		return "normal";
// 	}
// 	
// 	public static void main(String[] args) {
// 		Converter transformer = new Converter("dat/tok/regexp.txt");
// 		String[] tokens = {"H5N1", "H5N1;Np", "2006/11/12", "22/07/2012;D", "khanhlh@mail.com", "12:05", "299.2", "33/3", "Jean-Claude Juncker", "Lê Hồng Quang", "không có gì"};
// 		for (String token : tokens)
// 			System.out.println(token + "=>" + transformer.convert(token));
// 	}
// }


// Path: src/main/java/vn/vitk/tag/ContextExtractor.java
import java.io.Serializable;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.sql.Row;
import scala.Tuple2;
import vn.vitk.util.Converter;

package vn.vitk.tag;




/**
 * @author Phuong LE-HONG
 *         <p>
 *         May 11, 2016, 5:57:52 PM
 *         <p>
 *         A context extractor extracts all labeled contexts from a training dataset
 *         containing labeled sequences.
 *         
 */
public class ContextExtractor implements Serializable {

	private static final long serialVersionUID = 6481557600701316137L;
	
	private MarkovOrder markovOrder = MarkovOrder.FIRST;
	
	private Converter tokenConverter;