// Path: src/main/java/vn/vitk/util/Constants.java
// public interface Constants {
// 	public static final String REGEXP_FILE = "/export/dat/tok/regexp.txt";
// }


import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.ml.Model;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.feature.CountVectorizerModel;
import org.apache.spark.ml.feature.StringIndexerModel;
import org.apache.spark.ml.param.ParamMap;
import org.apache.spark.ml.util.DefaultParamsReader;
import org.apache.spark.ml.util.DefaultParamsWriter;
import org.apache.spark.ml.util.MLReader;
import org.apache.spark.ml.util.MLWritable;
import org.apache.spark.ml.util.MLWriter;
import org.apache.spark.ml.util.SchemaUtils;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.util.MLUtils;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
import scala.collection.Iterator;
import scala.collection.mutable.WrappedArray;
import vn.vitk.util.Constants;
package vn.vitk.tag;




/**
 * @author Phuong LE-HONG
 *         <p>
 *         May 9, 2016, 5:52:38 PM
 *         <p>
 *         A conditional Markov model (or Maximum-Entropy Markov model -- MEMM)
 *         for sequence labeling which is fitted by a CMM graphical model.
 * 
 */
public class CMMModel extends Model<CMMModel> implements MLWritable {

	private static final long serialVersionUID = -4855076361905836432L;
	
	private PipelineModel pipelineModel;
	private ContextExtractor contextExtractor;
	private final Vector weights;
	private final String[] tags;
	private final Map<String, Integer> featureMap;
	private final Map<String, Set<Integer>> tagDictionary; 
	
	/**
	 * Creates a conditional Markov model.
	 * @param pipelineModel
	 * @param weights
	 * @param markovOrder
	 */
	public CMMModel(PipelineModel pipelineModel, Vector weights, MarkovOrder markovOrder, Map<String, Set<Integer>> tagDictionary) {
		this.pipelineModel = pipelineModel;
// Path: src/main/java/vn/vitk/util/Constants.java
// public interface Constants {
// 	public static final String REGEXP_FILE = "/export/dat/tok/regexp.txt";
// }


// Path: src/main/java/vn/vitk/tag/CMMModel.java
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.ml.Model;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.ml.feature.CountVectorizerModel;
import org.apache.spark.ml.feature.StringIndexerModel;
import org.apache.spark.ml.param.ParamMap;
import org.apache.spark.ml.util.DefaultParamsReader;
import org.apache.spark.ml.util.DefaultParamsWriter;
import org.apache.spark.ml.util.MLReader;
import org.apache.spark.ml.util.MLWritable;
import org.apache.spark.ml.util.MLWriter;
import org.apache.spark.ml.util.SchemaUtils;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.util.MLUtils;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
import scala.collection.Iterator;
import scala.collection.mutable.WrappedArray;
import vn.vitk.util.Constants;

package vn.vitk.tag;




/**
 * @author Phuong LE-HONG
 *         <p>
 *         May 9, 2016, 5:52:38 PM
 *         <p>
 *         A conditional Markov model (or Maximum-Entropy Markov model -- MEMM)
 *         for sequence labeling which is fitted by a CMM graphical model.
 * 
 */
public class CMMModel extends Model<CMMModel> implements MLWritable {

	private static final long serialVersionUID = -4855076361905836432L;
	
	private PipelineModel pipelineModel;
	private ContextExtractor contextExtractor;
	private final Vector weights;
	private final String[] tags;
	private final Map<String, Integer> featureMap;
	private final Map<String, Set<Integer>> tagDictionary; 
	
	/**
	 * Creates a conditional Markov model.
	 * @param pipelineModel
	 * @param weights
	 * @param markovOrder
	 */
	public CMMModel(PipelineModel pipelineModel, Vector weights, MarkovOrder markovOrder, Map<String, Set<Integer>> tagDictionary) {
		this.pipelineModel = pipelineModel;
		this.contextExtractor = new ContextExtractor(markovOrder, Constants.REGEXP_FILE);