// Path: src/main/java/pitt/search/semanticvectors/LuceneUtils.java
// public enum TermWeight {
//   /** No term weighting: all terms have weight 1. */
//   NONE,
//   /** Use inverse document frequency: see {@link LuceneUtils#getIDF}. */
//   IDF,
//   /** Use log entropy: see {@link LuceneUtils#getEntropy}. */
//   LOGENTROPY,
//   /** Use term frequency: see {@link LuceneUtils#getGlobalTermFreq} */
//   FREQ,
//   /** Use square root of term frequency. */
//   SQRT,
//   /** Use log of term frequency. */
//   LOGFREQ,
// }
// 
// Path: src/main/java/pitt/search/semanticvectors/vectors/VectorType.java
// public enum VectorType {
//   /**
//    * Uses binary-valued vectors. May be slower for some operations, but highly accurate.
//    */
//   BINARY,
//   /**
//    * "Standard" real-valued vectors. Performs well for many operations though bind and release
//    * are (in some cases) either lossy with respect to argument structure and binding, or
//    * inexact with respect to inverse.
//    */
//   REAL,
//   /**
//    * Complex-valued vectors, default "polar" version, normalized to coordinates on the unit circle,
//    * and compared using angular differences.
//    */
//   COMPLEX,
//   /**
//    * Complex-valued vectors, normalized and compared using a hermitian scalar product.
//    */
//   COMPLEXFLAT,
//   /**
//    * Vector of integer values, describing a permutation
//    */
//   PERMUTATION
// }


import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import pitt.search.semanticvectors.LuceneUtils.TermWeight;
import pitt.search.semanticvectors.utils.VerbatimLogger;
import pitt.search.semanticvectors.vectors.RealVector;
import pitt.search.semanticvectors.vectors.Vector;
import pitt.search.semanticvectors.vectors.VectorType;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.util.logging.Logger;
import ch.akuhn.edu.mit.tedlab.*;
package pitt.search.semanticvectors;





/**
 * Interface to Adrian Kuhn and David Erni's implementation of SVDLIBJ, a native Java version
 * of Doug Rhodes' SVDLIBC, which was in turn based on SVDPACK, by Michael Berry, Theresa Do,
 * Gavin O'Brien, Vijay Krishna and Sowmini Varadhan.
 *
 * This class will produce two files, svd_termvectors.bin and svd_docvectors.bin from a Lucene index
 * Command line arguments are consistent with the rest of the Semantic Vectors Package
 */
public class LSA {
  private static final Logger logger = Logger.getLogger(LSA.class.getCanonicalName());

  public static String usageMessage = "\nLSA class in package pitt.search.semanticvectors"
      + "\nUsage: java pitt.search.semanticvectors.LSA [other flags] -luceneindexpath PATH_TO_LUCENE_INDEX"
      + "Use flags to configure dimension, min term frequency, etc. See online documentation for other available flags";

  private FlagConfig flagConfig;
  /** Stores the list of terms in the same order as rows in the matrix. */
  private String[] termList;
  /** The single contents field in the Lucene index: LSA only supports one. */
  private String contentsField;
  private LuceneUtils luceneUtils;
  
  /**
   * Basic constructor that tries to check up front that resources are available and
   * configurations are consistent.
   * 
   * @param luceneIndexDir Relative path to directory containing Lucene index.
   * @throws IOException 
   */
  private LSA(String luceneIndexDir, FlagConfig flagConfig) throws IOException {
    this.flagConfig = flagConfig;    
    this.luceneUtils = new LuceneUtils(flagConfig);

    if (flagConfig.contentsfields().length > 1) {
      logger.warning(
          "LSA implementation only supports a single -contentsfield. Only '"
          + flagConfig.contentsfields()[0] + "' will be indexed.");
    }

    this.contentsField = flagConfig.contentsfields()[0];
    
    // Find the number of docs, and if greater than dimension, set the dimension
    // to be this number.
    if (flagConfig.dimension() > this.luceneUtils.getNumDocs()) {
      logger.warning("Dimension for SVD cannot be greater than number of documents ... "
          + "Setting dimension to " + this.luceneUtils.getNumDocs());
      flagConfig.setDimension(this.luceneUtils.getNumDocs());
    }

// Path: src/main/java/pitt/search/semanticvectors/LuceneUtils.java
// public enum TermWeight {
//   /** No term weighting: all terms have weight 1. */
//   NONE,
//   /** Use inverse document frequency: see {@link LuceneUtils#getIDF}. */
//   IDF,
//   /** Use log entropy: see {@link LuceneUtils#getEntropy}. */
//   LOGENTROPY,
//   /** Use term frequency: see {@link LuceneUtils#getGlobalTermFreq} */
//   FREQ,
//   /** Use square root of term frequency. */
//   SQRT,
//   /** Use log of term frequency. */
//   LOGFREQ,
// }
// 
// Path: src/main/java/pitt/search/semanticvectors/vectors/VectorType.java
// public enum VectorType {
//   /**
//    * Uses binary-valued vectors. May be slower for some operations, but highly accurate.
//    */
//   BINARY,
//   /**
//    * "Standard" real-valued vectors. Performs well for many operations though bind and release
//    * are (in some cases) either lossy with respect to argument structure and binding, or
//    * inexact with respect to inverse.
//    */
//   REAL,
//   /**
//    * Complex-valued vectors, default "polar" version, normalized to coordinates on the unit circle,
//    * and compared using angular differences.
//    */
//   COMPLEX,
//   /**
//    * Complex-valued vectors, normalized and compared using a hermitian scalar product.
//    */
//   COMPLEXFLAT,
//   /**
//    * Vector of integer values, describing a permutation
//    */
//   PERMUTATION
// }


// Path: src/main/java/pitt/search/semanticvectors/LSA.java
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import pitt.search.semanticvectors.LuceneUtils.TermWeight;
import pitt.search.semanticvectors.utils.VerbatimLogger;
import pitt.search.semanticvectors.vectors.RealVector;
import pitt.search.semanticvectors.vectors.Vector;
import pitt.search.semanticvectors.vectors.VectorType;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.util.logging.Logger;
import ch.akuhn.edu.mit.tedlab.*;

package pitt.search.semanticvectors;





/**
 * Interface to Adrian Kuhn and David Erni's implementation of SVDLIBJ, a native Java version
 * of Doug Rhodes' SVDLIBC, which was in turn based on SVDPACK, by Michael Berry, Theresa Do,
 * Gavin O'Brien, Vijay Krishna and Sowmini Varadhan.
 *
 * This class will produce two files, svd_termvectors.bin and svd_docvectors.bin from a Lucene index
 * Command line arguments are consistent with the rest of the Semantic Vectors Package
 */
public class LSA {
  private static final Logger logger = Logger.getLogger(LSA.class.getCanonicalName());

  public static String usageMessage = "\nLSA class in package pitt.search.semanticvectors"
      + "\nUsage: java pitt.search.semanticvectors.LSA [other flags] -luceneindexpath PATH_TO_LUCENE_INDEX"
      + "Use flags to configure dimension, min term frequency, etc. See online documentation for other available flags";

  private FlagConfig flagConfig;
  /** Stores the list of terms in the same order as rows in the matrix. */
  private String[] termList;
  /** The single contents field in the Lucene index: LSA only supports one. */
  private String contentsField;
  private LuceneUtils luceneUtils;
  
  /**
   * Basic constructor that tries to check up front that resources are available and
   * configurations are consistent.
   * 
   * @param luceneIndexDir Relative path to directory containing Lucene index.
   * @throws IOException 
   */
  private LSA(String luceneIndexDir, FlagConfig flagConfig) throws IOException {
    this.flagConfig = flagConfig;    
    this.luceneUtils = new LuceneUtils(flagConfig);

    if (flagConfig.contentsfields().length > 1) {
      logger.warning(
          "LSA implementation only supports a single -contentsfield. Only '"
          + flagConfig.contentsfields()[0] + "' will be indexed.");
    }

    this.contentsField = flagConfig.contentsfields()[0];
    
    // Find the number of docs, and if greater than dimension, set the dimension
    // to be this number.
    if (flagConfig.dimension() > this.luceneUtils.getNumDocs()) {
      logger.warning("Dimension for SVD cannot be greater than number of documents ... "
          + "Setting dimension to " + this.luceneUtils.getNumDocs());
      flagConfig.setDimension(this.luceneUtils.getNumDocs());
    }

    if (flagConfig.termweight().equals(TermWeight.LOGENTROPY)) {