docs: @Override
    public void train(Collection<List<IN>> docs, DocumentReaderAndWriter<IN> readerAndWriter) {
      Timing timer = new Timing();
      timer.start();
      makeAnswerArraysAndTagIndex(docs);
      long elapsedMs = timer.stop();
      System.err.println("Time to convert docs to feature indices: " + Timing.toSecondsString(elapsedMs) + " seconds");
      if (flags.exportFeatures != null) {
        timer.start();
        CRFFeatureExporter<IN> featureExporter = new CRFFeatureExporter<IN>(this);
        featureExporter.printFeatures(flags.exportFeatures, docs);
        elapsedMs = timer.stop();
        System.err.println("Time to export features: " + Timing.toSecondsString(elapsedMs) + " seconds");
      }
      for (int i = 0; i <= flags.numTimesPruneFeatures; i++) {
        timer.start();
        Pair<int[][][][], int[][]> dataAndLabels = documentsToDataAndLabels(docs);
        elapsedMs = timer.stop();
        System.err.println("Time to convert docs to data/labels: " + Timing.toSecondsString(elapsedMs) + " seconds");
  
        Evaluator[] evaluators = null;
        if (flags.evaluateIters > 0) {
          List<Evaluator> evaluatorList = new ArrayList<Evaluator>();
          evaluatorList.add(new MemoryEvaluator());
          if (flags.evaluateTrain) {
            CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Train set", this);
            List<Pair<int[][][], int[]>> trainDataAndLabels = new ArrayList<Pair<int[][][], int[]>>();
            for (int j = 0; j < dataAndLabels.first().length; j++) {
              Pair<int[][][], int[]> p = new Pair<int[][][], int[]>(dataAndLabels.first()[j], dataAndLabels.second()[j]);
              trainDataAndLabels.add(p);
            }
            crfEvaluator.setTestData(docs, trainDataAndLabels);
            crfEvaluator.setEvalCmd(flags.evalCmd);
            evaluatorList.add(crfEvaluator);
          }
          if (flags.testFile != null) {
            CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + flags.testFile + ")",
                this);
            ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter);
            List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
            crfEvaluator.setTestData(testObjBank, testDataAndLabels);
            crfEvaluator.setEvalCmd(flags.evalCmd);
            evaluatorList.add(crfEvaluator);
          }
          if (flags.testFiles != null) {
            String[] testFiles = flags.testFiles.split(",");
            for (String testFile : testFiles) {
              CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set ("
                  + testFile + ")", this);
              ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter);
              List<Pair<int[][][], int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank);
              crfEvaluator.setTestData(testObjBank, testDataAndLabels);
              crfEvaluator.setEvalCmd(flags.evalCmd);
              evaluatorList.add(crfEvaluator);
            }
          }
          evaluators = new Evaluator[evaluatorList.size()];
          evaluatorList.toArray(evaluators);
        }
  
        if (flags.numTimesPruneFeatures == i) {
          docs = null; // hopefully saves memory
        }
        // save feature index to disk and read in later
        File featIndexFile = null;
  
        if (flags.saveFeatureIndexToDisk) {
          try {
            System.err.println("Writing feature index to temporary file.");
            featIndexFile = IOUtils.writeObjectToTempFile(featureIndex, "featIndex" + i + ".tmp");
            featureIndex = null;
          } catch (IOException e) {
            throw new RuntimeException("Could not open temporary feature index file for writing.");
          }
        }
  
        // first index is the number of the document
        // second index is position in the document also the index of the
        // clique/factor table
        // third index is the number of elements in the clique/window thase
        // features are for (starting with last element)
        // fourth index is position of the feature in the array that holds them
        // element in data[i][j][k][m] is the index of the mth feature occurring
        // in position k of the jth clique of the ith document
        int[][][][] data = dataAndLabels.first();
        // first index is the number of the document
        // second index is the position in the document
        // element in labels[i][j] is the index of the correct label (if it
        // exists) at position j in document i
        int[][] labels = dataAndLabels.second();
  
        if (flags.loadProcessedData != null) {
          List processedData = loadProcessedData(flags.loadProcessedData);
          if (processedData != null) {
            // enlarge the data and labels array
            int[][][][] allData = new int[data.length + processedData.size()][][][];
            int[][] allLabels = new int[labels.length + processedData.size()][];
            System.arraycopy(data, 0, allData, 0, data.length);
            System.arraycopy(labels, 0, allLabels, 0, labels.length);
            // add to the data and labels array
            addProcessedData(processedData, allData, allLabels, data.length);
            data = allData;
            labels = allLabels;
          }
        }
  
        if (flags.useFloat) {
          CRFLogConditionalObjectiveFloatFunction func = new CRFLogConditionalObjectiveFloatFunction(data, labels,
              featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
          func.crfType = flags.crfType;
  
          QNMinimizer minimizer;
          if (flags.interimOutputFreq != 0) {
            FloatFunction monitor = new ResultStoringFloatMonitor(flags.interimOutputFreq, flags.serializeTo);
            minimizer = new QNMinimizer(monitor);
          } else {
            minimizer = new QNMinimizer();
          }
  
          if (i == 0) {
            minimizer.setM(flags.QNsize);
          } else {
            minimizer.setM(flags.QNsize2);
          }
  
          float[] initialWeights;
          if (flags.initialWeights == null) {
            initialWeights = func.initial();
          } else {
            try {
              System.err.println("Reading initial weights from file " + flags.initialWeights);
              DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                  flags.initialWeights))));
              initialWeights = Convert.readFloatArr(dis);
            } catch (IOException e) {
              throw new RuntimeException("Could not read from float initial weight file " + flags.initialWeights);
            }
          }
          System.err.println("numWeights: " + initialWeights.length);
          float[] weights = minimizer.minimize(func, (float) flags.tolerance, initialWeights);
          this.weights = ArrayMath.floatArrayToDoubleArray(func.to2D(weights));
  
        } else {
  
          /*double[] estimate = null;
  
          if(flags.estimateInitial){
            int[][][][] approxData = new int[data.length/100][][][];
            int[][] approxLabels = new int[data.length/100][];
  
            Random generator = new Random(1);
            for(int k=0;k<approxData.length; k++){
              int thisInd = generator.nextInt(data.length);
              approxData[k] = data[ thisInd];
              approxLabels[k] = labels[ thisInd];
            }
  
            CRFLogConditionalObjectiveFunction approxFunc = new CRFLogConditionalObjectiveFunction(approxData, approxLabels, featureIndex, windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
            approxFunc.crfType = flags.crfType;
            minimizer = new QNMinimizer(10);
  
            if (flags.initialWeights == null) {
              estimate = approxFunc.initial();
            } else {
              try {
                System.err.println("Reading initial weights from file " + flags.initialWeights);
                DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(flags.initialWeights))));
                estimate = Convert.readDoubleArr(dis);
              } catch (IOException e) {
                throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
              }
            }
  
            estimate = minimizer.minimize(approxFunc, 1e-2, estimate);
  
          }
          */
          CRFLogConditionalObjectiveFunction func = new CRFLogConditionalObjectiveFunction(data, labels, featureIndex,
              windowSize, classIndex, labelIndices, map, flags.backgroundSymbol, flags.sigma);
          func.crfType = flags.crfType;
          if (evaluators != null) {
            for (Evaluator eval : evaluators) {
              if (eval instanceof CRFClassifierEvaluator) {
                ((CRFClassifierEvaluator) eval).setHelperFunction(func);
              }
            }
          }
          Minimizer minimizer = getMinimizer(i, evaluators);
  
          double[] initialWeights;
          if (flags.initialWeights == null) {
            initialWeights = func.initial();
          } else {
            try {
              System.err.println("Reading initial weights from file " + flags.initialWeights);
              DataInputStream dis = new DataInputStream(new BufferedInputStream(new GZIPInputStream(new FileInputStream(
                  flags.initialWeights))));
              initialWeights = Convert.readDoubleArr(dis);
            } catch (IOException e) {
              throw new RuntimeException("Could not read from double initial weight file " + flags.initialWeights);
            }
          }
          System.err.println("numWeights: " + initialWeights.length);
  
          if (flags.testObjFunction) {
            StochasticDiffFunctionTester tester = new StochasticDiffFunctionTester(func);
            if (tester.testSumOfBatches(initialWeights, 1e-4)) {
              System.err.println("Testing complete... exiting");
              System.exit(1);
            } else {
              System.err.println("Testing failed....exiting");
              System.exit(1);
            }
  
          }
          double[] weights = minimizer.minimize(func, flags.tolerance, initialWeights);
          this.weights = func.to2D(weights);
        }
  
        // save feature index to disk and read in later
        if (flags.saveFeatureIndexToDisk) {
          try {
            System.err.println("Reading temporary feature index file.");
            featureIndex = (Index<String>) IOUtils.readObjectFromFile(featIndexFile);
          } catch (Exception e) {
            throw new RuntimeException("Could not open temporary feature index file for reading.");
          }
        }
  
        if (i != flags.numTimesPruneFeatures) {
          dropFeaturesBelowThreshold(flags.featureDiffThresh);
          System.err.println("Removing features with weight below " + flags.featureDiffThresh + " and retraining...");
        }
  
      }
    }