static String generateXmlDescriptor(String engineId,
        List<EngineDef> engineDefs) throws SherlokException,
        ResourceInitializationException {

    EngineDef engineDef = findEngineDefById(engineId, engineDefs);

    // convert fields strings to primitives
    Map<String, Object> engineParameters = extractParameters(engineDef);
    Object[] flatParamsArray = MapOps.flattenParameters(engineParameters);

    // construct AE
    Class<? extends AnalysisComponent> classz = extractAnalysisComponentClass(engineDef);
    String engineDescription = engineDef
            .getIdForDescriptor(ENGINE_ID_SEPARATOR);
    AnalysisEngineDescription aed = AnalysisEngineFactory
            .createEngineDescription(classz, flatParamsArray);

    // generate XML descriptor
    try {
        File tmpEngine = new File(FileBased.ENGINE_CACHE_PATH
                + engineDescription + ".xml");
        tmpEngine.getParentFile().mkdirs();
        FileOutputStream fos = new FileOutputStream(tmpEngine);
        aed.toXML(fos);
    } catch (SAXException | IOException e) {
        throw new RuntimeException("could not write descriptor of "
                + engineId, e); // should not happen
    }

    return engineDescription;
} public static void validateId(String id, String variableName)
        throws SherlokException {
    if (id.indexOf(SEPARATOR) == -1) {
        throw new SherlokException(variableName + " '" + id
                + "' should have the format"//
                + " {name}:{version}, but no column was found.");

    } else if (id.split(SEPARATOR).length != 2) {
        throw new SherlokException(variableName + " '" + id
                + "' should have the format"//
                + " {name}:{version}, but more than one column was found.");
    } else {
        String[] splits = id.split(SEPARATOR);
        checkOnlyAlphanumDotUnderscore(splits[0], variableName);
        checkOnlyAlphanumDotUnderscore(splits[1], variableName);
    }
} public static <E> ArrayList<E> list() {
    return new ArrayList<E>();
} public static <K, V> HashMap<K, V> map() {
    return new HashMap<K, V>();
} public class FilteringTypeSystem extends TypeSystemImpl {
    private static final Logger LOG = getLogger(FilteringTypeSystem.class);

    /**
     * Add this type to the typesystem.
     * 
     * @param type
     *            the type to include in the JSON output
     */
    public void includeType(Type type) {
        int typeId = super.addType(type.getName(), annotBaseTypeCode);
        for (Feature f : type.getFeatures()) {
            try {
                super.addFeature(f.getShortName(), typeId, annotBaseTypeCode);
            } catch (CASAdminException cae) {
                if (!cae.getArguments()[0].equals("sofa")) {
                    throw new RuntimeException(cae);
                }
            } catch (Exception e) { // TODO xLATER why is this happening?
                LOG.warn("failed to add feature '{}' to type '{}': " //
                        + e.getMessage(), f, type);
            }
        }
    }

    // for debugging:
    /*-
    public Type getType(String typeName) {
        System.err.println("Type:: " + typeName);
        return super.getType(typeName);
    }

    public Type getArrayType(Type componentType) {
        System.err.println("ArrayType:: " + componentType);
        return super.getArrayType(componentType);
    }

    public Feature getFeatureByFullName(String featureName) {
        System.err.println("FeatureByFullName:: " + featureName);
        return super.getFeatureByFullName(featureName);
    }

    public Iterator<Type> getTypeIterator() {
        System.err.println("TypeIterator");
        return super.getTypeIterator();
    }

    public Type getTopType() {
        System.err.println("TopType");
        return super.getTopType();
    }

    public Vector<Type> getDirectlySubsumedTypes(Type type) {
        System.err.println("DirectlySubsumedTypes:: " + type);
        return super.getDirectlySubsumedTypes(type);
    }

    public List<Type> getDirectSubtypes(Type type) {
        System.err.println("DirectSubtypes:: " + type);
        return super.getDirectSubtypes(type);
    }

    public List<Type> getProperlySubsumedTypes(Type type) {
        System.err.println("ProperlySubsumedTypes:: " + type);
        return super.getProperlySubsumedTypes(type);
    }

    public Type getParent(Type type) {
        System.err.println("Parent:: " + type);
        return super.getParent(type);
    }

    public boolean subsumes(Type superType, Type subType) {
        System.err.println("subsumes:: " + subType);
        return super.subsumes(superType, subType);
    }

    public Iterator<Feature> getFeatures() {
        System.err.println("Features");
        return super.getFeatures();
    }

    public TypeNameSpace getTypeNameSpace(String name) {
        System.err.println("TypeNameSpace:: " + name);
        return super.getTypeNameSpace(name);
    }

    public LowLevelTypeSystem getLowLevelTypeSystem() {
        System.err.println("LowLevelTypeSystem");
        return super.getLowLevelTypeSystem();
    }
     */
} public static class TypeDTO {
    public final String typeName, description, supertypeName;
    private List<TypeFeatureDTO> typeFeatures = list();

    public TypeDTO(String typeName, String description,
            String supertypeName, List<TypeFeatureDTO> features) {
        this.typeName = typeName;
        this.description = description;
        this.supertypeName = supertypeName;
        typeFeatures.addAll(features);
    }

    public List<TypeFeatureDTO> getTypeFeatures() {
        return typeFeatures;
    }

    public void addTypeFeature(TypeFeatureDTO typeFeature) {
        this.typeFeatures.add(typeFeature);
    }

    @Override
    public String toString() {
        return typeName + ":" + supertypeName;
    }

    @Override
    public boolean equals(Object obj) {
        if (obj instanceof TypeDTO
                && ((TypeDTO) obj).typeName.equals(typeName))
            return true;
        return false;
    }

    @Override
    public int hashCode() {
        return typeName.hashCode();
    }
} public static class TypeFeatureDTO {
    public final String featureName, description, rangeTypeName;

    public TypeFeatureDTO(String featureName, String description,
            String rangeTypeName) {
        this.featureName = featureName;
        this.description = description;
        this.rangeTypeName = rangeTypeName;
    }

    public String getRangeTypeNameCleaned() {

        switch (rangeTypeName) {
        case "STRING":
            return "uima.cas.String";
        case "INT":
            return "uima.cas.Integer";
        case "FLOAT":
            return "uima.cas.Float";
        case "DOUBLE":
            return "uima.cas.Double";

        case "INTLIST":
            return "uima.cas.IntegerList";
        case "FLOATLIST":
            return "uima.cas.FloatList";
        case "DOUBLELIST":
            return "uima.cas.DoubleList";
        case "BOOLEANLIST":
            return "uima.cas.BooleanList";
            // TODO do more
        default:
            return rangeTypeName;
        }
    }

    @Override
    public String toString() {
        return featureName + ":" + rangeTypeName;
    }
} public class ConfigVariableManager {

    private static Logger LOG = getLogger(ConfigVariableManager.class);

    /**
     * Process the configuration variables in each value and return the
     * interpreted value. Works for both pipeline and engine definition
     * (actually for all class inheriting from Def).
     * 
     * <pre>
     * For example, with the following variables defined in the bundle of the
     * engine:
     *  - $bluima refers to a git repository
     *  - $chunker refers to the string 'opennlp/chunker'
     *  - $countries refers to the path 'file://config/resources/countries.txt'
     * then processing the value list:
     *  - "$bluima/$chunker/Chunker_Genia.bin.gz"
     *  - "$countries"
     * will return the following list:
     *  - "<a_path>/opennlp/chunker/Chunker_Genia.bin.gz"
     *  - "config/resources/countries.txt"
     * where <a_path> is some path to a local copy of bluima repository that  
     * Sherlok will download at runtime.
     * </pre>
     * 
     * @throws NoSuchVariableException
     *             when an unknown variable is used
     * @throws ProcessConfigVariableException
     *             if a variable could not be processed
     */
    public static List<String> processConfigVariables(List<String> values,
            Def def) throws NoSuchVariableException,
            ProcessConfigVariableException {
        List<String> processed = list();
        for (String value : values) {
            processed.add(processConfigVariables(value, def));
        }

        return processed;
    }

    // Accepts variable starting by '$' and containing only alpha-numerical
    // characters (+ underscore). The variable name can be accessed through
    // the named-capturing group "name", group name "var" include the '$'
    // and is an alias for group number 1.
    // NB: '$$$x' will be transformed into '$$x'
    private static final Pattern VARIABLE_PATTERN = Pattern
            .compile("(?:^|[^\\$])(?:\\{2})*(?<var>\\$(?<name>\\w+))");

    // Process configuration variables
    private static String processConfigVariables(String value, Def def)
            throws NoSuchVariableException, ProcessConfigVariableException {

        // Find all variables
        Matcher matcher = VARIABLE_PATTERN.matcher(value);
        Map<String, ConfigVariable> config = def.getConfigVariables();

        while (matcher.find()) {
            String name = matcher.group("name");
            String processed = processConfigVariable(name, config);

            // replace *this* occurrence of "$name" in the original string
            // and not all since we could have $$name somewhere.
            String prefix = value.substring(0, matcher.start(1));
            String postfix = value.substring(matcher.end());
            value = prefix + processed + postfix;

            // reset the matcher since we edited the value
            matcher = VARIABLE_PATTERN.matcher(value);
        }

        // Replace all double dollar signs
        value = value.replace("$$", "$");

        return value;
    }

    private static String processConfigVariable(String name,
            Map<String, ConfigVariable> config) throws NoSuchVariableException,
            ProcessConfigVariableException {
        ConfigVariable var = config.get(name);
        if (var == null) {
            LOG.debug("unknown variable '" + name + "'");
            throw new NoSuchVariableException(name);
        }
        return var.getProcessedValue();
    }

} public class NoSuchVariableException extends Exception {
    public NoSuchVariableException(String name) {
        super("unknown configuration variable '" + name + "'");
    }

    private static final long serialVersionUID = -5819642307257618387L;
} public class ProcessConfigVariableException extends Exception {

    public ProcessConfigVariableException(String msg, Throwable t) {
        super(msg, t);
    }

    public ProcessConfigVariableException(String msg) {
        super(msg);
    }

    private static final long serialVersionUID = -95682363642593272L;

} @JsonPropertyOrder({ "name", "class", "description", "parameters" })
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public static class EngineDef {

    /**
     * an optional unique name for this bundle. Letters, numbers and
     * underscore only
     */
    private String name,

    /** (optional) */
    description;

    /** the Java UIMA class name of this engine. */
    @JsonProperty("class")
    private String classz;

    /** UIMA parameters. To overwrite default parameters */
    // TODO serialize without [ ], by creating custom
    // serializer, see Def.LineSerializer
    private Map<String, List<String>> parameters = map();

    /** TRANSITIVE (JsonIgnore), dynamically set by the bundle. */
    @JsonIgnore
    private BundleDef bundle;

    // get/set

    /** @return the engine name. Falls back on {@link #classz's simple name}. */
    public String getName() {
        if (name != null) {
            return name;
        } else if (classz != null) {
            if (classz.contains(".")) {
                return classz.substring(classz.lastIndexOf(".") + 1,
                        classz.length());
            } else { // no dot in class name --> just return it
                return classz;
            }
        }
        return null;
    }

    public EngineDef setName(String name) {
        this.name = name;
        return this;
    }

    public String getDescription() {
        return description;
    }

    public EngineDef setDescription(String description) {
        this.description = description;
        return this;
    }

    public String getClassz() {
        return classz;
    }

    public EngineDef setClassz(String classz) {
        this.classz = classz;
        return this;
    }

    public Map<String, List<String>> getParameters() {
        return parameters;
    }

    public EngineDef setParameters(Map<String, List<String>> parameters) {
        this.parameters = parameters;
        return this;
    }

    public EngineDef addParameter(String key, List<String> value) {
        this.parameters.put(key, value);
        return this;
    }

    public List<String> getParameter(String key) {
        return this.parameters.get(key);
    }

    public boolean validate() throws SherlokException {
        checkOnlyAlphanumDotUnderscore(name,
                "EngineDef '" + this.toString() + "'' id");
        return true;
    }

    public BundleDef getBundle() {
        return bundle;
    }

    /** Is set at load time by {@link Controller#_load()}. */
    public EngineDef setBundle(final BundleDef bundle) {
        this.bundle = bundle;
        return this;
    }

    /** Needs bundle to be set (see {@link #setBundle()}). */
    @JsonIgnore
    public String getId() {
        return createId(name, bundle.getVersion());
    }

    /** @return the id without non-alphanumeric, separated by separator */
    public String getIdForDescriptor(final String separator) {
        return getName().replaceAll("[^A-Za-z0-9]", "_") + separator
                + bundle.getVersion().replaceAll("[^A-Za-z0-9]", "_");
    }

    @Override
    public String toString() {
        return name;
    }
} @JsonPropertyOrder(value = { "name", "version", "description", "language",
        "domain", "loadOnStartup", "scriptLines", "config", "output", "tests" }, alphabetic = true)
@JsonInclude(NON_DEFAULT)
public class PipelineDef extends Def {

    /** Which language this pipeline works for (ISO code). Defaults to 'en' */
    private String language = "en";

    /** The list of engine definitions */
    @JsonProperty("script")
    @JsonSerialize(using = ListSerializer.class)
    private List<String> scriptLines = list();

    /** Controls the output of this pipeline */
    private PipelineOutput output = new PipelineOutput();

    /** Embedded (integration) tests */
    @JsonInclude(ALWAYS)
    private List<PipelineTest> tests = list();

    /** Defines what kind of annotation this pipeline outputs */
    @JsonInclude(JsonInclude.Include.NON_DEFAULT)
    public static class PipelineOutput {

        @JsonProperty("filter_annotations")
        private List<String> annotationFilters = list();

        @JsonProperty("include_annotations")
        private List<String> annotationIncludes = list();

        public List<String> getAnnotationFilters() {
            return annotationFilters;
        }

        public PipelineOutput setAnnotationFilters(
                List<String> annotationFilters) {
            this.annotationFilters = annotationFilters;
            return this;
        }

        public List<String> getAnnotationIncludes() {
            return annotationIncludes;
        }

        public PipelineOutput setAnnotationIncludes(
                List<String> annotationIncludes) {
            this.annotationIncludes = annotationIncludes;
            return this;
        }
    }

    /**
     * Tests one single input string against a list of expected
     * {@link JsonAnnotation}s
     */
    @JsonIgnoreProperties(ignoreUnknown = true)
    @JsonInclude(NON_DEFAULT)
    @JsonPropertyOrder(value = { "comment", "input", "expected", "comparison" }, alphabetic = true)
    public static class PipelineTest {

        public enum Comparison {
            /** all expected {@link JsonAnnotation}s are present in system */
            atLeast,
            /** expected and system {@link JsonAnnotation}s are exactly equals */
            exact;
        }

        private String comment;
        private String input;
        private Map<String, List<JsonAnnotation>> expected = map();
        private Comparison comparison = Comparison.atLeast; // default

        public String getComment() {
            return comment;
        }

        public PipelineTest setComment(String comment) {
            this.comment = comment;
            return this;
        }

        public String getInput() {
            return input;
        }

        public PipelineTest setInput(String input) {
            this.input = input;
            return this;
        }

        public Map<String, List<JsonAnnotation>> getExpected() {
            return expected;
        }

        public PipelineTest setExpected(
                Map<String, List<JsonAnnotation>> expected) {
            this.expected = expected;
            return this;
        }

        public Comparison getComparison() {
            return comparison;
        }

        public void setComparison(Comparison comparison) {
            this.comparison = comparison;
        }

        @Override
        public String toString() {
            return input + "::" + expected;
        }

    }

    // Get/Set ////////////////////////////////////////////////////////////////

    public String getLanguage() {
        return language;
    }

    public PipelineDef setLanguage(String language) {
        this.language = language;
        return this;
    }

    public List<String> getScriptLines() {
        return scriptLines;
    }

    public PipelineDef setScriptLines(List<String> scriptLines) {
        this.scriptLines = scriptLines;
        return this;
    }

    public PipelineDef addScriptLine(String scriptLine) {
        this.scriptLines.add(scriptLine);
        return this;
    }

    public PipelineOutput getOutput() {
        return output;
    }

    public PipelineDef setOutput(PipelineOutput output) {
        this.output = output;
        return this;
    }

    public List<PipelineTest> getTests() {
        return tests;
    }

    public PipelineDef setTests(List<PipelineTest> tests) {
        this.tests = tests;
        return this;
    }

    public PipelineDef addTest(PipelineTest test) {
        this.tests.add(test);
        return this;
    }

    // Utilities //////////////////////////////////////////////////////////////

    public void validate(String context) throws SherlokException {
        try {
            super.validate();
            validateArgument(language != null, "'language' can not be null");
            validateArgument(language.length() > 0,
                    "'language' can not be empty");
            validateDomain(domain);

            // output
            validateArgument(
                    !(!output.annotationFilters.isEmpty() && !output.annotationIncludes
                            .isEmpty()),
                    "either provide 'filter_annotations' or 'include_annotations', but not both for pipeline '"
                            + getId() + "'");

            String typeMsg = "It should be a list of valid identifiers, separated by dots, "
                    + "and optionally ending with '.*' to match all subsequent types. "
                    + "Identifiers must be composed of letters, numbers, the underscore"
                    + " _ and the dollar sign $. Identifiers may only begin with a letter, "
                    + "the underscore or a dollar sign.";
            for (String type : output.annotationFilters) {
                validateTypeIdentifier(type, "output annotation filter '"
                        + type + "' is not valid." + typeMsg);
            }
            for (String type : output.annotationIncludes) {
                validateTypeIdentifier(type, "output annotation includes '"
                        + type + "' is not valid." + typeMsg);
            }
            if (tests.isEmpty()) {
                LOG.warn("no tests for pipeline '{}'", getId());
            }

            // TODO validate usage of variables in script and make sure no
            // unknown variable is used
        } catch (SherlokException e) {
            throw e.setObject(this.toString()).setWhen(context);
        }
    }

    /**
     * Ensures that this pipeline only uses engines defined in
     * {@link Controller}'s engines
     * 
     * @param engineIds
     *            the engine ids of the controller
     */
    public void validateEngines(Set<String> engineIds) throws SherlokException {
        for (String engineFromScript : getEnginesFromScript()) {
            if (!engineIds.contains(engineFromScript)) {
                throw new SherlokException("engine declared in pipeline '"
                        + getId() + "' but not found", engineFromScript);
            }
        }
    }

    /**
     * Extract all {@link EngineDef}s from theses script lines, by scanning for
     * lines starting with "ENGINE"
     */
    @JsonIgnore
    public List<String> getEnginesFromScript() {
        return getEnginesFromScript(getScriptLines());
    }

    /**
     * Extract all {@link EngineDef}s from theses script lines, by scanning for
     * lines starting with "ENGINE"
     */
    public static List<String> getEnginesFromScript(List<String> scriptLines) {
        List<String> engineIds = list();
        for (String scriptLine : scriptLines) {
            if (scriptLine.startsWith("ENGINE")) {
                String pengineId = scriptLine.trim()
                        .substring("ENGINE".length()).trim()
                        .replaceAll(";", "");
                engineIds.add(pengineId);
            }
        }
        return engineIds;
    }

    /** Adds newlines between entries */
    @SuppressWarnings("rawtypes")
    public static class ListSerializer extends JsonSerializer<List> {

        @Override
        public boolean isEmpty(final List value) {
            return value.isEmpty();
        }

        @Override
        public Class<List> handledType() {
            return List.class;
        }

        @Override
        public void serialize(final List value, final JsonGenerator jgen,
                final SerializerProvider provider) throws IOException,
                JsonProcessingException {
            jgen.writeStartArray();
            for (Object item : value) {
                jgen.writeRaw('\n');
                jgen.writeObject(item);
            }
            jgen.writeRaw('\n');
            jgen.writeEndArray();
        }
    }

} @JsonPropertyOrder(value = { "when", "message", "object", "remedy", "details" })
@JsonIgnoreProperties(value = { "stackTrace", "localizedMessage" })
@JsonInclude(NON_DEFAULT)
public class SherlokException extends Exception {
    private static final long serialVersionUID = 2101833257950070947L;

    public SherlokException() {
    }

    public SherlokException(String message) {
        this._message = message;
    }

    public SherlokException(String message, String object) {
        this._message = message;
        this.object = object;
    }

    /** In which circumstances did something go wrong? */
    @JsonProperty("when")
    private String when;

    /** What went wrong? */
    @JsonProperty("message")
    private String _message;

    /** More details about what went wrong? */
    private Object details;

    /** What was the object causing the error? */
    private String object;

    /** Suggestion on how to fix the error, if available */
    private String remedy;
    /** The REST route, if available */
    private String route;

    public String getWhen() {
        return when;
    }

    public SherlokException setWhen(String when) {
        this.when = when;
        return this;
    }

    public String getMessage() {
        return _message;
    }

    public SherlokException setMessage(String message) {
        this._message = message;
        return this;
    }

    public Object getDetails() {
        return details;
    }

    public SherlokException setDetails(String details) {
        this.details = details;
        return this;
    }

    public SherlokException setDetails(StackTraceElement[] elements) {

        StringBuilder msg = new StringBuilder();
        for (int i = 0, n = elements.length; i < n && n < 5; i++) {
            msg.append(elements[i].getFileName() + ":"
                    + elements[i].getLineNumber() + ">> "
                    + elements[i].getMethodName() + "(); ");
        }
        setDetails(msg.toString().trim());
        return this;
    }

    /**
     * @param detailsObj
     *            must be JSON seralizable
     */
    public SherlokException setDetails(Object detailsObj) {
        this.details = detailsObj;
        return this;
    }

    public String getObject() {
        return object;
    }

    public SherlokException setObject(String object) {
        this.object = object;
        return this;
    }

    public String getRemedy() {
        return remedy;
    }

    public SherlokException setRemedy(String remedy) {
        this.remedy = remedy;
        return this;
    }

    public SherlokException setRoute(String route) {
        this.route = route;
        return this;
    }

    public String getRoute() {
        return route;
    }

    /** To display errors nicely as JSON */
    public Object toJson() {
        return this;
    }

    @Override
    public String toString() {
        return (getMessage() != null ? getMessage() : "") //
                + (getObject() != null ? " ON OBJECT " + getObject() : "")//
                + (getWhen() != null ? " WHEN " + getWhen() : "")//
                + (getRoute() != null ? " and ROUTE " + getRoute() : "")//
                + (getDetails() != null ? "; DETAILS: " + getDetails() : "")//
                + (getRemedy() != null ? "; REMEDY: " + getRemedy() : "");
    }
}
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
import static org.apache.uima.ruta.engine.RutaEngine.PARAM_ADDITIONAL_ENGINES;
import static org.apache.uima.ruta.engine.RutaEngine.PARAM_DESCRIPTOR_PATHS;
import static org.apache.uima.ruta.engine.RutaEngine.PARAM_MAIN_SCRIPT;
import static org.apache.uima.ruta.engine.RutaEngine.PARAM_RESOURCE_PATHS;
import static org.apache.uima.ruta.engine.RutaEngine.PARAM_SCRIPT_PATHS;
import static org.apache.uima.ruta.engine.RutaEngine.SCRIPT_FILE_EXTENSION;
import static org.apache.uima.util.FileUtils.saveString2File;
import static org.sherlok.EngineOps.generateXmlDescriptor;
import static org.sherlok.utils.CheckThat.validateId;
import static org.sherlok.utils.Create.list;
import static org.sherlok.utils.Create.map;
import static org.slf4j.LoggerFactory.getLogger;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.StringWriter;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
import org.apache.uima.UIMAException;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.TypeSystem;
import org.apache.uima.cas.impl.FilteringTypeSystem;
import org.apache.uima.cas.impl.TypeSystemImpl;
import org.apache.uima.fit.component.NoOpAnnotator;
import org.apache.uima.fit.factory.AnalysisEngineFactory;
import org.apache.uima.fit.factory.TypeSystemDescriptionFactory;
import org.apache.uima.fit.pipeline.SimplePipeline;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.json.JsonCasSerializer;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeDescription;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.ruta.engine.RutaEngine;
import org.apache.uima.ruta.ontologies.OntoActionExtension;
import org.apache.uima.util.CasPool;
import org.sherlok.RutaHelper.TypeDTO;
import org.sherlok.RutaHelper.TypeFeatureDTO;
import org.sherlok.config.ConfigVariableManager;
import org.sherlok.config.NoSuchVariableException;
import org.sherlok.config.ProcessConfigVariableException;
import org.sherlok.mappings.BundleDef.EngineDef;
import org.sherlok.mappings.PipelineDef;
import org.sherlok.mappings.SherlokException;
import org.slf4j.Logger;
import org.xml.sax.SAXException;
    }

    public interface Annotate {
        public Object annotate(final CAS cas, final AnalysisEngine[] aes)
                throws AnalysisEngineProcessException;
    }

    /**
     * @param annotate
     *            visitor pattern
     * @return a payload, defined by 'annotate'
     */
    public Object annotate(Annotate annotate) throws UIMAException,
            SAXException, SherlokException {

        CAS cas = null;
        try {
            // TODO how long to wait?
            cas = casPool.getCas(0);// cas.reset done by casPool
            return annotate.annotate(cas, aes);
        } finally {
            casPool.releaseCas(cas);
        }
    }

    /**
     * @param text
     *            the text to annotate
     */
    public String annotate(String text) throws UIMAException, SherlokException {

        CAS cas = null;
        try {
            // TODO how long to wait?
            cas = casPool.getCas(0);// cas.reset done by casPool
            // for (TypeDescription td : tsd.getTypes())
            // LOG.debug("type: {} <<<< {}", td.getName(),
            // td.getSupertypeName());

            cas.setDocumentText(text);
            cas.setDocumentLanguage(language);

            LOG.trace("annotating: " + text);
            SimplePipeline.runPipeline(cas, aes);

            if (LOG.isTraceEnabled()) {
                FSIterator<Annotation> it = cas.getJCas().getAnnotationIndex()
                        .iterator();
                while (it.hasNext()) {
                    Annotation a = it.next();
                    StringBuffer sb = new StringBuffer();
                    a.prettyPrint(2, 2, sb, false);
                    LOG.trace("'{}'\t{}", a.getCoveredText(), sb.toString()
                            .replaceAll("[\r\n] *", "\t"));
                }
            }

            StringWriter sw = new StringWriter();
            jsonSerializer.serialize(cas, sw);

            String json = sw.toString();
            // rename JSON field, for readability
            json = json.replaceFirst("@cas_feature_structures", "annotations");
            return json;

        } catch (AnalysisEngineProcessException aepe) {
            Throwable cause = aepe.getCause();
            if (cause instanceof IllegalArgumentException) {
                throw new SherlokException("Failed to annotate " + text,
                        this.toString()).setDetails(cause.getMessage());
            } else {
                throw aepe;
            }
        } catch (IOException io) {
            throw new SherlokException("Failed to annotate " + text,
                    this.toString()).setDetails(io.getMessage());
        } finally {
            casPool.releaseCas(cas);
        }
    }

    public void close() {
        for (AnalysisEngine engine : aes) {
            engine.destroy();
        }
    }

    public PipelineDef getPipelineDef() {
        return pipelineDef;
    }

    @Override
    public String toString() {
        return pipelineDef.toString();
    }

    private void initScript(List<String> scriptLines, List<EngineDef> engineDefs)
            throws ResourceInitializationException, IOException,
            SherlokException {

        // Handle variables in script
        try {
            scriptLines = ConfigVariableManager.processConfigVariables(
                    scriptLines, pipelineDef);
        } catch (NoSuchVariableException | ProcessConfigVariableException e) {
            throw new SherlokException(e.getMessage()).setObject(pipelineDef
                    .toString());
        }

        // load engines
        List<String> engineDescriptions = list();
        for (int i = 0; i < scriptLines.size(); i++) {

            String scriptLine = scriptLines.get(i);

            if (scriptLine.startsWith("ENGINE")) {
                // find the corresponding engine description
                String engineId = extractEngineId(scriptLine);

                // create ae and write xml descriptor
                String engineDescription = generateXmlDescriptor(engineId,