public class AnnotationSearchConstants {

    public static final String FIELD_MOTIVATIONS = "motivations";

    public static final String PARAM_FIELD_MOTIVATION = "motivation";

    public static final String PARAM_FIELD_DATE = "date";
    public static final String PARAM_FIELD_USER = "user";

    public static final String PARAM_FIELD_QUERY = "q";

    public static final String PARAM_FIELD_PAGE = "page";

    public static final String PARAM_FIELD_WIDTH = "width";
    public static final String PARAM_FIELD_HEIGHT = "height";

    public static final int DEFAULT_PAGING_NUMBER = 100;

    public static final int MAX_NUMBER_OF_HITS_RETURNED = 1000;

    public static final String PAINTING_MOTIVATION = "painting";

    public static final String NON_PAINTING_MOTIVATION = "non-painting";

    public static final String EMPTY_QUERY_MESSAGE = "Please enter either a query, motivation, date or user to search";

    public static final String EMPTY_AUTOCOMPLETE_QUERY_MESSAGE = "Please enter a query to search";

    public static final String TEXT_INDEX_NAME = "text_index";

    public static final String W3C_INDEX_NAME = "w3cannotation";

    public static final String CONTEXT_MANIFEST_NAME = "manifest";

    public static final boolean USE_PARED_DOWN_OA = true;

    private AnnotationSearchConstants() {

    }
} public class TextSearchAnnotationMapper extends DefaultResultMapper {

    private static final Logger LOG = Logger.getLogger(TextSearchAnnotationMapper.class);

    @Override
    public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
	long totalHits = response.getHits().totalHits();
	LOG.info("total hits in mapResults are :" + totalHits);
	List<T> results = new ArrayList<>();
	for (SearchHit hit : response.getHits()) {
	    if (hit != null) {
		T result;
		if (StringUtils.isNotBlank(hit.sourceAsString())) {
		    result = mapEntity(hit.sourceAsString(), clazz);
		} else {
		    result = mapEntity(hit.getFields().values(), clazz);
		}
		setPersistentEntityId(result, hit.getId(), clazz);

		results.add(result);
	    }
	}
	return new AggregatedPageImpl<>(results, pageable, totalHits);
    }

    private <T> T mapEntity(Collection<SearchHitField> values, Class<T> clazz) {
	return mapEntity(buildJSONFromFields(values), clazz);
    }

    private String buildJSONFromFields(Collection<SearchHitField> values) {
	JsonFactory nodeFactory = new JsonFactory();
	try {
	    ByteArrayOutputStream stream = new ByteArrayOutputStream();
	    JsonGenerator generator = nodeFactory.createGenerator(stream, JsonEncoding.UTF8);
	    generator.writeStartObject();
	    for (SearchHitField value : values) {
		if (value.getValues().size() > 1) {
		    generator.writeArrayFieldStart(value.getName());
		    for (Object val : value.getValues()) {
			generator.writeObject(val);
		    }
		    generator.writeEndArray();
		} else {
		    generator.writeObjectField(value.getName(), value.getValue());
		}
	    }
	    generator.writeEndObject();
	    generator.flush();
	    return new String(stream.toByteArray(), Charset.forName("UTF-8"));
	} catch (IOException e) {
	    LOG.error("IOException in buildJSONFromFields ", e);
	    return null;
	}
    }

    /*
     * vastly changed method from the one in defaultResultMapper. Mappingcontext
     * was always null do we were never getting the _id field contents when we
     * has no _source in the results
     */
    private <T> void setPersistentEntityId(T result, String id, Class<T> clazz) {
	if (clazz.isAnnotationPresent(Document.class)) {
	    Method setter;
	    try {
		setter = clazz.getDeclaredMethod("setId", String.class);
		if (setter != null) {
		    try {
			setter.invoke(result, id);
		    } catch (Exception t) {
			LOG.error("Exception in setPersistentEntityId trying to invoke setter", t);
		    }
		}
	    } catch (NoSuchMethodException e) {
		LOG.error("NoSuchMethodException in setPersistentEntityId ", e);
	    } catch (SecurityException e) {
		LOG.error("SecurityException in setPersistentEntityId ", e);
	    }

	}
    }

} public class ImageHelperObject {

    private List<Map<String, Object>> imageJson;
    private Map <String, List<Positions>> positionsMap;
    private Map<String, Object> offsetPayloadMap;
    private Map<String,String> crossPageImageMap;

    public List<Map<String, Object>> getImageJson() {
	return imageJson;
    }

    public void setImageJson(List<Map<String, Object>> imageJson) {
	this.imageJson = imageJson;
    }

    public Map <String, List<Positions>> getPositionsMap() {
	return positionsMap;
    }

    public void setPositionsMap(Map <String, List<Positions>> positionsMap) {
	this.positionsMap = positionsMap;
    }

    public Map<String, Object> getOffsetPayloadMap() {
	return offsetPayloadMap;
    }

    public void setOffsetPayloadMap(Map<String, Object> offsetPayloadMap) {
	this.offsetPayloadMap = offsetPayloadMap;
    }

    public Map<String,String> getCrossPageImageMap() {
	return crossPageImageMap;
    }

    public void setCrossPageImageMap(Map<String,String> crossPageImageMap) {
	this.crossPageImageMap = crossPageImageMap;
    }
} public class Images {

    private List <Image> images;

    public Images(){

    }

    public Images(List<Image> images){
	this.images = images;
    }

    public List <Image> getImages() {
	return images;
    }

    public void setImages(List <Image> images) {
	this.images = images;
    }
} public class PageOverlapDetails {


    public PageOverlapDetails(){

    }



    private String canvasId;
    private String imageId;
    private String nextCanvasId;
    private String nextImageId;
    private int endPositionOfCurrentText;

    public String getCanvasId() {
	return canvasId;
    }
    public void setCanvasId(String canvasId) {
	this.canvasId = canvasId;
    }
    public String getImageId() {
	return imageId;
    }
    public void setImageId(String imageId) {
	this.imageId = imageId;
    }
    public String getNextCanvasId() {
	return nextCanvasId;
    }
    public void setNextCanvasId(String nextCanvasId) {
	this.nextCanvasId = nextCanvasId;
    }
    public String getNextImageId() {
	return nextImageId;
    }
    public void setNextImageId(String nextImageId) {
	this.nextImageId = nextImageId;
    }
    public int getEndPositionOfCurrentText() {
	return endPositionOfCurrentText;
    }
    public void setEndPositionOfCurrentText(int endPositionOfCurrentText) {
	this.endPositionOfCurrentText = endPositionOfCurrentText;
    }
} public class Positions {

    private int startPosition;
    private int endPosition;

    public Positions(int start, int end) {
	this.startPosition = start;
	this.endPosition = end;
    }

    public int getStartPosition() {
	return startPosition;
    }

    public void setStartPosition(int startPosition) {
	this.startPosition = startPosition;
    }

    public int getEndPosition() {
	return endPosition;
    }

    public void setEndPosition(int endPosition) {
	this.endPosition = endPosition;
    }

    @Override
    public String toString() {
	return "position start:end [" + startPosition + ":" + endPosition + "]";
    }
} public class ServiceResponse<T> implements Serializable{

    /**
     *
     */
    private static final long serialVersionUID = 8231671973667011469L;

    public enum Status {
        OK, NOT_FOUND, CACHE_KEY_MISS, ILLEGAL_MODIFICATION, NON_CONFORMANT
    }

    private final Status status;
    private final transient T obj;

    public ServiceResponse(Status status, T obj) {
        this.status = status;
        this.obj = obj;
    }

    public Status getStatus() {
        return status;
    }

    public T getObj() {
        return obj;
    }
} public class TermOffsetStart {

    private String term;
    private int start;

    public TermOffsetStart(){
	//use this for testing
    }

    public TermOffsetStart(String term, int start){
	this.term = term;
	this.start = start;
    }

    public int getStart() {
	return start;
    }
    public void setStart(int start) {
	this.start = start;
    }

    public String getTerm() {
	return term;
    }
    public void setTerm(String term) {
	this.term = term;
    }

    @Override
    public String toString(){
	return "term:" + term +" start: " + start;
    }
} public class TermOffsetsWithPosition extends TermOffsets{

    private int position;

    public int getPosition() {
	return position;
    }

    public void setPosition(int position) {
	this.position = position;
    }

    @Override
    public String toString(){
	return "position: " + position +" start:" + this.getStart() +" end:"+this.getStart();
    }
} public class TermWithTermOffsets {

    private String term;
    private List<TermOffsetsWithPosition> offsets;

    public String getTerm() {
	return term;
    }
    public void setTerm(String term) {
	this.term = term;
    }
    public List<TermOffsetsWithPosition> getOffsets() {
	return offsets;
    }
    public void setOffsets(List<TermOffsetsWithPosition> offsets) {
	this.offsets = offsets;
    }

    @Override
    public String toString(){
	return "term:" + term+ " postion offsets:"+offsets.toString();
    }
} @Document(indexName = "text_index", type = "plaintext")
public class TextAnnotation {

    @Id
    private String id;

    @Field(type = FieldType.keyword, index = false)
    private String imageId;

    @Field(type = FieldType.keyword)
    private List<String> manifestId;

    @Mapping(mappingPath = "/mappings/text-field-mappings.json")
    private String plaintext;

    @CompletionField
    @Mapping(mappingPath = "/mappings/suggest-field-mappings.json")
    private ContextCompletion suggest;

    @Field(type = FieldType.keyword, index = false)
    private String nextImageId;

    @Field(type = FieldType.keyword, index = false)
    private String nextCanvasId;

    @Field(type = FieldType.Integer, index = false)
    private int endPositionOfCurrentText;

    public String getId() {
	return id;
    }

    public void setId(String id) {
	this.id = id;
    }

    public String getImageId() {
	return imageId;
    }

    public void setImageId(String imageId) {
	this.imageId = imageId;
    }

    public String getPlaintext() {
	return plaintext;
    }

    public void setPlaintext(String plaintext) {
	this.plaintext = plaintext;
    }

    public ContextCompletion getSuggest() {
	return suggest;
    }

    public void setSuggest(ContextCompletion newSuggest) {
	this.suggest = newSuggest;
    }

    public List<String> getManifestId() {
	return manifestId;
    }

    public void setManifestId(List<String> manifestId) {
	this.manifestId = manifestId;
    }

    @Override
    public String toString() {
	return "Plaintext [(" + getId() + "),(" + getImageId() + "),(" + getPlaintext() + "),(" + getManifestId()
		+ "),(" + getSuggest().getInput() + ")" + ",(nextCanvasId: " + this.getNextCanvasId() + ")"
		+ ",(nextImageId: " + this.getNextImageId() + "), (endPositionOfCurrentText: "
		+ this.getEndPositionOfCurrentText() + "]" + "getSuggest:" + getSuggest();

    }

    public String getNextImageId() {
	return nextImageId;
    }

    public void setNextImageId(String nextImageId) {
	this.nextImageId = nextImageId;
    }

    public String getNextCanvasId() {
	return nextCanvasId;
    }

    public void setNextCanvasId(String nextCanvasId) {
	this.nextCanvasId = nextCanvasId;
    }

    public int getEndPositionOfCurrentText() {
	return endPositionOfCurrentText;
    }

    public void setEndPositionOfCurrentText(int endPositionOfCurrentText) {
	this.endPositionOfCurrentText = endPositionOfCurrentText;
    }
} public enum Status {
    OK, NOT_FOUND, CACHE_KEY_MISS, ILLEGAL_MODIFICATION, NON_CONFORMANT
} public class PageParameters {

    private int total;

    private String firstPageNumber;
    private String lastPageNumber;
    private String nextPageNumber;
    private int nextPage = 0;
    private int lastPage = 0;
    private String previousPageNumber;
    private String startIndexString;

    private int startIndex;

    public int getTotal() {
	return total;
    }
    public void setTotal(int total) {
	this.total = total;
    }

    public String getFirstPageNumber() {
	return firstPageNumber;
    }
    public void setFirstPageNumber(String firstPageNumber) {
	this.firstPageNumber = firstPageNumber;
    }
    public String getLastPageNumber() {
	return lastPageNumber;
    }
    public void setLastPageNumber(String lastPageNumber) {
	this.lastPageNumber = lastPageNumber;
    }
    public String getNextPageNumber() {
	return nextPageNumber;
    }
    public void setNextPageNumber(String nextPageNumber) {
	this.nextPageNumber = nextPageNumber;
    }
    public String getPreviousPageNumber() {
	return previousPageNumber;
    }
    public void setPreviousPageNumber(String previousPageNumber) {
	this.previousPageNumber = previousPageNumber;
    }
    public int getStartIndex() {
	return startIndex;
    }
    public void setStartIndex(int startIndex) {
	this.startIndex = startIndex;
    }
    public int getNextPage() {
	return nextPage;
    }
    public void setNextPage(int nextPage) {
	this.nextPage = nextPage;
    }
    public int getLastPage() {
	return lastPage;
    }
    public void setLastPage(int lastPage) {
	this.lastPage = lastPage;
    }
    public String getStartIndexString() {
	return startIndexString;
    }
    public void setStartIndexString(String startIndexString) {
	this.startIndexString = startIndexString;
    }


    @Override
    public String toString(){
	return "total:" +this.getTotal() + " startIndex:" +this.getStartIndex() + " nextPage:" + this.getNextPage() ;
    }

} public interface GetPayloadService {

    public String getJsonPayload(String url, String payload);

} public interface TextSearchService {



   public ServiceResponse<Map<String, Object>> getTextPositions(String query, String queryString, boolean isW3c, String page, boolean isMixedSearch, String within, String widthHeight);


    public long getTotalHits();

    public PageParameters getPageParameters();

    public TextUtils getTextUtils();

}
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
import org.elasticsearch.action.termvectors.TermVectorsRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import com.digirati.themathmos.AnnotationSearchConstants;
import com.digirati.themathmos.mapper.TextSearchAnnotationMapper;
import com.digirati.themathmos.model.ImageHelperObject;
import com.digirati.themathmos.model.Images;
import com.digirati.themathmos.model.PageOverlapDetails;
import com.digirati.themathmos.model.Positions;
import com.digirati.themathmos.model.ServiceResponse;
import com.digirati.themathmos.model.TermOffsetStart;
import com.digirati.themathmos.model.TermOffsetsWithPosition;
import com.digirati.themathmos.model.TermWithTermOffsets;
import com.digirati.themathmos.model.TextAnnotation;
import com.digirati.themathmos.model.ServiceResponse.Status;
import com.digirati.themathmos.model.annotation.page.PageParameters;
import com.digirati.themathmos.service.GetPayloadService;
import com.digirati.themathmos.service.TextSearchService;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.google.gson.internal.LinkedTreeMap;
	searchRequestBuilder.addDocValueField("nextImageId");
	searchRequestBuilder.addDocValueField("nextCanvasId");
	searchRequestBuilder.addDocValueField("endPositionOfCurrentText");

	LOG.info("doSearch query " + searchRequestBuilder.toString());
	SearchResponse response = searchRequestBuilder.execute().actionGet();

	totalHits = response.getHits().totalHits();

	LOG.info("Total hits are: " + totalHits);
	LOG.debug("response: " + response.toString());

	return resultsMapper.mapResults(response, TextAnnotation.class, pageable);
    }

    /**
     * Method to build a matchPhraseQuery {@code QueryBuilder}
     *
     * @param query
     *            - The {@code String} query e.g. turnips
     * @return {@code QueryBuilder}
     */
    private QueryBuilder buildQuery(String query) {
	return QueryBuilders.matchPhraseQuery(TEXT_FIELD_NAME, query);
    }

    /**
     * Method to build a matchPhraseQuery {@code QueryBuilder}
     *
     * @param query
     *            - The {@code String} query e.g. turnips
     * @param within
     *            - The {@code String} base64 encoded within String
     * @return {@code QueryBuilder}
     */
    private QueryBuilder buildQuery(String query, String within) {
	String decodedWithinUrl = textUtils.decodeWithinUrl(within);
	if (null != decodedWithinUrl) {
	    BoolQueryBuilder must = QueryBuilders.boolQuery();
	    QueryBuilder builder = QueryBuilders.matchPhraseQuery(TEXT_FIELD_NAME, query);
	    must.must(QueryBuilders.matchQuery("manifestId", decodedWithinUrl));
	    must.must(builder);
	    return must;

	} else {
	    LOG.info("Unable to decode the within " + within);
	    return null;
	}

    }

    /**
     * Method to create a {@code MultiTermVectorsRequest} by building up a
     * series of {@code TermVectorsRequest} and adding them to a
     * {@code MultiTermVectorsRequest}. Send this request to elasticsearch and
     * return either null or a {@code MultiTermVectorsResponse}
     *
     * @param index
     *            - The {@code String} index we are querying.
     * @param type
     *            - The {@code String} type we are querying.
     * @param ids
     *            - The {@code String[]} of text ids we wish to get termvectors
     *            for
     * @param field
     *            - The {@code String} name of the field we wish to get
     *            termvectors for e.g. the text field.
     * @return the {@code MultiTermVectorsResponse} or null if we have an empty
     *         {@code MultiTermVectorsRequest}
     */
    private MultiTermVectorsResponse getMultiTermVectors(String index, String type, String[] ids, String field) {
	MultiTermVectorsRequest multiTermVectorsRequest = new MultiTermVectorsRequest();

	for (String id : ids) {
	    if (null != id) {
		TermVectorsRequest termVectorsRequest = new TermVectorsRequest(index, type, id).selectedFields(field)
			.termStatistics(true);
		multiTermVectorsRequest.add(termVectorsRequest);
	    }
	}

	if (!multiTermVectorsRequest.isEmpty()) {
	    return client.multiTermVectors(multiTermVectorsRequest).actionGet();
	}
	return null;

    }

    private int removeDotZero(Double input) {
	return input.intValue();
    }

    private String removeDotZeroString(Double input) {
	return Integer.toString(input.intValue());
    }

    /**
     * Find the offsets for each query term
     *
     * @param query
     *            - The {@code String} query e.g. turnips
     * @param builder
     *            - The {@code XContentBuilder} representing the json of the
     *            {@code MultiTermVectorsResponse}.
     * @param termWithOffsets
     *            - Populated with the key = the lowercase query term and value
     *            a {@code List} of {@code TermOffsetsWithPosition} which are
     *            the position, and start and end offsets
     */
    private void findOffsetsForQuery(String query, String builder, TermWithTermOffsets termWithOffsets) {
	try {
	    Map<String, Object> javaRootBodyMapObject = new Gson().fromJson(builder, Map.class);

	    Map termVectors = (Map) javaRootBodyMapObject.get(TERM_VECTORS_FIELD_NAME);
	    LinkedTreeMap text = (LinkedTreeMap) termVectors.get(TEXT_FIELD_NAME);
	    LinkedTreeMap terms = (LinkedTreeMap) text.get(TERMS_FIELD_NAME);
	    LinkedTreeMap queryTerm = (LinkedTreeMap) terms.get(query.toLowerCase());

	    termWithOffsets.setTerm(query.toLowerCase());

	    List<TermOffsetsWithPosition> termOffsets = new ArrayList<>();