public abstract class AbstractDisambiguationAlgorithm {

	protected AbstractDisambiguationTask task;

	public void disambiguate(AbstractDisambiguationTask task)
			throws IllegalDisambiguationAlgorithmInputException {
		if (checkAndSetInputParameter(task)) {
			if (preDisambiguation()) {
				processAlgorithm();
			}
		} else {
			throw new IllegalDisambiguationAlgorithmInputException(
					"Check your input knowledge base and disambiguation task");
		}
	}

	public static String extractContext(int position, String text,
			int contextarea) {
		if(text == null || text.length() == 0) {
			return "";
		}
		
		long startArea = position - contextarea;
		long endArea = position + contextarea;
		if (startArea < 0) {
			startArea = 0;
		}
		if (endArea > text.length() - 1) {
			endArea = text.length() - 1;
		}
		String tempText = text.substring((int) startArea, (int) endArea);
		String[] splitter = tempText.split(" ");
		String result = "";
		for (int i = 1; i < splitter.length - 1; i++) {
			result += splitter[i] + " ";
		}
		return result;
	}

	protected abstract boolean checkAndSetInputParameter(AbstractDisambiguationTask task);

	protected abstract void processAlgorithm()
			throws IllegalDisambiguationAlgorithmInputException;

	protected abstract boolean preDisambiguation();
} public class IllegalDisambiguationAlgorithmInputException extends
		IllegalArgumentException {

	private static final long serialVersionUID = 1L;

	IllegalDisambiguationAlgorithmInputException() {
		super("Wrong Knowledge base!");
	}

	IllegalDisambiguationAlgorithmInputException(String text) {
		super(text);
	}

} public class SurfaceForm implements Comparable<SurfaceForm>, Cloneable {

	private int queryNr;
	private String surfaceForm;
	private String context;
	private List<String> candidates;
	private Integer ambiguity;
	private boolean isACandidate;
	private double difference;
	private int position;
	private boolean matchesInitial;
	private boolean initial;
	private boolean isRelevant;

	public SurfaceForm(String surfaceForm, String context, List<String> candidates, int qryNr, int position) {
		super();
		this.ambiguity = candidates.size();
		this.surfaceForm = surfaceForm;
		this.context = context;
		this.candidates = candidates;
		this.queryNr = qryNr;
		this.isACandidate = true;
		this.difference = 0;
		this.position = position;
		this.initial = false;
		this.isRelevant = true;
	}

	public boolean isRelevant() {
		return isRelevant;
	}

	public void setRelevant(boolean isRelevant) {
		this.isRelevant = isRelevant;
	}

	public boolean isMatchesInitial() {
		return matchesInitial;
	}

	public void setMatchesInitial(boolean matchesInitial) {
		this.matchesInitial = matchesInitial;
	}

	public boolean isInitial() {
		return initial;
	}

	public void setInitial(boolean initial) {
		this.initial = initial;
	}

	public void setCandidates(List<String> candidates) {
		this.candidates = candidates;
	}

	public List<String> getCandidates() {
		return candidates;
	}

	public void setACandidate(boolean can) {
		this.isACandidate = can;
	}

	public String getSurfaceForm() {
		return surfaceForm;
	}

	public boolean isACandidate() {
		return isACandidate;
	}

	public String getContext() {
		return context;
	}

	public int getQueryNr() {
		return queryNr;
	}

	public int getAmbiguity() {
		return this.ambiguity;
	}

	public void setDisambiguatedEntity(String url) {
		candidates.clear();
		candidates.add(url);
	}

	public void clearList() {
		candidates.clear();
	}

	public void addCandidate(String can) {
		candidates.add(can);
	}

	public double getDifference() {
		return difference;
	}

	public void setDifference(double difference) {
		this.difference = difference;
	}

	public int getPosition() {
		return position;
	}

	public void setPosition(int position) {
		this.position = position;
	}

	@Override
	public int compareTo(SurfaceForm o) {
		if (this.difference < o.getDifference()) {
			return 1;
		} else if (this.difference > o.getDifference()) {
			return -1;
		} else {
			return 0;
		}
	}

	public Object clone() {
		ArrayList<String> newCandidates = new ArrayList<String>();
		for (String s : candidates) {
			newCandidates.add(s);
		}

		SurfaceForm n = new SurfaceForm(new String(this.surfaceForm), new String(this.context), newCandidates,
				this.queryNr, this.position);
		n.setACandidate(this.isACandidate);
		n.setInitial(this.initial);
		n.setMatchesInitial(this.matchesInitial);
		n.setRelevant(this.isRelevant);
		return n;
	}
} public abstract class AbstractDisambiguationTask {

	protected int returnNr;

	protected AbstractKnowledgeBase kb;

	protected KnowledgeBaseIdentifiers kbIdentifier;
	
	protected boolean retrieveDocClasses;
	
	protected List<Response> responses;

	public int getReturnNr() {
		return returnNr;
	}

	public void setReturnNr(int returnNr) {
		this.returnNr = returnNr;
	}

	public AbstractKnowledgeBase getKb() {
		return kb;
	}

	public void setKb(AbstractKnowledgeBase kb) {
		this.kb = kb;
	}

	public KnowledgeBaseIdentifiers getKbIdentifier() {
		return this.kbIdentifier;
	}
	
	public boolean isRetrieveDocClasses() {
		return retrieveDocClasses;
	}

	public void setRetrieveDocClasses(boolean retrieveDocClasses) {
		this.retrieveDocClasses = retrieveDocClasses;
	}
	
	public List<Response> getResponse() {
		return responses;
	}

	public void setResponse(List<Response> responses) {
		this.responses = responses;
	}
	
	public abstract void setKbIdentifier(String kbversion, String setting);
} public class DisambiguationTaskCollective extends AbstractDisambiguationTask {

	private List<EntityDisambiguationDPO> entitiesToDis;
	
	/* A maintopic e.g. the column identifier in a table */
	private String mainTopic;

	public DisambiguationTaskCollective(final List<EntityDisambiguationDPO> entityToDis, String mainTopic) {
		super();
		this.entitiesToDis = entityToDis;
		this.mainTopic = mainTopic;
	}

	public List<EntityDisambiguationDPO> getEntityToDisambiguate() {
		return this.entitiesToDis;
	}
	
	public String getMainTopic() {
		return this.mainTopic;
	}

	public void setSurfaceForm(final List<EntityDisambiguationDPO> surfaceForm) {
		this.entitiesToDis = surfaceForm;
	}

	/**
	 * Assignment function to determine the used knowledge base
	 * 
	 * @param kbversion
	 * @param setting
	 */
	@Override
	public void setKbIdentifier(String kbversion, String setting) {
		if(setting == null) {
			this.kbIdentifier = KnowledgeBaseIdentifiers.Standard;
		} else if(setting.equalsIgnoreCase("DocumentCentric")) {
			if(kbversion.equalsIgnoreCase("default")) {
				this.kbIdentifier = KnowledgeBaseIdentifiers.DocumentCentricDefault;
			} else {
				this.kbIdentifier = KnowledgeBaseIdentifiers.DocumentCentricDefault;
			}
		} else if(setting.equalsIgnoreCase("EntityCentric")) {
			if(kbversion.equalsIgnoreCase("default")) {
				this.kbIdentifier = KnowledgeBaseIdentifiers.Standard;
			} else if(kbversion.equalsIgnoreCase("cstable")) {
				this.kbIdentifier = KnowledgeBaseIdentifiers.CSTable;
			} else if(kbversion.equalsIgnoreCase("biomed")) {
				this.kbIdentifier = KnowledgeBaseIdentifiers.Biomed;
			} else {
				this.kbIdentifier = KnowledgeBaseIdentifiers.Standard;
			}
		} else {
			this.kbIdentifier = KnowledgeBaseIdentifiers.Standard;
		}
	}	
} public class EntityDisambiguationDPO {

	private String documentId;
	private String context;
	private String selectedText;
	private String setting;
	private String kbversion;
	private int startPosition;

	public EntityDisambiguationDPO() {
		super();
	}

	public String getContext() {
		return this.context;
	}

	public String getSelectedText() {
		return this.selectedText;
	}

	public void setContext(final String context) {
		this.context = context;
	}

	public void setSelectedText(final String selectedText) {
		this.selectedText = selectedText;
	}

	public void setSetting(final String setting) {
		this.setting = setting;
	}

	public String getSetting() {
		return setting;
	}
	
	public void setDocumentId(final String documentId) {
		this.documentId = documentId;
	}

	public String getDocumentId() {
		return this.documentId;
	}

	public void setInternSetting(final String setting) {
		this.setting = setting;
	}
	
	public String getKbversion() {
		return kbversion;
	}

	public void setKbversion(String kbversion) {
		this.kbversion = kbversion;
	}

	public int getStartPosition() {
		return startPosition;
	}

	public void setStartPosition(int startPosition) {
		this.startPosition = startPosition;
	}
} public class Response {

	private List<DisambiguatedEntity> disEntities;
	private String selectedText;
	private int documentId;

	public Response() {
		super();
		this.disEntities = new LinkedList<DisambiguatedEntity>();
	}

	public List<DisambiguatedEntity> getDisEntities() {
		return this.disEntities;
	}

	public String getSelectedText() {
		return this.selectedText;
	}

	public void setDisEntities(final List<DisambiguatedEntity> disEntities) {
		this.disEntities = disEntities;
	}

	public void setSelectedText(final String selectedText) {
		this.selectedText = selectedText;
	}

	public int getDocumentId() {
		return documentId;
	}

	public void setDocumentId(int documentId) {
		this.documentId = documentId;
	}
} public class EntityCentricKnowledgeBase extends AbstractKnowledgeBase {

	private final static Logger logger = LoggerFactory.getLogger(EntityCentricKnowledgeBase.class);
	
	protected static final String TRIMLABELAMOUNT = ";;;";
	protected static final String TRIMOCCOCC = ":::";
	protected static final String KBOCCURRENCESFIELD = "Occurrences";
	protected static final String KBMAINLINK = "Mainlink";

	/**
	 * This hashmap stores the Prior values of the standard DbPedia, CalbC and
	 * eHealth index. Key: Lucene intern document id Value: Amount of
	 * occurrences of this entity
	 */
	protected Map<String, Integer> indexpriorHashMap;

	/**
	 * This hashmap stores the Sense Prior values of the standard DbPedia, CalbC
	 * and eHealth index.
	 * <p>
	 * <li>- Key: Lucene intern document id <br>
	 * - Value: HashMap storing the label appearances <br>
	 * 
	 * HashMap2:<br>
	 * - Key: Hash value of the appearing label <br>
	 * - Value: Number of occurrences of this label</li>
	 */
	protected Map<Integer, HashMap<Integer, Integer>> indexsensePriorHashMap;
	protected Map<String, HashMap<Integer, Integer>> indexsensePriorHashMapBlanc;

	/**
	 * This map stores the relations the entities can be associated with. Key:
	 * The source entity uri Value: HashSet of other entities that form a binary
	 * relation with the source entity.
	 */
	protected Map<Integer, HashSet<String>> indexRelation;

	protected IEntityCentricExtFeatures externFeatureDef;
	
	public EntityCentricKnowledgeBase(String uri, boolean dynamic, Similarity sim) {
		super(uri, dynamic, sim);
		this.externFeatureDef = new ECExternFeatures();
	}

	public EntityCentricKnowledgeBase(String uri, boolean dynamic) {
		super(uri, dynamic);
		this.externFeatureDef = new ECExternFeatures();
	}

	/**
	 * Returns the feature definition class.
	 * 
	 * @return
	 */
	public IEntityCentricExtFeatures getFeatureDefinition() {
		return this.externFeatureDef;
	}

	@Override
	public void initialize() {
		indexpriorHashMap = new HashMap<String, Integer>();
		indexsensePriorHashMap = new HashMap<Integer, HashMap<Integer, Integer>>();
		indexRelation = new HashMap<Integer, HashSet<String>>();
		indexsensePriorHashMapBlanc = new HashMap<String, HashMap<Integer, Integer>>();
		try {
			final IndexReader iReader = super.getSearcher().getIndexReader();
			final int maxDoc = iReader.numDocs();
			for (int i = 0; i < maxDoc; i++) {
				if ((i % 50000) == 0) {
					logger.info(this.kbName()+ " Loaded Entities: " + i);
				}

				final String val = iReader.document(i).get(KBOCCURRENCESFIELD);
				String entity = iReader.document(i).get("Mainlink").replaceAll("http://dbpedia.org/resource/", "");
				if ((val != null) && !val.equalsIgnoreCase("")) {
					final String[] splitter = val.split(TRIMLABELAMOUNT);
					final HashMap<Integer, Integer> hash = new HashMap<Integer, Integer>();
					for (final String element : splitter) {
						final String[] value = element.split(TRIMOCCOCC);
						int check = 1;
						try {
							check = Integer.valueOf(value[1]);
						} catch (final NumberFormatException e) {
							logger.warn(this.kbName()+ " Warning NumberFormatException while Initialization:" + val);
						}
						hash.put(value[0].toLowerCase(Locale.US).hashCode(), check);
					}
					indexsensePriorHashMapBlanc.put(entity, hash);
				}
			}
			iReader.close();
		} catch (final IOException e) {
			logger.error("IOException in "+EntityCentricKnowledgeBase.class.getName(), e);
		}
	}
	
	protected String kbName() {
		return "General KB";
	}

	class ECExternFeatures implements IEntityCentricExtFeatures {

		private ECExternFeatures() {
			super();
		}

		@Override
		public float getPriorOfDocument(int docId) {
			float res = 0.0f;
			if (indexpriorHashMap.containsKey(docId)) {
				final float prior = (float) Math.log(indexpriorHashMap.get(docId));
				res = prior;
			}
			return res;
		}

		@Override
		public float getSensePriorOfDocument(String keyword, int docId) {
			float res = 0.0f;
			if (indexsensePriorHashMap.containsKey(docId)) {
				final HashMap<Integer, Integer> hash = indexsensePriorHashMap.get(docId);
				if (hash.containsKey(keyword.toLowerCase(Locale.US).hashCode())) {
					final int value = hash.get(keyword.toLowerCase(Locale.US).hashCode());
					res = (float) Math.log(value + 1);
				}
			}
			return res;
		}

		@Override
		public int getOccurrences(String sf, String uri) {
			String entity = uri.replaceAll("http://dbpedia.org/resource/", "");
			int res = 0;
			if (indexsensePriorHashMapBlanc.containsKey(entity)) {
				final HashMap<Integer, Integer> hash = indexsensePriorHashMapBlanc.get(entity);
				if (hash.containsKey(sf.toLowerCase().hashCode())) {
					res = hash.get(sf.toLowerCase().hashCode());
				}
			}
			return (res + 1);
		}
		
		@Override
		public Set<String> getRelations(String url) {
			if (indexRelation.containsKey(url.hashCode())) {
				return indexRelation.get(url.hashCode());
			}
			return new HashSet<String>();
		}
	}
} public abstract class AbstractKnowledgeBase extends TimerTask {

	private final static Logger logger = LoggerFactory.getLogger(AbstractKnowledgeBase.class);
	
	private String indexUri;
	
	private boolean dynamic;

	private SearcherManager manager;

	private IndexSearcher searcher;
	
	AbstractKnowledgeBase(String uri, boolean dynamic) {
		this(uri, dynamic, new DefaultSimilarity());
	}

	AbstractKnowledgeBase(String uri, boolean dynamic, Similarity sim) {
		super();
		this.indexUri = uri;
		this.dynamic = dynamic;

		File indexDir = new File(indexUri);
		Directory dir;
		try {
			dir = FSDirectory.open(indexDir);
			this.manager = new SearcherManager(dir, new SearcherFactory());
		} catch (IOException e) {
			logger.error("IOException in "+AbstractKnowledgeBase.class.getName(), e);
		}
	}

	public String getIndexUri() {
		return indexUri;
	}


	public IndexSearcher getSearcher() {
		try {
			this.searcher = manager.acquire();
		} catch (IOException e) {
			logger.error("IOException in "+AbstractKnowledgeBase.class.getName(), e);
		}
		return this.searcher;
	}

	public void release() {
		try {
			manager.release(searcher);
		} catch (IOException e) {
			logger.error("IOException in "+AbstractKnowledgeBase.class.getName(), e);
		}
	}

	/**
	 * Periodically reopens the Indexreader, if and only if this is an dynamic
	 * knowledge base. The changed knowledge base will be live within a few moments.
	 */
	@Override
	public void run() {
		if (dynamic) {
			try {
				manager.maybeRefresh();
			} catch (IOException e) {
				logger.error("IOException in "+AbstractKnowledgeBase.class.getName(), e);
			}
		}
	}
	
	public abstract void initialize();
} public abstract class AbstractEntityCentricKBGeneral extends EntityCentricKnowledgeBase {

	private final static Logger logger = LoggerFactory.getLogger(AbstractEntityCentricKBGeneral.class);
	
	private static Cache<String, Float> w2vCache;
	private static Cache<String, Float> d2vCache;

	static {
		w2vCache = CacheBuilder.newBuilder().maximumSize(5000).expireAfterWrite(60, TimeUnit.MINUTES).build();
		d2vCache = CacheBuilder.newBuilder().maximumSize(5000).expireAfterWrite(60, TimeUnit.MINUTES).build();
	}

	public AbstractEntityCentricKBGeneral(String uri, boolean dynamic) {
		super(uri, dynamic);
	}

	public AbstractEntityCentricKBGeneral(String uri, boolean dynamic, Similarity sim) {
		super(uri, dynamic, sim);
	}
	
	/**
	 * Given a set of word2vec queries, this methods retrieves the corresponding
	 * word2vec similarities. If the similarities of a query is not cashed, we
	 * query the word2vec server and compute the similarities from scratch.
	 *
	 * @param set
	 *            A set of query strings
	 * @return Returns a map containing the word2vec similarities of the given
	 *         queries
	 */
	public Map<String, Float> getWord2VecSimilarities(Set<String> set) {
		Map<String, Float> map = new HashMap<String, Float>();
		Set<String> neededSimilarities = new HashSet<String>();
		for (String s : set) {
			Float val = w2vCache.getIfPresent(s);
			if (val != null) {
				map.put(s, val);
			} else {
				neededSimilarities.add(s);
			}
		}
		if (neededSimilarities.size() > 0) {
			Map<String, Float> computedSimilarities = queryWord2VecSimilarities(neededSimilarities);
			w2vCache.putAll(computedSimilarities);
			map.putAll(computedSimilarities);
		}
		return map;
	}

	public float getDoc2VecSimilarity(String sf, String context, String entity) {
		String key = sf + context + entity;
		Float val = d2vCache.getIfPresent(key);
		if (val != null) {
			return val + 1.0f;
		} else {
			return 0;
		}
	}

	/**
	 * Retrieves the word2vec similarities of a set of entity pairs
	 *
	 *
	 * @param neededSimilarities
	 *            A set of entity pairs whose word2vec similarities are needed.
	 *            The strings are already in the appropriate format
	 * @return a map that contains the query strings as keys and the
	 *         similarities as values
	 */
	private Map<String, Float> queryWord2VecSimilarities(Set<String> neededSimilarities) {
		Word2VecJsonFormat format = new Word2VecJsonFormat();
		format.setData(neededSimilarities);
		format.setDomain(generateDomainName());
		JSONArray res = Word2VecJsonFormat.performquery(format, "w2vsim");
		Map<String, Float> map = new HashMap<String, Float>();
		for (int i = 0; i < res.length(); i++) {
			try {
				JSONObject obj = res.getJSONObject(i);
				String ents = obj.getString("ents");
				float sim = (float) obj.getDouble("sim") + 1;
				map.put(ents, sim);
			} catch (JSONException e) {
				logger.error("IOException in "+AbstractEntityCentricKBGeneral.class.getName(), e);
			}
		}
		return map;
	}

	public void precomputeDoc2VecSimilarities(List<SurfaceForm> rep, int contextSize) {
		Doc2VecJsonFormat format = new Doc2VecJsonFormat();
		for (SurfaceForm sf : rep) {
			String context = AbstractDisambiguationAlgorithm.extractContext(
					sf.getPosition(), sf.getContext(), contextSize);

			context = context.toLowerCase();
			context = context.replaceAll("[\\.\\,\\!\\? ]+", " ");

			Data doc = new Data();
			List<String> candidates = sf.getCandidates();
			List<String> toDoCandidates = new ArrayList<String>();
			for(String can : candidates) {
				if(!isInCache(sf.getSurfaceForm(), context, can)) {
					toDoCandidates.add(can);
				}
			}
//			if(!toDoCandidates.isEmpty()) {
				String[] cans = new String[toDoCandidates.size()];
				sf.getCandidates().toArray(cans);
				doc.setCandidates(cans);
				doc.setContext(context);
				doc.setSurfaceForm(sf.getSurfaceForm());
				doc.getQryNr();
				format.addData(doc);
//			}
		}
		JSONArray res = Word2VecJsonFormat.performquery(format, "d2vsim");

		// We obtain the same order of surface forms
		for (int i = 0; i < res.length(); i++) {
			SurfaceForm c = rep.get(i);
			try {
				JSONObject obj = res.getJSONObject(i);
				JSONArray simArray = obj.getJSONArray("sim");
				for (int j = 0; j < simArray.length(); j++) {
					float sim = (float) simArray.getDouble(j);
					String entity = c.getCandidates().get(j);
					d2vCache.put(c.getSurfaceForm() + c.getContext() + entity, sim);
				}
			} catch (JSONException e) {
				logger.error("JSONException in "+AbstractEntityCentricKBGeneral.class.getName(), e);
			}
		}
	}

	private boolean isInCache(String surfaceForm, String context, String entity) {
		String key = surfaceForm + context + entity;
		return d2vCache.getIfPresent(key) != null;
	}

	protected abstract String generateDomainName();
	
	public abstract String generateWord2VecFormatString(String source, String target);

	public abstract String generateWord2VecFormatString(List<String> source, String target);
	
}
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import doser.entitydisambiguation.algorithms.AbstractDisambiguationAlgorithm;
import doser.entitydisambiguation.algorithms.IllegalDisambiguationAlgorithmInputException;
import doser.entitydisambiguation.algorithms.SurfaceForm;
import doser.entitydisambiguation.backend.AbstractDisambiguationTask;
import doser.entitydisambiguation.backend.DisambiguationTaskCollective;
import doser.entitydisambiguation.dpo.EntityDisambiguationDPO;
import doser.entitydisambiguation.dpo.Response;
import doser.entitydisambiguation.knowledgebases.EntityCentricKnowledgeBase;
import doser.lucene.query.TermQuery;
import doser.entitydisambiguation.knowledgebases.AbstractKnowledgeBase;
import doser.entitydisambiguation.knowledgebases.AbstractEntityCentricKBGeneral;
package doser.entitydisambiguation.algorithms.collective.general;




public class CollectiveDisambiguationGeneralEntities extends AbstractDisambiguationAlgorithm {

	private final static Logger logger = LoggerFactory.getLogger(CollectiveDisambiguationGeneralEntities.class);
	
	private AbstractEntityCentricKBGeneral eckb;
	
	private DisambiguationTaskCollective task;
	
	@Override
	protected boolean checkAndSetInputParameter(AbstractDisambiguationTask task) {