public final class FieldNames {
    public static final String ID = "_id";
    public static final String TEXT = "text";
    public static final String READING_PREFIX = "reading_";
    public static final String SCORE = "score";
    public static final String QUERY_FREQ = "queryFreq";
    public static final String DOC_FREQ = "docFreq";
    public static final String USER_BOOST = "userBoost";
    public static final String KINDS = "kinds";
    public static final String TIMESTAMP = "@timestamp";
    public static final String TAGS = "tags";
    public static final String ROLES = "roles";
    public static final String FIELDS = "fields";
    public static final String LANGUAGES = "languages";

    public static final String ARRAY_KEY = "key";
    public static final String ARRAY_VALUE = "value";

    public static final String ANALYZER_SETTINGS_TYPE = "settingsType";
    public static final String ANALYZER_SETTINGS_FIELD_NAME = "fieldName";
    public static final String ANALYZER_SETTINGS_READING_ANALYZER = "readingAnalyzer";
    public static final String ANALYZER_SETTINGS_READING_TERM_ANALYZER = "readingTermAnalyzer";
    public static final String ANALYZER_SETTINGS_NORMALIZE_ANALYZER = "normalizeAnalyzer";
    public static final String ANALYZER_SETTINGS_CONTENTS_ANALYZER = "contentsAnalyzer";
    public static final String ANALYZER_SETTINGS_CONTENTS_READING_ANALYZER = "contentsReadingAnalyzer";

    private FieldNames() {
    }
} public final class SuggestConstants {
    private SuggestConstants() {
    }

    public static final String EMPTY_STRING = "";

    public static final String USER_DICT_ENCODING = "fess.user.dict.encoding";

    public static final String USER_DICT_PATH = "fess.user.dict.path";

    public static final String TEXT_SEPARATOR = " ";

    public static final String DEFAULT_ROLE = "_guest_";

    public static final String DEFAULT_TYPE = "_doc";
} public class ElevateWord {

    protected final String elevateWord;
    protected final float boost;
    protected final List<String> readings;
    protected final List<String> fields;
    protected final List<String> tags;
    protected final List<String> roles;

    public ElevateWord(final String elevateWord, final float boost, final List<String> readings, final List<String> fields,
            final List<String> tags, final List<String> roles) {
        this.elevateWord = elevateWord;
        this.boost = boost;
        this.readings = readings;
        this.fields = fields;
        if (tags == null) {
            this.tags = Collections.emptyList();
        } else {
            this.tags = tags;
        }
        if (roles == null) {
            this.roles = Collections.emptyList();
        } else {
            this.roles = roles;
        }
    }

    public String getElevateWord() {
        return elevateWord;
    }

    public float getBoost() {
        return boost;
    }

    public List<String> getReadings() {
        return readings;
    }

    public List<String> getFields() {
        return fields;
    }

    public List<String> getTags() {
        return tags;
    }

    public List<String> getRoles() {
        return roles;
    }

    public SuggestItem toSuggestItem() {
        final String[][] readingArray =
                this.getReadings().stream().map(reading -> new String[] { reading }).toArray(count -> new String[count][]);
        return new SuggestItem(new String[] { this.getElevateWord() }, readingArray, fields.toArray(new String[fields.size()]), 1, 0,
                this.getBoost(), this.getTags().toArray(new String[this.getTags().size()]),
                this.getRoles().toArray(new String[this.getRoles().size()]), null, SuggestItem.Kind.USER);
    }
} public class SuggestItem {

    public enum Kind {
        DOCUMENT("document"), QUERY("query"), USER("user");

        private final String kind;

        Kind(final String kind) {
            this.kind = kind;
        }

        @Override
        public String toString() {
            return kind;
        }
    }

    private String text;

    private ZonedDateTime timestamp;

    private long queryFreq;

    private long docFreq;

    private float userBoost;

    private String[][] readings;

    private String[] fields;

    private String[] tags;

    private String[] roles;

    private String[] languages;

    private Kind[] kinds;

    private Map<String, Object> emptySource;

    private String id;

    private SuggestItem() {
    }

    public SuggestItem(final String[] text, final String[][] readings, final String[] fields, final long docFreq, final long queryFreq,
            final float userBoost, @Nullable final String[] tags, @Nullable final String[] roles, @Nullable final String[] languages,
            final Kind kind) {
        this.text = String.join(SuggestConstants.TEXT_SEPARATOR, text);
        this.readings = readings;
        this.fields = fields != null ? fields : new String[] {};
        this.tags = tags != null ? tags : new String[] {};

        if (roles == null || roles.length == 0) {
            this.roles = new String[] { SuggestConstants.DEFAULT_ROLE };
        } else {
            this.roles = new String[roles.length];
            System.arraycopy(roles, 0, this.roles, 0, roles.length);
        }

        this.languages = languages != null ? languages : new String[] {};

        this.kinds = new Kind[] { kind };
        if (userBoost > 1) {
            this.userBoost = userBoost;
        } else {
            this.userBoost = 1;
        }
        this.docFreq = docFreq;
        this.queryFreq = queryFreq;
        this.timestamp = ZonedDateTime.now();
        this.emptySource = createEmptyMap();
        this.id = SuggestUtil.createSuggestTextId(this.text);
    }

    public String getText() {
        return text;
    }

    public String[][] getReadings() {
        return readings;
    }

    public String[] getTags() {
        return tags;
    }

    public String[] getRoles() {
        return roles;
    }

    public String[] getLanguages() {
        return languages;
    }

    public String[] getFields() {
        return fields;
    }

    public Kind[] getKinds() {
        return kinds;
    }

    public long getQueryFreq() {
        return queryFreq;
    }

    public long getDocFreq() {
        return docFreq;
    }

    public float getUserBoost() {
        return userBoost;
    }

    public ZonedDateTime getTimestamp() {
        return timestamp;
    }

    public void setText(final String text) {
        this.text = text;
    }

    public void setTimestamp(final ZonedDateTime timestamp) {
        this.timestamp = timestamp;
    }

    public void setQueryFreq(final long queryFreq) {
        this.queryFreq = queryFreq;
    }

    public void setDocFreq(final long docFreq) {
        this.docFreq = docFreq;
    }

    public void setUserBoost(final float userBoost) {
        this.userBoost = userBoost;
    }

    public void setReadings(final String[][] readings) {
        this.readings = readings;
    }

    public void setFields(final String[] fields) {
        this.fields = fields;
    }

    public void setTags(final String[] tags) {
        this.tags = tags;
    }

    public void setRoles(final String[] roles) {
        this.roles = roles;
    }

    public void setLanguages(final String[] languages) {
        this.languages = languages;
    }

    public void setKinds(final Kind[] kinds) {
        this.kinds = kinds;
    }

    public void setEmptySource(final Map<String, Object> emptySource) {
        this.emptySource = emptySource;
    }

    public void setId(final String id) {
        this.id = id;
    }

    public Map<String, Object> toEmptyMap() {
        return emptySource;
    }

    protected Map<String, Object> createEmptyMap() {
        final Map<String, Object> map = new HashMap<>();
        map.put(FieldNames.TEXT, StringUtil.EMPTY);

        for (int i = 0; i < readings.length; i++) {
            map.put(FieldNames.READING_PREFIX + i, new String[] {});
        }

        map.put(FieldNames.FIELDS, new String[] {});
        map.put(FieldNames.TAGS, new String[] {});
        map.put(FieldNames.ROLES, new String[] {});
        map.put(FieldNames.LANGUAGES, new String[] {});
        map.put(FieldNames.KINDS, new String[] {});
        map.put(FieldNames.SCORE, 1.0F);
        map.put(FieldNames.QUERY_FREQ, 0L);
        map.put(FieldNames.DOC_FREQ, 0L);
        map.put(FieldNames.USER_BOOST, 1.0F);
        map.put(FieldNames.TIMESTAMP, DateTimeFormatter.ISO_INSTANT.format(ZonedDateTime.now()));
        return map;
    }

    public String getId() {
        return id;
    }

    public Map<String, Object> getSource() {
        final Map<String, Object> map = new HashMap<>();
        map.put(FieldNames.TEXT, text);

        for (int i = 0; i < readings.length; i++) {
            map.put(FieldNames.READING_PREFIX + i, readings[i]);
        }

        map.put(FieldNames.FIELDS, fields);
        map.put(FieldNames.TAGS, tags);
        map.put(FieldNames.ROLES, roles);
        map.put(FieldNames.LANGUAGES, languages);
        map.put(FieldNames.KINDS, Stream.of(kinds).map(Kind::toString).toArray());
        map.put(FieldNames.QUERY_FREQ, queryFreq);
        map.put(FieldNames.DOC_FREQ, docFreq);
        map.put(FieldNames.USER_BOOST, userBoost);
        map.put(FieldNames.SCORE, (queryFreq + docFreq) * userBoost);
        map.put(FieldNames.TIMESTAMP, timestamp.toInstant().toEpochMilli());
        return map;
    }

    public static SuggestItem parseSource(final Map<String, Object> source) {
        final String text = source.get(FieldNames.TEXT).toString();
        final List<String[]> readings = new ArrayList<>();
        for (int i = 0;; i++) {
            final Object readingObj = source.get(FieldNames.READING_PREFIX + i);
            if (!(readingObj instanceof List)) {
                break;
            }
            @SuppressWarnings("unchecked")
            final List<String> list = (List<String>) readingObj;
            readings.add(list.toArray(new String[list.size()]));
        }
        final List<String> fields = SuggestUtil.getAsList(source.get(FieldNames.FIELDS));
        final long docFreq = Long.parseLong(source.get(FieldNames.DOC_FREQ).toString());
        final long queryFreq = Long.parseLong(source.get(FieldNames.QUERY_FREQ).toString());
        final float userBoost = Float.parseFloat(source.get(FieldNames.USER_BOOST).toString());
        final List<String> tags = SuggestUtil.getAsList(source.get(FieldNames.TAGS));
        final List<String> roles = SuggestUtil.getAsList(source.get(FieldNames.ROLES));
        final List<String> languages = SuggestUtil.getAsList(source.get(FieldNames.LANGUAGES));
        final List<String> kinds = SuggestUtil.getAsList(source.get(FieldNames.KINDS));
        final long timestamp = Long.parseLong(source.get(FieldNames.TIMESTAMP).toString());

        final SuggestItem item = new SuggestItem();
        item.text = text;
        item.readings = readings.toArray(new String[readings.size()][]);
        item.fields = fields.toArray(new String[fields.size()]);
        item.docFreq = docFreq;
        item.queryFreq = queryFreq;
        item.userBoost = userBoost;
        item.tags = tags.toArray(new String[tags.size()]);
        item.roles = roles.toArray(new String[roles.size()]);
        item.languages = languages.toArray(new String[languages.size()]);

        item.kinds = new Kind[kinds.size()];
        for (int i = 0; i < kinds.size(); i++) {
            final String kind = kinds.get(i);
            if (kind.equals(Kind.DOCUMENT.toString())) {
                item.kinds[i] = Kind.DOCUMENT;
            } else if (kind.equals(Kind.QUERY.toString())) {
                item.kinds[i] = Kind.QUERY;
            } else if (kind.equals(Kind.USER.toString())) {
                item.kinds[i] = Kind.USER;
            }
        }

        item.id = SuggestUtil.createSuggestTextId(item.text);
        item.timestamp = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), Clock.systemDefaultZone().getZone());
        return item;
    }

    public Map<String, Object> getUpdatedSource(final Map<String, Object> existingSource) {
        final Map<String, Object> map = new HashMap<>();
        map.put(FieldNames.TEXT, text);

        for (int i = 0; i < readings.length; i++) {
            final Object readingObj = existingSource.get(FieldNames.READING_PREFIX + i);
            if (readingObj instanceof List) {
                @SuppressWarnings("unchecked")
                final List<String> existingValues = (List<String>) readingObj;
                concatValues(existingValues, readings[i]);
                map.put(FieldNames.READING_PREFIX + i, existingValues);
            } else {
                map.put(FieldNames.READING_PREFIX + i, readings[i]);
            }
        }

        final Object fieldsObj = existingSource.get(FieldNames.FIELDS);
        if (fieldsObj instanceof List) {
            @SuppressWarnings("unchecked")
            final List<String> existingValues = (List<String>) fieldsObj;
            concatValues(existingValues, fields);
            map.put(FieldNames.FIELDS, existingValues);
        } else {
            map.put(FieldNames.FIELDS, fields);
        }

        final Object tagsObj = existingSource.get(FieldNames.TAGS);
        if (tagsObj instanceof List) {
            @SuppressWarnings("unchecked")
            final List<String> existingValues = (List<String>) tagsObj;
            concatValues(existingValues, tags);
            map.put(FieldNames.TAGS, existingValues);
        } else {
            map.put(FieldNames.TAGS, tags);
        }

        final Object rolesObj = existingSource.get(FieldNames.ROLES);
        if (rolesObj instanceof List) {
            @SuppressWarnings("unchecked")
            final List<String> existingValues = (List<String>) rolesObj;
            concatValues(existingValues, roles);
            map.put(FieldNames.ROLES, existingValues);
        } else {
            map.put(FieldNames.ROLES, roles);
        }

        final Object langsObj = existingSource.get(FieldNames.LANGUAGES);
        if (langsObj instanceof List) {
            @SuppressWarnings("unchecked")
            final List<String> existingValues = (List<String>) langsObj;
            concatValues(existingValues, languages);
            map.put(FieldNames.LANGUAGES, existingValues);
        } else {
            map.put(FieldNames.LANGUAGES, languages);
        }

        final Object kindsObj = existingSource.get(FieldNames.KINDS);
        if (kindsObj instanceof List) {
            @SuppressWarnings("unchecked")
            final List<String> existingFields = (List<String>) kindsObj;
            concatValues(existingFields, Stream.of(kinds).map(Kind::toString).toArray(count -> new String[count]));
            map.put(FieldNames.KINDS, existingFields);
        } else {
            map.put(FieldNames.KINDS, Stream.of(kinds).map(Kind::toString).toArray());
        }

        final long updatedQueryFreq;
        final Object queryFreqObj = existingSource.get(FieldNames.QUERY_FREQ);
        if (queryFreqObj == null) {
            updatedQueryFreq = queryFreq;
        } else {
            final Long existingValue = Long.parseLong(queryFreqObj.toString());
            updatedQueryFreq = queryFreq + existingValue;
        }
        map.put(FieldNames.QUERY_FREQ, updatedQueryFreq);

        final long updatedDocFreq;
        final Object docFreqObj = existingSource.get(FieldNames.DOC_FREQ);
        if (docFreqObj == null) {
            updatedDocFreq = docFreq;
        } else {
            final Long existingValue = Long.parseLong(docFreqObj.toString());
            updatedDocFreq = docFreq + existingValue;
        }
        map.put(FieldNames.DOC_FREQ, updatedDocFreq);

        map.put(FieldNames.USER_BOOST, userBoost);
        map.put(FieldNames.SCORE, (updatedQueryFreq + updatedDocFreq) * userBoost);
        map.put(FieldNames.TIMESTAMP, timestamp.toInstant().toEpochMilli());
        return map;
    }

    protected static <T> void concatValues(final List<T> dest, final T... newValues) {
        for (final T value : newValues) {
            if (!dest.contains(value)) {
                dest.add(value);
            }
        }
    }

    protected static Kind[] concatKinds(final Kind[] kinds, final Kind... newKinds) {
        if (kinds == null) {
            return newKinds;
        }
        if (newKinds == null) {
            return kinds;
        }

        final List<Kind> list = new ArrayList<>(kinds.length + newKinds.length);
        list.addAll(Arrays.asList(kinds));
        for (final Kind kind : newKinds) {
            if (!list.contains(kind)) {
                list.add(kind);
            }
        }
        return list.toArray(new Kind[list.size()]);
    }

    public static SuggestItem merge(final SuggestItem item1, final SuggestItem item2) {
        if (!item1.getId().equals(item2.getId())) {
            throw new IllegalArgumentException("Item id is mismatch.");
        }

        final SuggestItem mergedItem = new SuggestItem();

        mergedItem.id = item1.getId();
        mergedItem.text = item1.getText();

        mergedItem.readings = new String[mergedItem.text.split(SuggestConstants.TEXT_SEPARATOR).length][];
        for (int i = 0; i < mergedItem.readings.length; i++) {
            final List<String> list = new ArrayList<>();
            if (item1.getReadings().length > i) {
                Collections.addAll(list, item1.getReadings()[i]);
            }
            if (item2.getReadings().length > i) {
                for (final String reading : item2.getReadings()[i]) {
                    if (!list.contains(reading)) {
                        list.add(reading);
                    }
                }
            }
            mergedItem.readings[i] = list.toArray(new String[list.size()]);
        }

        final List<String> fieldList = new ArrayList<>(item1.getFields().length + item2.getFields().length);
        Collections.addAll(fieldList, item1.getFields());
        for (final String field : item2.getFields()) {
            if (!fieldList.contains(field)) {
                fieldList.add(field);
            }
        }
        mergedItem.fields = fieldList.toArray(new String[fieldList.size()]);

        final List<String> tagList = new ArrayList<>(item1.getTags().length + item2.getTags().length);
        Collections.addAll(tagList, item1.getTags());
        for (final String tag : item2.getTags()) {
            if (!tagList.contains(tag)) {
                tagList.add(tag);
            }
        }
        mergedItem.tags = tagList.toArray(new String[tagList.size()]);

        final List<String> langList = new ArrayList<>(item1.getLanguages().length + item2.getLanguages().length);
        Collections.addAll(langList, item1.getLanguages());
        for (final String lang : item2.getLanguages()) {
            if (!langList.contains(lang)) {
                langList.add(lang);
            }
        }
        mergedItem.languages = langList.toArray(new String[langList.size()]);

        final List<String> roleList = new ArrayList<>(item1.getRoles().length + item2.getRoles().length);
        Collections.addAll(roleList, item1.getRoles());
        for (final String role : item2.getRoles()) {
            if (!roleList.contains(role)) {
                roleList.add(role);
            }
        }
        mergedItem.roles = roleList.toArray(new String[roleList.size()]);

        mergedItem.kinds = concatKinds(item1.kinds, item2.kinds);
        mergedItem.timestamp = item2.timestamp;
        mergedItem.queryFreq = item1.queryFreq + item2.queryFreq;
        mergedItem.docFreq = item1.docFreq + item2.docFreq;
        mergedItem.userBoost = item2.userBoost;
        mergedItem.emptySource = item2.emptySource;

        return mergedItem;
    }

    public boolean isBadWord(final String[] badWords) {
        for (final String badWord : badWords) {
            if (text.contains(badWord)) {
                return true;
            }
        }
        return false;
    }

    @Override
    public String toString() {
        return "SuggestItem [text=" + text + ", timestamp=" + timestamp + ", queryFreq=" + queryFreq + ", docFreq=" + docFreq
                + ", userBoost=" + userBoost + ", readings=" + Arrays.toString(readings) + ", fields=" + Arrays.toString(fields) + ", tags="
                + Arrays.toString(tags) + ", roles=" + Arrays.toString(roles) + ", languages=" + Arrays.toString(languages) + ", kinds="
                + Arrays.toString(kinds) + ", emptySource=" + emptySource + ", id=" + id + "]";
    }
} public class SuggestIndexResponse implements Response {
    protected final int numberOfSuggestDocs;
    protected final int numberOfInputDocs;
    protected final boolean hasError;
    protected final List<Throwable> errors = new ArrayList<>();
    protected final long took;

    protected SuggestIndexResponse(final int numberOfSuggestDocs, final int numberOfInputDocs, final List<Throwable> errors,
            final long took) {
        this.numberOfSuggestDocs = numberOfSuggestDocs;
        this.numberOfInputDocs = numberOfInputDocs;
        this.took = took;
        if (errors == null || errors.isEmpty()) {
            hasError = false;
        } else {
            hasError = true;
            errors.forEach(this.errors::add);
        }
    }

    public int getNumberOfSuggestDocs() {
        return numberOfSuggestDocs;
    }

    public int getNumberOfInputDocs() {
        return numberOfInputDocs;
    }

    public boolean hasError() {
        return hasError;
    }

    public List<Throwable> getErrors() {
        return errors;
    }

    public long getTook() {
        return took;
    }
} public class ESSourceReader implements DocumentReader {
    private static final Logger logger = Logger.getLogger(ESSourceReader.class.getName());

    protected final Queue<Map<String, Object>> queue = new ConcurrentLinkedQueue<>();
    protected final AtomicBoolean isFinished = new AtomicBoolean(false);
    protected final Random random = new Random();

    protected final Client client;
    protected final SuggestSettings settings;
    protected final String indexName;
    protected final String typeName;
    protected final String[] supportedFields;

    protected int scrollSize = 1;
    protected int maxRetryCount = 5;
    protected long limitOfDocumentSize = 50000;
    protected QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
    protected int limitPercentage = 100;
    protected long limitNumber = -1;
    protected List<SortBuilder<?>> sortList = new ArrayList<>();

    protected String scrollId = null;

    protected final AtomicLong docCount = new AtomicLong(0);
    protected final long totalDocNum;

    public ESSourceReader(final Client client, final SuggestSettings settings, final String indexName, final String typeName) {
        this.client = client;
        this.settings = settings;
        this.indexName = indexName;
        this.typeName = typeName;
        this.supportedFields = settings.array().get(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
        this.totalDocNum = getTotal();
    }

    @Override
    public synchronized Map<String, Object> read() {
        while (!isFinished.get() && queue.isEmpty()) {
            addDocumentToQueue();
        }

        return queue.poll();
    }

    @Override
    public void close() {
        isFinished.set(true);
        queue.clear();
    }

    public void setScrollSize(final int scrollSize) {
        this.scrollSize = scrollSize;
    }

    public void setLimitOfDocumentSize(final long limitOfDocumentSize) {
        if (logger.isLoggable(Level.INFO)) {
            logger.info(() -> String.format("Set document limit:%s", limitOfDocumentSize));
        }
        this.limitOfDocumentSize = limitOfDocumentSize;
    }

    public void setQuery(final QueryBuilder queryBuilder) {
        this.queryBuilder = queryBuilder;
    }

    public void addSort(final SortBuilder<?> sortBuilder) {
        this.sortList.add(sortBuilder);
    }

    public void setLimitDocNumPercentage(final String limitPercentage) {
        if (logger.isLoggable(Level.INFO)) {
            logger.info(() -> String.format("Set document limitPercentage:%s", limitPercentage));
        }

        if (limitPercentage.endsWith("%")) {
            this.limitPercentage = Integer.parseInt(limitPercentage.substring(0, limitPercentage.length() - 1));
        } else {
            this.limitPercentage = Integer.parseInt(limitPercentage);
        }

        if (this.limitPercentage > 100) {
            this.limitPercentage = 100;
        } else if (this.limitPercentage < 1) {
            this.limitPercentage = 1;
        }
    }

    public void setLimitNumber(final long limitNumber) {
        this.limitNumber = limitNumber;
    }

    protected void addDocumentToQueue() {
        if (docCount.get() > getLimitDocNum(totalDocNum, limitPercentage, limitNumber)) {
            isFinished.set(true);
            return;
        }

        RuntimeException exception = null;

        for (int i = 0; i < maxRetryCount; i++) {
            try {
                final SearchResponse response;
                if (scrollId == null) {
                    final SearchRequestBuilder builder = client.prepareSearch().setIndices(indexName).setScroll(settings.getScrollTimeout())
                            .setQuery(queryBuilder).setSize(scrollSize);
                    for (final SortBuilder<?> sortBuilder : sortList) {
                        builder.addSort(sortBuilder);
                    }
                    response = builder.execute().actionGet(settings.getSearchTimeout());
                } else {
                    response = client.prepareSearchScroll(scrollId).setScroll(settings.getScrollTimeout()).execute()
                            .actionGet(settings.getSearchTimeout());
                    if (!scrollId.equals(response.getScrollId())) {
                        SuggestUtil.deleteScrollContext(client, scrollId);
                    }
                }
                scrollId = response.getScrollId();
                final SearchHit[] hits = response.getHits().getHits();
                if (scrollId == null || hits.length == 0) {
                    SuggestUtil.deleteScrollContext(client, scrollId);
                    isFinished.set(true);
                }

                for (final SearchHit hit : hits) {
                    final Map<String, Object> source = hit.getSourceAsMap();
                    if (limitOfDocumentSize > 0) {
                        long size = 0;
                        for (final String field : supportedFields) {
                            final Object value = source.get(field);
                            if (value != null) {
                                size += value.toString().length();
                            }
                        }

                        if (size <= limitOfDocumentSize) {
                            queue.add(source);
                        }
                    } else {
                        queue.add(source);
                    }
                }
                exception = null;
                break;
            } catch (final Exception e) {
                exception = new RuntimeException(e);
                scrollId = null;
            }
        }

        docCount.getAndAdd(queue.size());

        if (exception != null) {
            throw exception;
        }
    }

    protected static long getLimitDocNum(final long total, final long limitPercentage, final long limitNumber) {
        final long percentNum = (long) (total * (limitPercentage / 100f));
        if (limitNumber < 0) {
            return percentNum;
        }
        return percentNum < limitNumber ? percentNum : limitNumber;
    }

    protected long getTotal() {
        final SearchResponse response = client.prepareSearch().setIndices(indexName).setQuery(queryBuilder).setSize(0)
                .setTrackTotalHits(true).execute().actionGet(settings.getSearchTimeout());
        return response.getHits().getTotalHits().value;
    }

} public class QueryLog {
    private final String q;
    private final String fq;

    public QueryLog(final String queryString, @Nullable final String filterQueryString) {
        this.q = queryString;
        this.fq = filterQueryString;
    }

    public String getQueryString() {
        return q;
    }

    public String getFilterQueryString() {
        return fq;
    }
} public interface QueryLogReader extends Closeable {
    QueryLog read();

    @Override
    void close();
} public class PopularWordsResponse implements Response {
    protected final String index;

    protected final long tookMs;

    protected final List<String> words;

    protected final int num;

    protected final long total;

    protected final List<SuggestItem> items;

    public PopularWordsResponse(final String index, final long tookMs, final List<String> words, final long total,
            final List<SuggestItem> items) {
        this.index = index;
        this.tookMs = tookMs;
        this.words = words;
        this.num = words.size();
        this.total = total;
        this.items = items;
    }

    public String getIndex() {
        return index;
    }

    public long getTookMs() {
        return tookMs;
    }

    public List<String> getWords() {
        return words;
    }

    public int getNum() {
        return num;
    }

    public long getTotal() {
        return total;
    }

    public List<SuggestItem> getItems() {
        return items;
    }
} public class SuggestResponse implements Response {
    protected final String index;

    protected final long tookMs;

    protected final List<String> words;

    protected final int num;

    protected final long total;

    protected final List<SuggestItem> items;

    public SuggestResponse(final String index, final long tookMs, final List<String> words, final long total,
            final List<SuggestItem> items) {
        this.index = index;
        this.tookMs = tookMs;
        this.words = words;
        this.num = words.size();
        this.total = total;
        this.items = items;
    }

    public String getIndex() {
        return index;
    }

    public long getTookMs() {
        return tookMs;
    }

    public List<String> getWords() {
        return words;
    }

    public int getNum() {
        return num;
    }

    public long getTotal() {
        return total;
    }

    public List<SuggestItem> getItems() {
        return items;
    }
} public class AnalyzerSettings {
    public static final String READING_ANALYZER = "reading_analyzer";
    public static final String READING_TERM_ANALYZER = "reading_term_analyzer";
    public static final String NORMALIZE_ANALYZER = "normalize_analyzer";
    public static final String CONTENTS_ANALYZER = "contents_analyzer";
    public static final String CONTENTS_READING_ANALYZER = "contents_reading_analyzer";
    public static final String FIELD_ANALYZER_MAPPING = "fieldAnalyzerMapping";

    public static final String DOC_TYPE_NAME = "_doc";

    protected final Client client;
    protected final String analyzerSettingsIndexName;
    private final SuggestSettings settings;

    protected static Map<String, Set<String>> analyzerMap = new ConcurrentHashMap<>();
    protected static Map<String, Map<String, FieldAnalyzerMapping>> fieldAnalyzerMappingMap = new ConcurrentHashMap<>();

    protected static final String[] SUPPORTED_LANGUAGES = { "ar", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es", "et", "fa", "fi",
            "fr", "gu", "he", "hi", "hr", "hu", "id", "it", "ja", "ko", "lt", "lv", "mk", "ml", "nl", "no", "pa", "pl", "pt", "ro", "ru",
            "si", "sq", "sv", "ta", "te", "th", "tl", "tr", "uk", "ur", "vi", "zh-cn", "zh-tw" };

    public AnalyzerSettings(final Client client, final SuggestSettings settings, final String settingsIndexName) {
        this.client = client;
        this.settings = settings;
        analyzerSettingsIndexName = createAnalyzerSettingsIndexName(settingsIndexName);
    }

    public synchronized void init() {
        try {
            final IndicesExistsResponse response =
                    client.admin().indices().prepareExists(analyzerSettingsIndexName).execute().actionGet(settings.getIndicesTimeout());
            if (!response.isExists()) {
                createAnalyzerSettings(loadIndexSettings(), loadIndexMapping());
            }
            analyzerMap.put(analyzerSettingsIndexName, getAnalyzerNames());
            fieldAnalyzerMappingMap.put(analyzerSettingsIndexName, getFieldAnalyzerMapping());
        } catch (final IOException e) {
            throw new SuggestSettingsException("Failed to create mappings.");
        }
    }

    public String getAnalyzerSettingsIndexName() {
        return analyzerSettingsIndexName;
    }

    public String getReadingAnalyzerName(final String field, final String lang) {
        final Map<String, FieldAnalyzerMapping> fieldAnalyzerMapping = fieldAnalyzerMappingMap.get(analyzerSettingsIndexName);
        final Set<String> analyzerNames = analyzerMap.get(analyzerSettingsIndexName);
        final String analyzerName;
        if (StringUtil.isNotBlank(field) && fieldAnalyzerMapping.containsKey(field)
                && fieldAnalyzerMapping.get(field).readingAnalyzer != null) {
            analyzerName = fieldAnalyzerMapping.get(field).readingAnalyzer;
        } else {
            analyzerName = READING_ANALYZER;
        }
        final String analyzerNameWithlang = isSupportedLanguage(lang) ? analyzerName + '_' + lang : analyzerName;
        if (analyzerNames.contains(analyzerNameWithlang)) {
            return analyzerNameWithlang;
        }
        return analyzerName;
    }

    public String getReadingTermAnalyzerName(final String field, final String lang) {
        final Map<String, FieldAnalyzerMapping> fieldAnalyzerMapping = fieldAnalyzerMappingMap.get(analyzerSettingsIndexName);
        final Set<String> analyzerNames = analyzerMap.get(analyzerSettingsIndexName);
        final String analyzerName;
        if (StringUtil.isNotBlank(field) && fieldAnalyzerMapping.containsKey(field)
                && fieldAnalyzerMapping.get(field).readingTermAnalyzer != null) {
            analyzerName = fieldAnalyzerMapping.get(field).readingTermAnalyzer;
        } else {
            analyzerName = READING_TERM_ANALYZER;
        }
        final String analyzerNameWithlang = isSupportedLanguage(lang) ? analyzerName + '_' + lang : analyzerName;
        if (analyzerNames.contains(analyzerNameWithlang)) {
            return analyzerNameWithlang;
        }
        return analyzerName;
    }

    public String getNormalizeAnalyzerName(final String field, final String lang) {
        final Map<String, FieldAnalyzerMapping> fieldAnalyzerMapping = fieldAnalyzerMappingMap.get(analyzerSettingsIndexName);
        final Set<String> analyzerNames = analyzerMap.get(analyzerSettingsIndexName);
        final String analyzerName;
        if (StringUtil.isNotBlank(field) && fieldAnalyzerMapping.containsKey(field)
                && fieldAnalyzerMapping.get(field).normalizeAnalyzer != null) {
            analyzerName = fieldAnalyzerMapping.get(field).normalizeAnalyzer;
        } else {
            analyzerName = NORMALIZE_ANALYZER;
        }
        final String analyzerNameWithlang = isSupportedLanguage(lang) ? analyzerName + '_' + lang : analyzerName;
        if (analyzerNames.contains(analyzerNameWithlang)) {
            return analyzerNameWithlang;
        }
        return analyzerName;
    }

    public String getContentsAnalyzerName(final String field, final String lang) {
        final Map<String, FieldAnalyzerMapping> fieldAnalyzerMapping = fieldAnalyzerMappingMap.get(analyzerSettingsIndexName);
        final Set<String> analyzerNames = analyzerMap.get(analyzerSettingsIndexName);
        final String analyzerName;
        if (StringUtil.isNotBlank(field) && fieldAnalyzerMapping.containsKey(field)
                && fieldAnalyzerMapping.get(field).contentsAnalyzer != null) {
            analyzerName = fieldAnalyzerMapping.get(field).contentsAnalyzer;
        } else {
            analyzerName = CONTENTS_ANALYZER;
        }
        final String analyzerNameWithlang = isSupportedLanguage(lang) ? analyzerName + '_' + lang : analyzerName;
        if (analyzerNames.contains(analyzerNameWithlang)) {
            return analyzerNameWithlang;
        }
        return analyzerName;
    }

    public String getContentsReadingAnalyzerName(final String field, final String lang) {
        final Map<String, FieldAnalyzerMapping> fieldAnalyzerMapping = fieldAnalyzerMappingMap.get(analyzerSettingsIndexName);
        final Set<String> analyzerNames = analyzerMap.get(analyzerSettingsIndexName);
        final String analyzerName;
        if (StringUtil.isNotBlank(field) && fieldAnalyzerMapping.containsKey(field)
                && fieldAnalyzerMapping.get(field).contentsReadingAnalyzer != null) {
            analyzerName = fieldAnalyzerMapping.get(field).contentsReadingAnalyzer;
        } else {
            analyzerName = CONTENTS_READING_ANALYZER;
        }
        final String analyzerNameWithlang = isSupportedLanguage(lang) ? analyzerName + '_' + lang : analyzerName;
        if (analyzerNames.contains(analyzerNameWithlang)) {
            return analyzerNameWithlang;
        }
        return analyzerName;
    }

    public void updateAnalyzer(final Map<String, Object> settings) {
        client.admin().indices().prepareCreate(analyzerSettingsIndexName).setSettings(settings).execute()
                .actionGet(this.settings.getIndicesTimeout());
    }

    protected void deleteAnalyzerSettings() {
        client.admin().indices().prepareDelete(analyzerSettingsIndexName).execute().actionGet(settings.getIndicesTimeout());
    }

    protected void createAnalyzerSettings(final String settings, final String mappings) {
        client.admin().indices().prepareCreate(analyzerSettingsIndexName).setSettings(settings, XContentType.JSON)
                .addMapping("_doc", mappings, XContentType.JSON).execute().actionGet(this.settings.getIndicesTimeout());
    }

    protected void createAnalyzerSettings(final Map<String, Object> settings, final Map<String, Object> mappings) {
        client.admin().indices().prepareCreate(analyzerSettingsIndexName).setSettings(settings).execute()
                .actionGet(this.settings.getIndicesTimeout());
    }

    protected String createAnalyzerSettingsIndexName(final String settingsIndexName) {
        return settingsIndexName + "_analyzer";
    }

    protected String loadIndexSettings() throws IOException {
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br =
                new BufferedReader(new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream(getSuggestAnalyzerPath())))) {

            String line;
            while ((line = br.readLine()) != null) {
                sb.append(line);
            }
        }
        return sb.toString().replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath);
    }

    protected String getSuggestAnalyzerPath() {
        final Object typeObj = this.settings.get("elasticsearch.type");
        if (typeObj != null) {
            final String path = "suggest_indices/_" + typeObj.toString() + "/suggest_analyzer.json";
            if (ResourceUtil.getResourceNoException(path) != null) {
                return path;
            }
        }
        return "suggest_indices/suggest_analyzer.json";
    }

    protected String loadIndexMapping() throws IOException {
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br = new BufferedReader(new InputStreamReader(
                this.getClass().getClassLoader().getResourceAsStream("suggest_indices/analyzer/mapping-default.json")));) {

            String line;
            while ((line = br.readLine()) != null) {
                sb.append(line);
            }
        }
        return sb.toString();
    }

    public class DefaultContentsAnalyzer implements SuggestAnalyzer {

        private final int maxContentLenth = settings.getAsInt(SuggestSettings.DefaultKeys.MAX_CONTENT_LENGTH, 50000);

        @Override
        public List<AnalyzeToken> analyze(final String text, final String field, final String lang) {
            if (text == null || text.length() > maxContentLenth) {
                return Collections.emptyList();
            }
            final AnalyzeAction.Response analyzeResponse = client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text)
                    .setAnalyzer(getContentsAnalyzerName(field, lang)).execute().actionGet(settings.getIndicesTimeout());
            return analyzeResponse.getTokens();
        }

        @Override
        public List<AnalyzeToken> analyzeAndReading(final String text, final String field, final String lang) {
            try {
                final String contentsReadingAnalyzerName = getContentsReadingAnalyzerName(field, lang);
                if (StringUtil.isBlank(contentsReadingAnalyzerName)) {
                    return null;
                }
                final AnalyzeAction.Response analyzeResponse = client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text)
                        .setAnalyzer(contentsReadingAnalyzerName).execute().actionGet(settings.getIndicesTimeout());
                return analyzeResponse.getTokens();
            } catch (final IllegalArgumentException e) {
                return analyze(text, field, lang);
            }
        }
    }

    public static boolean isSupportedLanguage(final String lang) {
        return StringUtil.isNotBlank(lang) && Stream.of(SUPPORTED_LANGUAGES).anyMatch(lang::equals);
    }

    protected Set<String> getAnalyzerNames() {
        final GetSettingsResponse response =
                client.admin().indices().prepareGetSettings().setIndices(analyzerSettingsIndexName).execute().actionGet();
        final Settings settings = response.getIndexToSettings().get(analyzerSettingsIndexName);
        final Settings analyzerSettings = settings.getAsSettings("index.analysis.analyzer");
        return analyzerSettings.getAsGroups().keySet();
    }

    protected Map<String, FieldAnalyzerMapping> getFieldAnalyzerMapping() {
        final Map<String, FieldAnalyzerMapping> mappingMap = new HashMap<>();
        SearchResponse response = client.prepareSearch(analyzerSettingsIndexName)
                .setQuery(QueryBuilders.termQuery(FieldNames.ANALYZER_SETTINGS_TYPE, FIELD_ANALYZER_MAPPING))
                .setScroll(settings.getScrollTimeout()).execute().actionGet(settings.getSearchTimeout());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {
                final SearchHit[] hits = response.getHits().getHits();
                if (hits.length == 0) {
                    break;
                }
                for (final SearchHit hit : hits) {
                    final Map<String, Object> source = hit.getSourceAsMap();
                    final String fieldReadingAnalyzer = source.get(FieldNames.ANALYZER_SETTINGS_READING_ANALYZER) == null ? null
                            : source.get(FieldNames.ANALYZER_SETTINGS_READING_ANALYZER).toString();
                    final String fieldReadingTermAnalyzer = source.get(FieldNames.ANALYZER_SETTINGS_READING_TERM_ANALYZER) == null ? null
                            : source.get(FieldNames.ANALYZER_SETTINGS_READING_TERM_ANALYZER).toString();
                    final String fieldNormalizeAnalyzer = source.get(FieldNames.ANALYZER_SETTINGS_NORMALIZE_ANALYZER) == null ? null
                            : source.get(FieldNames.ANALYZER_SETTINGS_NORMALIZE_ANALYZER).toString();
                    final String fieldContentsAnalyzer = source.get(FieldNames.ANALYZER_SETTINGS_CONTENTS_ANALYZER) == null ? null
                            : source.get(FieldNames.ANALYZER_SETTINGS_CONTENTS_ANALYZER).toString();
                    final String fieldContentsReadingAnalyzer =
                            source.get(FieldNames.ANALYZER_SETTINGS_CONTENTS_READING_ANALYZER) == null ? null
                                    : source.get(FieldNames.ANALYZER_SETTINGS_CONTENTS_READING_ANALYZER).toString();

                    mappingMap.put(source.get(FieldNames.ANALYZER_SETTINGS_FIELD_NAME).toString(),
                            new FieldAnalyzerMapping(fieldReadingAnalyzer, fieldReadingTermAnalyzer, fieldNormalizeAnalyzer,
                                    fieldContentsAnalyzer, fieldContentsReadingAnalyzer));
                }
                response = client.prepareSearchScroll(scrollId).setScroll(settings.getScrollTimeout()).execute()
                        .actionGet(settings.getSearchTimeout());
                if (!scrollId.equals(response.getScrollId())) {
                    SuggestUtil.deleteScrollContext(client, scrollId);
                }
                scrollId = response.getScrollId();
            }
        } finally {
            SuggestUtil.deleteScrollContext(client, scrollId);
        }
        return mappingMap;
    }

    public Set<String> checkAnalyzer() {
        final String text = "text";
        final Set<String> undefinedAnalyzerSet = new HashSet<>();
        for (final String lang : SUPPORTED_LANGUAGES) {
            final String readingAnalyzer = getReadingAnalyzerName("", lang);
            try {
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(readingAnalyzer).execute()
                        .actionGet(settings.getIndicesTimeout());
            } catch (final IllegalArgumentException e) {
                undefinedAnalyzerSet.add(readingAnalyzer);
            }

            final String readingTermAnalyzer = getReadingTermAnalyzerName("", lang);
            try {
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(readingTermAnalyzer).execute()
                        .actionGet(settings.getIndicesTimeout());
            } catch (final IllegalArgumentException e) {
                undefinedAnalyzerSet.add(readingTermAnalyzer);
            }

            final String normalizeAnalyzer = getNormalizeAnalyzerName("", lang);
            try {
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(normalizeAnalyzer).execute()
                        .actionGet(settings.getIndicesTimeout());
            } catch (final IllegalArgumentException e) {
                undefinedAnalyzerSet.add(normalizeAnalyzer);
            }

            final String contentsAnalyzer = getContentsAnalyzerName("", lang);
            try {
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(contentsAnalyzer).execute()
                        .actionGet(settings.getIndicesTimeout());
            } catch (final IllegalArgumentException e) {
                undefinedAnalyzerSet.add(contentsAnalyzer);
            }

            final String contentsReadingAnalyzer = getContentsReadingAnalyzerName("", lang);
            try {
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(contentsReadingAnalyzer).execute()
                        .actionGet(settings.getIndicesTimeout());
            } catch (final IllegalArgumentException e) {
                undefinedAnalyzerSet.add(contentsReadingAnalyzer);
            }
        }

        return undefinedAnalyzerSet;
    }

    protected static class FieldAnalyzerMapping {
        protected final String readingAnalyzer;
        protected final String readingTermAnalyzer;
        protected final String normalizeAnalyzer;
        protected final String contentsAnalyzer;
        protected final String contentsReadingAnalyzer;

        public FieldAnalyzerMapping(final String readingAnalyzer, final String readingTermAnalyzer, final String normalizeAnalyzer,
                final String contentsAnalyzer, final String contentsReadingAnalyzer) {
            this.readingAnalyzer = readingAnalyzer;
            this.readingTermAnalyzer = readingTermAnalyzer;
            this.normalizeAnalyzer = normalizeAnalyzer;
            this.contentsAnalyzer = contentsAnalyzer;
            this.contentsReadingAnalyzer = contentsReadingAnalyzer;
        }
    }
} public class SuggestSettings {
    private static final Logger logger = Logger.getLogger(SuggestSettings.class.getName());

    protected final String settingsId;

    protected final Client client;

    protected final String settingsIndexName;

    protected final Map<String, Object> initialSettings;

    protected boolean initialized = false;

    protected final String badWordIndexName;
    protected final String elevateWordIndexName;

    protected TimeoutSettings timeoutSettings;

    public static class TimeoutSettings {
        protected String searchTimeout = "15s";
        protected String indexTimeout = "1m";
        protected String bulkTimeout = "1m";
        protected String indicesTimeout = "1m";
        protected String clusterTimeout = "1m";
        protected String scrollTimeout = "1m";
    }

    public SuggestSettings(final Client client, final String settingsId, final Map<String, Object> initialSettings,
            final String settingsIndexName, final TimeoutSettings timeoutSettings) {
        this.client = client;
        this.settingsId = settingsId;
        this.settingsIndexName = settingsIndexName;
        this.initialSettings = initialSettings;
        this.timeoutSettings = timeoutSettings;

        this.badWordIndexName = settingsIndexName + "-badword";
        this.elevateWordIndexName = settingsIndexName + "-elevateword";
    }

    public void init() {
        if (initialized) {
            return;
        }
        initialized = true;
        initialize(initialSettings);
        new AnalyzerSettings(client, this, settingsIndexName).init();
    }

    private void initialize(final Map<String, Object> initialSettings) {
        boolean doIndexCreate = false;
        boolean doCreate = false;
        try {
            final GetResponse getResponse =
                    client.prepareGet().setIndex(settingsIndexName).setId(settingsId).execute().actionGet(getSearchTimeout());

            if (!getResponse.isExists()) {
                doCreate = true;
            }
        } catch (final IndexNotFoundException e) {
            doIndexCreate = true;
            doCreate = true;
        }

        if (doCreate) {
            if (doIndexCreate) {
                try {
                    client.admin().indices().prepareCreate(settingsIndexName).setSettings(loadIndexSettings(), XContentType.JSON).execute()
                            .actionGet(getIndicesTimeout());
                } catch (final IOException e) {
                    throw new SuggesterException(e);
                }
            }

            final List<Tuple<String, Object>> arraySettings = new ArrayList<>();
            final Map<String, Object> defaultSettings = defaultSettings();
            initialSettings.forEach((key, value) -> {
                if (value instanceof Collection) {
                    @SuppressWarnings("unchecked")
                    final Collection<Object> collection = (Collection<Object>) value;
                    collection.forEach(element -> arraySettings.add(new Tuple<>(key, element)));
                } else if (value instanceof Object[]) {
                    for (final Object element : (Object[]) value) {
                        arraySettings.add(new Tuple<>(key, element));
                    }
                } else {
                    defaultSettings.put(key, value);
                }
            });
            set(defaultSettings);

            final List<Tuple<String, Object>> defaultArraySettings = defaultArraySettings();
            defaultArraySettings.addAll(arraySettings);
            defaultArraySettings.forEach(t -> array().add(t.v1(), t.v2()));
        }
    }

    public Object get(final String key) {
        final GetResponse getResponse =
                client.prepareGet().setIndex(settingsIndexName).setId(settingsId).execute().actionGet(getSearchTimeout());
        if (!getResponse.isExists()) {
            return null;
        }
        final Map<String, Object> map = getResponse.getSource();
        return map.get(key);
    }

    public String getAsString(final String key, final String defaultValue) {
        final Object obj = get(key);

        final String value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = obj.toString();
        }

        return value;
    }

    public int getAsInt(final String key, final int defaultValue) {
        final Object obj = get(key);

        final int value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Integer.parseInt(obj.toString());
        }

        return value;
    }

    public long getAsLong(final String key, final long defaultValue) {
        final Object obj = get(key);

        final long value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Long.parseLong(obj.toString());
        }

        return value;
    }

    public float getAsFloat(final String key, final float defaultValue) {
        final Object obj = get(key);

        final float value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Float.parseFloat(obj.toString());
        }

        return value;
    }

    public boolean getAsBoolean(final String key, final boolean defaultValue) {
        final Object obj = get(key);

        final boolean value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Boolean.parseBoolean(obj.toString());
        }

        return value;
    }

    public void set(final String key, final Object value) {
        if (logger.isLoggable(Level.FINER)) {
            logger.finer(() -> String.format("Set suggest settings. %s key: %s value: %s", settingsIndexName, key, value));
        }
        try {
            client.prepareUpdate().setIndex(settingsIndexName).setId(settingsId).setDocAsUpsert(true).setDoc(key, value)
                    .setRetryOnConflict(5).execute().actionGet(getIndexTimeout());
            client.admin().indices().prepareRefresh().setIndices(settingsIndexName).execute().actionGet(getIndicesTimeout());
        } catch (final Exception e) {
            throw new SuggestSettingsException("Failed to update suggestSettings.", e);
        }
    }

    public void set(final Map<String, Object> map) {
        if (logger.isLoggable(Level.FINER)) {
            logger.finer(() -> String.format("Set suggest settings. %s %s", settingsIndexName, map.toString()));
        }
        try {
            final XContentBuilder builder = JsonXContent.contentBuilder().map(map);
            builder.flush();
            client.prepareUpdate().setIndex(settingsIndexName).setId(settingsId).setDocAsUpsert(true).setDoc(builder).setRetryOnConflict(5)
                    .execute().actionGet(getIndexTimeout());
            client.admin().indices().prepareRefresh().setIndices(settingsIndexName).execute().actionGet(getIndicesTimeout());
        } catch (final Exception e) {
            throw new SuggestSettingsException("Failed to update suggestSettings.", e);
        }
    }

    public ArraySettings array() {
        return new ArraySettings(this, client, settingsIndexName, settingsId);
    }

    public AnalyzerSettings analyzer() {
        return new AnalyzerSettings(client, this, settingsIndexName);
    }

    public BadWordSettings badword() {
        return new BadWordSettings(this, client, settingsIndexName, settingsId);
    }

    public ElevateWordSettings elevateWord() {
        return new ElevateWordSettings(this, client, settingsIndexName, settingsId);
    }

    public String getSettingsIndexName() {
        return settingsIndexName;
    }

    public boolean isInitialized() {
        return initialized;
    }

    public String getSettingsId() {
        return settingsId;
    }

    private Map<String, Object> defaultSettings() {
        final Map<String, Object> defaultSettings = new HashMap<>();
        defaultSettings.put(DefaultKeys.INDEX, (settingsId + ".suggest").toLowerCase());
        defaultSettings.put(DefaultKeys.TAG_FIELD_NAME, "label,virtual_host");
        defaultSettings.put(DefaultKeys.ROLE_FIELD_NAME, "role");
        defaultSettings.put(DefaultKeys.LANG_FIELD_NAME, "lang");
        defaultSettings.put(DefaultKeys.PARALLEL_PROCESSING, Boolean.FALSE);
        return defaultSettings;
    }

    private List<Tuple<String, Object>> defaultArraySettings() {
        final List<Tuple<String, Object>> tuples = new ArrayList<>();
        tuples.add(new Tuple<>(DefaultKeys.SUPPORTED_FIELDS, "content"));
        return tuples;
    }

    protected String loadIndexSettings() throws IOException {
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br = new BufferedReader(
                new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream("suggest_indices/suggest_settings.json")));) {

            String line;
            while ((line = br.readLine()) != null) {
                sb.append(line);
            }
        }
        return sb.toString().replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath);
    }

    public static SuggestSettingsBuilder builder() {
        return new SuggestSettingsBuilder();
    }

    public static class DefaultKeys {
        public static final String INDEX = "index";
        public static final String SUPPORTED_FIELDS = "supportedFields";
        public static final String TAG_FIELD_NAME = "tagFieldName";
        public static final String ROLE_FIELD_NAME = "roleFieldName";
        public static final String LANG_FIELD_NAME = "langFieldName";
        public static final String PARALLEL_PROCESSING = "parallel";
        public static final String MAX_CONTENT_LENGTH = "maxContextLength";

        private DefaultKeys() {
        }
    }

    public String getScrollTimeout() {
        return timeoutSettings.scrollTimeout;
    }

    public String getSearchTimeout() {
        return timeoutSettings.searchTimeout;
    }

    public String getIndexTimeout() {
        return timeoutSettings.indexTimeout;
    }

    public String getIndicesTimeout() {
        return timeoutSettings.indicesTimeout;
    }

    public String getBulkTimeout() {
        return timeoutSettings.bulkTimeout;
    }

    public String getClusterTimeout() {
        return timeoutSettings.clusterTimeout;
    }
}
import static org.codelibs.opensearch.runner.OpenSearchRunner.newConfigs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import org.codelibs.fess.suggest.constants.FieldNames;
import org.codelibs.fess.suggest.constants.SuggestConstants;
import org.codelibs.fess.suggest.entity.ElevateWord;
import org.codelibs.fess.suggest.entity.SuggestItem;
import org.codelibs.fess.suggest.index.SuggestIndexResponse;
import org.codelibs.fess.suggest.index.contents.document.ESSourceReader;
import org.codelibs.fess.suggest.index.contents.querylog.QueryLog;
import org.codelibs.fess.suggest.index.contents.querylog.QueryLogReader;
import org.codelibs.fess.suggest.request.popularwords.PopularWordsResponse;
import org.codelibs.fess.suggest.request.suggest.SuggestResponse;
import org.codelibs.fess.suggest.settings.AnalyzerSettings;
import org.codelibs.fess.suggest.settings.SuggestSettings;
import org.codelibs.opensearch.runner.OpenSearchRunner;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.opensearch.action.admin.indices.get.GetIndexResponse;
import org.opensearch.action.bulk.BulkRequestBuilder;
import org.opensearch.action.index.IndexAction;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.client.Client;
/*
 * Copyright 2012-2022 CodeLibs Project and the Others.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.suggest;




public class SuggesterTest {
    static Suggester suggester;

    static OpenSearchRunner runner;

    @BeforeClass
    public static void beforeClass() throws Exception {
        runner = new OpenSearchRunner();
        runner.onBuild((number, settingsBuilder) -> {
            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("discovery.type", "single-node");
            // settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301");
            // settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
        }).build(newConfigs().clusterName("ArraySettingsTest").numOfNode(1)
                .pluginTypes("org.codelibs.opensearch.extension.ExtensionPlugin"));
        runner.ensureYellow();
    }

    @AfterClass
    public static void afterClass() throws Exception {
        runner.close();
        runner.clean();
    }

    @Before
    public void before() throws Exception {
        runner.admin().indices().prepareDelete("_all").execute().actionGet();
        runner.refresh();
        suggester = Suggester.builder().build(runner.client(), "SuggesterTest");
        suggester.createIndexIfNothing();
    }

    @Test
    public void test_indexAndSuggest() throws Exception {
        SuggestItem[] items = getItemSet1();
        suggester.indexer().index(items);
        suggester.refresh();

        SuggestResponse response = suggester.suggest().setQuery("kensaku").setSuggestDetail(true).execute().getResponse();