public interface SuggestAnalyzer {
    List<AnalyzeToken> analyze(String text, String field, String lang);

    List<AnalyzeToken> analyzeAndReading(String text, String field, String lang);
} public final class FieldNames {
    public static final String ID = "_id";
    public static final String TEXT = "text";
    public static final String READING_PREFIX = "reading_";
    public static final String SCORE = "score";
    public static final String QUERY_FREQ = "queryFreq";
    public static final String DOC_FREQ = "docFreq";
    public static final String USER_BOOST = "userBoost";
    public static final String KINDS = "kinds";
    public static final String TIMESTAMP = "@timestamp";
    public static final String TAGS = "tags";
    public static final String ROLES = "roles";
    public static final String FIELDS = "fields";
    public static final String LANGUAGES = "languages";

    public static final String ARRAY_KEY = "key";
    public static final String ARRAY_VALUE = "value";

    public static final String ANALYZER_SETTINGS_TYPE = "settingsType";
    public static final String ANALYZER_SETTINGS_FIELD_NAME = "fieldName";
    public static final String ANALYZER_SETTINGS_READING_ANALYZER = "readingAnalyzer";
    public static final String ANALYZER_SETTINGS_READING_TERM_ANALYZER = "readingTermAnalyzer";
    public static final String ANALYZER_SETTINGS_NORMALIZE_ANALYZER = "normalizeAnalyzer";
    public static final String ANALYZER_SETTINGS_CONTENTS_ANALYZER = "contentsAnalyzer";
    public static final String ANALYZER_SETTINGS_CONTENTS_READING_ANALYZER = "contentsReadingAnalyzer";

    private FieldNames() {
    }
} public final class SuggestConstants {
    private SuggestConstants() {
    }

    public static final String EMPTY_STRING = "";

    public static final String USER_DICT_ENCODING = "fess.user.dict.encoding";

    public static final String USER_DICT_PATH = "fess.user.dict.path";

    public static final String TEXT_SEPARATOR = " ";

    public static final String DEFAULT_ROLE = "_guest_";

    public static final String DEFAULT_TYPE = "_doc";
} public interface ReadingConverter {
    default int getMaxReadingNum() {
        return 10;
    }

    void init() throws IOException;

    List<String> convert(String text, final String field, String... langs) throws IOException;
} public class SuggesterException extends RuntimeException {

    private static final long serialVersionUID = 1L;

    public SuggesterException(final String msg) {
        super(msg);
    }

    public SuggesterException(final Throwable cause) {
        super(cause);
    }

    public SuggesterException(final String msg, final Throwable cause) {
        super(msg, cause);
    }
} public class SuggestIndexer {
    private final static Logger logger = Logger.getLogger(SuggestIndexer.class.getName());

    protected final Client client;
    protected String index;
    protected SuggestSettings settings;

    protected String[] supportedFields;
    protected String[] tagFieldNames;
    protected String roleFieldName;
    protected String langFieldName;
    protected String[] badWords;
    protected boolean parallel;

    protected ReadingConverter readingConverter;
    protected ReadingConverter contentsReadingConverter;
    protected Normalizer normalizer;
    protected SuggestAnalyzer analyzer;

    protected ContentsParser contentsParser;
    protected SuggestWriter suggestWriter;

    protected ExecutorService threadPool;

    public SuggestIndexer(final Client client, final String index, final ReadingConverter readingConverter,
            final ReadingConverter contentsReadingConverter, final Normalizer normalizer, final SuggestAnalyzer analyzer,
            final SuggestSettings settings, final ExecutorService threadPool) {
        this.client = client;
        this.index = index;

        this.supportedFields = settings.array().get(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
        this.badWords = settings.badword().get(true);
        this.tagFieldNames = settings.getAsString(SuggestSettings.DefaultKeys.TAG_FIELD_NAME, StringUtil.EMPTY).split(",");
        this.roleFieldName = settings.getAsString(SuggestSettings.DefaultKeys.ROLE_FIELD_NAME, StringUtil.EMPTY);
        this.langFieldName = settings.getAsString(SuggestSettings.DefaultKeys.LANG_FIELD_NAME, StringUtil.EMPTY);
        this.parallel = settings.getAsBoolean(SuggestSettings.DefaultKeys.PARALLEL_PROCESSING, false);
        this.readingConverter = readingConverter;
        this.contentsReadingConverter = contentsReadingConverter;
        this.normalizer = normalizer;
        this.analyzer = analyzer;
        this.settings = settings;

        this.contentsParser = new DefaultContentsParser();
        this.suggestWriter = new SuggestIndexWriter();

        this.threadPool = threadPool;
    }

    // TODO return result
    public SuggestIndexResponse index(final SuggestItem item) {
        return index(new SuggestItem[] { item });
    }

    // TODO return result
    public SuggestIndexResponse index(final SuggestItem[] items) {
        // TODO parallel?
        final SuggestItem[] array = Stream.of(items).filter(item -> !item.isBadWord(badWords)).toArray(n -> new SuggestItem[n]);

        try {
            final long start = System.currentTimeMillis();
            final SuggestWriterResult result = suggestWriter.write(client, settings, index, array, true);
            return new SuggestIndexResponse(items.length, items.length, result.getFailures(), System.currentTimeMillis() - start);
        } catch (final Exception e) {
            throw new SuggestIndexException("Failed to write items[" + items.length + "] to " + index, e);
        }
    }

    public SuggestDeleteResponse delete(final String id) {
        final long start = System.currentTimeMillis();
        final SuggestWriterResult result = suggestWriter.delete(client, settings, index, id);
        return new SuggestDeleteResponse(result.getFailures(), System.currentTimeMillis() - start);
    }

    public SuggestDeleteResponse deleteByQuery(final String queryString) {
        return deleteByQuery(QueryBuilders.queryStringQuery(queryString).defaultOperator(Operator.AND));
    }

    public SuggestDeleteResponse deleteByQuery(final QueryBuilder queryBuilder) {
        final long start = System.currentTimeMillis();
        final SuggestWriterResult result = suggestWriter.deleteByQuery(client, settings, index, queryBuilder);
        return new SuggestDeleteResponse(result.getFailures(), System.currentTimeMillis() - start);
    }

    public SuggestDeleteResponse deleteAll() {
        final SuggestDeleteResponse response = deleteByQuery(QueryBuilders.matchAllQuery());
        restoreElevateWord();
        return response;
    }

    public SuggestDeleteResponse deleteDocumentWords() {
        final long start = System.currentTimeMillis();

        final SuggestDeleteResponse deleteResponse =
                deleteByQuery(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(FieldNames.DOC_FREQ).gte(1))
                        .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()))
                        .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())));
        if (deleteResponse.hasError()) {
            throw new SuggestIndexException(deleteResponse.getErrors().get(0));
        }

        final List<SuggestItem> updateItems = new ArrayList<>();
        SearchResponse response = client.prepareSearch(index).setSize(500).setScroll(settings.getScrollTimeout())
                .setQuery(QueryBuilders.rangeQuery(FieldNames.DOC_FREQ).gte(1)).execute().actionGet(settings.getSearchTimeout());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {
                final SearchHit[] hits = response.getHits().getHits();
                if (hits.length == 0) {
                    break;
                }
                for (final SearchHit hit : hits) {
                    final SuggestItem item = SuggestItem.parseSource(hit.getSourceAsMap());
                    item.setDocFreq(0);
                    item.setKinds(Stream.of(item.getKinds()).filter(kind -> kind != SuggestItem.Kind.DOCUMENT)
                            .toArray(count -> new SuggestItem.Kind[count]));
                    updateItems.add(item);
                }
                final SuggestWriterResult result =
                        suggestWriter.write(client, settings, index, updateItems.toArray(new SuggestItem[updateItems.size()]), false);
                if (result.hasFailure()) {
                    throw new SuggestIndexException(result.getFailures().get(0));
                }

                response = client.prepareSearchScroll(scrollId).execute().actionGet(settings.getSearchTimeout());
                if (!scrollId.equals(response.getScrollId())) {
                    SuggestUtil.deleteScrollContext(client, scrollId);
                }
                scrollId = response.getScrollId();
            }
        } finally {
            SuggestUtil.deleteScrollContext(client, scrollId);
        }

        return new SuggestDeleteResponse(null, System.currentTimeMillis() - start);
    }

    public SuggestDeleteResponse deleteQueryWords() {
        final long start = System.currentTimeMillis();

        final SuggestDeleteResponse deleteResponse =
                deleteByQuery(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(FieldNames.QUERY_FREQ).gte(1))
                        .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()))
                        .mustNot(QueryBuilders.matchPhraseQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())));
        if (deleteResponse.hasError()) {
            throw new SuggestIndexException(deleteResponse.getErrors().get(0));
        }

        final List<SuggestItem> updateItems = new ArrayList<>();
        SearchResponse response = client.prepareSearch(index).setSize(500).setScroll(settings.getScrollTimeout())
                .setQuery(QueryBuilders.rangeQuery(FieldNames.QUERY_FREQ).gte(1)).execute().actionGet(settings.getSearchTimeout());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {
                final SearchHit[] hits = response.getHits().getHits();
                if (hits.length == 0) {
                    break;
                }
                for (final SearchHit hit : hits) {
                    final SuggestItem item = SuggestItem.parseSource(hit.getSourceAsMap());
                    item.setQueryFreq(0);
                    item.setKinds(Stream.of(item.getKinds()).filter(kind -> kind != SuggestItem.Kind.QUERY)
                            .toArray(count -> new SuggestItem.Kind[count]));
                    updateItems.add(item);
                }
                final SuggestWriterResult result =
                        suggestWriter.write(client, settings, index, updateItems.toArray(new SuggestItem[updateItems.size()]), false);
                if (result.hasFailure()) {
                    throw new SuggestIndexException(result.getFailures().get(0));
                }

                response = client.prepareSearchScroll(scrollId).execute().actionGet(settings.getSearchTimeout());
                if (!scrollId.equals(response.getScrollId())) {
                    SuggestUtil.deleteScrollContext(client, scrollId);
                }
                scrollId = response.getScrollId();
            }
        } finally {
            SuggestUtil.deleteScrollContext(client, scrollId);
        }

        return new SuggestDeleteResponse(null, System.currentTimeMillis() - start);
    }

    public SuggestIndexResponse indexFromQueryLog(final QueryLog queryLog) {
        return indexFromQueryLog(new QueryLog[] { queryLog });
    }

    public SuggestIndexResponse indexFromQueryLog(final QueryLog[] queryLogs) {
        if (logger.isLoggable(Level.INFO)) {
            logger.info(() -> String.format("Index from querylog. num:%s", queryLogs.length));
        }
        try {
            final long start = System.currentTimeMillis();
            final Stream<QueryLog> stream = Stream.of(queryLogs);
            if (parallel) {
                stream.parallel();
            }
            final SuggestItem[] array = stream
                    .flatMap(queryLog -> contentsParser
                            .parseQueryLog(queryLog, supportedFields, tagFieldNames, roleFieldName, readingConverter, normalizer).stream())
                    .toArray(n -> new SuggestItem[n]);
            final SuggestIndexResponse response = index(array);
            return new SuggestIndexResponse(array.length, queryLogs.length, response.getErrors(), System.currentTimeMillis() - start);
        } catch (final Exception e) {
            throw new SuggestIndexException("Failed to index from query_string.", e);
        }
    }

    // TODO replace queryLogReader with lambda reader
    public Deferred<SuggestIndexResponse>.Promise indexFromQueryLog(final QueryLogReader queryLogReader, final int docPerReq,
            final long requestInterval) {
        final Deferred<SuggestIndexResponse> deferred = new Deferred<>();
        threadPool.execute(() -> {
            final long start = System.currentTimeMillis();
            int numberOfSuggestDocs = 0;
            int numberOfInputDocs = 0;
            final List<Throwable> errors = new ArrayList<>();

            final List<QueryLog> queryLogs = new ArrayList<>(docPerReq);
            try {
                QueryLog queryLog = queryLogReader.read();
                while (queryLog != null) {
                    if (Thread.currentThread().isInterrupted()) {
                        break;
                    }
                    queryLogs.add(queryLog);
                    queryLog = queryLogReader.read();
                    if (queryLog == null && !queryLogs.isEmpty() || queryLogs.size() >= docPerReq) {
                        final SuggestIndexResponse res = indexFromQueryLog(queryLogs.toArray(new QueryLog[queryLogs.size()]));
                        errors.addAll(res.getErrors());
                        numberOfSuggestDocs += res.getNumberOfSuggestDocs();
                        numberOfInputDocs += res.getNumberOfInputDocs();
                        queryLogs.clear();

                        Thread.sleep(requestInterval);
                    }
                }
                deferred.resolve(
                        new SuggestIndexResponse(numberOfSuggestDocs, numberOfInputDocs, errors, System.currentTimeMillis() - start));
            } catch (final Throwable t) {
                deferred.reject(t);
            } finally {
                queryLogReader.close();
            }
        });
        return deferred.promise();
    }

    public SuggestIndexResponse indexFromDocument(final Map<String, Object>[] documents) {
        final long start = System.currentTimeMillis();
        try {
            final Stream<Map<String, Object>> stream = Stream.of(documents);
            if (parallel) {
                stream.parallel();
            }
            final SuggestItem[] array = stream.flatMap(document -> {
                try {
                    return contentsParser.parseDocument(document, supportedFields, tagFieldNames, roleFieldName, langFieldName,
                            readingConverter, contentsReadingConverter, normalizer, analyzer).stream();
                } catch (OpenSearchStatusException | IllegalStateException e) {
                    final String msg = e.getMessage();
                    if (StringUtil.isNotEmpty(msg) || msg.contains("index.analyze.max_token_count")) {
                        if (logger.isLoggable(Level.WARNING)) {
                            logger.warning(() -> String.format("Failed to parse document. %s", msg));
                        }
                        return Stream.empty();
                    }
                    throw e;
                }
            }).toArray(n -> new SuggestItem[n]);
            final long parseTime = System.currentTimeMillis();
            final SuggestIndexResponse response = index(array);
            final long indexTime = System.currentTimeMillis();

            if (logger.isLoggable(Level.INFO)) {
                final OperatingSystemMXBean operatingSystemMXBean =
                        (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
                final double cpuLoad = operatingSystemMXBean.getProcessCpuLoad();
                final long maxMemory = Runtime.getRuntime().maxMemory();
                logger.info(String.format(
                        "Index %d words from %d documents. (parse:{%dmsec}, index:{%dmsec}, Cpu:{%f}, Mem:{heap:%dMB, used:%dMB)",
                        array.length, documents.length, parseTime - start, indexTime - start, cpuLoad, maxMemory / (1024 * 1024),
                        (maxMemory - Runtime.getRuntime().freeMemory()) / (1024 * 1024)));
            }
            return new SuggestIndexResponse(array.length, documents.length, response.getErrors(), indexTime - start);
        } catch (final Exception e) {
            logger.log(Level.SEVERE, "Failed to index from document", e);
            throw new SuggestIndexException("Failed to index from document", e);
        }
    }

    @Deprecated
    public Deferred<SuggestIndexResponse>.Promise indexFromDocument(final Supplier<DocumentReader> reader, final int docPerReq,
            final long requestInterval) {
        return indexFromDocument(reader, docPerReq, () -> ThreadUtil.sleep(requestInterval));
    }

    public Deferred<SuggestIndexResponse>.Promise indexFromDocument(final Supplier<DocumentReader> reader, final int docPerReq,
            final Runnable waitController) {
        if (logger.isLoggable(Level.INFO)) {
            logger.info(() -> String.format("Start index by DocumentReader. docPerReq:%s", docPerReq));
        }
        final Deferred<SuggestIndexResponse> deferred = new Deferred<>();
        threadPool.execute(() -> {
            final long start = System.currentTimeMillis();
            int numberOfSuggestDocs = 0;
            int numberOfInputDocs = 0;

            final List<Throwable> errors = new ArrayList<>();
            final List<Map<String, Object>> docs = new ArrayList<>(docPerReq);
            try (final DocumentReader documentReader = reader.get()) {
                Map<String, Object> doc = documentReader.read();
                while (doc != null) {
                    if (Thread.currentThread().isInterrupted()) {
                        break;
                    }
                    docs.add(doc);
                    doc = documentReader.read();
                    if (doc == null || docs.size() >= docPerReq) {
                        final SuggestIndexResponse res = indexFromDocument(docs.toArray(new Map[docs.size()]));
                        errors.addAll(res.getErrors());
                        numberOfSuggestDocs += res.getNumberOfSuggestDocs();
                        numberOfInputDocs += res.getNumberOfInputDocs();
                        client.admin().indices().prepareRefresh(index).execute().actionGet(settings.getIndicesTimeout());
                        docs.clear();

                        waitController.run();
                    }
                }

                deferred.resolve(
                        new SuggestIndexResponse(numberOfSuggestDocs, numberOfInputDocs, errors, System.currentTimeMillis() - start));
            } catch (final Throwable t) {
                deferred.reject(t);
            }
        });
        return deferred.promise();
    }

    public SuggestIndexResponse indexFromSearchWord(final String searchWord, final String[] fields, final String[] tags,
            final String[] roles, final int num, final String[] langs) {
        if (logger.isLoggable(Level.FINE)) {
            logger.fine(() -> String.format("Index from searchWord. word:%s", searchWord));
        }

        final long start = System.currentTimeMillis();
        final StringBuilder buf = new StringBuilder(searchWord.length());
        char prev = 0;
        for (final char c : searchWord.toCharArray()) {
            if (!Character.isWhitespace(c)) {
                buf.append(c);
            } else if (!Character.isWhitespace(prev)) {
                buf.append(' ');
            }
            prev = c;
        }
        final String[] words = buf.toString().trim().split(" ");
        try {
            final SuggestItem item =
                    contentsParser.parseSearchWords(words, null, fields, tags, roles, num, readingConverter, normalizer, analyzer, langs);
            if (item == null) {
                return new SuggestIndexResponse(0, 1, null, System.currentTimeMillis() - start);
            }
            final SuggestIndexResponse response = index(item);
            return new SuggestIndexResponse(1, 1, response.getErrors(), System.currentTimeMillis() - start);
        } catch (final Exception e) {
            final String msg = "Failed to index from document: searchWord=" + searchWord//
                    + ", fields=" + Arrays.toString(fields)//
                    + ", tags=" + Arrays.toString(tags)//
                    + ", roles=" + Arrays.toString(roles)//
                    + ", langs=" + Arrays.toString(langs)//
                    + ", num=" + num;
            if (logger.isLoggable(Level.WARNING)) {
                logger.warning(msg);
            }
            throw new SuggestIndexException(msg, e);
        }
    }

    public SuggestDeleteResponse addBadWord(final String badWord, final boolean apply) {
        final String normalized = normalizer.normalize(badWord, "");
        settings.badword().add(normalized);
        badWords = settings.badword().get(true);
        if (apply) {
            return deleteByQuery(QueryBuilders.wildcardQuery(FieldNames.TEXT, "*" + normalized + "*"));
        }
        return new SuggestDeleteResponse(null, 0);
    }

    public void deleteBadWord(final String badWord) {
        settings.badword().delete(normalizer.normalize(badWord, ""));
    }

    public SuggestIndexResponse addElevateWord(final ElevateWord elevateWord, final boolean apply) {
        final String normalizedWord = normalizer.normalize(elevateWord.getElevateWord(), "");
        final List<String> normalizedReadings =
                elevateWord.getReadings().stream().map(reading -> normalizer.normalize(reading, "")).collect(Collectors.toList());
        final ElevateWord normalized = new ElevateWord(normalizedWord, elevateWord.getBoost(), normalizedReadings, elevateWord.getFields(),
                elevateWord.getTags(), elevateWord.getRoles());
        settings.elevateWord().add(normalized);
        if (apply) {
            return index(normalized.toSuggestItem());
        }
        return new SuggestIndexResponse(0, 0, null, 0);
    }

    public SuggestDeleteResponse deleteElevateWord(final String elevateWord, final boolean apply) {
        final String normalized = normalizer.normalize(elevateWord, "");
        settings.elevateWord().delete(normalized);
        if (apply) {
            return delete(SuggestUtil.createSuggestTextId(normalized));
        }
        return new SuggestDeleteResponse(null, 0);
    }

    public SuggestIndexResponse restoreElevateWord() {
        final long start = System.currentTimeMillis();
        int numberOfSuggestDocs = 0;
        int numberOfInputDocs = 0;

        final ElevateWord[] elevateWords = settings.elevateWord().get();
        final List<Throwable> errors = new ArrayList<>(elevateWords.length);
        for (final ElevateWord elevateWord : elevateWords) {
            final SuggestIndexResponse res = addElevateWord(elevateWord, true);
            numberOfSuggestDocs += res.getNumberOfSuggestDocs();
            numberOfInputDocs += res.getNumberOfInputDocs();
            errors.addAll(res.getErrors());
        }
        return new SuggestIndexResponse(numberOfSuggestDocs, numberOfInputDocs, errors, System.currentTimeMillis() - start);
    }

    public SuggestDeleteResponse deleteOldWords(final ZonedDateTime threshold) {
        final long start = System.currentTimeMillis();
        final String query = FieldNames.TIMESTAMP + ":[* TO " + threshold.toInstant().toEpochMilli() + "] NOT " + FieldNames.KINDS + ':'
                + SuggestItem.Kind.USER;
        deleteByQuery(query);
        return new SuggestDeleteResponse(null, System.currentTimeMillis() - start);
    }

    public SuggestIndexer setIndexName(final String index) {
        this.index = index;
        return this;
    }

    public SuggestIndexer setSupportedFields(final String[] supportedFields) {
        this.supportedFields = supportedFields;
        return this;
    }

    public SuggestIndexer setTagFieldNames(final String[] tagFieldNames) {
        this.tagFieldNames = tagFieldNames;
        return this;
    }

    public SuggestIndexer setRoleFieldName(final String roleFieldName) {
        this.roleFieldName = roleFieldName;
        return this;
    }

    public SuggestIndexer setReadingConverter(final ReadingConverter readingConverter) {
        this.readingConverter = readingConverter;
        return this;
    }

    public SuggestIndexer setNormalizer(final Normalizer normalizer) {
        this.normalizer = normalizer;
        return this;
    }

    public SuggestIndexer setAnalyzer(final SuggestAnalyzer analyzer) {
        this.analyzer = analyzer;
        return this;
    }

    public SuggestIndexer setContentsParser(final ContentsParser contentsParser) {
        this.contentsParser = contentsParser;
        return this;
    }

    public SuggestIndexer setSuggestWriter(final SuggestWriter suggestWriter) {
        this.suggestWriter = suggestWriter;
        return this;
    }

} public interface Normalizer {
    String normalize(String text, String field, String... langs);
} public class PopularWordsRequestBuilder extends RequestBuilder<PopularWordsRequest, PopularWordsResponse> {
    public PopularWordsRequestBuilder(final Client client) {
        super(client, new PopularWordsRequest());
    }

    public PopularWordsRequestBuilder setIndex(final String index) {
        request.setIndex(index);
        return this;
    }

    public PopularWordsRequestBuilder setType(final String type) {
        request.setType(type);
        return this;
    }

    public PopularWordsRequestBuilder setSize(final int size) {
        request.setSize(size);
        return this;
    }

    public PopularWordsRequestBuilder addTag(final String tag) {
        request.addTag(tag);
        return this;
    }

    public PopularWordsRequestBuilder addRole(final String role) {
        request.addRole(role);
        return this;
    }

    public PopularWordsRequestBuilder addField(final String field) {
        request.addField(field);
        return this;
    }

    public PopularWordsRequestBuilder addLanguage(final String lang) {
        request.addLanguage(lang);
        return this;
    }

    public PopularWordsRequestBuilder setSeed(final String seed) {
        request.setSeed(seed);
        return this;
    }

    public PopularWordsRequestBuilder setWindowSize(final int windowSize) {
        request.setWindowSize(windowSize);
        return this;
    }

    public PopularWordsRequestBuilder addExcludeWord(final String excludeWord) {
        request.addExcludeWord(excludeWord);
        return this;
    }

    public PopularWordsRequestBuilder setQueryFreqThreshold(final int queryFreqThreshold) {
        request.setQueryFreqThreshold(queryFreqThreshold);
        return this;
    }
} public class SuggestRequestBuilder extends RequestBuilder<SuggestRequest, SuggestResponse> {
    public SuggestRequestBuilder(final Client client, final ReadingConverter readingConverter, final Normalizer normalizer) {
        super(client, new SuggestRequest());
        request.setReadingConverter(readingConverter);
        request.setNormalizer(normalizer);
    }

    public SuggestRequestBuilder setIndex(final String index) {
        request.setIndex(index);
        return this;
    }

    public SuggestRequestBuilder setType(final String type) {
        request.setType(type);
        return this;
    }

    public SuggestRequestBuilder setSize(final int size) {
        request.setSize(size);
        return this;
    }

    public SuggestRequestBuilder setQuery(final String query) {
        request.setQuery(query);
        return this;
    }

    public SuggestRequestBuilder addTag(final String tag) {
        request.addTag(tag);
        return this;
    }

    public SuggestRequestBuilder addRole(final String role) {
        request.addRole(role);
        return this;
    }

    public SuggestRequestBuilder addField(final String field) {
        request.addField(field);
        return this;
    }

    public SuggestRequestBuilder addKind(final String kind) {
        request.addKind(kind);
        return this;
    }

    public SuggestRequestBuilder setSuggestDetail(final boolean suggestDetail) {
        request.setSuggestDetail(suggestDetail);
        return this;
    }

    public SuggestRequestBuilder setPrefixMatchWeight(final float prefixMatchWeight) {
        request.setPrefixMatchWeight(prefixMatchWeight);
        return this;
    }

    public SuggestRequestBuilder setMatchWordFirst(final boolean matchWordFirst) {
        request.setMatchWordFirst(matchWordFirst);
        return this;
    }

    public SuggestRequestBuilder setSkipDuplicateWords(final boolean skipDuplicateWords) {
        request.setSkipDuplicateWords(skipDuplicateWords);
        return this;
    }

    public SuggestRequestBuilder addLang(final String lang) {
        request.addLang(lang);
        return this;
    }
} public class SuggestSettings {
    private static final Logger logger = Logger.getLogger(SuggestSettings.class.getName());

    protected final String settingsId;

    protected final Client client;

    protected final String settingsIndexName;

    protected final Map<String, Object> initialSettings;

    protected boolean initialized = false;

    protected final String badWordIndexName;
    protected final String elevateWordIndexName;

    protected TimeoutSettings timeoutSettings;

    public static class TimeoutSettings {
        protected String searchTimeout = "15s";
        protected String indexTimeout = "1m";
        protected String bulkTimeout = "1m";
        protected String indicesTimeout = "1m";
        protected String clusterTimeout = "1m";
        protected String scrollTimeout = "1m";
    }

    public SuggestSettings(final Client client, final String settingsId, final Map<String, Object> initialSettings,
            final String settingsIndexName, final TimeoutSettings timeoutSettings) {
        this.client = client;
        this.settingsId = settingsId;
        this.settingsIndexName = settingsIndexName;
        this.initialSettings = initialSettings;
        this.timeoutSettings = timeoutSettings;

        this.badWordIndexName = settingsIndexName + "-badword";
        this.elevateWordIndexName = settingsIndexName + "-elevateword";
    }

    public void init() {
        if (initialized) {
            return;
        }
        initialized = true;
        initialize(initialSettings);
        new AnalyzerSettings(client, this, settingsIndexName).init();
    }

    private void initialize(final Map<String, Object> initialSettings) {
        boolean doIndexCreate = false;
        boolean doCreate = false;
        try {
            final GetResponse getResponse =
                    client.prepareGet().setIndex(settingsIndexName).setId(settingsId).execute().actionGet(getSearchTimeout());

            if (!getResponse.isExists()) {
                doCreate = true;
            }
        } catch (final IndexNotFoundException e) {
            doIndexCreate = true;
            doCreate = true;
        }

        if (doCreate) {
            if (doIndexCreate) {
                try {
                    client.admin().indices().prepareCreate(settingsIndexName).setSettings(loadIndexSettings(), XContentType.JSON).execute()
                            .actionGet(getIndicesTimeout());
                } catch (final IOException e) {
                    throw new SuggesterException(e);
                }
            }

            final List<Tuple<String, Object>> arraySettings = new ArrayList<>();
            final Map<String, Object> defaultSettings = defaultSettings();
            initialSettings.forEach((key, value) -> {
                if (value instanceof Collection) {
                    @SuppressWarnings("unchecked")
                    final Collection<Object> collection = (Collection<Object>) value;
                    collection.forEach(element -> arraySettings.add(new Tuple<>(key, element)));
                } else if (value instanceof Object[]) {
                    for (final Object element : (Object[]) value) {
                        arraySettings.add(new Tuple<>(key, element));
                    }
                } else {
                    defaultSettings.put(key, value);
                }
            });
            set(defaultSettings);

            final List<Tuple<String, Object>> defaultArraySettings = defaultArraySettings();
            defaultArraySettings.addAll(arraySettings);
            defaultArraySettings.forEach(t -> array().add(t.v1(), t.v2()));
        }
    }

    public Object get(final String key) {
        final GetResponse getResponse =
                client.prepareGet().setIndex(settingsIndexName).setId(settingsId).execute().actionGet(getSearchTimeout());
        if (!getResponse.isExists()) {
            return null;
        }
        final Map<String, Object> map = getResponse.getSource();
        return map.get(key);
    }

    public String getAsString(final String key, final String defaultValue) {
        final Object obj = get(key);

        final String value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = obj.toString();
        }

        return value;
    }

    public int getAsInt(final String key, final int defaultValue) {
        final Object obj = get(key);

        final int value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Integer.parseInt(obj.toString());
        }

        return value;
    }

    public long getAsLong(final String key, final long defaultValue) {
        final Object obj = get(key);

        final long value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Long.parseLong(obj.toString());
        }

        return value;
    }

    public float getAsFloat(final String key, final float defaultValue) {
        final Object obj = get(key);

        final float value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Float.parseFloat(obj.toString());
        }

        return value;
    }

    public boolean getAsBoolean(final String key, final boolean defaultValue) {
        final Object obj = get(key);

        final boolean value;
        if (obj == null) {
            value = defaultValue;
        } else {
            value = Boolean.parseBoolean(obj.toString());
        }

        return value;
    }

    public void set(final String key, final Object value) {
        if (logger.isLoggable(Level.FINER)) {
            logger.finer(() -> String.format("Set suggest settings. %s key: %s value: %s", settingsIndexName, key, value));
        }
        try {
            client.prepareUpdate().setIndex(settingsIndexName).setId(settingsId).setDocAsUpsert(true).setDoc(key, value)
                    .setRetryOnConflict(5).execute().actionGet(getIndexTimeout());
            client.admin().indices().prepareRefresh().setIndices(settingsIndexName).execute().actionGet(getIndicesTimeout());
        } catch (final Exception e) {
            throw new SuggestSettingsException("Failed to update suggestSettings.", e);
        }
    }

    public void set(final Map<String, Object> map) {
        if (logger.isLoggable(Level.FINER)) {
            logger.finer(() -> String.format("Set suggest settings. %s %s", settingsIndexName, map.toString()));
        }
        try {
            final XContentBuilder builder = JsonXContent.contentBuilder().map(map);
            builder.flush();
            client.prepareUpdate().setIndex(settingsIndexName).setId(settingsId).setDocAsUpsert(true).setDoc(builder).setRetryOnConflict(5)
                    .execute().actionGet(getIndexTimeout());
            client.admin().indices().prepareRefresh().setIndices(settingsIndexName).execute().actionGet(getIndicesTimeout());
        } catch (final Exception e) {
            throw new SuggestSettingsException("Failed to update suggestSettings.", e);
        }
    }

    public ArraySettings array() {
        return new ArraySettings(this, client, settingsIndexName, settingsId);
    }

    public AnalyzerSettings analyzer() {
        return new AnalyzerSettings(client, this, settingsIndexName);
    }

    public BadWordSettings badword() {
        return new BadWordSettings(this, client, settingsIndexName, settingsId);
    }

    public ElevateWordSettings elevateWord() {
        return new ElevateWordSettings(this, client, settingsIndexName, settingsId);
    }

    public String getSettingsIndexName() {
        return settingsIndexName;
    }

    public boolean isInitialized() {
        return initialized;
    }

    public String getSettingsId() {
        return settingsId;
    }

    private Map<String, Object> defaultSettings() {
        final Map<String, Object> defaultSettings = new HashMap<>();
        defaultSettings.put(DefaultKeys.INDEX, (settingsId + ".suggest").toLowerCase());
        defaultSettings.put(DefaultKeys.TAG_FIELD_NAME, "label,virtual_host");
        defaultSettings.put(DefaultKeys.ROLE_FIELD_NAME, "role");
        defaultSettings.put(DefaultKeys.LANG_FIELD_NAME, "lang");
        defaultSettings.put(DefaultKeys.PARALLEL_PROCESSING, Boolean.FALSE);
        return defaultSettings;
    }

    private List<Tuple<String, Object>> defaultArraySettings() {
        final List<Tuple<String, Object>> tuples = new ArrayList<>();
        tuples.add(new Tuple<>(DefaultKeys.SUPPORTED_FIELDS, "content"));
        return tuples;
    }

    protected String loadIndexSettings() throws IOException {
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br = new BufferedReader(
                new InputStreamReader(this.getClass().getClassLoader().getResourceAsStream("suggest_indices/suggest_settings.json")));) {

            String line;
            while ((line = br.readLine()) != null) {
                sb.append(line);
            }
        }
        return sb.toString().replaceAll(Pattern.quote("${fess.dictionary.path}"), dictionaryPath);
    }

    public static SuggestSettingsBuilder builder() {
        return new SuggestSettingsBuilder();
    }

    public static class DefaultKeys {
        public static final String INDEX = "index";
        public static final String SUPPORTED_FIELDS = "supportedFields";
        public static final String TAG_FIELD_NAME = "tagFieldName";
        public static final String ROLE_FIELD_NAME = "roleFieldName";
        public static final String LANG_FIELD_NAME = "langFieldName";
        public static final String PARALLEL_PROCESSING = "parallel";
        public static final String MAX_CONTENT_LENGTH = "maxContextLength";

        private DefaultKeys() {
        }
    }

    public String getScrollTimeout() {
        return timeoutSettings.scrollTimeout;
    }

    public String getSearchTimeout() {
        return timeoutSettings.searchTimeout;
    }

    public String getIndexTimeout() {
        return timeoutSettings.indexTimeout;
    }

    public String getIndicesTimeout() {
        return timeoutSettings.indicesTimeout;
    }

    public String getBulkTimeout() {
        return timeoutSettings.bulkTimeout;
    }

    public String getClusterTimeout() {
        return timeoutSettings.clusterTimeout;
    }
}
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.suggest.analysis.SuggestAnalyzer;
import org.codelibs.fess.suggest.constants.FieldNames;
import org.codelibs.fess.suggest.constants.SuggestConstants;
import org.codelibs.fess.suggest.converter.ReadingConverter;
import org.codelibs.fess.suggest.exception.SuggesterException;
import org.codelibs.fess.suggest.index.SuggestIndexer;
import org.codelibs.fess.suggest.normalizer.Normalizer;
import org.codelibs.fess.suggest.request.popularwords.PopularWordsRequestBuilder;
import org.codelibs.fess.suggest.request.suggest.SuggestRequestBuilder;
import org.codelibs.fess.suggest.settings.SuggestSettings;
import org.opensearch.action.admin.indices.alias.Alias;
import org.opensearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.opensearch.action.admin.indices.get.GetIndexResponse;
import org.opensearch.action.admin.indices.refresh.RefreshResponse;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
                throw new SuggesterException("Could not create next index: " + indexName);
            }
            client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(suggestSettings.getClusterTimeout());

            final IndicesAliasesRequestBuilder aliasesRequestBuilder =
                    client.admin().indices().prepareAliases().addAlias(indexName, getUpdateAlias(index));
            for (final String prevIndex : prevIndices) {
                aliasesRequestBuilder.removeAlias(prevIndex, getUpdateAlias(index));
            }
            aliasesRequestBuilder.execute().actionGet(suggestSettings.getIndicesTimeout());
        } catch (final Exception e) {
            logger.log(Level.SEVERE, "Failed to replace with new index.", e);
            throw new SuggesterException("Failed to replace with new index.", e);
        }
    }

    public void switchIndex() {
        try {
            final List<String> updateIndices = new ArrayList<>();
            final String updateAlias = getUpdateAlias(index);
            final IndicesExistsResponse updateIndicesResponse =
                    client.admin().indices().prepareExists(updateAlias).execute().actionGet(suggestSettings.getIndicesTimeout());
            if (updateIndicesResponse.isExists()) {
                final GetAliasesResponse getAliasesResponse =
                        client.admin().indices().prepareGetAliases(updateAlias).execute().actionGet(suggestSettings.getIndicesTimeout());
                getAliasesResponse.getAliases()
                        .forEach(x -> x.value.stream().filter(y -> updateAlias.equals(y.alias())).forEach(y -> updateIndices.add(x.key)));
            }
            if (updateIndices.size() != 1) {
                logger.severe(() -> String.format("Unexpected update indices num:%s", updateIndices.size()));
                throw new SuggesterException("Unexpected update indices num:" + updateIndices.size());
            }
            final String updateIndex = updateIndices.get(0);

            final List<String> searchIndices = new ArrayList<>();
            final String searchAlias = getSearchAlias(index);
            final IndicesExistsResponse searchIndicesResponse =
                    client.admin().indices().prepareExists(searchAlias).execute().actionGet(suggestSettings.getIndicesTimeout());
            if (searchIndicesResponse.isExists()) {
                final GetAliasesResponse getAliasesResponse =
                        client.admin().indices().prepareGetAliases(searchAlias).execute().actionGet(suggestSettings.getIndicesTimeout());
                getAliasesResponse.getAliases()
                        .forEach(x -> x.value.stream().filter(y -> searchAlias.equals(y.alias())).forEach(y -> searchIndices.add(x.key)));
            }
            if (searchIndices.size() != 1) {
                logger.severe(() -> String.format("Unexpected update indices num:%s", searchIndices.size()));
                throw new SuggesterException("Unexpected search indices num:" + searchIndices.size());
            }
            final String searchIndex = searchIndices.get(0);

            if (updateIndex.equals(searchIndex)) {
                return;
            }

            if (logger.isLoggable(Level.INFO)) {
                logger.info(() -> String.format("Switch suggest.search index. %s => %s", searchIndex, updateIndex));
            }
            client.admin().indices().prepareAliases().removeAlias(searchIndex, searchAlias).addAlias(updateIndex, searchAlias).execute()
                    .actionGet(suggestSettings.getIndicesTimeout());
        } catch (final Exception e) {
            logger.log(Level.SEVERE, "Failed to switch index.", e);
            throw new SuggesterException("Failed to switch index.", e);
        }
    }

    public void removeDisableIndices() {
        final GetIndexResponse response =
                client.admin().indices().prepareGetIndex().addIndices("*").execute().actionGet(suggestSettings.getIndicesTimeout());
        Stream.of(response.getIndices()).filter(s -> {
            if (!isSuggestIndex(s)) {
                return false;
            }
            final List<AliasMetadata> list = response.getAliases().get(s);
            if (list == null) {
                return true;
            }
            return list.isEmpty();
        }).forEach(s -> {
            if (logger.isLoggable(Level.INFO)) {
                logger.info(() -> String.format("Delete index: %s", s));
            }
            client.admin().indices().prepareDelete(s).execute().actionGet(suggestSettings.getIndicesTimeout());
        });
    }

    public SuggestIndexer indexer() {
        return createDefaultIndexer();
    }

    public static SuggesterBuilder builder() {
        return new SuggesterBuilder();
    }

    // getter
    public SuggestSettings settings() {
        return suggestSettings;
    }

    public ReadingConverter getReadingConverter() {
        return readingConverter;
    }

    public Normalizer getNormalizer() {
        return normalizer;
    }

    protected SuggestIndexer createDefaultIndexer() {
        return new SuggestIndexer(client, getUpdateAlias(index), readingConverter, contentsReadingConverter, normalizer, analyzer,
                suggestSettings, threadPool);
    }

    public String getIndex() {
        return index;
    }

    public long getAllWordsNum() {
        return getNum(QueryBuilders.matchAllQuery());
    }

    public long getDocumentWordsNum() {
        return getNum(QueryBuilders.rangeQuery(FieldNames.DOC_FREQ).gte(1));