// Path: src/test/java/org/codelibs/opensearch/extension/kuromoji/index/analysis/KuromojiTokenizerFactory.java
// public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
// 
//     private static final String USER_DICT_PATH_OPTION = "user_dictionary";
//     private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules";
//     private static final String NBEST_COST = "nbest_cost";
//     private static final String NBEST_EXAMPLES = "nbest_examples";
// 
//     private final UserDictionary userDictionary;
//     private final Mode mode;
//     private final String nBestExamples;
//     private final int nBestCost;
// 
//     private boolean discartPunctuation;
// 
//     public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
//         super(indexSettings, settings, name);
//         mode = getMode(settings);
//         userDictionary = getUserDictionary(env, settings);
//         discartPunctuation = settings.getAsBoolean("discard_punctuation", true);
//         nBestCost = settings.getAsInt(NBEST_COST, -1);
//         nBestExamples = settings.get(NBEST_EXAMPLES);
//     }
// 
//     public static UserDictionary getUserDictionary(Environment env, Settings settings) {
//         if (settings.get(USER_DICT_PATH_OPTION) != null && settings.get(USER_DICT_RULES_OPTION) != null) {
//             throw new IllegalArgumentException(
//                     "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]");
//         }
//         try {
//             List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false);
//             if (ruleList == null || ruleList.isEmpty()) {
//                 return null;
//             }
//             Set<String> dup = new HashSet<>();
//             int lineNum = 0;
//             for (String line : ruleList) {
//                 // ignore comments
//                 if (line.startsWith("#") == false) {
//                     String[] values = CSVUtil.parse(line);
//                     if (dup.add(values[0]) == false) {
//                         throw new IllegalArgumentException(
//                                 "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + lineNum + "]");
//                     }
//                 }
//                 ++lineNum;
//             }
//             StringBuilder sb = new StringBuilder();
//             for (String line : ruleList) {
//                 sb.append(line).append(System.lineSeparator());
//             }
//             return UserDictionary.open(new StringReader(sb.toString()));
//         } catch (IOException e) {
//             throw new OpenSearchException("failed to load kuromoji user dictionary", e);
//         }
//     }
// 
//     public static JapaneseTokenizer.Mode getMode(Settings settings) {
//         JapaneseTokenizer.Mode mode = JapaneseTokenizer.DEFAULT_MODE;
//         String modeSetting = settings.get("mode", null);
//         if (modeSetting != null) {
//             if ("search".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.SEARCH;
//             } else if ("normal".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.NORMAL;
//             } else if ("extended".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.EXTENDED;
//             }
//         }
//         return mode;
//     }
// 
//     @Override
//     public Tokenizer create() {
//         JapaneseTokenizer t = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
//         int nBestCost = this.nBestCost;
//         if (nBestExamples != null) {
//             nBestCost = Math.max(nBestCost, t.calcNBestCost(nBestExamples));
//         }
//         t.setNBestCost(nBestCost);
//         return t;
//     }
// 
// }


import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.EnumMap;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
import org.apache.lucene.analysis.ja.dict.Dictionary;
import org.apache.lucene.analysis.ja.dict.TokenInfoFST;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.util.AttributeSource;
import org.codelibs.opensearch.extension.kuromoji.index.analysis.KuromojiTokenizerFactory;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.env.Environment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.analysis.AbstractTokenizerFactory;

    protected final Field userFSTField;

    protected final Field userFSTReaderField;

    protected final Field dictionaryMapField;

    private final Environment env;

    private final Settings settings;

    private File reloadableFile = null;

    protected volatile long dictionaryTimestamp;

    private volatile long lastChecked;

    private long reloadInterval;

    private volatile UserDictionary userDictionary;

    private final Mode mode;

    private final boolean discartPunctuation;

    public ReloadableKuromojiTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name,
            final Settings settings) {
        super(indexSettings, settings, name);
        this.env = env;
        this.settings = settings;
// Path: src/test/java/org/codelibs/opensearch/extension/kuromoji/index/analysis/KuromojiTokenizerFactory.java
// public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
// 
//     private static final String USER_DICT_PATH_OPTION = "user_dictionary";
//     private static final String USER_DICT_RULES_OPTION = "user_dictionary_rules";
//     private static final String NBEST_COST = "nbest_cost";
//     private static final String NBEST_EXAMPLES = "nbest_examples";
// 
//     private final UserDictionary userDictionary;
//     private final Mode mode;
//     private final String nBestExamples;
//     private final int nBestCost;
// 
//     private boolean discartPunctuation;
// 
//     public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
//         super(indexSettings, settings, name);
//         mode = getMode(settings);
//         userDictionary = getUserDictionary(env, settings);
//         discartPunctuation = settings.getAsBoolean("discard_punctuation", true);
//         nBestCost = settings.getAsInt(NBEST_COST, -1);
//         nBestExamples = settings.get(NBEST_EXAMPLES);
//     }
// 
//     public static UserDictionary getUserDictionary(Environment env, Settings settings) {
//         if (settings.get(USER_DICT_PATH_OPTION) != null && settings.get(USER_DICT_RULES_OPTION) != null) {
//             throw new IllegalArgumentException(
//                     "It is not allowed to use [" + USER_DICT_PATH_OPTION + "] in conjunction" + " with [" + USER_DICT_RULES_OPTION + "]");
//         }
//         try {
//             List<String> ruleList = Analysis.getWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, false);
//             if (ruleList == null || ruleList.isEmpty()) {
//                 return null;
//             }
//             Set<String> dup = new HashSet<>();
//             int lineNum = 0;
//             for (String line : ruleList) {
//                 // ignore comments
//                 if (line.startsWith("#") == false) {
//                     String[] values = CSVUtil.parse(line);
//                     if (dup.add(values[0]) == false) {
//                         throw new IllegalArgumentException(
//                                 "Found duplicate term [" + values[0] + "] in user dictionary " + "at line [" + lineNum + "]");
//                     }
//                 }
//                 ++lineNum;
//             }
//             StringBuilder sb = new StringBuilder();
//             for (String line : ruleList) {
//                 sb.append(line).append(System.lineSeparator());
//             }
//             return UserDictionary.open(new StringReader(sb.toString()));
//         } catch (IOException e) {
//             throw new OpenSearchException("failed to load kuromoji user dictionary", e);
//         }
//     }
// 
//     public static JapaneseTokenizer.Mode getMode(Settings settings) {
//         JapaneseTokenizer.Mode mode = JapaneseTokenizer.DEFAULT_MODE;
//         String modeSetting = settings.get("mode", null);
//         if (modeSetting != null) {
//             if ("search".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.SEARCH;
//             } else if ("normal".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.NORMAL;
//             } else if ("extended".equalsIgnoreCase(modeSetting)) {
//                 mode = JapaneseTokenizer.Mode.EXTENDED;
//             }
//         }
//         return mode;
//     }
// 
//     @Override
//     public Tokenizer create() {
//         JapaneseTokenizer t = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
//         int nBestCost = this.nBestCost;
//         if (nBestExamples != null) {
//             nBestCost = Math.max(nBestCost, t.calcNBestCost(nBestExamples));
//         }
//         t.setNBestCost(nBestCost);
//         return t;
//     }
// 
// }


// Path: src/test/java/org/codelibs/opensearch/extension/analysis/ReloadableKuromojiTokenizerFactory.java
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.EnumMap;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
import org.apache.lucene.analysis.ja.dict.Dictionary;
import org.apache.lucene.analysis.ja.dict.TokenInfoFST;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.util.AttributeSource;
import org.codelibs.opensearch.extension.kuromoji.index.analysis.KuromojiTokenizerFactory;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.env.Environment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.analysis.AbstractTokenizerFactory;


    protected final Field userFSTField;

    protected final Field userFSTReaderField;

    protected final Field dictionaryMapField;

    private final Environment env;

    private final Settings settings;

    private File reloadableFile = null;

    protected volatile long dictionaryTimestamp;

    private volatile long lastChecked;

    private long reloadInterval;

    private volatile UserDictionary userDictionary;

    private final Mode mode;

    private final boolean discartPunctuation;

    public ReloadableKuromojiTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name,
            final Settings settings) {
        super(indexSettings, settings, name);
        this.env = env;
        this.settings = settings;
        mode = KuromojiTokenizerFactory.getMode(settings);