public class SenFactory {
  
  private static final Map<String,SenFactory> map = new ConcurrentHashMap<String,SenFactory>();
  private static final String EMPTY_DICTIONARYDIR_KEY = "NO_DICTIONARY_INSTANCE";

  public static final String unknownPOS = "未知語";

  private final String[] posIndex, conjTypeIndex, conjFormIndex;
  private final ByteBuffer costs, pos, tokens, trie;

  /**
   * Get the singleton factory instance
   *
   * @param dictionaryDir a directory of dictionaries
   * @return the factory instance
     */
  public synchronized static SenFactory getInstance(String dictionaryDir) {
    
    String key = (dictionaryDir == null || dictionaryDir.trim().length() == 0) ? EMPTY_DICTIONARYDIR_KEY : dictionaryDir; 
    SenFactory instance = map.get(key);
    if (instance == null) {
      try {
        instance = new SenFactory(dictionaryDir);
        map.put(key, instance);
      } catch (IOException ex) {
        throw new RuntimeException(ex);
      }
    }
    
    return instance;
  }

  /**
   * Constructor for Sen, which is an Japanese Morphological Analyzer
   *
   * @param dictionaryDir
   * @throws IOException
   */
  private SenFactory(String dictionaryDir) throws IOException {
    InputStream in = null;
    DataInputStream din = null;

    // read main data files
    try {
      in = getInputStream("header.sen", dictionaryDir);
      din = new DataInputStream(in);
      costs = loadBuffer("connectionCost.sen", din.readInt(), dictionaryDir).asReadOnlyBuffer();
      pos = loadBuffer("partOfSpeech.sen", din.readInt(), dictionaryDir).asReadOnlyBuffer();
      tokens = loadBuffer("token.sen", din.readInt(), dictionaryDir).asReadOnlyBuffer();
      trie = loadBuffer("trie.sen", din.readInt(), dictionaryDir).asReadOnlyBuffer();
    } finally {
      IOUtils.closeWhileHandlingException(din, in);
    }

    // read index files
    try {
      in = getInputStream("posIndex.sen", dictionaryDir);
      din = new DataInputStream(in);
      posIndex = new String[din.readChar()];
      for (int i = 0; i < posIndex.length; i++) {
        posIndex[i] = din.readUTF();
      }
    
      conjTypeIndex = new String[din.readChar()];
      for (int i = 0; i < conjTypeIndex.length; i++) {
        conjTypeIndex[i] = din.readUTF();
      }
    
      conjFormIndex = new String[din.readChar()];
      for (int i = 0; i < conjFormIndex.length; i++) {
        conjFormIndex[i] = din.readUTF();
      }
    } finally {
      IOUtils.closeWhileHandlingException(din, in);
    }
  }

  /**
   * @param name
   * @param dictionaryDir
   * @return
   * @throws IOException
   */
  private static InputStream getInputStream(String name, String dictionaryDir) throws IOException{
    InputStream in = null;
    if(dictionaryDir == null || dictionaryDir.trim().length() == 0){
      in = SenFactory.class.getResourceAsStream(name);
    }else{
      in = new FileInputStream(new File(dictionaryDir, name));
    }
    if(in == null){
      throw new RuntimeException("Not found resource["+name+"]. dictionaryDir=["+dictionaryDir+"]");
    }
    return in;
  }

  /**
   * Load specified dictionary data into ByteBuffer
   *
   * @param resource
   * @param size
   * @param dictionaryDir
   * @return
   * @throws IOException
   */
  private static ByteBuffer loadBuffer(String resource, int size, String dictionaryDir) throws IOException {
    InputStream in = null;
    try {
      in = getInputStream(resource, dictionaryDir);
    ByteBuffer buffer = ByteBuffer.allocateDirect(size);
    buffer.limit(size);
    
    byte[] buf = new byte[1024];
    
    while (true) {
      int numBytes = in.read(buf);
      if (numBytes == -1) break;
      
      buffer.put(buf, 0, numBytes);
    }
    
    buffer.rewind();
    
    return buffer;
    } finally {
      IOUtils.closeWhileHandlingException(in);
    }
  }
  
  /**
   * Builds a Tokenizer for the given dictionary configuration
   *
   * @param dictionaryDir The dictionary configuration filename
   * @param tokenizeUnknownKatakana
   * @return The constructed Tokenizer
   */
  private static Tokenizer getTokenizer(String dictionaryDir, boolean tokenizeUnknownKatakana) {
    SenFactory localInstance = SenFactory.getInstance(dictionaryDir);
    
    return new JapaneseTokenizer(
            new Dictionary(
                    localInstance.costs.asShortBuffer(),
                    localInstance.pos.duplicate(),
                    localInstance.tokens.duplicate(),
                    localInstance.trie.asIntBuffer(),
                    localInstance.posIndex,
                    localInstance.conjTypeIndex,
                    localInstance.conjFormIndex
            ),
            unknownPOS,
            tokenizeUnknownKatakana);
  }
  
  /**
   * Creates a Viterbi from the given configuration
   *
   * @param dictionaryDir a directory of dictionary
   * @param tokenizeUnknownKatakana
   * @return A Viterbi
   */
  static Viterbi getViterbi(String dictionaryDir, boolean tokenizeUnknownKatakana) {
    // for test only
    return new Viterbi(getTokenizer(dictionaryDir, tokenizeUnknownKatakana));
  }
  
  /**
   * Creates a StringTagger from the given configuration
   *
   * @param dictionaryDir a directory of dictionary
   * @param tokenizeUnknownKatakana
   * @return A StringTagger
     */
  public static StringTagger getStringTagger(String dictionaryDir, boolean tokenizeUnknownKatakana) {
    return new StringTagger(getTokenizer(dictionaryDir, tokenizeUnknownKatakana));
  }
  
  /**
   * Creates a ReadingProcessor from the given configuration
   *
   * @param dictionaryDir
   * @param tokenizeUnknownKatakana
   * @return The reading processor
   */
  static ReadingProcessor getReadingProcessor(String dictionaryDir, boolean tokenizeUnknownKatakana) {
    //for test only
    return new ReadingProcessor(getTokenizer(dictionaryDir, tokenizeUnknownKatakana));
  }

} public class StringTagger {
  
  /**
   * The Viterbi analyser used to decompose strings
   */
  private final Viterbi viterbi;
  
  /**
   * {@link StreamFilter}s to apply during analysis
   */
  private List<StreamFilter> filterList = new ArrayList<StreamFilter>();
  
  /**
   * Apply the pre-processing phase of all attached {@link StreamFilter}s to
   * the input sentence
   *
   * @param sentence The input sentence
   */
  private void filterPreProcess(Sentence sentence) {
    for (int i = 0; i < filterList.size(); i++) {
      filterList.get(i).preProcess(sentence);
    }
  }
  
  /**
   * Apply the post-processing phase of all attached {@link StreamFilter}s to
   * the analysed {@link Token}s
   *
   * @param tokens The analysed {@link Token}s
   * @return The filtered {@link Token}s
   */
  private List<Token> filterPostProcess(List<Token> tokens) {
    for (int i = filterList.size() - 1; i >= 0; i--) {
      StreamFilter filter = filterList.get(i);
      tokens = filter.postProcess(tokens);
    }
    
    return tokens;
  }
  
  /**
   * Add a {@link StreamFilter} to be applied during analysis
   *
   * @param filter The {@link StreamFilter} to add
   */
  public void addFilter(StreamFilter filter) {
    filterList.add(filter);
  }
  
  /**
   * Remove all current {@link StreamFilter}s
   */
  public void removeFilters() {
    filterList.clear();
  }
  
  /**
   * Decompose a string into its most likely constituent morphemes
   * 
   * @param surface The string to analyse
   * @return An array of {@link Token}s representing the most likely morphemes
   * @throws IOException 
   */
  public List<Token> analyze(String surface, List<Token> reuse) throws IOException {
    Sentence sentence = new Sentence(surface.toCharArray());
    filterPreProcess(sentence);
    
    List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
    
    tokens = filterPostProcess(tokens);
    
    return tokens;
  }
  
  /**
   * @deprecated use {@link #analyze(String, List)} instead.
   */
  @Deprecated
  public List<Token> analyze(String surface) throws IOException {
    return analyze(surface, new ArrayList<Token>());
  }
  
  /**
   * Decompose a string into its most likely constituent morphemes
   * 
   * @param surface The string to analyse
   * @return An array of {@link Token}s representing the most likely morphemes
   * @throws IOException 
   */
  public List<Token> analyze(char[] surface, List<Token> reuse) throws IOException {
    Sentence sentence = new Sentence(surface);
    filterPreProcess(sentence);
    
    List<Token> tokens = viterbi.getBestTokens(sentence, reuse);
    
    tokens = filterPostProcess(tokens);
    
    return tokens;
  }
  
  /**
   * @deprecated use {@link #analyze(char[], List)} instead.
   */
  @Deprecated
  public List<Token> analyze(char[] surface) throws IOException {
    return analyze(surface, new ArrayList<Token>());
  }
  
  /**
   * @param tokenizer The Tokenizer to use for analysis 
   */
  public StringTagger(Tokenizer tokenizer) {
    this.viterbi = new Viterbi(tokenizer);
  }
} public class Morpheme {
  
  /**
   * The {@link Dictionary} that contains this <code>Morpheme</code>
   */
  private Dictionary dictionary;
  
  /**
   * The index of this part-of-speech within the part-of-speech information
   * file
   */
  private int partOfSpeechIndex;
  
  /**
   * Indicates if the part-of-speech data has been loaded
   */
  private LoadState loaded;
  
  private static enum LoadState {
    NONE,
    BASIC,
    FULL
  }
  
  /**
   * The conjugation type of the morpheme
   */
  private String conjugationalType = null;
  
  /**
   * The conjugation form of the morpheme
   */
  private String conjugationalForm = null;
  
  /**
   * The unconjugated form of the morpheme
   */
  private String basicForm = null;
  
  /**
   * The readings of the morpheme
   */
  private List<String> readings = null;
  
  /**
   * The pronunciations of the morpheme
   */
  private List<String> pronunciations = null;
  
  /**
   * The part-of-speech in Chasen format
   */
  private String partOfSpeech = null;
  
  /**
   * Arbitrary additional information
   */
  private String additionalInformation = null;
  
  /**
   * Loads the part-of-speech data from the {@link Dictionary}
   *
   */
  private void load(LoadState requested) {
    if (loaded.compareTo(requested) >= 0) {
      return; // we already loaded what we need
    }
    
    ByteBuffer buffer = dictionary.getPartOfSpeechInfoBuffer();
    buffer.position(partOfSpeechIndex);
    char[] temp = new char[512];
    int length;
    
    if (loaded == LoadState.NONE) {
      partOfSpeech = dictionary.posIndex[DictionaryUtil.readVInt(buffer)];
      conjugationalType = dictionary.conjTypeIndex[DictionaryUtil.readVInt(buffer)];
      conjugationalForm = dictionary.conjFormIndex[DictionaryUtil.readVInt(buffer)];
      
      length = DictionaryUtil.readVInt(buffer);
      if (length == 0) {
        basicForm = "*"; // basic form is the same as surface form
      } else {
        DictionaryUtil.readString(buffer, temp, 0, length);
        basicForm = new String(temp, 0, length);
      }
    } else {
      // we've already loaded the basic data, but not the full, just skip over bytes
      DictionaryUtil.readVInt(buffer); // POS
      DictionaryUtil.readVInt(buffer); // conjType
      DictionaryUtil.readVInt(buffer); // conjForm
      length = DictionaryUtil.readVInt(buffer); // length
      if (length != 0)
        DictionaryUtil.readString(buffer, temp, 0, length); // basicForm
    }
    
    loaded = LoadState.BASIC;
    
    if (requested == LoadState.FULL) {
      int readingData = DictionaryUtil.readVInt(buffer);
      int numReadings = readingData >>> 1;
      
      readings = new ArrayList<String>(numReadings);
      pronunciations = new ArrayList<String>(numReadings);
      
      for (int i = 0; i < numReadings; i++) {
        length = DictionaryUtil.readVInt(buffer);
        final int readingLength = length >>> 1;
        if ((readingData & 1) == 0) {
          DictionaryUtil.readKatakana(buffer, temp, 0, readingLength);
        } else {
          DictionaryUtil.readString(buffer, temp, 0, readingLength);
        }
        String reading = new String(temp, 0, readingLength);
        readings.add(reading);
        if ((length & 1) != 0) {
          final int pronunciationLength = DictionaryUtil.readVInt(buffer);
          if ((readingData & 1) == 0) {
            DictionaryUtil.readKatakana(buffer, temp, 0, pronunciationLength);
          } else {
            DictionaryUtil.readString(buffer, temp, 0, pronunciationLength);
          }
          pronunciations.add(new String(temp, 0, pronunciationLength));
        } else {
          pronunciations.add(reading);
        }
      }
      loaded = LoadState.FULL;
    }
  }
  
  /**
   * Gets the conjugation type of the morpheme
   * 
   * @return The conjugation type
   */
  public String getConjugationalType() {
    load(LoadState.BASIC);
    return conjugationalType;
  }
  
  /**
   * Gets the conjugation form of the morpheme
   * 
   * @return The conjugation form
   */
  public String getConjugationalForm() {
    load(LoadState.BASIC);
    return conjugationalForm;
  }
  
  /**
   * Gets the unconjugated form of the morpheme
   * 
   * @return The unconjugated form
   */
  public String getBasicForm() {
    load(LoadState.BASIC);
    return basicForm;
  }
  
  /**
   * Gets the readings of the morpheme
   * 
   * @return The readings
   */
  public List<String> getReadings() {
    load(LoadState.FULL);
    return readings;
  }
  
  /**
   * Gets the pronunciations of the morpheme
   * 
   * @return The pronunciations
   */
  public List<String> getPronunciations() {
    load(LoadState.FULL);
    return pronunciations;
  }
  
  /**
   * Gets the part-of-speech in Chasen format
   * 
   * @return The part-of-speech in Chasen format
   */
  public String getPartOfSpeech() {
    load(LoadState.BASIC);
    return partOfSpeech;
  }
  
  /**
   * Gets the additional information string
   * 
   * @return The additional information string
   */
  public String getAdditionalInformation() {
    return additionalInformation;
  }
  
  /**
   * Sets an arbitrary string of additional information
   * 
   * @param additionalInformation The additional information to set
   */
  public void setAdditionalInformation(String additionalInformation) {
    this.additionalInformation = additionalInformation;
  }
  
  /**
   * Compare two lists of strings
   *
   * @param list1 The first list
   * @param list2 The second list
   * @return <code>true</code> if the lists' contents are exactly equal
   */
  private boolean stringListsEqual(List<String> list1, List<String> list2) {
    
    if (list1 == list2) {
      return true;
    }
    
    if ((list1 == null) || (list2 == null)) {
      return false;
    }
    
    if (list1.size() != list2.size()) {
      return false;
    }
    
    for (int i = 0; i < list1.size(); i++) {
      if (!list1.get(i).equals(list2.get(i))) {
        return false;
      }
    }
    
    return true;
    
  }
  
  @Override
  public boolean equals(Object object) {
    
    if (object instanceof Morpheme) {
      
      Morpheme morpheme = (Morpheme) object;
      
      String partOfSpeech = this.getPartOfSpeech();
      String conjugationalType = this.getConjugationalType();
      String conjugationalForm = this.getConjugationalForm();
      String basicForm = this.getBasicForm();
      List<String> pronunciations = this.getPronunciations();
      List<String> readings = this.getReadings();
      String additionalInformation = this.getAdditionalInformation();
      
      String otherPartOfSpeech = morpheme.getPartOfSpeech();
      String otherConjugationalType = morpheme.getConjugationalType();
      String otherConjugationalForm = morpheme.getConjugationalForm();
      String otherBasicForm = morpheme.getBasicForm();
      List<String> otherPronunciations = morpheme.getPronunciations();
      List<String> otherReadings = morpheme.getReadings();
      String otherAdditionalInformation = morpheme.getAdditionalInformation();
      
      if (
          ((basicForm == otherBasicForm) || (basicForm != null && basicForm.equals(otherBasicForm)))
          && ((conjugationalType == otherConjugationalType) || (conjugationalType != null && conjugationalType.equals(otherConjugationalType)))
          && ((conjugationalForm == otherConjugationalForm) || (conjugationalForm != null && conjugationalForm.equals(otherConjugationalForm)))
          && ((partOfSpeech == otherPartOfSpeech) || (partOfSpeech != null && partOfSpeech.equals(otherPartOfSpeech)))
          && (stringListsEqual(pronunciations, otherPronunciations))
          && (stringListsEqual(readings, otherReadings))
          && ((additionalInformation == otherAdditionalInformation) || (additionalInformation != null && additionalInformation.equals(otherAdditionalInformation)))					
      )
      {
        return true;
      }
      
    }
    
    return false;
  }
  
  @Override
  public String toString() {
    load(LoadState.FULL);
    
    StringBuilder partOfSpeechData = new StringBuilder();
    String[] parts = partOfSpeech.split("-");
    for (int i = 0; i < 4; i++) {
      if (i < parts.length) {
        partOfSpeechData.append(parts[i]);
      } else {
        partOfSpeechData.append("*");
      }
      partOfSpeechData.append(",");
      
    }
    
    partOfSpeechData.append(conjugationalType);
    partOfSpeechData.append(",");
    partOfSpeechData.append(conjugationalForm);
    partOfSpeechData.append(",");
    partOfSpeechData.append(basicForm);
    partOfSpeechData.append(",");
    partOfSpeechData.append((readings.size() > 0) ? readings.get(0) : "null");
    partOfSpeechData.append(",");
    partOfSpeechData.append((pronunciations.size() > 0) ? pronunciations.get(0) : "null");
    return partOfSpeechData.toString();
  }

  /**
   * Builds a lazy proxy onto a part-of-speech stored in a Dictionary
   * 
   * @param dictionary The dictionary to proxy upon
   * @param partOfSpeechIndex The index into the part-of-speech file
   */
  public Morpheme(Dictionary dictionary, int partOfSpeechIndex) {
    this.dictionary = dictionary;
    this.partOfSpeechIndex = partOfSpeechIndex;
    this.loaded = LoadState.NONE;
  }
  
  /**
   * Creates a literal <code>Morpheme</code> that does not link to any
   * Dictionary
   * @param partOfSpeech The Chasen-format part-of-speech
   * @param conjugationalType The conjugational type
   * @param conjugationalForm The conjugational form
   * @param basicForm The unconjugated form
   * @param readings The readings
   * @param pronunciations The pronunciations
   * @param additionalInformation Arbitrary additional information
   */
  public Morpheme(String partOfSpeech, String conjugationalType, String conjugationalForm, String basicForm, String[] readings, String[] pronunciations, String additionalInformation) {
    this.basicForm = basicForm;
    this.conjugationalType = conjugationalType;
    this.conjugationalForm = conjugationalForm;
    this.readings = new ArrayList<String>(Arrays.asList(readings));
    this.pronunciations = new ArrayList<String>(Arrays.asList(pronunciations));
    this.partOfSpeech = partOfSpeech;
    this.additionalInformation = additionalInformation;
    this.loaded = LoadState.FULL;
  }
  
  /**
   * Creates a blank, modifiable <code>Morpheme</code> that does not link
   * to any Dictionary
   */
  public Morpheme() {
    this.readings = Collections.emptyList();
    this.pronunciations = Collections.emptyList();
    this.loaded = LoadState.FULL;
  }
} public interface StreamFilter {
  /**
   * Pre-processes a sentence
   *
   * @param sentence The sentence be pre-processed
   */
  public void preProcess(Sentence sentence);
  
  /**
   * Post-processes analysed tokens
   * 
   * @param tokens The analysed tokens
   * @return The post-processed tokens
   */
  public List<Token> postProcess(List<Token> tokens);
} public interface BasicFormAttribute extends Attribute {
  public String getBasicForm();
  public void setMorpheme(Morpheme morpheme);
} public interface ConjugationAttribute extends Attribute {
  public String getConjugationalForm();
  public String getConjugationalType();
  public void setMorpheme(Morpheme morpheme);
} public interface CostAttribute extends Attribute {
  public int getCost();
  public void setCost(int cost);
} public interface PartOfSpeechAttribute extends Attribute {
  public String getPartOfSpeech();
  public void setMorpheme(Morpheme morpheme);
} public interface PronunciationsAttribute extends Attribute {
  public List<String> getPronunciations();
  public void setMorpheme(Morpheme morpheme);
} public interface ReadingsAttribute extends Attribute {
  public List<String> getReadings();
  public void setMorpheme(Morpheme morpheme);
} public interface SentenceStartAttribute extends Attribute {
  public boolean getSentenceStart();
  public void setSentenceStart(boolean sentenceStart);
}
import org.apache.lucene.analysis.gosen.tokenAttributes.PronunciationsAttribute;
import org.apache.lucene.analysis.gosen.tokenAttributes.ReadingsAttribute;
import org.apache.lucene.analysis.gosen.tokenAttributes.SentenceStartAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.util.AttributeFactory;
import java.io.IOException;
import net.java.sen.SenFactory;
import net.java.sen.StringTagger;
import net.java.sen.dictionary.Morpheme;
import net.java.sen.dictionary.Token;
import net.java.sen.filter.StreamFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.gosen.tokenAttributes.BasicFormAttribute;
import org.apache.lucene.analysis.gosen.tokenAttributes.ConjugationAttribute;
import org.apache.lucene.analysis.gosen.tokenAttributes.CostAttribute;
import org.apache.lucene.analysis.gosen.tokenAttributes.PartOfSpeechAttribute;
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.analysis.gosen;




/**
 * This is a Japanese tokenizer which uses "Sen" morphological
 * analyzer.
 * <p>
 * sets the surface form as the term text, but also sets these attributes:
 * <ul>
 *   <li>{@link BasicFormAttribute}
 *   <li>{@link ConjugationAttribute}
 *   <li>{@link PartOfSpeechAttribute}
 *   <li>{@link PronunciationsAttribute}
 *   <li>{@link ReadingsAttribute}
 *   <li>{@link CostAttribute}
 *   <li>{@link SentenceStartAttribute}
 * </ul>
 */
public final class GosenTokenizer extends Tokenizer {
  private final StreamTagger2 tagger;

  // Term attributes
  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
  
  // morphological attributes
  private final BasicFormAttribute basicFormAtt = addAttribute(BasicFormAttribute.class);
  private final ConjugationAttribute conjugationAtt = addAttribute(ConjugationAttribute.class);
  private final PartOfSpeechAttribute partOfSpeechAtt = addAttribute(PartOfSpeechAttribute.class);
  private final PronunciationsAttribute pronunciationsAtt = addAttribute(PronunciationsAttribute.class);
  private final ReadingsAttribute readingsAtt = addAttribute(ReadingsAttribute.class);
  
  // sentence increment
  private final SentenceStartAttribute sentenceAtt = addAttribute(SentenceStartAttribute.class);

  // viterbi cost
  private final CostAttribute costAtt = addAttribute(CostAttribute.class);
  // viterbi costs from Token.getCost() are cumulative,
  // so we accumulate this so we can then subtract to present an absolute cost.
  private int accumulatedCost = 0;

  // Default value for UNKNOWN Katakana tokenization
  public static final boolean DEFAULT_UNKNOWN_KATAKANA_TOKENIZATION = false;


  /**
   * Constructors
   */
  public GosenTokenizer(StreamFilter filter, String dictionaryDir, boolean tokenizeUnknownKatakana) {
    this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, filter, dictionaryDir, tokenizeUnknownKatakana);
  }

  /**
   * Create A new GosenTokenizer
   *
   * @param factory the AttributeFactory to use
   * @param filter stream filter
   * @param dictionaryDir lucene-gosen dictionary directory
   * @param tokenizeUnknownKatakana determine whether segmenting unknown katakana or not
   */
  public GosenTokenizer(AttributeFactory factory, StreamFilter filter, String dictionaryDir, boolean tokenizeUnknownKatakana) {
    super(factory);
    StringTagger stringTagger = SenFactory.getStringTagger(dictionaryDir, tokenizeUnknownKatakana);