// Path: src/main/java/org/oscii/lex/Definition.java
// public class Definition {
//     public final String text;
//     public final List<String> pos;
//     public final String languageTag;
//     public final String dataSource;
// 
//     public Definition(String text, List<String> pos, String languageTag, String dataSoure) {
//         this.text = text;
//         this.pos = pos;
//         this.languageTag = languageTag;
//         this.dataSource = dataSoure;
//     }
// 
//     @Override
//     public String toString() {
//         return "Definition{" +
//                 "text='" + text + '\'' +
//                 ", pos=" + pos +
//                 ", languageTag='" + languageTag + '\'' +
//                 ", dataSource='" + dataSource + '\'' +
//                 '}';
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Expression.java
// public class Expression {
//     public final String text;
//     public final String degraded_text; // Lowercased, etc.
//     public final String language; // ISO-639-1 code (2-letter), e.g., "zh"
//     // Tag inventory: http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
//     public String languageTag; // RFC5654 language tag, e.g., "zh-cmn-Hans-CN"
//     public final String source;
// 
//     public Expression(String text, String degraded_text, String languageTag) {
//         this(text, degraded_text, languageTag, "");
//     }
// 
//     public Expression(String text, String degraded_text, String languageTag, String source) {
//         this.text = text;
//         this.degraded_text = degraded_text == null ? Lexicon.degrade(text) : degraded_text;
//         this.languageTag = languageTag;
//         // TODO(denero) Split language tag to open language
//         this.language = languageTag;
//         this.source = source;
//     }
// 
//     public Expression(String text, String languageTag) {
//         this(text, Lexicon.degrade(text), languageTag);
//     }
// 
//     @Override
//     public String toString() {
//         return "Expression{" +
//                 "text='" + text + '\'' +
//                 "language='" + language + '\'' +
//                 "source='" + source + '\'' +
//                 '}';
//     }
// 
//     @Override
//     public boolean equals(Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
// 
//         Expression that = (Expression) o;
// 
//         if (!language.equals(that.language)) return false;
//         if (!text.equals(that.text)) return false;
// 
//         return true;
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = text.hashCode();
//         result = 31 * result + language.hashCode();
//         return result;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Meaning.java
// public class Meaning {
//     public final Expression expression;
//     public final List<String> pos = new ArrayList<>();
//     public final List<Definition> definitions = new ArrayList<>();
//     public final List<Translation> translations = new ArrayList<>();
//     public final List<Expression> synonyms = new ArrayList<>();
// 
//     @Override
//     public String toString() {
//         return "Meaning{" +
//                 "expression=" + expression +
//                 ", definitions=" + definitions +
//                 ", translations=" + translations +
//                 '}';
//     }
// 
//     public Meaning(Expression expression) {
//         this.expression = expression;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Translation.java
// public class Translation {
//     public final Expression translation;
//     public final List<String> pos;
//     public double frequency;
// 
//     public Translation(Expression expression, List<String> pos) {
//         this.translation = expression;
//         this.pos = pos;
//         this.frequency = 0.0;
//     }
// 
//     @Override
//     public String toString() {
//         return "Translation{" +
//                 "translation=" + translation +
//                 ", frequency=" + frequency +
//                 '}';
//     }
// }


import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import gnu.trove.THashMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.lex.Definition;
import org.oscii.lex.Expression;
import org.oscii.lex.Meaning;
import org.oscii.lex.Translation;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toSet;
                record = gson.fromJson(reader, type);
                if (process.test(record)) {
                    accepted++;
                }
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        log.info(String.format("Parsed %d %s records", accepted, name));
    }

    /*
     * For all words with a shared meaning, process Meanings by language.
     */
    public void forEachMeaning(Consumer<Meaning> process) {
      denotations.values().stream().collect(groupingBy(dn -> dn.mn)).values()
      .forEach(denotationsForMeaning -> {
       Map<String, List<Meaning>> byLanguage = denotationsForMeaning.stream().map(this::createMeaning)
             .collect(groupingBy(m -> m.expression.languageTag));
           processMeanings(process, byLanguage);
      });
    }

    /*
     * Create a meaning for a denotation containing in-language definitions.
     */
    private Meaning createMeaning(Models.Dn dn) {
        Models.Ex ex = expressions.get(dn.ex);
        String languageTag = languageTags.get(ex.lv);
// Path: src/main/java/org/oscii/lex/Definition.java
// public class Definition {
//     public final String text;
//     public final List<String> pos;
//     public final String languageTag;
//     public final String dataSource;
// 
//     public Definition(String text, List<String> pos, String languageTag, String dataSoure) {
//         this.text = text;
//         this.pos = pos;
//         this.languageTag = languageTag;
//         this.dataSource = dataSoure;
//     }
// 
//     @Override
//     public String toString() {
//         return "Definition{" +
//                 "text='" + text + '\'' +
//                 ", pos=" + pos +
//                 ", languageTag='" + languageTag + '\'' +
//                 ", dataSource='" + dataSource + '\'' +
//                 '}';
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Expression.java
// public class Expression {
//     public final String text;
//     public final String degraded_text; // Lowercased, etc.
//     public final String language; // ISO-639-1 code (2-letter), e.g., "zh"
//     // Tag inventory: http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
//     public String languageTag; // RFC5654 language tag, e.g., "zh-cmn-Hans-CN"
//     public final String source;
// 
//     public Expression(String text, String degraded_text, String languageTag) {
//         this(text, degraded_text, languageTag, "");
//     }
// 
//     public Expression(String text, String degraded_text, String languageTag, String source) {
//         this.text = text;
//         this.degraded_text = degraded_text == null ? Lexicon.degrade(text) : degraded_text;
//         this.languageTag = languageTag;
//         // TODO(denero) Split language tag to open language
//         this.language = languageTag;
//         this.source = source;
//     }
// 
//     public Expression(String text, String languageTag) {
//         this(text, Lexicon.degrade(text), languageTag);
//     }
// 
//     @Override
//     public String toString() {
//         return "Expression{" +
//                 "text='" + text + '\'' +
//                 "language='" + language + '\'' +
//                 "source='" + source + '\'' +
//                 '}';
//     }
// 
//     @Override
//     public boolean equals(Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
// 
//         Expression that = (Expression) o;
// 
//         if (!language.equals(that.language)) return false;
//         if (!text.equals(that.text)) return false;
// 
//         return true;
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = text.hashCode();
//         result = 31 * result + language.hashCode();
//         return result;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Meaning.java
// public class Meaning {
//     public final Expression expression;
//     public final List<String> pos = new ArrayList<>();
//     public final List<Definition> definitions = new ArrayList<>();
//     public final List<Translation> translations = new ArrayList<>();
//     public final List<Expression> synonyms = new ArrayList<>();
// 
//     @Override
//     public String toString() {
//         return "Meaning{" +
//                 "expression=" + expression +
//                 ", definitions=" + definitions +
//                 ", translations=" + translations +
//                 '}';
//     }
// 
//     public Meaning(Expression expression) {
//         this.expression = expression;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Translation.java
// public class Translation {
//     public final Expression translation;
//     public final List<String> pos;
//     public double frequency;
// 
//     public Translation(Expression expression, List<String> pos) {
//         this.translation = expression;
//         this.pos = pos;
//         this.frequency = 0.0;
//     }
// 
//     @Override
//     public String toString() {
//         return "Translation{" +
//                 "translation=" + translation +
//                 ", frequency=" + frequency +
//                 '}';
//     }
// }


// Path: src/main/java/org/oscii/panlex/PanLexJSONParser.java
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import gnu.trove.THashMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.lex.Definition;
import org.oscii.lex.Expression;
import org.oscii.lex.Meaning;
import org.oscii.lex.Translation;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toSet;

                record = gson.fromJson(reader, type);
                if (process.test(record)) {
                    accepted++;
                }
            }
            reader.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        log.info(String.format("Parsed %d %s records", accepted, name));
    }

    /*
     * For all words with a shared meaning, process Meanings by language.
     */
    public void forEachMeaning(Consumer<Meaning> process) {
      denotations.values().stream().collect(groupingBy(dn -> dn.mn)).values()
      .forEach(denotationsForMeaning -> {
       Map<String, List<Meaning>> byLanguage = denotationsForMeaning.stream().map(this::createMeaning)
             .collect(groupingBy(m -> m.expression.languageTag));
           processMeanings(process, byLanguage);
      });
    }

    /*
     * Create a meaning for a denotation containing in-language definitions.
     */
    private Meaning createMeaning(Models.Dn dn) {
        Models.Ex ex = expressions.get(dn.ex);
        String languageTag = languageTags.get(ex.lv);
        Expression expression = new Expression(ex.tt, ex.td, languageTag);