// Path: src/main/java/org/oscii/lex/Expression.java
// public class Expression {
//     public final String text;
//     public final String degraded_text; // Lowercased, etc.
//     public final String language; // ISO-639-1 code (2-letter), e.g., "zh"
//     // Tag inventory: http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
//     public String languageTag; // RFC5654 language tag, e.g., "zh-cmn-Hans-CN"
//     public final String source;
// 
//     public Expression(String text, String degraded_text, String languageTag) {
//         this(text, degraded_text, languageTag, "");
//     }
// 
//     public Expression(String text, String degraded_text, String languageTag, String source) {
//         this.text = text;
//         this.degraded_text = degraded_text == null ? Lexicon.degrade(text) : degraded_text;
//         this.languageTag = languageTag;
//         // TODO(denero) Split language tag to open language
//         this.language = languageTag;
//         this.source = source;
//     }
// 
//     public Expression(String text, String languageTag) {
//         this(text, Lexicon.degrade(text), languageTag);
//     }
// 
//     @Override
//     public String toString() {
//         return "Expression{" +
//                 "text='" + text + '\'' +
//                 "language='" + language + '\'' +
//                 "source='" + source + '\'' +
//                 '}';
//     }
// 
//     @Override
//     public boolean equals(Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
// 
//         Expression that = (Expression) o;
// 
//         if (!language.equals(that.language)) return false;
//         if (!text.equals(that.text)) return false;
// 
//         return true;
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = text.hashCode();
//         result = 31 * result + language.hashCode();
//         return result;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Meaning.java
// public class Meaning {
//     public final Expression expression;
//     public final List<String> pos = new ArrayList<>();
//     public final List<Definition> definitions = new ArrayList<>();
//     public final List<Translation> translations = new ArrayList<>();
//     public final List<Expression> synonyms = new ArrayList<>();
// 
//     @Override
//     public String toString() {
//         return "Meaning{" +
//                 "expression=" + expression +
//                 ", definitions=" + definitions +
//                 ", translations=" + translations +
//                 '}';
//     }
// 
//     public Meaning(Expression expression) {
//         this.expression = expression;
//     }
// }


import com.codepoetics.protonpack.StreamUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.lex.Expression;
import org.oscii.lex.Meaning;
import java.io.IOException;
import java.nio.file.Files;
import java.util.*;
import java.util.function.Function;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static java.util.stream.Collectors.*;
            if (all == null) {
                all = new ArrayList<>(e.getValue().size());
                sentences.put(e.getKey(), all);
            }
            all.addAll(e.getValue());
        });
    }

    @Override
    public void tally() {
        sentences.keySet().stream().forEach(language -> {
            log.info("Indexing words for " + language);
            Map<String, List<Location>> indexForLanguage = indexTokens(sentences.get(language));
            index.put(language, indexForLanguage);
        });
    }

    /*
     * Index tokens of sentences by their type.
     */
    private Map<String, List<Location>> indexTokens(List<AlignedSentence> ss) {
        return ss.stream()
                .flatMap(s -> IntStream.range(0, s.tokens.length).mapToObj(j -> new Location(s, j)))
                .collect(groupingBy(Location::token));
    }

    /*
     * Return a function that takes words in another language and returns translation frequencies.
     */
    @Override
// Path: src/main/java/org/oscii/lex/Expression.java
// public class Expression {
//     public final String text;
//     public final String degraded_text; // Lowercased, etc.
//     public final String language; // ISO-639-1 code (2-letter), e.g., "zh"
//     // Tag inventory: http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
//     public String languageTag; // RFC5654 language tag, e.g., "zh-cmn-Hans-CN"
//     public final String source;
// 
//     public Expression(String text, String degraded_text, String languageTag) {
//         this(text, degraded_text, languageTag, "");
//     }
// 
//     public Expression(String text, String degraded_text, String languageTag, String source) {
//         this.text = text;
//         this.degraded_text = degraded_text == null ? Lexicon.degrade(text) : degraded_text;
//         this.languageTag = languageTag;
//         // TODO(denero) Split language tag to open language
//         this.language = languageTag;
//         this.source = source;
//     }
// 
//     public Expression(String text, String languageTag) {
//         this(text, Lexicon.degrade(text), languageTag);
//     }
// 
//     @Override
//     public String toString() {
//         return "Expression{" +
//                 "text='" + text + '\'' +
//                 "language='" + language + '\'' +
//                 "source='" + source + '\'' +
//                 '}';
//     }
// 
//     @Override
//     public boolean equals(Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
// 
//         Expression that = (Expression) o;
// 
//         if (!language.equals(that.language)) return false;
//         if (!text.equals(that.text)) return false;
// 
//         return true;
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = text.hashCode();
//         result = 31 * result + language.hashCode();
//         return result;
//     }
// }
// 
// Path: src/main/java/org/oscii/lex/Meaning.java
// public class Meaning {
//     public final Expression expression;
//     public final List<String> pos = new ArrayList<>();
//     public final List<Definition> definitions = new ArrayList<>();
//     public final List<Translation> translations = new ArrayList<>();
//     public final List<Expression> synonyms = new ArrayList<>();
// 
//     @Override
//     public String toString() {
//         return "Meaning{" +
//                 "expression=" + expression +
//                 ", definitions=" + definitions +
//                 ", translations=" + translations +
//                 '}';
//     }
// 
//     public Meaning(Expression expression) {
//         this.expression = expression;
//     }
// }


// Path: src/main/java/org/oscii/concordance/IndexedAlignedCorpus.java
import com.codepoetics.protonpack.StreamUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.oscii.lex.Expression;
import org.oscii.lex.Meaning;
import java.io.IOException;
import java.nio.file.Files;
import java.util.*;
import java.util.function.Function;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static java.util.stream.Collectors.*;

            if (all == null) {
                all = new ArrayList<>(e.getValue().size());
                sentences.put(e.getKey(), all);
            }
            all.addAll(e.getValue());
        });
    }

    @Override
    public void tally() {
        sentences.keySet().stream().forEach(language -> {
            log.info("Indexing words for " + language);
            Map<String, List<Location>> indexForLanguage = indexTokens(sentences.get(language));
            index.put(language, indexForLanguage);
        });
    }

    /*
     * Index tokens of sentences by their type.
     */
    private Map<String, List<Location>> indexTokens(List<AlignedSentence> ss) {
        return ss.stream()
                .flatMap(s -> IntStream.range(0, s.tokens.length).mapToObj(j -> new Location(s, j)))
                .collect(groupingBy(Location::token));
    }

    /*
     * Return a function that takes words in another language and returns translation frequencies.
     */
    @Override
    public Function<Expression, Double> translationFrequencies(Expression source) {