File size: 1,656 Bytes
4e8b6c1 949c5da 4e8b6c1 949c5da |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
import os
import fasttext
from huggingface_hub import hf_hub_download
import matplotlib.pyplot as plt
import seaborn as sns
def load_model(repo_id: str) -> fasttext.FastText._FastText:
model_path = hf_hub_download(
repo_id,
filename="model.bin",
cache_dir="fasttext/models",
)
return fasttext.load_model(model_path)
# repo_id="cis-lmu/glotlid"
repo_id = "facebook/fasttext-language-identification"
model = load_model(repo_id)
def identify_languages(title):
predictions = model.predict(title, k=5) # Get top 5 predictions
labels = predictions[0]
confidences = predictions[1]
results = []
for label, confidence in zip(labels, confidences):
language = label.replace("__label__", "")
results.append((language, confidence))
return results
def plot_confidences(results):
languages = [result[0] for result in results]
confidences = [result[1] for result in results]
plt.figure(figsize=(10, 6))
sns.barplot(x=confidences, y=languages)
plt.xlabel("Confidence")
plt.ylabel("Language")
plt.title("Language Identification Confidence")
return plt
def identify_and_plot(title):
results = identify_languages(title)
plot = plot_confidences(results)
return results, plot
LID_EXAMPLES = [
"ပိုၼ်းၵႅပ်ႈလိၵ်ႈလၢႆးတႆး ဢၼ်ၶူးပွင်သွၼ်လူင်လိၵ်ႈလၢႆးတႆး",
"ယု၀တီဂျင်းဖောမယ်၊ ရှမ်းစာပေသမိုင်းနှင့်",
"Hello World, မႂ်ႇသုင်ၶႃႈ",
]
|