# Application of model
import pickle
import joblib
import sys

import cv2
import mediapipe as mp
import numpy as np

sys.path.insert(1, "./datasets")
from transformations import minimum

# controls min probability to classify class
MIN_PROB = 0.0

# Loads model
model_dict = pickle.load(open('./models/minimum_model.p', 'rb'))
model = model_dict['model']

# Creates panel and hand application
cap = cv2.VideoCapture(0)
mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles

# Only one hand will be detected per frame
hands = mp_hands.Hands(static_image_mode=True, max_num_hands=1, min_detection_confidence=0.3)

while True:
    # Creates frame and changed color system to RGB
    ret, frame = cap.read()
    H, W, _ = frame.shape
    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # if there's a hand, tries to detect symbol
    results = hands.process(frame_rgb)
    if results.multi_hand_landmarks:
        hand_landmarks = results.multi_hand_landmarks[0]

        # Draws hand's coordinates
        mp_drawing.draw_landmarks(
            frame,  # image to draw
            hand_landmarks,  # model output
            mp_hands.HAND_CONNECTIONS,  # hand connections
            mp_drawing_styles.get_default_hand_landmarks_style(),
            mp_drawing_styles.get_default_hand_connections_style())

        # Gets values for prediction and to draw box around hand
        data_aux = []
        x_min, x_max = hand_landmarks.landmark[0].x, hand_landmarks.landmark[0].x
        y_min, y_max = hand_landmarks.landmark[0].y, hand_landmarks.landmark[0].y
        for i in range(len(hand_landmarks.landmark)):
            x = hand_landmarks.landmark[i].x
            y = hand_landmarks.landmark[i].y
            z = hand_landmarks.landmark[i].z
            data_aux.append(x)
            data_aux.append(y)
            data_aux.append(z)

            x_min = min(x_min, x)
            x_max = max(x_max, x)
            y_min = min(y_min, y)
            y_max = max(y_max, y)

        x1 = int(x_min * W) - 10
        y1 = int(y_min * H) - 10

        x2 = int(x_max * W) - 10
        y2 = int(y_max * H) - 10
        data_aux.append(results.multi_handedness[0].classification[0].label == "Left")

        # prediction
        input_values = minimum(data_aux)
        scaler = joblib.load("models/scaler.joblib")
        input_values = scaler.transform(np.reshape(input_values, (1,-1)))[0]
        if max(model.predict_proba([np.asarray(input_values)])[0]) < MIN_PROB:
            predicted_character = ''
        else:
            prediction = model.predict([np.asarray(input_values)])
            predicted_character = prediction[0]

        cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 0), 4)
        cv2.putText(frame, predicted_character, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 0, 0), 3,
                    cv2.LINE_AA)

    cv2.imshow('frame', frame)
    cv2.waitKey(1)


# Destroys panel
cap.release()
cv2.destroyAllWindows()

application.py
PETComputacaoUFPR/tradutor_libras
# Collects images from user to create datasets
import os
import cv2

DATASET_SIZE = 10  # how many images will be collected

# get symbols from "symbols" file
with open("symbols", "r") as file:
    lines = file.read().splitlines()
    symbols = lines[0].strip()

# creates images path
DATA_DIR = './images'
if not os.path.exists(DATA_DIR):
    os.makedirs(DATA_DIR)


# for some reason, the code prints an error the first time it runs the frame
# that's literally the only usage of this code block
cap = cv2.VideoCapture(0)
ret, frame = cap.read()
cv2.imshow('frame', frame)

# loops through symbols
for symbol in symbols:
    if not os.path.exists(os.path.join(DATA_DIR, symbol)):
        os.makedirs(os.path.join(DATA_DIR, symbol))

    # path of symbol
    symbol_path = os.path.join(DATA_DIR, symbol)

    # only collects images if there are less images than DATASET_SIZE
    if len(symbol_path) >= DATASET_SIZE:
        continue

    # collects remaining images
    print('Collecting data for class {}'.format(symbol))
    counter = len(os.listdir(symbol_path)) + 1
    while counter <= dataset_size:
        ret, frame = cap.read()
        # the output for pressing ENTER is 13
        if cv2.waitKey(1) == 13:
            print('image {} collected'.format(counter))
            cv2.imwrite(os.path.join(DATA_DIR, symbol, '_{}.jpg'.format(counter)), frame)
            counter += 1
        cv2.imshow('frame', frame)

# ends application after capturing all images
print("Images successfully captured!")
cap.release()
cv2.destroyAllWindows()

collect_images.py
PETComputacaoUFPR/tradutor_libras
# Creates dataset with hand coordinates positions
# Is used as a base dataset for other datasets
import os
import pickle

import mediapipe as mp
import cv2
import matplotlib.pyplot as plt

# name of file
FILE_NAME = input("file name (without .pickle): ")
FILE_NAME += ".pickle"

# get symbols from "symbols" file
with open("../symbols", "r") as file:
    lines = file.read().splitlines()
    symbols = lines[0].strip()

# Used to get hand's coordinates
mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles

# change confidence detection if necessary
hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)

# path of images
IMAGES_DIR = '../images'

# destiny directory
DATA_DIR = "./data"

# dictionary that will save data for dataset
data = {"features": [], "labels": []}

# loops through directories (each named with a label)
for dir_ in symbols:
    # loops through images
    for img_path in os.listdir(os.path.join(IMAGES_DIR, dir_)):
        data_aux = []

        # reads image and converts it to RGB
        img = cv2.imread(os.path.join(IMAGES_DIR, dir_, img_path))
        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        results = hands.process(img_rgb)

        # only adds data if there is only one hand detected
        if len(results.multi_hand_landmarks) == 1:
            hand_landmarks = results.multi_hand_landmarks[0]
            for i in range(len(hand_landmarks.landmark)):
                x = hand_landmarks.landmark[i].x
                y = hand_landmarks.landmark[i].y
                z = hand_landmarks.landmark[i].z
                data_aux.append(x)
                data_aux.append(y)
                data_aux.append(z)
            # if hand is left or right
            is_left = (results.multi_handedness[0].classification[0].label == "Left")
            data_aux.append(int(is_left))
            data["features"].append(data_aux)
            data["labels"].append(dir_)

# Creates dataset
with open(os.path.join(DATA_DIR, FILE_NAME), "wb") as dataset:
    pickle.dump(data, dataset)

print("Dataset successfully created!")

datasets/create_dataset.py
PETComputacaoUFPR/tradutor_libras
# gambiarra para poder unir arquivos pickle de diferentes pessoas

import os
import pickle

# final dataset
data = {"features": [], "labels": []}

DATA_DIR = "./data"

# loops through datasets
for dataset in os.listdir(DATA_DIR):
    # only merges pickles that start with "partial"
    if ("partial" not in dataset or "pickle" not in dataset):
       continue
    with open(os.path.join(DATA_DIR, dataset), "rb") as new_data:
        data.update(pickle.load(new_data))
    print(f"{dataset} merged")

# saves as new dataset
with open("base_dataset.pickle", "wb") as dataset:
    pickle.dump(data, dataset)

print("Dataset merged sucessfuly!")

datasets/merge_data.py
PETComputacaoUFPR/tradutor_libras
# Functions that take a pickle file and makes transformations such that
# can create a new dataset
import os
import pickle
from math import sqrt

# substracts each coordinate by the smallest value (ignores z values)
def minimum (features):
    # get smallest x and y
    min_x, min_y = features[0], features[1]
    for i, value in enumerate(features[0:-1]):
        # x value
        if i % 3 == 0:
            min_x = min(min_x, value)
        # y value
        elif i % 3 == 1:
            min_y = min(min_y, value)

    # new features
    new_features = []
    for i, value in enumerate(features):
        # x value
        if i % 3 == 0:
            new_features.append(value - min_x)
        # y value
        elif i % 3 == 1:
            new_features.append(value - min_y)
    # adds back the orientation of hand (left/right)
    new_features.append(features[-1])
    return new_features


# saves coordinates as unitary vectors (from point 0) and their distances
# uses 3D space
def vectorial3D (features):
    # gets values from origin (point 0)
    x_0, y_0, z_0 = features[0], features[1], features[2]

    new_features = []
    # loops through other points (last one is left/right hand)
    for i, value in enumerate(features[3:-1:3]):
        x, y, z = features[i], features[i+1], features[i+2]
        distance = sqrt((x - x_0)**2 + (y - y_0)**2 + (z - z_0)**2)
        new_features.append(x - x_0)
        new_features.append(y - y_0)
        new_features.append(z - z_0)
        new_features.append(distance)
    new_features.append(features[-1])
    return new_features

# saves coordinates as unitary vectors (from point 0) and their distances
# uses 2D space
def vectorial2D (features):
    # gets values from origin (point 0)
    x_0, y_0, z_0 = features[0], features[1], features[2]

    new_features = []
    # loops through other points (last one is left/right hand), ignoring z coordinate
    for i, value in enumerate(features[3:-1:3]):
        x, y = features[i], features[i+1]
        distance = sqrt((x - x_0)**2 + (y - y_0)**2)
        new_features.append(x - x_0)
        new_features.append(y - y_0)
        new_features.append(distance)
    new_features.append(features[-1])
    return new_features

datasets/transformations.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(data["features"], 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = RandomForestClassifier()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Close Dataset
f = open('model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()

models/forest_base_classifier.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle
import joblib

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import MinMaxScaler
import numpy as np

import sys
sys.path.insert(1, "../datasets")
from transformations import minimum

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))


# Transforms dataset
new_features = []
for observation in data["features"]:
    new_features.append(minimum(observation))

# MinMax Scaling
scaler = MinMaxScaler()
new_features = scaler.fit_transform(new_features)

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(new_features, 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = RandomForestClassifier()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Save Model and Scaler
f = open('minimum_model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()
joblib.dump(scaler, "scaler.joblib")

models/forest_minimum_classifier.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np

import sys
sys.path.insert(1, "../datasets")
from transformations import vectorial2D

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))


# Transforms dataset
new_features = []
for observation in data["features"]:
    new_features.append(vectorial2D(observation))

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(new_features, 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = RandomForestClassifier()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Save Model
f = open('minimum_model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()

models/forest_vectorial2D_classifier.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle
import joblib

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import MinMaxScaler
import numpy as np

import sys
sys.path.insert(1, "../datasets")
from transformations import vectorial3D

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))


# Transforms dataset
new_features = []
for observation in data["features"]:
    new_features.append(vectorial3D(observation))

# MinMaxScaling
scaler = MinMaxScaler()
new_features = scaler.fit_transform(new_features)

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(new_features, 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = RandomForestClassifier()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Save Model and Scaler
f = open('minimum_model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()
joblib.dump(scaler, "scaler_vectorial3D.joblib")

models/forest_vectorial3D_classifier.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np

import sys
sys.path.insert(1, "../datasets")
from transformations import minimum

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))


# Transforms dataset
new_features = []
for observation in data["features"]:
    new_features.append(minimum(observation))

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(new_features, 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = LogisticRegression()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Save Model
f = open('minimum_model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()

models/logreg_minimum_classifier.py
PETComputacaoUFPR/tradutor_libras
# Trains the classifier (RandomForest)
import pickle
import joblib

from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import MinMaxScaler
import numpy as np

import sys
sys.path.insert(1, "../datasets")
from transformations import vectorial3D

# Loads dataset
data = pickle.load(open('../datasets/base_dataset.pickle', 'rb'))


# Transforms dataset
new_features = []
for observation in data["features"]:
    new_features.append(vectorial3D(observation))

# MinMax Scaling
scaler = MinMaxScaler()
new_features = scaler.fit_transform(new_features)

# Train/Test split
x_train, x_test, y_train, y_test = train_test_split(new_features, 
    data["labels"], test_size=0.2, shuffle=True, stratify=data["labels"])

# Trains Model
model = LogisticRegression()
model.fit(x_train, y_train)

# Tests Model
y_predict = model.predict(x_test)
score = accuracy_score(y_predict, y_test)
print('{}% of samples were classified correctly !'.format(score * 100))

# Save Model and Scaler
f = open('minimum_model.p', 'wb')
pickle.dump({'model': model}, f)
f.close()
joblib.dump(scaler, "scaler_vectorial3D.joblib")

models/logreg_vectorial3D_classifier.py
PETComputacaoUFPR/tradutor_libras