app.py

from deepsparse import Pipeline
import time
import gradio as gr

markdownn = '''
# Named Entity Recognition Pipeline with DeepSparse
DeepSparse is sparsity-aware inference runtime offering GPU-class performance on CPUs and APIs to integrate ML into your application. DeepSparse provides sparsified pipelines for computer vision and NLP. 
The pipelines are similar to Hugging Face pipelines but are faster because they have been pruned and quantized. SparseML Named Entity Recognition Pipelines integrate with Hugging Face’s Transformers library to enable the sparsification of a large set of transformers models. Here is sample code for a token classification pipeline: 
```
from deepsparse import Pipeline
pipeline = Pipeline.create(task="ner", model_path="zoo:nlp/token_classification/distilbert-none/pytorch/huggingface/conll2003/pruned80_quant-none-vnni")
inference = pipeline(text)
print(inference)
```
'''
task = "ner"
dense_qa_pipeline = Pipeline.create(
        task=task,
        model_path="zoo:nlp/token_classification/bert-base/pytorch/huggingface/conll2003/base-none",
    )

sparse_qa_pipeline = Pipeline.create(
        task=task,
        model_path="zoo:nlp/token_classification/bert-base/pytorch/huggingface/conll2003/12layer_pruned80_quant-none-vnni",
    )
def run_pipeline(text):
    dense_start = time.perf_counter()
    dense_entities = []
    sparse_entities = []
    
    dense_output = dense_qa_pipeline(text)
    for item in dict(dense_output)['predictions'][0]:
        dictionary = dict(item)
        entity = dictionary['entity']
        
        if entity == "LABEL_0":
            value = "O"
        elif entity == "LABEL_1":
            value = "B-PER"
        elif entity == "LABEL_2":
            value = "I-PER"
        elif entity == "LABEL_3":
            value = "-ORG"
        elif entity == "LABEL_4":
            value = "I-ORG"
        elif entity == "LABEL_5":
            value = "B-LOC"
        elif entity == "LABEL_6":
            value = "I-LOC"
        elif entity == "LABEL_7":
            value = "B-MISC"
        else:
            value = "I-MISC"
        dictionary['entity'] = value
        dense_entities.append(dictionary)
        
    dense_output = {"text": text, "entities": dense_entities}
        
    dense_end = time.perf_counter()
    dense_duration = (dense_end - dense_start) * 1000.0

    sparse_start = time.perf_counter()
    
    sparse_output = sparse_qa_pipeline(text)
    for item in dict(sparse_output)['predictions'][0]:
        sparse_dictionary = dict(item)
        entity = dictionary['entity']
        
        if entity == "LABEL_0":
            value = "O"
        elif entity == "LABEL_1":
            value = "B-PER"
        elif entity == "LABEL_2":
            value = "I-PER"
        elif entity == "LABEL_3":
            value = "-ORG"
        elif entity == "LABEL_4":
            value = "I-ORG"
        elif entity == "LABEL_5":
            value = "B-LOC"
        elif entity == "LABEL_6":
            value = "I-LOC"
        elif entity == "LABEL_7":
            value = "B-MISC"
        else:
            value = "I-MISC"
        dictionary['entity'] = value
        sparse_entities.append(sparse_dictionary)

    sparse_output = {"text": text, "entities": sparse_entities}
        
    sparse_result = dict(sparse_output)
    sparse_end = time.perf_counter()
    sparse_duration = (sparse_end - sparse_start) * 1000.0
    
    return sparse_output, sparse_duration, dense_output, dense_duration


with gr.Blocks() as demo:
    gr.Markdown(markdownn)
    
    with gr.Row():
        text = gr.Text(label="Text")
       
    with gr.Row():
        with gr.Column():
            dense_duration = gr.Number(label="Dense Latency (ms):")
            dense_answers = gr.HighlightedText(label="Dense model answer")
            
        with gr.Column():
            sparse_duration = gr.Number(label="Sparse Latency (ms):")
            sparse_answers = gr.HighlightedText(label="Sparse model answers")

    btn = gr.Button("Submit")
    btn.click(
        run_pipeline,
        inputs=[text],
        outputs=[sparse_answers,sparse_duration,dense_answers,dense_duration],
    )

   
    gr.Examples(
        [
                ["We are flying from Texas to California"]        
        ],
        inputs=[text],
    )


if __name__ == "__main__":
    demo.launch()