Upload 5 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,235 @@
+import streamlit as st
+import sparknlp
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from annotated_text import annotated_text
+from streamlit_tags import st_tags
+
+# Page configuration
+st.set_page_config(layout="wide", initial_sidebar_state="auto")
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(task, model_name):
+    document_assembler = DocumentAssembler().setInputCol('text').setOutputCol('document')
+    tokenizer = Tokenizer().setInputCols(['document']).setOutputCol('token')
+
+    if task == "Token Classification":
+        token_classifier = AlbertForTokenClassification() \
+            .pretrained(model_name, 'en') \
+            .setInputCols(['token', 'document']) \
+            .setOutputCol('ner') \
+            .setCaseSensitive(False) \
+            .setMaxSentenceLength(512)
+
+        ner_converter = NerConverter() \
+            .setInputCols(['document', 'token', 'ner']) \
+            .setOutputCol('entities')
+
+        pipeline = Pipeline(stages=[document_assembler, tokenizer, token_classifier, ner_converter])
+
+    elif task == "Sequence Classification":
+        sequence_classifier = AlbertForSequenceClassification() \
+            .pretrained(model_name, 'en') \
+            .setInputCols(['token', 'document']) \
+            .setOutputCol('class') \
+            .setCaseSensitive(False) \
+            .setMaxSentenceLength(512)
+
+        pipeline = Pipeline(stages=[document_assembler, tokenizer, sequence_classifier])
+
+    elif task == "Question Answering":
+        document_assembler = MultiDocumentAssembler() \
+            .setInputCols(["question", "context"]) \
+            .setOutputCols(["document_question", "document_context"])
+
+        span_classifier = AlbertForQuestionAnswering() \
+            .pretrained(model_name, "en") \
+            .setInputCols(["document_question", "document_context"]) \
+            .setOutputCol("answer") \
+            .setCaseSensitive(False)
+
+        pipeline = Pipeline(stages=[document_assembler, span_classifier])
+
+    return pipeline
+
+def fit_data(pipeline, task, data, question=None, context=None):
+    if task in ['Token Classification', 'Sequence Classification']:
+        empty_df = spark.createDataFrame([['']]).toDF('text')
+        pipeline_model = pipeline.fit(empty_df)
+        model = LightPipeline(pipeline_model)
+        result = model.fullAnnotate(data)
+    elif task == "Question Answering":
+        df = spark.createDataFrame([[question, context]]).toDF("question", "context")
+        result = pipeline.fit(df).transform(df)
+        result = result.select('answer.result').collect()
+    return result
+
+def annotate_text(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+tasks_models_descriptions = {
+    "Token Classification": {
+        "models": ["albert_base_token_classifier_conll03"],
+        "description": "The 'albert_base_token_classifier_conll03' model excels in identifying and classifying tokens within text. Ideal for tasks like named entity recognition (NER), it accurately extracts entities such as names, dates, and locations."
+    },
+    "Sequence Classification": {
+        "models": ["albert_base_sequence_classifier_ag_news", "albert_base_sequence_classifier_imdb"],
+        "description": "The 'albert_base_sequence_classifier_ag_news' model specializes in sentiment analysis and document classification. It accurately assesses the mood of customer reviews, classifies emails, and sorts text corpora."
+    },
+    "Question Answering": {
+        "models": ["albert_base_qa_squad2"],
+        "description": "The 'albert_base_qa_squad2' model is designed for answering questions based on provided context. Ideal for chatbots and virtual assistants, it delivers precise answers, improving user interaction and support system efficiency."
+    }
+}
+
+# Sidebar content
+task = st.sidebar.selectbox("Choose the task", list(tasks_models_descriptions.keys()))
+model_name = st.sidebar.selectbox("Choose the pretrained model", tasks_models_descriptions[task]["models"], help="For more info about the models visit: https://sparknlp.org/models")
+
+# Reference notebook link in sidebar
+colab_link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/DistilBertForTokenClassification.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(colab_link, unsafe_allow_html=True)
+
+# Page content
+st.markdown(f'<div class="main-title">Albert for {task}</div>', unsafe_allow_html=True)
+st.write(tasks_models_descriptions[task]["description"])
+
+if model == 'albert_base_sequence_classifier_ag_news':
+  examples_to_select = [
+    "The Prime Minister of the country announced new policies aimed at reducing carbon emissions by 40% over the next decade. These measures are part of a global effort to combat climate change.",
+    "A devastating earthquake hit the coastal city, leaving thousands homeless. International aid is pouring in to assist in the relief efforts as rescue teams continue to search for survivors.",
+    "The basketball star signed a record-breaking contract, making him the highest-paid player in the league. Fans are excited to see how he will perform in the upcoming season.",
+    "The Olympic Games concluded with a spectacular closing ceremony. The host nation topped the medal tally with an impressive number of gold medals across various sports.",
+    "The stock market saw significant gains today, with the Dow Jones reaching an all-time high. Investors are optimistic about the economic recovery following positive job growth data.",
+    "A major retail chain announced the closure of 150 stores nationwide due to declining sales. The company plans to focus more on its online presence to adapt to changing consumer behavior.",
+    "Scientists have discovered a new exoplanet that could potentially support life. The planet, located in a nearby star system, has conditions similar to those on Earth.",
+    "A leading technology firm unveiled its latest smartphone model, featuring a revolutionary camera system and a powerful new processor. The device is expected to set new standards in mobile computing.",
+    "In a historic move, the two rival nations signed a peace treaty, ending decades of hostility. The agreement is seen as a major step toward lasting stability in the region.",
+    "The tennis legend announced her retirement after a career spanning two decades, during which she won numerous Grand Slam titles. Tributes are pouring in from across the sporting world.",
+    "The energy company reported a significant increase in profits this quarter, driven by higher oil prices. Analysts are predicting continued growth in the coming months.",
+    "A new study has revealed that a diet high in fiber can significantly reduce the risk of heart disease. The findings are based on a large-scale analysis of dietary habits over the past decade."
+    ]
+else:
+  examples_to_select = [
+    "This movie was absolutely fantastic! The storyline was gripping, the characters were well-developed, and the cinematography was stunning. I was on the edge of my seat the entire time.",
+    "A heartwarming and beautiful film. The performances were top-notch, and the direction was flawless. This is easily one of the best movies I've seen this year.",
+    "What a delightful surprise! The humor was spot on, and the plot was refreshingly original. The cast did an amazing job bringing the characters to life. Highly recommended!",
+    "This was one of the worst movies I’ve ever seen. The plot was predictable, the acting was wooden, and the pacing was painfully slow. I couldn’t wait for it to end.",
+    "A complete waste of time. The movie lacked any real substance or direction, and the dialogue was cringe-worthy. I wouldn’t recommend this to anyone.",
+    "I had high hopes for this film, but it turned out to be a huge disappointment. The story was disjointed, and the special effects were laughably bad. Don’t bother watching this one.",
+    "The movie was okay, but nothing special. It had a few good moments, but overall, it felt pretty average. Not something I would watch again, but it wasn’t terrible either.",
+    "An average film with a decent plot. The acting was passable, but it didn't leave much of an impression on me. It's a movie you might watch once and forget about.",
+    "This movie was neither good nor bad, just kind of there. It had some interesting ideas, but they weren’t executed very well. It’s a film you could take or leave."
+  ]
+
+# Load examples
+examples_mapping = {
+    "Token Classification": [
+        "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+        "The Mona Lisa is a 16th century oil painting created by Leonardo. It's held at the Louvre in Paris.",
+        "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously. “I can tell you very senior CEOs of major American car companies would shake my hand and turn away because I wasn’t worth talking to,” said Thrun, now the co-founder and CEO of online higher education startup Udacity, in an interview with Recode earlier this week.",
+        "Facebook is a social networking service launched as TheFacebook on February 4, 2004. It was founded by Mark Zuckerberg with his college roommates and fellow Harvard University students Eduardo Saverin, Andrew McCollum, Dustin Moskovitz and Chris Hughes. The website's membership was initially limited by the founders to Harvard students, but was expanded to other colleges in the Boston area, the Ivy League, and gradually most universities in the United States and Canada.",
+        "The history of natural language processing generally started in the 1950s, although work can be found from earlier periods. In 1950, Alan Turing published an article titled 'Computing Machinery and Intelligence' which proposed what is now called the Turing test as a criterion of intelligence",
+        "Geoffrey Everest Hinton is an English Canadian cognitive psychologist and computer scientist, most noted for his work on artificial neural networks. Since 2013 he divides his time working for Google and the University of Toronto. In 2017, he cofounded and became the Chief Scientific Advisor of the Vector Institute in Toronto.",
+        "When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there.",
+        "Steven Paul Jobs was an American business magnate, industrial designer, investor, and media proprietor. He was the chairman, chief executive officer (CEO), and co-founder of Apple Inc., the chairman and majority shareholder of Pixar, a member of The Walt Disney Company's board of directors following its acquisition of Pixar, and the founder, chairman, and CEO of NeXT. Jobs is widely recognized as a pioneer of the personal computer revolution of the 1970s and 1980s, along with Apple co-founder Steve Wozniak. Jobs was born in San Francisco, California, and put up for adoption. He was raised in the San Francisco Bay Area. He attended Reed College in 1972 before dropping out that same year, and traveled through India in 1974 seeking enlightenment and studying Zen Buddhism.",
+        "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+        "Other than being the king of the north, John Snow is a an english physician and a leader in the development of anaesthesia and medical hygiene. He is considered for being the first one using data to cure cholera outbreak in 1834."
+    ],
+    "Sequence Classification": examples_to_select,
+    "Question Answering": {
+        """What does increased oxygen concentrations in the patient’s lungs displace?""": """Hyperbaric (high-pressure) medicine uses special oxygen chambers to increase the partial pressure of O 2 around the patient and, when needed, the medical staff. Carbon monoxide poisoning, gas gangrene, and decompression sickness (the ’bends’) are sometimes treated using these devices. Increased O 2 concentration in the lungs helps to displace carbon monoxide from the heme group of hemoglobin. Oxygen gas is poisonous to the anaerobic bacteria that cause gas gangrene, so increasing its partial pressure helps kill them. Decompression sickness occurs in divers who decompress too quickly after a dive, resulting in bubbles of inert gas, mostly nitrogen and helium, forming in their blood. Increasing the pressure of O 2 as soon as possible is part of the treatment.""",
+        """What category of game is Legend of Zelda: Twilight Princess?""": """The Legend of Zelda: Twilight Princess (Japanese: ゼルダの伝説 トワイライトプリンセス, Hepburn: Zeruda no Densetsu: Towairaito Purinsesu?) is an action-adventure game developed and published by Nintendo for the GameCube and Wii home video game consoles. It is the thirteenth installment in the The Legend of Zelda series. Originally planned for release on the GameCube in November 2005, Twilight Princess was delayed by Nintendo to allow its developers to refine the game, add more content, and port it to the Wii. The Wii version was released alongside the console in North America in November 2006, and in Japan, Europe, and Australia the following month. The GameCube version was released worldwide in December 2006.""",
+        """Who is founder of Alibaba Group?""": """Alibaba Group founder Jack Ma has made his first appearance since Chinese regulators cracked down on his business empire. His absence had fuelled speculation over his whereabouts amid increasing official scrutiny of his businesses. The billionaire met 100 rural teachers in China via a video meeting on Wednesday, according to local government media. Alibaba shares surged 5% on Hong Kong's stock exchange on the news.""",
+        """For what instrument did Frédéric write primarily for?""": """Frédéric François Chopin (/ˈʃoʊpæn/; French pronunciation: ​[fʁe.de.ʁik fʁɑ̃.swa ʃɔ.pɛ̃]; 22 February or 1 March 1810 – 17 October 1849), born Fryderyk Franciszek Chopin,[n 1] was a Polish and French (by citizenship and birth of father) composer and a virtuoso pianist of the Romantic era, who wrote primarily for the solo piano. He gained and has maintained renown worldwide as one of the leading musicians of his era, whose "poetic genius was based on a professional technique that was without equal in his generation." Chopin was born in what was then the Duchy of Warsaw, and grew up in Warsaw, which after 1815 became part of Congress Poland. A child prodigy, he completed his musical education and composed his earlier works in Warsaw before leaving Poland at the age of 20, less than a month before the outbreak of the November 1830 Uprising.""",
+        """The most populated city in the United States is which city?""": """New York—often called New York City or the City of New York to distinguish it from the State of New York, of which it is a part—is the most populous city in the United States and the center of the New York metropolitan area, the premier gateway for legal immigration to the United States and one of the most populous urban agglomerations in the world. A global power city, New York exerts a significant impact upon commerce, finance, media, art, fashion, research, technology, education, and entertainment, its fast pace defining the term New York minute. Home to the headquarters of the United Nations, New York is an important center for international diplomacy and has been described as the cultural and financial capital of the world."""
+    }
+}
+
+# Input Handling
+if task == 'Question Answering':
+    st.subheader('Try it yourself!')
+    question_input = st.text_input('Enter a question')
+    context_input = st.text_area("Enter the context")
+    st.subheader('Selected Text')
+    if question_input and context_input:
+        question = question_input
+        context = context_input
+    else:
+        st.error("Please enter both question and context.")
+
+else:
+    st.subheader('Try it yourself!')
+    user_input = st.text_input("Enter a sentence")
+    text_to_analyze = user_input if user_input else "Example text"
+    st.markdown(f"**Text to Analyze:** {text_to_analyze}")
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(task, model_name)
+
+# Process Data
+output = fit_data(pipeline, task, text_to_analyze, question, context)
+
+st.subheader("Processed Output:")
+
+if task == 'Token Classification':
+    results = {
+        'Document': output[0]['document'][0].result,
+        'NER Chunk': [n.result for n in output[0]['entities']],
+        "NER Label": [n.metadata['entity'] for n in output[0]['entities']]
+    }
+    annotate_text(results)
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)
+
+elif task == 'Sequence Classification':
+    st.markdown(f"Classification: **{output[0]['class'][0].result}**")
+
+elif task == "Question Answering":
+    output_text = "".join(output[0][0])
+    st.markdown(f"Answer: **{output_text}**")

Dockerfile

@@ -0,0 +1,72 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean
+
+# Set up JAVA_HOME
+RUN echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> /etc/profile \
+    && echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /etc/profile
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set up PySpark to use Python 3.8 for both driver and workers
+ENV PYSPARK_PYTHON=/usr/bin/python3.8
+ENV PYSPARK_DRIVER_PYTHON=/usr/bin/python3.8
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,610 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+        .benchmark-table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 20px;
+        }
+        .benchmark-table th, .benchmark-table td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        .benchmark-table th {
+            background-color: #4A90E2;
+            color: white;
+        }
+        .benchmark-table td {
+            background-color: #f2f2f2;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Title
+st.markdown('<div class="main-title">Introduction to ALBERT Annotators in Spark NLP</div>', unsafe_allow_html=True)
+
+# Subtitle
+st.markdown("""
+<div class="section">
+    <p>ALBERT (A Lite BERT) offers a more efficient alternative to BERT by implementing two parameter-reduction techniques: splitting the embedding matrix and using repeating layers. It maintains high performance while being more memory-efficient. Below, we provide an overview of the ALBERT annotator for token classification:</p>
+</div>
+""", unsafe_allow_html=True)
+
+tab1, tab2, tab3 = st.tabs(["ALBERT for Token Classification", "ALBERT for Sequence Classification", "ALBERT for Question Answering"])
+
+with tab1:
+    st.markdown("""
+    <div class="section">
+        <h2>ALBERT for Token Classification</h2>
+        <p>The <strong>AlbertForTokenClassification</strong> annotator is designed for Named Entity Recognition (NER) tasks using ALBERT. This model efficiently handles token classification, enabling the identification and classification of entities in text. The ALBERT model, with its parameter-reduction techniques, achieves state-of-the-art performance while being more lightweight compared to BERT.</p>
+        <p>Token classification with ALBERT enables:</p>
+        <ul>
+            <li><strong>Named Entity Recognition (NER):</strong> Identifying and classifying entities such as names, organizations, locations, and other predefined categories.</li>
+            <li><strong>Information Extraction:</strong> Extracting key information from unstructured text for further analysis.</li>
+            <li><strong>Text Categorization:</strong> Enhancing document retrieval and categorization based on entity recognition.</li>
+        </ul>
+        <p>Here is an example of how ALBERT token classification works:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Entity</th>
+                <th>Label</th>
+            </tr>
+            <tr>
+                <td>Google</td>
+                <td>ORG</td>
+            </tr>
+            <tr>
+                <td>Satya Nadella</td>
+                <td>PER</td>
+            </tr>
+            <tr>
+                <td>Seattle</td>
+                <td>LOC</td>
+            </tr>
+        </table>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # ALBERT Token Classification - NER CoNLL
+    st.markdown('<div class="sub-title">ALBERT Token Classification - NER CoNLL</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>The <strong>albert_base_token_classifier_conll03</strong> is a fine-tuned ALBERT model for token classification tasks, specifically adapted for Named Entity Recognition (NER) on the CoNLL-03 dataset. It recognizes four types of entities: location (LOC), organizations (ORG), person (PER), and Miscellaneous (MISC).</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # How to Use the Model - Token Classification
+    st.markdown('<div class="sub-title">How to Use the Model</div>', unsafe_allow_html=True)
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    document_assembler = DocumentAssembler() \\
+        .setInputCol('text') \\
+        .setOutputCol('document')
+
+    tokenizer = Tokenizer() \\
+        .setInputCols(['document']) \\
+        .setOutputCol('token')
+
+    tokenClassifier = AlbertForTokenClassification \\
+        .pretrained('albert_base_token_classifier_conll03', 'en') \\
+        .setInputCols(['token', 'document']) \\
+        .setOutputCol('ner') \\
+        .setCaseSensitive(True) \\
+        .setMaxSentenceLength(512)
+
+    # Convert NER labels to entities
+    ner_converter = NerConverter() \\
+        .setInputCols(['document', 'token', 'ner']) \\
+        .setOutputCol('entities')
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        tokenizer,
+        tokenClassifier,
+        ner_converter
+    ])
+
+    example = spark.createDataFrame([["My name is John!"]]).toDF("text")
+    result = pipeline.fit(example).transform(example)
+
+    result.select(
+        expr("explode(entities) as ner_chunk")
+    ).select(
+        col("ner_chunk.result").alias("chunk"),
+        col("ner_chunk.metadata.entity").alias("ner_label")
+    ).show(truncate=False)
+    ''', language='python')
+
+    # Results
+    st.text("""
+    +-----+---------+
+    |chunk|ner_label|
+    +-----+---------+
+    |John |PER      |
+    +-----+---------+
+    """)
+
+    # Performance Metrics
+    st.markdown('<div class="sub-title">Performance Metrics</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Here are the detailed performance metrics for the ALBERT token classification model:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Entity</th>
+                <th>Precision</th>
+                <th>Recall</th>
+                <th>F1-Score</th>
+                <th>Support</th>
+            </tr>
+            <tr>
+                <td>B-LOC</td>
+                <td>0.95</td>
+                <td>0.97</td>
+                <td>0.96</td>
+                <td>1837</td>
+            </tr>
+            <tr>
+                <td>B-MISC</td>
+                <td>0.87</td>
+                <td>0.86</td>
+                <td>0.87</td>
+                <td>922</td>
+            </tr>
+            <tr>
+                <td>B-ORG</td>
+                <td>0.90</td>
+                <td>0.91</td>
+                <td>0.90</td>
+                <td>1341</td>
+            </tr>
+            <tr>
+                <td>B-PER</td>
+                <td>0.91</td>
+                <td>0.97</td>
+                <td>0.94</td>
+                <td>1842</td>
+            </tr>
+            <tr>
+                <td>I-LOC</td>
+                <td>0.88</td>
+                <td>0.86</td>
+                <td>0.87</td>
+                <td>257</td>
+            </tr>
+            <tr>
+                <td>I-MISC</td>
+                <td>0.78</td>
+                <td>0.76</td>
+                <td>0.77</td>
+                <td>346</td>
+            </tr>
+            <tr>
+                <td>I-ORG</td>
+                <td>0.84</td>
+                <td>0.85</td>
+                <td>0.85</td>
+                <td>751</td>
+            </tr>
+            <tr>
+                <td>I-PER</td>
+                <td>0.97</td>
+                <td>0.92</td>
+                <td>0.94</td>
+                <td>1307</td>
+            </tr>
+            <tr>
+                <td>O</td>
+                <td>0.99</td>
+                <td>0.99</td>
+                <td>0.99</td>
+                <td>42759</td>
+            </tr>
+            <tr>
+                <td>average</td>
+                <td>0.92</td>
+                <td>0.92</td>
+                <td>0.92</td>
+                <td>52000</td>
+            </tr>
+        </table>
+    </div>
+                
+    """, unsafe_allow_html=True)
+    # Model Info Section
+    st.markdown('<div class="sub-title">Model Info</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><strong>Model Name:</strong> ALBERT for Token Classification</li>
+            <li><strong>Pretrained Model:</strong> albert_base_token_classifier_conll03</li>
+            <li><strong>Training Dataset:</strong> CoNLL-03</li>
+            <li><strong>Languages Supported:</strong> English</li>
+            <li><strong>Use Cases:</strong>
+                <ul>
+                    <li>Named Entity Recognition (NER)</li>
+                    <li>Information Extraction</li>
+                    <li>Text Categorization</li>
+                </ul>
+            </li>
+            <li><strong>Performance:</strong> High accuracy with a focus on memory efficiency</li>
+            <li><strong>Implementation:</strong> Spark NLP</li>
+            <li><strong>Resource Requirements:</strong> Moderate computational resources; suitable for production environments with optimization</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://arxiv.org/abs/1909.11942" target="_blank">Lan, Z., Chen, J., Goodman, S., Gimpel, K., Sharma, P., & Soricut, R. (2019). ALBERT: A Lite BERT for Self-supervised Learning of Language Representations. arXiv preprint arXiv:1909.11942.</a></li>
+            <li><a class="link" href="https://github.com/google-research/albert" target="_blank">Google Research's ALBERT GitHub Repository</a></li>
+            <li><a class="link" href="https://sparknlp.org/2022/06/15/albert_base_qa_squad2_en_3_0.html" target="_blank">Spark NLP Model - albert_base_qa_squad2</a></li>
+            <li><a class="link" href="https://nlp.stanford.edu/projects/conll2003/" target="_blank">CoNLL-03 Named Entity Recognition Dataset</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+with tab2:
+    st.markdown("""
+    <div class="section">
+        <h2>ALBERT for Sequence Classification</h2>
+        <p>The <strong>AlbertForSequenceClassification</strong> annotator is tailored for tasks like sentiment analysis or multi-class text classification using the ALBERT model. This model efficiently handles sequence classification, achieving state-of-the-art performance with reduced parameters compared to BERT.</p>
+        <p>Sequence classification with ALBERT enables:</p>
+        <ul>
+            <li><strong>Sentiment Analysis:</strong> Determining the sentiment expressed in text, such as positive, negative, or neutral.</li>
+            <li><strong>Multi-Class Text Classification:</strong> Categorizing text into predefined classes, such as news categories or topics.</li>
+            <li><strong>Document Classification:</strong> Enhancing search and categorization of documents based on content classification.</li>
+        </ul>
+        <p>Here is an example of how ALBERT sequence classification works:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Text</th>
+                <th>Label</th>
+            </tr>
+            <tr>
+                <td>Disney Comics was a comic book publishing company operated by The Walt Disney Company which ran from 1990 to 1993.</td>
+                <td>Business</td>
+            </tr>
+        </table>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # ALBERT Sequence Classification - AG News
+    st.markdown('<div class="sub-title">ALBERT Sequence Classification - AG News</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>The <strong>albert_base_sequence_classifier_ag_news</strong> is a fine-tuned ALBERT model for sequence classification tasks, specifically adapted for text classification on the AG News dataset. It recognizes four categories: Business, Sci/Tech, Sports, and World.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # How to Use the Model - Sequence Classification
+    st.markdown('<div class="sub-title">How to Use the Model</div>', unsafe_allow_html=True)
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    document_assembler = DocumentAssembler() \\
+        .setInputCol('text') \\
+        .setOutputCol('document')
+
+    tokenizer = Tokenizer() \\
+        .setInputCols(['document']) \\
+        .setOutputCol('token')
+
+    sequenceClassifier = AlbertForSequenceClassification \\
+        .pretrained('albert_base_sequence_classifier_ag_news', 'en') \\
+        .setInputCols(['token', 'document']) \\
+        .setOutputCol('class') \\
+        .setCaseSensitive(False) \\
+        .setMaxSentenceLength(512)
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        tokenizer,
+        sequenceClassifier    
+    ])
+
+    example = spark.createDataFrame([["Disney Comics was a comic book publishing company operated by The Walt Disney Company which ran from 1990 to 1993."]]).toDF("text")
+    result = pipeline.fit(example).transform(example)
+
+    result.select(
+        expr("explode(class) as classification_result")
+    ).select(
+        col("classification_result.result").alias("category")
+    ).show(truncate=False)
+    ''', language='python')
+
+    # Results
+    st.text("""
+    +---------+
+    |category |
+    +---------+
+    |Business |
+    +---------+
+    """)
+
+    # Performance Metrics
+    st.markdown('<div class="sub-title">Performance Metrics</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>Here are the detailed performance metrics for the ALBERT sequence classification model on the AG News dataset:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Metric</th>
+                <th>Score</th>
+            </tr>
+            <tr>
+                <td>Accuracy</td>
+                <td>0.9472</td>
+            </tr>
+            <tr>
+                <td>F1-Score</td>
+                <td>0.9472</td>
+            </tr>
+            <tr>
+                <td>Precision</td>
+                <td>0.9472</td>
+            </tr>
+            <tr>
+                <td>Recall</td>
+                <td>0.9472</td>
+            </tr>
+            <tr>
+                <td>Evaluation Loss</td>
+                <td>0.1882</td>
+            </tr>
+        </table>
+    </div>
+                
+    """, unsafe_allow_html=True)
+    
+    # Model Info Section
+    st.markdown('<div class="sub-title">Model Info</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><strong>Model Name:</strong> ALBERT for Sequence Classification</li>
+            <li><strong>Pretrained Model:</strong> albert_base_sequence_classifier_ag_news</li>
+            <li><strong>Training Dataset:</strong> AG News</li>
+            <li><strong>Languages Supported:</strong> English</li>
+            <li><strong>Use Cases:</strong>
+                <ul>
+                    <li>Sentiment Analysis</li>
+                    <li>Multi-Class Text Classification</li>
+                    <li>Document Classification</li>
+                </ul>
+            </li>
+            <li><strong>Performance:</strong> High accuracy with a focus on memory efficiency</li>
+            <li><strong>Implementation:</strong> Spark NLP</li>
+            <li><strong>Resource Requirements:</strong> Moderate computational resources; suitable for production environments with optimization</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://arxiv.org/abs/1909.11942" target="_blank">Lan, Z., Chen, J., Goodman, S., Gimpel, K., Sharma, P., & Soricut, R. (2019). ALBERT: A Lite BERT for Self-supervised Learning of Language Representations. arXiv preprint arXiv:1909.11942.</a></li>
+            <li><a class="link" href="https://github.com/google-research/albert" target="_blank">Google Research's ALBERT GitHub Repository</a></li>
+            <li><a class="link" href="https://sparknlp.org/2021/12/16/albert_base_sequence_classifier_ag_news_en.html" target="_blank">Spark NLP Model - albert_base_sequence_classifier_ag_news</a></li>
+            <li><a class="link" href="https://huggingface.co/datasets/ag_news" target="_blank">AG News Dataset</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+with tab3:
+    st.markdown("""
+    <div class="section">
+        <h2>ALBERT for Question Answering</h2>
+        <p>The <strong>AlbertForQuestionAnswering</strong> annotator is specialized for tasks involving Question Answering (QA) using the ALBERT model. This model efficiently processes question-context pairs to provide accurate answers, making it ideal for QA systems and information retrieval applications.</p>
+        <p>Question Answering with ALBERT enables:</p>
+        <ul>
+            <li><strong>Information Retrieval:</strong> Extracting precise answers from large text corpora based on user queries.</li>
+            <li><strong>Knowledge Management:</strong> Enhancing customer support and information systems by providing accurate answers.</li>
+            <li><strong>Contextual Understanding:</strong> Leveraging ALBERT’s capabilities to understand the context of questions and provide relevant answers.</li>
+        </ul>
+        <p>Here is an example of how ALBERT question answering works:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Question</th>
+                <th>Context</th>
+                <th>Answer</th>
+            </tr>
+            <tr>
+                <td>What is my name?</td>
+                <td>My name is Clara and I live in Berkeley.</td>
+                <td>Clara</td>
+            </tr>
+        </table>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # ALBERT Question Answering - SQuAD2
+    st.markdown('<div class="sub-title">ALBERT Question Answering - SQuAD2</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>The <strong>albert_base_qa_squad2</strong> is a fine-tuned ALBERT model for Question Answering tasks, specifically adapted for the SQuAD2 dataset. It is capable of answering questions based on the provided context with high accuracy.</p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # How to Use the Model - Question Answering
+    st.markdown('<div class="sub-title">How to Use the Model</div>', unsafe_allow_html=True)
+    st.code('''
+    from sparknlp.base import *
+    from sparknlp.annotator import *
+    from pyspark.ml import Pipeline
+    from pyspark.sql.functions import col, expr
+
+    documentAssembler = MultiDocumentAssembler() \\
+        .setInputCols(["question", "context"]) \\
+        .setOutputCols(["document_question", "document_context"])
+
+    spanClassifier = AlbertForQuestionAnswering.pretrained("albert_base_qa_squad2","en") \\
+        .setInputCols(["document_question", "document_context"]) \\
+        .setOutputCol("answer") \\
+        .setCaseSensitive(False)
+
+    pipeline = Pipeline(stages=[documentAssembler, spanClassifier])
+
+    data = spark.createDataFrame([["What is my name?", "My name is Clara and I live in Berkeley."]]).toDF("question", "context")
+    result = pipeline.fit(data).transform(data)
+
+    result.select(
+        col("answer.result").alias("predicted_answer")
+    ).show(truncate=False)
+    ''', language='python')
+
+    # Results
+    st.text("""
+    +----------------+
+    |predicted_answer|
+    +----------------+
+    |[clara]         |
+    +----------------+
+    """)
+
+    # Performance Metrics
+    st.markdown('<div class="sub-title">Performance Metrics</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <p>The performance metrics of the ALBERT question answering model on a development subset of the SQuAD2 dataset are:</p>
+        <table class="benchmark-table">
+            <tr>
+                <th>Metric</th>
+                <th>Score</th>
+            </tr>
+            <tr>
+                <td>Exact Match</td>
+                <td>78.71%</td>
+            </tr>
+            <tr>
+                <td>F1 Score</td>
+                <td>81.89%</td>
+            </tr>
+            <tr>
+                <td>Total</td>
+                <td>6078</td>
+            </tr>
+            <tr>
+                <td>HasAns Exact Match</td>
+                <td>75.40%</td>
+            </tr>
+            <tr>
+                <td>HasAns F1 Score</td>
+                <td>82.04%</td>
+            </tr>
+            <tr>
+                <td>HasAns Total</td>
+                <td>2910</td>
+            </tr>
+            <tr>
+                <td>NoAns Exact Match</td>
+                <td>81.76%</td>
+            </tr>
+            <tr>
+                <td>NoAns F1 Score</td>
+                <td>81.76%</td>
+            </tr>
+            <tr>
+                <td>NoAns Total</td>
+                <td>3168</td>
+            </tr>
+            <tr>
+                <td>Best Exact Match</td>
+                <td>78.73%</td>
+            </tr>
+            <tr>
+                <td>Best F1 Score</td>
+                <td>81.91%</td>
+            </tr>
+        </table>
+    </div>
+    """, unsafe_allow_html=True)
+    
+    # Model Info Section
+    st.markdown('<div class="sub-title">Model Info</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><strong>Model Name:</strong> ALBERT for Question Answering</li>
+            <li><strong>Pretrained Model:</strong> albert_base_qa_squad2</li>
+            <li><strong>Training Dataset:</strong> SQuAD2</li>
+            <li><strong>Languages Supported:</strong> English</li>
+            <li><strong>Use Cases:</strong>
+                <ul>
+                    <li>Information Retrieval</li>
+                    <li>Knowledge Management</li>
+                    <li>Contextual Understanding</li>
+                </ul>
+            </li>
+            <li><strong>Performance:</strong> High accuracy with optimized resource usage</li>
+            <li><strong>Implementation:</strong> Spark NLP</li>
+            <li><strong>Resource Requirements:</strong> Moderate computational resources; suitable for production environments</li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+    # References Section
+    st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+    st.markdown("""
+    <div class="section">
+        <ul>
+            <li><a class="link" href="https://arxiv.org/abs/1909.11942" target="_blank">Lan, Z., Chen, J., Goodman, S., Gimpel, K., Sharma, P., & Soricut, R. (2019). ALBERT: A Lite BERT for Self-supervised Learning of Language Representations. arXiv preprint arXiv:1909.11942.</a></li>
+            <li><a class="link" href="https://sparknlp.org/2022/06/15/albert_base_qa_squad2_en_3_0.html" target="_blank">Spark NLP Model - albert_base_qa_squad2</a></li>
+        </ul>
+    </div>
+    """, unsafe_allow_html=True)
+
+# Community & Support
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="content">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,7 @@
+streamlit
+st-annotated-text
+streamlit-tags
+pandas
+numpy
+spark-nlp
+pyspark