Secpar is a Python command-line tool designed to scrape code submissions from various online programming platforms and store them in a GitHub repository. It supports platforms such as Codeforces, CSES (University of Helsinki), and Vjudge
from secpar.lib.Input.args import *
from secpar.lib.Commands.CommandFactory import *


def main():
    print("""
          _____                                
         / ___/ ___   _____ ____   ____ _ _____
         \__ \ / _ \ / ___// __ \ / __ `// ___/
        ___/ //  __// /__ / /_/ // /_/ // /    
       /____/ \___/ \___// .___/ \__,_//_/     
                        /_/    
      ==========================================        
        """)

    input_data = parse_args()
    command = CommandFactory(input_data).create()
    command.execute()

if __name__ == "__main__":
    main()
secpar/__main__.py
MostafaGalal1/Secpar
# Import the 'abstractmethod' decorator from the 'abc' module.
from abc import abstractmethod

# Define an abstract base class called 'AbstractCommand'.
class AbstractCommand:
    # Declare an abstract method 'execute'.
    @abstractmethod
    def execute(self):
        pass  # Placeholder 'pass' statement indicating no implementation.

# End of the 'AbstractCommand' class definition.

secpar/lib/Commands/AbstractCommand.py
MostafaGalal1/Secpar
# Import necessary modules.
import os
from secpar.lib.Commands.InitCommand import InitCommand
from secpar.lib.Commands.ScrapeCommand import ScrapeCommand
from secpar.lib.Commands.AbstractCommand import AbstractCommand

# Define a class called 'CommandFactory' responsible for creating command objects.
class CommandFactory:
    def __init__(self, data):
        self.input_data = data  # Initialize the factory with input data.

    def create(self):
        # Check the 'command' attribute in the input data to determine the type of command to create.
        if self.input_data.command == "init":
            return InitCommand()  # Return an 'InitCommand' object if the command is 'init'.
        elif self.input_data.scrape is not None:
            return ScrapeCommand(self.input_data.scrape)  # Return a 'ScrapCommand' object if 'scrap' is not None.
        elif self.input_data.command is None:
            # If 'command' is None, check if a specific file exists and provide feedback to the user.
            path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "Resources", "User", "UserData.json")
            if os.path.exists(path):
                print("Please specify the platform to scrap, example: `secpar -s codeforces`")
                return AbstractCommand()  # Return an 'AbstractCommand' object.
            else:
                return InitCommand()  # Return an 'InitCommand' object if the file does not exist.

# End of the 'CommandFactory' class definition.

secpar/lib/Commands/CommandFactory.py
MostafaGalal1/Secpar
# Import necessary modules.
import os
import json
from secpar.lib.Commands.AbstractCommand import AbstractCommand

# Define a class called 'InitCommand' that inherits from 'AbstractCommand'.
class InitCommand(AbstractCommand):
    def __init__(self):
        # Define the path to the 'UserData.json' file.
        self.path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "Resources", "User", "UserData.json")

        # If the file exists, remove it to start fresh.
        if os.path.exists(self.path):
            os.remove(self.path)

    def execute(self):
        # Prompt the user for GitHub-related information.
        github_name = input("Enter your GitHub username: ")
        github_repo_name = input("Enter the name of your GitHub repository: ")
        access_token_github = input("Enter your GitHub access token: ")

        # Create a dictionary containing user data.
        user_data = {
            "repo_owner": github_name,
            "repo_name": github_repo_name,
            "access_token": access_token_github,
        }

        try:
            # Write the user data to the 'UserData.json' file.
            with open(self.path, 'w') as json_file:
                json.dump(user_data, json_file, indent=4)
        except Exception as e:
            # Handle any exceptions that may occur during file writing.
            print(f"Error writing to 'UserData.json': {e}")

# End of the 'InitCommand' class definition.

secpar/lib/Commands/InitCommand.py
MostafaGalal1/Secpar
import os
import json
import getpass

from secpar.lib.Commands.AbstractCommand import *
from secpar.lib.Scrapers.ScraperFactory import ScraperFactory
from secpar.lib.Formatters.ReadMeBuilder import ReadMeBuilder


class ScrapeCommand(AbstractCommand):
    def __init__(self, scraper_name):
        self.scraper_name = scraper_name.lower()
        self.path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "Resources", "User", "UserData.json")
        try:
            with open(self.path, 'r') as file:
                self.data = json.load(file)
        except FileNotFoundError:
            raise ValueError("You need to initialize your data first")

    def execute(self):
        if self.scraper_name == "cses" or self.scraper_name == "vjudge":
            self.data["user_name"] = input(f"Enter your {self.scraper_name} username: ")
            self.data["password"] = getpass.getpass("Enter your password: ")
        elif self.scraper_name == "codeforces":
            self.data["user_name"] = input("Enter your Codeforces handle: ")
            use_tor_input = input("Do you want to use tor? (y/n): ").strip().lower()
            self.data["use_tor"] = True if use_tor_input =="y" else False

        ScraperFactory(self.scraper_name, self.data).create().scrape()
        readme_builder = ReadMeBuilder(self.data)
        readme_content = readme_builder.build()
        readme_builder.update_readme(readme_content)

secpar/lib/Commands/ScrapeCommand.py
MostafaGalal1/Secpar
import json
from abc import ABC, abstractmethod

# Define an abstract base class called 'AbstractFormatter' that inherits from 'ABC' (Abstract Base Class).
class AbstractFormatter(ABC):
    def __init__(self, repo):
        # Initialize common attributes for formatters.
        self.platform = self.__class__.__name__.split("Formatter")[0]  # Get the platform name from the class name.
        self.repo = repo
        self.data = self.get_data()  # Load data from a specific file.

    @abstractmethod
    def format(self):
        pass  # Declare an abstract 'format' method that should be implemented in concrete formatter classes.

    def get_data(self):
        try:
            # Attempt to retrieve and parse data from a specific file in the repository.
            file_contents = self.repo.get_contents(f"submissions/{self.platform}Submissions.json")
            data = json.loads(file_contents.decoded_content.decode('utf-8'))
            return data
        except:
            pass  # Handle exceptions gracefully (e.g., if the file doesn't exist).

    def get_header(self):
        return self.data.get("Header")  # Get the "Header" information from the loaded data.

# End of the 'AbstractFormatter' class definition.

secpar/lib/Formatters/AbstractFormatter.py
MostafaGalal1/Secpar
# Import the base class 'AbstractFormatter'.
from secpar.lib.Formatters.AbstractFormatter import AbstractFormatter

# Define a concrete formatter class 'CodeforcesFormatter' that inherits from 'AbstractFormatter'.
class CodeforcesFormatter(AbstractFormatter):
    def format(self):
        # Check if there is no data; return an empty string.
        if self.data is None:
            return ""

        # Initialize the readme content with the header.
        readme_content = self.get_header()

        # Initialize a count for numbering submissions.
        count = len(self.data["Submissions"])

        # Iterate over each submission in the data.
        for submission in self.data["Submissions"]:
            name = submission.get("name")
            problem_link = submission.get("problem_link")
            language = submission.get("language")
            solution_link = submission.get("directory_link")
            tags = submission.get("tags")
            date = submission.get("date")

            # Append formatted submission information to the readme content.
            readme_content += f'{count} | [{name}]({problem_link}) | [{language}]({solution_link}) | {tags} | {date} |\n'
            count -= 1  # Decrement the count.

        readme_content += '\n'  # Add a newline at the end.
        return readme_content  # Return the formatted readme content.

# End of the 'CodeforcesFormatter' class definition.

secpar/lib/Formatters/CodeforcesFormatter.py
MostafaGalal1/Secpar
# Import the base class 'AbstractFormatter'.
from secpar.lib.Formatters.AbstractFormatter import AbstractFormatter

# Define a concrete formatter class 'CsesFormatter' that inherits from 'AbstractFormatter'.
class CsesFormatter(AbstractFormatter):

    def format(self):
        # Check if there is no data; return an empty string.
        if self.data is None:
            return ""

        # Initialize the readme content with the header.
        readme_content = self.get_header()

        # Initialize a count for numbering submissions.
        count = len(self.data["Submissions"])

        # Iterate over each submission in the data.
        for submission in self.data["Submissions"]:
            name = submission.get("name")
            problem_link = submission.get("problem_link")
            language = submission.get("language")
            solution_link = submission.get("directory_link")
            tags = submission.get("tags")
            date = submission.get("date")

            # Append formatted submission information to the readme content.
            readme_content += f'{count} | [{name}]({problem_link}) | [{language}]({solution_link}) | {tags} | {date} |\n'
            count -= 1  # Decrement the count.

        readme_content += '\n'  # Add a newline at the end.
        return readme_content  # Return the formatted readme content.

# End of the 'CsesFormatter' class definition.

secpar/lib/Formatters/CsesFormatter.py
MostafaGalal1/Secpar
# Import necessary modules.
from github import Github
from secpar.lib.Formatters.CodeforcesFormatter import CodeforcesFormatter
from secpar.lib.Formatters.CsesFormatter import CsesFormatter
from secpar.lib.Formatters.VjudgeFormatter import VjudgeFormatter

# Define a class called 'ReadMeBuilder'.
class ReadMeBuilder():
    def __init__(self, data):
        # List of supported platforms.
        self.platforms = ["Codeforces", "CSES", "Vjudge"]

        # Create a GitHub instance using the provided access token.
        git = Github(data.get("access_token"))
        self.repo = git.get_user(data.get("repo_owner")).get_repo(data.get("repo_name"))

    def get_formatter(self, name):
        # Return the appropriate formatter based on the platform name.
        if name == "Codeforces":
            return CodeforcesFormatter(self.repo)
        elif name == "CSES":
            return CsesFormatter(self.repo)
        elif name == "Vjudge":
            return VjudgeFormatter(self.repo)

    def update_readme(self, readme_content):
        readme_path = 'README.md'
        try:
            # Try to update the existing README file.
            existing_readme = self.repo.get_contents(readme_path)
            self.repo.update_file(readme_path, "Update README.md", readme_content, existing_readme.sha)
            print("README.md updated.")
        except Exception as e:
            # If the README doesn't exist, create it.
            self.repo.create_file(readme_path, "Create README.md", readme_content, 'main')
            print("README.md created.")

    def build(self):
        readme_intro = "Submissions\n======================\n\n> *Auto-generated using [Secpar](https://github.com/MostafaGalal1/Secpar)*\n\n"
        table_of_platforms = "## Platforms\n"
        readme_content = ""

        for platform in self.platforms:
            # Generate formatted content for each platform and append it to the overall readme content.
            formatted_platform_content = self.get_formatter(platform).format()
            if len(formatted_platform_content) != 0:
                table_of_platforms += f"* [{platform}](#{platform.lower()})\n"
            readme_content += formatted_platform_content

        table_of_platforms += '\n'
        readme_content += '\n'
        readme_content = readme_intro + table_of_platforms + readme_content
        return readme_content

# End of the 'ReadMeBuilder' class definition.

secpar/lib/Formatters/ReadMeBuilder.py
MostafaGalal1/Secpar
# Import the base class 'AbstractFormatter'.
from secpar.lib.Formatters.AbstractFormatter import AbstractFormatter

# Define a concrete formatter class 'VjudgeFormatter' that inherits from 'AbstractFormatter'.
class VjudgeFormatter(AbstractFormatter):

    def format(self):
        # Check if there is no data; return an empty string.
        if self.data is None:
            return ""

        # Initialize the readme content with the header.
        readme_content = self.get_header()

        # Initialize a count for numbering submissions.
        count = len(self.data.get("Submissions"))

        # Iterate over each submission in the data.
        for submission in self.data.get("Submissions"):
            name = submission.get("name")
            problem_link = submission.get("problem_link")
            language = submission.get("language")
            solution_link = submission.get("directory_link")
            date = submission.get("date")

            # Append formatted submission information to the readme content.
            readme_content += f'{count} | [{name}]({problem_link}) | [{language}]({solution_link}) | {date} |\n'
            count -= 1  # Decrement the count.

        readme_content += '\n'  # Add a newline at the end.
        return readme_content  # Return the formatted readme content.

# End of the 'VjudgeFormatter' class definition.

secpar/lib/Formatters/VjudgeFormatter.py
MostafaGalal1/Secpar
import argparse

def parse_args():
    # Create an ArgumentParser instance with a description and an epilog for usage examples.
    parser = argparse.ArgumentParser(
        description="Creates a repository of all the submissions from a given platform",
        epilog="Example usage: `secpar -c init`"
    )

    # Create argument groups for commands and platforms.
    command_group = parser.add_argument_group('Commands', 'Commands to initialize')
    platform_group = parser.add_argument_group('Platforms', 'Platforms to scrape data from')

    # Add arguments for the "command" option (-c or --command).
    command_group.add_argument(
        "-c",
        "--command",
        choices=["init"],  # Specify valid choices for the "command" argument.
        type=str,
        help="Choose a command: init"
    )

    # Add arguments for the "scrape" option (-s or --scrape).
    platform_group.add_argument(
        "-s",
        "--scrape",
        choices=["codeforces", "cses", "vjudge"],  # Specify valid choices for the "scrape" argument.
        type=str,
        help="Specify the platform name (codeforces, cses, or vjudge)"
    )

    # Parse the command-line arguments.
    args = parser.parse_args()

    # Check if both "scrape" and "command" options are provided, which is invalid.
    if args.scrape and args.command:
        parser.error("The --scrape option should be used alone, without specifying --command.")

    return args  # Return the parsed arguments.

# End of the 'parse_args' function.

secpar/lib/Input/args.py
MostafaGalal1/Secpar
import json
import os
from time import sleep

import requests
from abc import ABC, abstractmethod
from github import Github

# Disable SSL/TLS warnings from urllib3
requests.packages.urllib3.disable_warnings()

# Define an abstract base class for scrapers
class AbstractScraper(ABC):
    def __init__(self, platform, username, password, repo_owner, repo_name, access_token, platform_header):
        # Create a Github object using an access token
        git = Github(access_token)

        # Initialize class attributes
        self.platform = platform
        self.username = username
        self.password = password
        self.headers = {
            # Define HTTP headers for requests
            "accept": "application/json, text/javascript, */*; q=0.01",
            "accept-language": "en-GB,en;q=0.9,ar-EG;q=0.8,ar;q=0.7,en-US;q=0.6",
            "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
            "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-origin",
            "x-requested-with": "XMLHttpRequest",
            "Referer": self.username,
            "Referrer-Policy": "strict-origin-when-cross-origin",
        }
        self.platform_header = platform_header
        self.current_submissions = {}
        self.extensions = {}
        self.repo = git.get_user(repo_owner).get_repo(repo_name)

    def scrape(self):
        # Perform the scraping workflow by calling various methods
        self.load_extensions()
        self.load_already_added()
        if self.login():
            self.get_submissions()
            self.update_submission_json()
        else:
            print("Login failed, please try again")

    def load_extensions(self):
        # Load extensions from a JSON file
        path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "Resources", "Extensions", f"{self.platform}Extensions.json")
        with open(path, 'r') as json_file:
            self.extensions = json.load(json_file)

    def load_already_added(self):
        # Load previously added submissions from a JSON file in the repository
        submissions_path = f'submissions/{self.platform}Submissions.json'
        try:
            json_string = self.repo.get_contents(submissions_path).decoded_content.decode('utf-8')
            self.current_submissions = json.loads(json_string)['Submissions']
            self.current_submissions = {obj['id']: obj for obj in self.current_submissions}
        except:
            self.current_submissions = {}

    @abstractmethod
    def login(self):
        pass

    @abstractmethod
    def get_submissions(self):
        pass

    @abstractmethod
    def get_submission_html(self, submission):
        pass

    def check_already_added(self, submission_id):
        # Check if a submission with the given ID is already added
        if str(submission_id) in self.current_submissions:
            return True
        return False

    @abstractmethod
    def update_already_added(self, submission_id):
        pass

    @abstractmethod
    def generate_directory_link(self, submission):
        pass

    @staticmethod
    def print_progress_bar(progress, end):
        # Print a progress bar with percentage
        print("[{0}{1}] {2}%    ".format("█" * int((progress/end)*50), "-" * int(50-(progress/end)*50), int((progress/end)*100)), end="\r")

    def update_submission_json(self):
        # Update the JSON file in the repository with the latest submissions
        submissions = list(self.current_submissions.values())
        submissions = sorted(submissions, key=lambda item: item['date'], reverse=True)

        try:
            # Update the file if it exists
            self.repo.get_contents(f'submissions/{self.platform}Submissions.json')
            self.repo.update_file(f'submissions/{self.platform}Submissions.json', f"Update {self.platform}Submissions.json", json.dumps({'Header': self.platform_header, 'Submissions': submissions}), self.repo.get_contents(f'submissions/{self.platform}Submissions.json').sha)
        except:
            # Create the file if it doesn't exist
            self.repo.create_file(f'submissions/{self.platform}Submissions.json', f"Create {self.platform}Submissions.json", json.dumps({'Header': self.platform_header, 'Submissions': submissions}), 'main')
            sleep(2)

# You can now create subclasses of AbstractScraper and implement the abstract methods for specific scraping tasks.

secpar/lib/Scrapers/AbstractScraper.py
MostafaGalal1/Secpar
from stem import Signal
from stem.control import Controller
import requests
from datetime import datetime
from bs4 import BeautifulSoup
from secpar.lib.Scrapers.AbstractScraper import AbstractScraper

# Tor settings
CONTROL_PORT = 9051
SOCKS_PORT = 9050
MAX_REQUESTS = 8  # Number of requests before renewing the Tor circuit
SUBMISSIONS_PER_UPDATE = 100  # Number of submissions to process before updating the storage

# Function to create a Tor session
def get_tor_session():
    session = requests.session()
    session.proxies = {
        'http': 'socks5h://localhost:{}'.format(SOCKS_PORT),
        'https': 'socks5h://localhost:{}'.format(SOCKS_PORT)
    }
    return session

# Function to renew the Tor circuit
def renew_connection():
    with Controller.from_port(port=CONTROL_PORT) as controller:
        controller.authenticate()
        controller.signal(Signal.NEWNYM)

# Various functions to extract information from Codeforces submission JSON objects

# Function to get the submission's contest ID
def get_contest_id(submission):
    return submission.get('contestId')

# Function to get the problem name from a submission
def get_problem_name(submission):
    return submission.get('problem').get('name')

# Function to generate a unique hash key for a problem based on contest ID and index
def get_problem_hashkey(submission):
    return str(submission.get('problem').get('contestId')) + submission.get('problem').get('index')

# Function to get the tags associated with a problem
def get_problem_tags(submission):
    return submission.get('problem').get('tags')

# Function to get the problem index (e.g., 'A', 'B', 'C') from a submission
def get_problem_index(submission):
    return submission.get("problem").get("index")

# Function to get the problem rating from a submission
def get_problem_rating(submission):
    return submission.get('problem').get('rating')

# Function to generate a link to the problem on Codeforces
def get_problem_link(submission):
    contest_id = get_contest_id(submission)
    problem_index = get_problem_index(submission)
    return f'https://codeforces.com/contest/{contest_id}/problem/{problem_index}'

# Function to check if a submission was accepted
def get_submission_verdict(submission):
    return submission.get('verdict') == "OK"

# Function to get the submission ID
def get_submission_id(submission):
    return submission.get('id')

# Function to check if a submission is valid (accepted and not from a gym)
def is_valid_submission(submission):
    return get_submission_verdict(submission) and not is_gym_submission(submission)

# Function to get the programming language used in a submission
def get_submission_language(submission):
    return submission.get('programmingLanguage')

# Function to get the submission date and time
def get_submission_date(submission):
    submission_creation_date = datetime.utcfromtimestamp(submission.get('creationTimeSeconds'))
    return submission_creation_date.strftime('%Y-%m-%d %H:%M')

# Function to generate a link to view the submission on Codeforces
def get_submission_link(submission):
    contest_id = get_contest_id(submission)
    submission_id = get_submission_id(submission)
    return f'https://codeforces.com/contest/{contest_id}/submission/{submission_id}'

# Function to get the code of a submission
def get_submission_code(submission):
    return submission.find('pre').text

# Function to check if a submission is from a gym contest
def is_gym_submission(submission):
    contest_id = get_contest_id(submission)
    return not contest_id or contest_id >= 100000  # Check that the submission isn't in a gym

# CodeforcesScraper class
class CodeforcesScraper(AbstractScraper):

    def __init__(self, user_name, repo_owner, repo_name, access_token, use_tor=False):
        self.platform = 'Codeforces'
        self.platform_header = '''## Codeforces<a name="codeforces"></a>
| # | Problem | Solution | Tags | Submitted |
| - |  -----  | -------- | ---- | --------- |\n'''
        super().__init__(self.platform, user_name, '', repo_owner, repo_name, access_token, self.platform_header)

        self.session = get_tor_session() if use_tor else requests.session()
        self.use_tor = use_tor
        self.request_count = 0

    def login(self):
        return True

    def get_new_submissions(self, submissions):
        new_submissions = []
        submissions_hash = {}

        for submission in submissions:
            if not is_valid_submission(submission):
                continue
            problem_key = get_problem_hashkey(submission)
            if self.use_tor and self.check_already_added(problem_key):
                problem_link = self.current_submissions.get(problem_key).get('directory_link')
                if problem_link != self.repo.html_url + '/blob/main/' + self.generate_directory_link(submission):
                    self.current_submissions.pop(problem_key)
            if problem_key not in submissions_hash and not self.check_already_added(problem_key):
                new_submissions.append(submission)
                submissions_hash[problem_key] = True

        return new_submissions

    # Function to retrieve new submissions and process them
    def get_submissions(self):
        user_submissions_url = f'https://codeforces.com/api/user.status?handle={self.username}'
        response = self.session.get(user_submissions_url, verify=False, headers=self.headers, timeout=20)
        submissions = response.json().get("result")

        submissions_per_update = 100
        progress_count = 0
        new_submissions = self.get_new_submissions(submissions)
        end = len(new_submissions)

        for submission in new_submissions:
            progress_count += 1
            self.print_progress_bar(progress_count, end)
            if self.use_tor:
                self.push_code(submission)
            self.update_already_added(submission)

            if progress_count % submissions_per_update == 0:
                self.update_submission_json()
        self.print_progress_bar(1, 1)
        print("")

    # Function to push the code of a submission to a GitHub repository
    def push_code(self, submission):
        submission_html = self.get_submission_html(submission)
        name = get_problem_name(submission)
        code = get_submission_code(submission_html)

        if code is not None:
            directory = self.generate_directory_link(submission)
            try:
                self.repo.create_file(directory, f"Add problem `{name}`", code, 'main')
            except:
                pass

    # Function to update the record of added submissions
    def update_already_added(self, submission):
        problem_key = get_problem_hashkey(submission)
        name = get_problem_name(submission)
        problem_link = get_problem_link(submission)
        directory_link = self.repo.html_url + '/blob/main/' + self.generate_directory_link(
            submission) if self.use_tor else get_submission_link(submission)
        language = get_submission_language(submission)
        tags = get_problem_tags(submission)
        rating = get_problem_rating(submission)
        date = get_submission_date(submission)
        tags = " ".join([f"`{tag}`" for tag in tags])

        self.current_submissions[problem_key] = {'id': problem_key, 'name': name,
                                                  'problem_link': problem_link, 'language': language,
                                                  'directory_link': directory_link, 'tags': f'{tags} `{rating}`',
                                                  'date': date}

    # Function to retrieve the HTML page of a submission from Codeforces
    def get_submission_html(self, submission):
        submission_url = get_submission_link(submission)

        while True:
            self.request_count += 1
            if self.request_count == MAX_REQUESTS:
                self.session = get_tor_session()
                renew_connection()
                self.request_count = 0
            try:
                response = self.session.get(submission_url, verify=False, headers=self.headers, timeout=20)
                if response.status_code == 200:
                    return BeautifulSoup(response.text, 'html.parser')
            except Exception as e:
                print(e)

    # Function to generate a link to the submission in a GitHub repository
    def generate_directory_link(self, submission):
        contest_id = get_contest_id(submission)
        submission_id = get_submission_id(submission)
        language = get_submission_language(submission)
        return f'{self.platform}/{contest_id}/{submission_id}.{self.extensions[language]}'

secpar/lib/Scrapers/CodeforcesScraper.py
MostafaGalal1/Secpar
import requests
from datetime import datetime
from bs4 import BeautifulSoup
from secpar.lib.Scrapers.AbstractScraper import AbstractScraper

# Function to get the count of accepted submissions from a CSES account
def get_accepted_submissions_count(account):
        response = requests.get(account)
        soup = BeautifulSoup(response.text, 'html.parser')
        table = soup.findAll('table', class_='narrow')[1]
        return int(table.find('a').text)

# Function to extract the problem name from a submission
def get_problem_name(submission):
    return submission.find('td', string='Task:').next_sibling.text

# Function to extract problem tags from a submission
def get_problem_tags(submission):
    return submission.find('div', class_='nav sidebar').find('h4').text

# Function to generate a link to the problem on CSES
def get_problem_link(submission):
    return 'https://cses.fi' + submission.find('td', string='Task:').find_next('a').get('href')

# Function to get the submission ID from a problem
def get_submission_id(problem):
    return problem.find('a').text.strip()

# Function to get the programming language used in a submission
def get_submission_language(submission):
    return submission.find('td', string='Language:').next_sibling.text

# Function to extract the submission date and time
def get_submission_date(submission):
    timestamp_str = submission.find('td', string='Submission time:').next_sibling.text
    timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S %z")
    return timestamp.strftime("%Y-%m-%d %H:%M")

# Function to extract the code of a submission
def get_submission_code(submission):
    return submission.find('pre', class_='prettyprint').text

# Function to check if a problem is solved (has a full score)
def check_problem_solved(problem):
    return problem.find('span', class_='task-score icon full')

# Function to fix cascaded HTML problems
def fix_cascaded_html(topic):
    cascaded_problems = topic.find_all('li')
    corrected_soup = BeautifulSoup("<ul></ul>", "html.parser")

    for cascaded_problem in cascaded_problems:
        corrected_soup.ul.append(cascaded_problem)

    return corrected_soup.prettify()

# CsesScraper class
class CsesScraper(AbstractScraper):

    def __init__(self, username, password, repo_owner, repo_name, access_token):
        self.platform = 'Cses'
        self.platform_header = '''## CSES<a name="cses"></a>
| # | Problem | Solution | Tags | Submitted |
| - |  -----  | -------- | ---- | --------- |\n'''
        super().__init__(self.platform, username, password, repo_owner, repo_name, access_token, self.platform_header)

        self.session = requests.session()
        self.credits = {
            'csrf_token': '',
            'nick': self.username,
            'pass': self.password
        }

    def login(self):
        login_url = 'https://cses.fi/login'
        response = self.session.get(login_url, verify=False, headers=self.headers, timeout=20)
        soup = BeautifulSoup(response.text, 'html.parser')
        self.credits['csrf_token'] = soup.find('input', {'name': 'csrf_token'})['value']
        response = self.session.post(login_url, self.credits)
        return response.url != login_url

    def get_submissions(self):
        user_submissions_url = 'https://cses.fi/problemset'
        response = self.session.get(user_submissions_url, verify=False, headers=self.headers, timeout=20)
        soup = BeautifulSoup(response.text, 'html.parser')
        topics = soup.find_all('ul', class_='task-list')
        account = 'https://cses.fi' + soup.find('a', class_='account').get('href')

        submissions_per_update = 100
        progress_count = 0
        end = get_accepted_submissions_count(account)

        for topic in topics:
            topic = fix_cascaded_html(topic)
            soup = BeautifulSoup(topic, 'html.parser')
            problems = soup.find_all('li', class_='task')

            for problem in problems:
                if check_problem_solved(problem):
                    progress_count += 1
                    self.print_progress_bar(progress_count, end)
                    if self.check_already_added(get_submission_id(problem)):
                        continue
                    submission = self.get_submission_html(problem)
                    self.push_code(submission)
                    self.update_already_added(submission)
                    if progress_count % submissions_per_update == 0:
                        self.update_submission_json()
        self.print_progress_bar(1, 1)
        print("")

    def push_code(self, submission):
        name = get_problem_name(submission)
        code = get_submission_code(submission)

        if code is not None:
            directory = self.generate_directory_link(submission)
            try:
                self.repo.create_file(directory, f"Add problem {name}", code, 'main')
            except:
                pass

    def update_already_added(self, submission):
        name = get_problem_name(submission)
        problem_link = get_problem_link(submission)
        directory_link = self.repo.html_url + '/blob/main/' + self.generate_directory_link(submission)
        language = get_submission_language(submission)
        tags = get_problem_tags(submission)
        date = get_submission_date(submission)

        self.current_submissions[str(name)] = {'id': str(name), 'name': name, 'problem_link': problem_link,
                                               'language': language, 'directory_link': directory_link,
                                               'tags': f'{tags}', 'date': date}

    def get_submission_html(self, problem):
        response = self.session.get('https://cses.fi' + problem.find('a').get('href'), verify=False, headers=self.headers, timeout=20)
        soup = BeautifulSoup(response.text, 'html.parser')
        submission_link = soup.find('h4', string='Your submissions').find_next_sibling('a')
        while not submission_link.find('span', class_='task-score icon full'):
            submission_link = submission_link.find_next_sibling('a')
        else:
            submission_link = submission_link.find('span', class_='task-score icon full').parent.get('href')

        response = self.session.get('https://cses.fi' + submission_link, verify=False, headers=self.headers, timeout=20)
        return BeautifulSoup(response.text, 'html.parser')

    def generate_directory_link(self, submission):
        name = get_problem_name(submission)
        topic = get_problem_tags(submission)
        language = get_submission_language(submission)
        return f'{self.platform.upper()}/{topic}/{name}.{self.extensions[language]}'.replace(' ', '_')

secpar/lib/Scrapers/CsesScraper.py
MostafaGalal1/Secpar
from secpar.lib.Scrapers.CsesScraper import CsesScraper
from secpar.lib.Scrapers.CodeforcesScraper import CodeforcesScraper
from secpar.lib.Scrapers.VjudgeScraper import VjudgeScraper

# Factory class to create scraper instances based on the provided scraper name
class ScraperFactory:
    def __init__(self, scraper_name, data):
        self.scraper_name = scraper_name  # The name of the scraper to create
        self.data = data  # Data required for scraper initialization

    # Method to create a scraper instance based on the provided scraper name
    def create(self):
        if self.scraper_name == "cses":
            # Create and return an instance of the CsesScraper
            return CsesScraper(self.data["user_name"], self.data["password"], self.data["repo_owner"], self.data["repo_name"], self.data["access_token"])
        elif self.scraper_name == "codeforces":
            # Create and return an instance of the CodeforcesScraper
            return CodeforcesScraper(self.data["user_name"], self.data["repo_owner"], self.data["repo_name"], self.data["access_token"], self.data["use_tor"])
        elif self.scraper_name == "vjudge":
            # Create and return an instance of the VjudgeScraper
            return VjudgeScraper(self.data["user_name"], self.data["password"], self.data["repo_owner"], self.data["repo_name"], self.data["access_token"])
        else:
            # Raise an error if the provided scraper name is unsupported
            raise ValueError(f"Unsupported scraper name: {self.scraper_name}")


secpar/lib/Scrapers/ScraperFactory.py
MostafaGalal1/Secpar
import requests
from datetime import datetime
from secpar.lib.Scrapers.AbstractScraper import AbstractScraper

# Function to get the problem number from a submission
def get_problem_number(submission):
    return submission.get('probNum')

# Function to get the problem hashkey from a submission
def get_problem_hashkey(submission):
    return submission.get('oj') + submission.get('probNum')


# Function to get the name of the online judge (OJ) from a submission
def get_oj_name(submission):
    return submission.get('oj')

# Function to generate a link to the problem on vjudge
def get_problem_link(submission):
    oj = get_oj_name(submission)
    problem_name = get_problem_number(submission)
    return f'https://vjudge.net/problem/{oj}-{problem_name}'

# Function to get the submission ID from a submission
def get_submission_id(submission):
    return submission.get('runId')

# Function to generate a link to the submission on vjudge
def get_submission_url(submission):
    submission_id = get_submission_id(submission)
    return f'https://vjudge.net/solution/data/{submission_id}'

# Function to get the programming language used in a submission
def get_submission_language(submission):
    return submission.get('language')

# Function to get the submission date and time
def get_submission_date(submission):
    timestamp = datetime.fromtimestamp(submission.get('time') / 1000)
    return timestamp.strftime("%Y:%m:%d %H:%M")

# Function to get the code of a submission
def get_submission_code(submission):
    return submission.get('code')

# VjudgeScraper class
class VjudgeScraper(AbstractScraper):

    def __init__(self, username, password, repo_owner, repo_name, access_token):
        self.platform = 'Vjudge'
        self.platform_header = '''## Vjudge<a name="vjudge"></a>
| # | Problem | Solution | Submitted |
| - |  -----  | -------- | --------- |\n'''
        super().__init__(self.platform, username, password, repo_owner, repo_name, access_token, self.platform_header)

        self.session = requests.session()
        self.credits = {
            'username': self.username,
            'password': self.password,
            'captcha': '',
        }

    def login(self):
        login_url = 'https://vjudge.net/user/login'
        response = self.session.post(login_url, self.credits)
        return response.text == 'success'

    # Function to get the count of accepted submissions from multiple platforms
    def get_accepted_submissions_count(self, submissions):
        count = 0
        for platform in submissions:
            for submission in submissions.get(platform):
                if not self.check_already_added(platform + submission):
                    count += 1
        return count

    # Function to retrieve new submissions from vjudge
    def get_new_submissions(self):
        submissions_per_page = 20
        page_count = 0
        new_submissions = []
        submissions_hash = {}

        while True:
            page_submissions = self.get_page_submissions(page_count, submissions_per_page)
            if len(page_submissions) == 0:
                break
            for submission in page_submissions:
                problem_key = get_problem_hashkey(submission)
                if problem_key not in submissions_hash and not self.check_already_added(problem_key):
                    new_submissions.append(submission)
                    submissions_hash[problem_key] = True
            page_count += 1

        return new_submissions

    # Function to retrieve submissions from a specific page
    def get_submissions(self):
        submissions_per_update = 100
        progress_count = 0
        new_submissions = self.get_new_submissions()
        end = len(new_submissions)

        for submission in new_submissions:
            progress_count += 1
            self.print_progress_bar(progress_count, end)
            self.push_code(submission)
            self.update_already_added(submission)

            if progress_count % submissions_per_update == 0:
                self.update_submission_json()
        self.print_progress_bar(1, 1)
        print("")

    def get_problem_name(self, submission):
        problem_number = get_problem_number(submission)
        response = self.session.get(
            f'https://vjudge.net/problem/data?draw=0&start=0&length=20&OJId=All&probNum={problem_number}&source=&category=all',
            verify=False, timeout=20)
        return response.json().get("data")[0].get('title')

    def get_page_submissions(self, page, submissions_per_page):
        response = self.session.get(f'https://vjudge.net/status/data?draw={page}&start={page * submissions_per_page}'
                                    f'&length=20&un={self.username}&OJId=All&res=1&orderBy=run_id', verify=False, headers=self.headers, timeout=20)
        return response.json().get("data")

    # Function to push code to the repository
    def push_code(self, submission):
        submission_html = self.get_submission_html(submission)
        name = self.get_problem_name(submission_html)
        code = get_submission_code(submission_html)

        if code is not None:
            directory = self.generate_directory_link(submission)
            try:
                self.repo.create_file(directory, f"Add problem `{name}`", code, 'main')
            except:
                pass

    # Function to update the record of already added submissions
    def update_already_added(self, submission):
        problem_key = get_problem_hashkey(submission)
        name = self.get_problem_name(submission)
        problem_link = get_problem_link(submission)
        directory_link = self.repo.html_url + '/blob/main/' + self.generate_directory_link(submission)
        language = get_submission_language(submission)
        date = get_submission_date(submission)

        self.current_submissions[problem_key] = {'id': problem_key, 'name': name,
                                                        'problem_link': problem_link, 'language': language,
                                                        'directory_link': directory_link, 'date': date}

    # Function to retrieve the HTML of a submission
    def get_submission_html(self, submission):
        submission_url = get_submission_url(submission)
        response = self.session.get(submission_url, verify=False, headers=self.headers, timeout=20)
        return response.json()

    # Function to generate the directory link for a submission
    def generate_directory_link(self, submission):
        problem_number = get_problem_number(submission)
        oj = get_oj_name(submission)
        return f'{self.platform}/{oj}/{problem_number}.cpp'

secpar/lib/Scrapers/VjudgeScraper.py
MostafaGalal1/Secpar
from setuptools import setup, find_packages
import codecs
import os

# Get the current directory
here = os.path.abspath(os.path.dirname(__file__))

# Read the README file as long description
with codecs.open(os.path.join(here, "README.md"), encoding="utf-8") as fh:
    long_description = fh.read()

VERSION = '1.1.3'
DESCRIPTION = 'Scrape problems submissions from different platforms'

# Define the authors
author_name_1 = 'Mostafa Galal'
author_email_1 = 'mostafam.galal82@gmail.com'
author_name_2 = 'Mahmoud Goda'
author_email_2 = 'mahmoden17@gmail.com'

# Combine author information
authors = f"{author_name_1} <{author_email_1}>, {author_name_2} <{author_email_2}>"

# Setting up
setup(
    name="Secpar",
    version=VERSION,
    author=authors,
    description=DESCRIPTION,
    long_description_content_type="text/markdown",
    long_description=long_description,
    packages=find_packages(),
    package_data={'secpar': ['*.json', '*.py']},
    include_package_data=True,
    license="MIT",
    entry_points={
        "console_scripts": ['secpar = secpar.__main__:main']
    },
    url="https://github.com/MostafaGalal1/Secpar",
    python_requires='>=3.5',
    install_requires=[
        "PyGithub>=1.54.0",
        "requests>=2.0.0",
        "beautifulsoup4>=4.0.0",
        "stem>=1.8.0",
        "PySocks>=1.7.1,<2.0.0",
        "argparse>=1.4.0",
    ],
    keywords=['CP', 'scraper', 'secpar', 'problem solving', 'python', 'tools'],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: End Users/Desktop",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python :: 3",
        "Operating System :: OS Independent",
    ]
)

setup.py
MostafaGalal1/Secpar