scrape https://tenders.ppa.gov.gh/contracts/
#!/usr/bin/env python3
import csv
import glob
import os

import requests
from bs4 import BeautifulSoup

script_directory = os.path.dirname(os.path.abspath(__file__))
folder_path = f"{script_directory}/alltenders"


all_titles = []
all_values = []
allfiles = glob.glob(f"{folder_path}/*")
allfiles.sort()

headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
    "sec-ch-ua": "Chromium;v=116, Not)A;Brand;v=24, Google Chrome;v=116",
}
urls = [
    "https://www.ghaneps.gov.gh/epps/viewAllAwardedContracts.do?d-3998960-p=1&selectedItem=viewAllAwardedContracts.do&T01_ps=50000"
]

for url in urls:
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.content, "html.parser")
    table_header_elements = soup.select("table>thead>tr>*")
    table_body_elements = soup.select("table>tbody>tr")
    all_titles.append(
        [
            i.text.replace("\n", "").replace("\t", "").replace("\r", "")
            for i in table_header_elements
        ]
    )
    for tb in table_body_elements:
        tb_item = tb.findAll("td")
        values = [
            i.text.replace("\n", "").replace("\t", "").replace("\r", "")
            for i in tb_item
        ]
        all_values.append(values)
    all_titles.append(table_header_elements)
all_values.sort()

csv_file_path = f"{script_directory}/csv/tenders-ghanaeps.csv"
with open(csv_file_path, "w") as csv_file:
    csv_writer = csv.writer(csv_file, dialect="unix")
    csv_writer.writerow(all_titles[0])
    for values in all_values:
        csv_writer.writerow(values)
print(f"CSV file saved as {csv_file_path}")

ghanepsscrape.py
retornam/gh-ppa-scrape
#!/usr/bin/env python3
import csv
import glob
import os
from multiprocessing import Pool

from bs4 import BeautifulSoup


def parse_html_file(file_path):
    with open(file_path, "r") as html_file:
        html_content = html_file.read()
    soup = BeautifulSoup(html_content, "html.parser")
    dt_elements = soup.find_all("dt")
    dd_elements = soup.find_all("dd")
    ttemp = [s.get_text().strip() for s in dt_elements]
    vtemp = [
        s.get_text().replace("\n", " ").replace("\t", " ").replace("\r", " ").strip()
        for s in dd_elements
    ]
    return ttemp, vtemp


def write_csv_file(filename, titles, values):
    with open(filename, "w") as csv_file:
        csv_writer = csv.writer(csv_file, dialect="unix")
        csv_writer.writerow(titles)
        for value in values:
            csv_writer.writerow(value)
    print(f"CSV file saved as {filename}")


def process_folder(folder_path):
    allfiles = glob.glob(f"{folder_path}/*")
    allfiles.sort()
    titles = None
    tender_types = {"Open Tender": [], "Restricted Tender": [], "Single Sourced": []}
    for file_path in allfiles:
        ttemp, vtemp = parse_html_file(file_path)
        for key in tender_types.keys():
            if key in vtemp:
                tender_types[key].append(vtemp)
                if key == "Open Tender":
                    open_tender_titles = ttemp
                else:
                    titles = ttemp
    return titles, open_tender_titles, tender_types


if __name__ == "__main__":
    script_directory = os.path.dirname(os.path.abspath(__file__))
    folder_path = f"{script_directory}/alltenders"
    with Pool(processes=os.cpu_count()) as pool:
        results = pool.map(process_folder, [folder_path])
    titles, open_tender_titles, tender_types = results[0]

    restricted_file_path = f"{script_directory}/csv/tenders-ppa-restricted.csv"
    single_sourced_file_path = f"{script_directory}/csv/tenders-ppa-single-sourced.csv"
    open_file_path = f"{script_directory}/csv/tenders-ppa-open.csv"

    write_csv_file(restricted_file_path, titles, tender_types["Restricted Tender"])
    write_csv_file(single_sourced_file_path, titles, tender_types["Single Sourced"])
    write_csv_file(open_file_path, open_tender_titles, tender_types["Open Tender"])

ppascrape.py
retornam/gh-ppa-scrape