Upload 9 files

collect_data_es.py

@@ -0,0 +1,226 @@
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+import re
+
+#################### Spanish Wikipedia ####################
+
+###############
+# Title based #
+###############
+
+def extract_result_resultado(sentence):
+    match = re.search(r"(RESULTADO:|El resultado fue)\s*(\w+)", sentence, flags=re.IGNORECASE)
+    return match.group(2).strip() if match else None
+
+def extract_result(sentence):
+    #print(f"Extracting result from sentence: {sentence}")  
+    match = re.search(r"se\s+decidió\s+(\w+)", sentence, flags=re.IGNORECASE)
+    if match:
+        #print(f"Match found for 'se decidió': {match.groups()}")  
+        return match.group(1).strip()
+    #print("No match found for 'se decidió'.") 
+    return None
+
+def clean_comments_with_no_text_after_timestamp(content_div):
+    for ol in content_div.find_all('ol'): 
+        for li in ol.find_all('li'):  
+            li_text = li.get_text(strip=True)
+            if "(CEST)" in li_text or "(CET)" in li_text:
+                match = re.search(r"\(C[ES]T\)\s*(.*)", li_text)
+                if match:
+                    after_timestamp = match.group(1).strip()
+                    if not after_timestamp:  
+                        li.decompose()
+            else:
+                li.decompose()
+    return content_div
+
+def extract_cleaned_spanish_discussion_and_result(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Received status code {response.status_code} for URL: {url}")
+        return pd.DataFrame(columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    title = url.split('/')[-1].replace('_', ' ').replace(':', '')
+    text_url = f"https://es.wikipedia.org/wiki/{url.split('/')[-1]}"
+    discussion_url = url
+
+    content_div = soup.find('div', class_='mw-content-ltr mw-parser-output')
+    if not content_div:
+        print("Error: Main discussion container not found")
+        return pd.DataFrame(columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+
+    discussion_uncleaned = content_div.prettify()
+    discussion = ''
+    result_sentence = ''
+    result = None
+
+    try:
+        result_p = next(
+            (p for p in content_div.find_all('p') if "El resultado fue" in p.get_text() or "RESULTADO:" in p.get_text()), None
+        )
+
+        if result_p:
+            result_sentence = result_p.get_text(strip=True)
+            bold_tag = result_p.find('b')
+            if bold_tag:
+                result = bold_tag.get_text(strip=True)
+            else:
+                match = re.search(r"(El resultado fue|RESULTADO:)\s*(.+?)\.", result_sentence, flags=re.IGNORECASE)
+                result = match.group(2).strip() if match else None
+                #print(f"Extracted result from sentence: {result}")
+
+        content_div = clean_comments_with_no_text_after_timestamp(content_div)
+        discussion_text_parts = content_div.find_all(recursive=False)
+        cleaned_text_parts = []
+        for part in discussion_text_parts:
+            cleaned_text_parts.append(part.get_text(strip=True))
+        discussion = "\n".join(cleaned_text_parts)
+
+        if not result:
+            result_div = content_div.find('div', class_='messagebox')
+            if result_div:
+                result_dl = result_div.find('dl')
+                if result_dl:
+                    result_sentence = result_dl.get_text(strip=True)
+                    #print(f"Extracted result sentence from messagebox: {result_sentence}")  
+                    result = extract_result(result_sentence)
+            if not result and not result_sentence:
+                    result_p = next((p for p in content_div.find_all('p') if "RESULTADO:" in p.get_text() or "se decidió" in p.get_text()), None)
+                    if result_p:
+                        result_sentence = result_p.get_text(strip=True)
+                        #print(f"Extracted result sentence from paragraph: {result_sentence}")  
+                        result = extract_result(result_sentence)
+
+            if not result and not result_sentence:
+                    voting_sentence = next((p for p in content_div.find_all('p') if "se decidió" in p.get_text()), None)
+                    if voting_sentence:
+                        result_sentence = voting_sentence.get_text(strip=True)
+                        #print(f"Extracted voting sentence: {result_sentence}")  
+                        result = extract_result(result_sentence)
+
+        # if result:
+        #     print(f"Final extracted result: {result}")  
+
+        if "Votación" in discussion:
+            discussion = discussion.split("Votación", 1)[1].strip()
+
+    except Exception as e:
+        print(f"Error processing discussion: {e}")
+    data = [[title, discussion_uncleaned, discussion, result_sentence, result, text_url, discussion_url]]
+    df = pd.DataFrame(data, columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+    df['result'] = df['result'].apply(lambda x: extract_result_resultado(x) if isinstance(x, str) and len(x.split()) > 1 else x)
+    return df
+
+# url = 'https://es.wikipedia.org/wiki/Wikipedia:Consultas_de_borrado/!Hispahack' #'https://es.wikipedia.org/wiki/Wikipedia:Consultas_de_borrado/:Country_Club_La_Planicie'
+# df = extract_cleaned_spanish_discussion_and_result(url)
+# df
+
+###############
+# Date based #
+###############
+
+
+def extract_result(sentence):
+    match = re.search(r"(El resultado fue|RESULTADO:)\s*(\w+)", sentence, flags=re.IGNORECASE)
+    return match.group(2).strip() if match else None
+
+def extract_multiple_discussions(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error: Received status code {response.status_code} for URL: {url}")
+        return pd.DataFrame(columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    content_div = soup.find('div', class_='mw-content-ltr mw-parser-output')
+    if not content_div:
+        print("Error: Main discussion container not found")
+        return pd.DataFrame(columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+    data = []
+    headings = content_div.find_all('div', class_='mw-heading mw-heading3')
+    for idx, heading in enumerate(headings):
+        try:
+            title_tag = heading.find('a', class_='new') or heading.find('a')
+            if title_tag:
+                title = title_tag.text.strip()
+                text_url = f"https://es.wikipedia.org{title_tag['href']}"
+            else:
+                title = f"{url.split('/')[-1]}_{idx + 1}"
+                text_url = f"https://es.wikipedia.org/wiki/{title}"
+            previous_sibling = heading.find_previous_sibling()
+            result_sentence = None
+            result = None
+            while previous_sibling:
+                if previous_sibling.name == 'p' and "El resultado fue" in previous_sibling.get_text():
+                    normalized_text = previous_sibling.get_text(separator=" ", strip=True)
+                    result_sentence = normalized_text
+                    result = extract_result(result_sentence)
+                    break
+                previous_sibling = previous_sibling.find_previous_sibling()
+            if not result_sentence:
+                result_p = content_div.find('p', string=lambda text: text and "RESULTADO:" in text)
+                if result_p:
+                    result_sentence = result_p.get_text(strip=True)
+                    result = extract_result(result_sentence)
+            discussion_html = ""
+            current = heading.find_next_sibling()
+            while current and not (current.name == 'div' and 'mw-heading mw-heading3' in current.get('class', [])):
+                discussion_html += str(current)
+                current = current.find_next_sibling()
+
+            discussion_uncleaned = discussion_html
+            discussion = BeautifulSoup(discussion_html, 'html.parser').get_text(strip=True)
+            data.append([title, discussion_uncleaned, discussion, result_sentence, result, text_url, url])
+        except Exception as e:
+            print(f"Error processing heading: {e}")
+    df = pd.DataFrame(data, columns=['title', 'discussion_uncleaned', 'discussion', 'result_sentence', 'result', 'text_url', 'discussion_url'])
+    return df
+
+# url = 'https://es.wikipedia.org/wiki/Wikipedia:Consultas_de_borrado/Registro/10_de_septiembre_de_2009'
+# df = extract_multiple_discussions(url)
+# df
+
+###############
+# Collect ES #
+###############
+
+def collect_es(mode='title', title='', url = '',date=''):
+    if mode not in ['title', 'year', 'url']:
+        raise ValueError("mode must be either 'title' or 'year'")
+
+    if mode == 'title':
+        if not title or date:
+            raise ValueError("For 'title' mode, 'title' must be provided and 'date' must be empty.")
+        url = f"https://es.wikipedia.org/wiki/Wikipedia:Consultas_de_borrado/{title}"
+        df = extract_cleaned_spanish_discussion_and_result(url)
+        if df.empty:
+            print(f"No data found for url: {url}")
+        return df
+    elif mode == 'url':
+        if title or date:
+            raise ValueError("For 'url' mode, 'url' must be provided and 'title' must be empty.")
+        df = extract_cleaned_spanish_discussion_and_result(url)
+        return df
+
+    elif mode == 'year':
+        if title or not date:
+            raise ValueError("For 'year' mode, 'date' must be provided and 'title' must be empty.")
+        month_map = {
+            '01': 'enero', '02': 'febrero', '03': 'marzo', '04': 'abril', '05': 'mayo', '06': 'junio',
+            '07': 'julio', '08': 'agosto', '09': 'septiembre', '10': 'octubre', '11': 'noviembre', '12': 'diciembre'
+        }
+
+        match = re.match(r'(\d{2})/(\d{2})/(\d{4})', date)
+        if not match:
+            raise ValueError("Date must be in the format dd/mm/yyyy")
+
+        day, month, year = match.groups()
+        if month not in month_map:
+            raise ValueError("Invalid month in date")
+
+        date_str = f"{int(day)}_de_{month_map[month]}_de_{year}"
+        url = f"https://es.wikipedia.org/wiki/Wikipedia:Consultas_de_borrado/Registro/{date_str}"
+        df = extract_multiple_discussions(url)
+        return df

collect_data_gr.py

@@ -0,0 +1,378 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import re
+import pysbd
+
+###############################################
+# Functions from Code 1 (collapsible approach)#
+###############################################
+
+def extract_result(sentence):
+    match = re.search(r"(Διαγραφή|Παραμονή|Άλλο αποτέλεσμα|διαγραφή|Συγχώνευση|Διατήρηση)", sentence, flags=re.IGNORECASE)
+    delete_cases = [
+        'Μη εγκυκλοπαιδικό', 'Πράγματι δεν φαίνεται πως το λήμμα είναι εγκυκλοπαιδικό',
+        'Δεν διαπιστώθηκε εγκυκλοπαιδικότητα', 'Μη εγκυκλοπαιδικό λήμμα',
+        'Το λήμμα κρίθηκε ότι είναι καταλληλότερο για κάποιο άλλο αδελφό εγχείρημα, παρά για την Βικιπαίδεια + ατεκμηρίωτο.',
+        'Δεν υπάρχουν επαρκείς αναφορές για την βιογραφούμενη'
+    ]
+    if match:
+        outcome = match.group(1).strip()
+    elif sentence in delete_cases:
+        outcome = 'Διαγραφή'
+    else:
+        outcome = 'Δεν υπάρχει συναίνεση'
+    return normalize_outcome(outcome)
+
+def normalize_outcome(o):
+    lowered = o.lower()
+    if 'διαγρ' in lowered:  # covers 'διαγραφή'
+        return 'Διαγραφή'
+    elif 'διατήρη' in lowered or 'παραμονή' in lowered:
+        return 'Διατήρηση'
+    elif 'συγχών' in lowered:
+        return 'συγχώνευση'
+    else:
+        # Covers 'Άλλο αποτέλεσμα' and unknown cases
+        return 'Δεν υπάρχει συναίνεση'
+
+def extract_discussions_from_page_collapsible(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_sections = soup.find_all('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    titles = []
+    for section in discussion_sections:
+        try:
+            h2_tag = section.find('h2')
+            if not h2_tag:
+                continue
+            title_link = h2_tag.find('a')
+            title = title_link.text.strip() if title_link else h2_tag.get_text(strip=True)
+            titles.append(title)
+        except:
+            pass
+
+    discussion_tables = soup.find_all('table')
+    if not discussion_tables:
+        return pd.DataFrame(columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+    data = []
+    for idx, table in enumerate(discussion_tables):
+        try:
+            decision_row = table.find('tr')
+            decision_cell = decision_row.find('th') if decision_row else None
+            if decision_cell:
+                result_match = re.search(
+                    r"Η συζήτηση τελείωσε, το αποτέλεσμα ήταν: <i>(.*?)</i>", str(decision_cell), re.DOTALL
+                )
+                result_sentence = result_match.group(1).strip() if result_match else "No result found"
+            else:
+                result_sentence = "No result found"
+
+            discussion_row = decision_row.find_next_sibling('tr') if decision_row else None
+            discussion_cell = discussion_row.find('td', class_='plainlinks') if discussion_row else None
+            discussion_content = discussion_cell.get_text(separator="\n") if discussion_cell else "No discussion content found"
+            discussion_content = discussion_content.split('\nμητρώο\n)\n\n\n\n\n')[-1].replace('\n','')
+
+            title = titles[idx] if idx < len(titles) else f"Discussion {idx + 1}"
+            data.append({
+                "title": title,
+                "discussion": discussion_content,
+                "result_sentence": result_sentence,
+                "result": extract_result(result_sentence),
+                "text_url": url
+            })
+        except:
+            pass
+
+    return pd.DataFrame(data, columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+###########################################
+# Functions from Code 2 (non-collapsible) #
+###########################################
+
+def extract_discussions_from_page_non_collapsible(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_sections = soup.find_all('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    titles = []
+    for section in discussion_sections:
+        try:
+            h2_tag = section.find('h2')
+            if not h2_tag:
+                continue
+            title_link = h2_tag.find('a')
+            title = title_link.text.strip() if title_link else h2_tag.get_text(strip=True)
+            titles.append(title)
+        except:
+            pass
+
+    discussion_tables = soup.find_all('table', class_='pagediscussion')
+    if not discussion_tables:
+        return pd.DataFrame(columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+    data = []
+    for idx, table in enumerate(discussion_tables):
+        try:
+            decision_row = table.find('tr')
+            decision_cell = decision_row.find('th') if decision_row else None
+            if decision_cell:
+                result_match = re.search(
+                    r"Η συζήτηση τελείωσε, το αποτέλεσμα ήταν: <i>(.*?)</i>", str(decision_cell), re.DOTALL
+                )
+                result_sentence = result_match.group(1).strip() if result_match else "No result found"
+            else:
+                result_sentence = "No result found"
+
+            discussion_row = decision_row.find_next_sibling('tr') if decision_row else None
+            discussion_cell = discussion_row.find('td', class_='plainlinks') if discussion_row else None
+            discussion_content = discussion_cell.get_text(separator="\n") if discussion_cell else "No discussion content found"
+            discussion_content = discussion_content.split('\nμητρώο\n)\n\n\n\n\n')[-1].replace('\n','')
+
+            title = titles[idx] if idx < len(titles) else f"Discussion {idx + 1}"
+            data.append({
+                "title": title,
+                "discussion": discussion_content,
+                "result_sentence": result_sentence,
+                "result": extract_result(result_sentence),
+                "text_url": url
+            })
+        except:
+            pass
+
+    return pd.DataFrame(data, columns=['title', 'discussion', 'result_sentence', 'result', 'text_url'])
+
+###########################################
+# Title-based extraction with fallback    #
+###########################################
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl', 'ul']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip()])
+    return text
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="el", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences)
+
+def clean_discussion_text(text):
+    return text.strip()
+
+def extract_outcome_from_text(text):
+    outcomes = ['Διαγραφή', 'Παραμονή', 'διαγραφή', 'Συγχώνευση', 'Διατήρηση', 'Άλλο αποτέλεσμα']
+    lowered = text.lower()
+    found_outcome = None
+    for outcome in outcomes:
+        if outcome.lower() in lowered:
+            found_outcome = outcome
+            break
+    if not found_outcome:
+        found_outcome = 'Δεν υπάρχει συναίνεση'
+    return normalize_outcome(found_outcome)
+
+def extract_discussion_section(soup, title):
+    t = title.replace(' ', '_')
+    h2_tag = soup.find('h2', id=t)
+    if not h2_tag:
+        return '', '', ''
+
+    heading_div = h2_tag.find_parent('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    if not heading_div:
+        return '', '', ''
+
+    next_heading_div = heading_div.find_next('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+
+    html_fragments = []
+    current = heading_div.next_sibling
+    while current and current != next_heading_div:
+        if hasattr(current, 'prettify'):
+            html_fragments.append(current.prettify())
+        else:
+            html_fragments.append(str(current))
+        current = current.next_sibling
+
+    discussion_html = ''.join(html_fragments).strip()
+    if not discussion_html:
+        return '', '', ''
+
+    sub_soup = BeautifulSoup(discussion_html, 'html.parser')
+    discussion_tags = sub_soup.find_all(['p', 'ul', 'dl'])
+
+    if not discussion_tags:
+        return '', '', ''
+
+    cleaned_parts = []
+    for tag in discussion_tags:
+        for unwanted in tag.find_all(['span', 'img', 'a', 'div', 'table'], recursive=True):
+            unwanted.decompose()
+        text = tag.get_text(separator=' ', strip=True)
+        if text:
+            cleaned_parts.append(text)
+
+    cleaned_discussion = ' '.join(cleaned_parts)
+    label = extract_outcome_from_text(cleaned_discussion)
+
+    return discussion_html, label, cleaned_discussion
+
+def extract_fallback_discussion(url, title):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return '', None
+
+    soup = BeautifulSoup(response.text, 'html.parser')
+    discussion_tables = soup.find_all('table')
+    if not discussion_tables:
+        return '', None
+    for table in discussion_tables:
+        table_text = table.get_text(separator='\n', strip=True)
+        if title in table_text:
+            decision_row = table.find('tr')
+            decision_cell = decision_row.find('th') if decision_row else None
+            if decision_cell:
+                result_match = re.search(r"Η συζήτηση τελείωσε, το αποτέλεσμα ήταν: <i>(.*?)</i>", str(decision_cell), re.DOTALL)
+                result_sentence = result_match.group(1).strip() if result_match else "No result found"
+            else:
+                result_sentence = "No result found"
+
+            discussion_row = decision_row.find_next_sibling('tr') if decision_row else None
+            discussion_cell = discussion_row.find('td', class_='plainlinks') if discussion_row else None
+            discussion_content = ''
+            if discussion_cell:
+                discussion_content = discussion_cell.get_text(separator=' ', strip=True)
+
+            if discussion_content:
+                outcome = extract_result(result_sentence)
+                return discussion_content, outcome
+
+    return '', None
+
+def extract_div_from_title_with_fallback(title, url ='', date=''):
+    if not date:
+        raise ValueError("For 'title' mode, 'date' must be provided in the format: mm/yyyy")
+
+    month_map = {
+        '01': 'Ιανουαρίου', '02': 'Φεβρουαρίου', '03': 'Μαρτίου', '04': 'Απριλίου', '05': 'Μαΐου', '06': 'Ιουνίου',
+        '07': 'Ιουλίου', '08': 'Αυγούστου', '09': 'Σεπτεμβρίου', '10': 'Οκτωβρίου', '11': 'Νοεμβρίου', '12': 'Δεκεμβρίου'
+    }
+    if '_' in date and date.split('_')[0] in month_map.values():
+        # If date is already in 'Month_Year' format
+        date_str = date
+    else:
+        # Try to parse date in 'mm/yyyy' format
+        match = re.match(r'(\d{2})/(\d{4})', date)
+        if not match:
+            raise ValueError("Date must be in the format mm/yyyy or Month_Year")
+        mm, yyyy = match.groups()
+        if mm not in month_map:
+            raise ValueError(f"Invalid month: {mm}")
+        
+        date_str = f"{month_map[mm]}_{yyyy}"  # Convert to 'Month_Year' format
+    base_url = 'https://el.wikipedia.org/wiki/Βικιπαίδεια:Σελίδες_για_διαγραφή'
+    url = f"{base_url}/{date_str}#{title}"
+
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'discussion_url', 'discussion', 'outcome'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_html, label, cleaned_discussion = extract_discussion_section(soup, title)
+
+    text_url = f"{base_url}/{date_str}"
+    discussion_url = text_url + '#' + title
+
+    cleaned_discussion = html_to_plaintext(cleaned_discussion)
+    cleaned_discussion = split_text_into_sentences(cleaned_discussion)
+    cleaned_discussion = clean_discussion_text(cleaned_discussion)
+
+    if not cleaned_discussion.strip():
+        fallback_url = f"{base_url}/{date_str}"
+        discussion_content, outcome = extract_fallback_discussion(fallback_url, title)
+        cleaned_discussion = html_to_plaintext(discussion_content)
+        cleaned_discussion = split_text_into_sentences(cleaned_discussion)
+        cleaned_discussion = clean_discussion_text(cleaned_discussion)
+        if outcome:
+            label = normalize_outcome(outcome)
+
+    df = pd.DataFrame([[title, discussion_url, cleaned_discussion, label]],
+                      columns=['title', 'discussion_url', 'discussion', 'outcome'])
+    return df
+
+def normalize_outcome(o):
+    lowered = o.lower()
+    if 'διαγρ' in lowered:
+        return 'Διαγραφή'
+    elif 'διατήρη' in lowered or 'παραμονή' in lowered:
+        return 'Διατήρηση'
+    elif 'συγχών' in lowered:
+        return 'συγχώνευση'
+    else:
+        return 'Δεν υπάρχει συναίνεση'
+
+###################################
+# The collect_gr() function       #
+###################################
+
+def collect_gr(mode='url', title='', url = '', years=[]):
+    if mode not in ['title', 'year', 'url']:
+        raise ValueError("mode must be either 'title' or 'year' or 'url'.")
+
+    if mode == 'title':
+        if not title or not years or len(years) != 1:
+            raise ValueError("For 'title' mode, 'title' must be provided and 'years' must be a single-element list like ['mm/yyyy'].")
+        date = years[0]
+        df = extract_div_from_title_with_fallback(title, date=date)
+        return df[['title', 'discussion_url', 'discussion', 'outcome']]
+    
+    elif mode == 'url':
+        if title or years:
+            raise ValueError("For 'url' mode, 'title' must be empty and 'years' must be empty.")
+        #collect the title and date from the url like: base_url = 'https://el.wikipedia.org/wiki/Βικιπαίδεια:Σελίδες_για_διαγραφή'/{date_str}#{title}
+        match = re.search(r'Βικιπαίδεια:Σελίδες_γι��_διαγραφή/([^#]+)#(.+)', url)
+        if not match:
+            raise ValueError("URL format is incorrect.")
+        date_str, title = match.groups()
+        print(date_str, title)
+        df = extract_div_from_title_with_fallback(title, date=date_str)
+        return df[['title', 'discussion_url', 'discussion', 'outcome']]
+
+
+    elif mode == 'year':
+        if title or not years:
+            raise ValueError("For 'year' mode, 'title' must be empty and 'years' must be provided.")
+        if len(years) == 1:
+            start_year = end_year = years[0]
+        elif len(years) == 2:
+            start_year, end_year = min(years), max(years)
+        else:
+            raise ValueError("Invalid years input. Provide one year or two years for a range.")
+
+        all_data = []
+        for year in range(start_year, end_year + 1):
+            url = f"https://el.wikipedia.org/wiki/Βικιπαίδεια:Σελίδες_για_διαγραφή/Ιανουαρίου_{year}"
+            df = extract_discussions_from_page_collapsible(url)
+            if df.empty:
+                df = extract_discussions_from_page_non_collapsible(url)
+
+            if not df.empty:
+                df['result'] = df['result'].apply(normalize_outcome)
+                df['discussion_url'] = df.apply(lambda row: row['text_url'] + '#' + row['title'].replace(' ', '_'), axis=1)
+                df = df.rename(columns={'result':'outcome'})
+                all_data.append(df[['title', 'discussion_url', 'discussion', 'outcome']])
+
+        if all_data:
+            return pd.concat(all_data, ignore_index=True)
+        else:
+            return pd.DataFrame(columns=['title', 'discussion_url', 'discussion', 'outcome'])

collect_data_wikidata_ent.py

@@ -0,0 +1,302 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import pysbd
+import re
+
+
+########################
+## Year based search   ##
+########################
+
+BASE_URL = "https://www.wikidata.org/wiki/Wikidata:Requests_for_deletions/Archive"
+
+def get_soup(url):
+    response = requests.get(url)
+    response.raise_for_status()
+    return BeautifulSoup(response.text, 'html.parser')
+
+def get_year_urls():
+    soup = get_soup(BASE_URL)
+    year_urls = {}
+    for link in soup.select('a[href^="/wiki/Wikidata:Requests_for_deletions/Archive/"]'):
+        year_url = link['href']
+        if year_url.endswith(tuple(str(year) for year in range(2012, 2025))):
+            year = year_url.split('/')[-1]
+            full_year_url = "https://www.wikidata.org" + year_url
+            year_urls[year] = full_year_url
+    return year_urls
+
+def get_month_day_urls(year_url):
+    soup = get_soup(year_url)
+    month_day_urls = []
+    for link in soup.select('a[href^="/wiki/Wikidata:Requests_for_deletions/Archive/"]'):
+        date_url = link['href']
+        if len(date_url.split('/')) >= 7:
+            full_date_url = "https://www.wikidata.org" + date_url
+            if full_date_url not in month_day_urls:
+                month_day_urls.append(full_date_url)
+    return month_day_urls
+
+def extract_outcome_from_dd(dd):
+    try:
+        result_tag = dd.find('b')
+        if result_tag:
+            return result_tag.get_text().strip()
+        return 'unknown'
+    except:
+        return 'unknown'
+
+def extract_discussions(url):
+    soup = get_soup(url)
+    discussions = []
+    for h2 in soup.find_all('h2'):
+        title_tag = h2.find('a')
+        if title_tag and 'Q' in title_tag.get_text():
+            title = title_tag.get_text().strip()
+            discussion_parts = []
+            last_dd = None
+            for sibling in h2.find_all_next():
+                if sibling.name == 'h2':
+                    break
+                if sibling.name == 'p':
+                    discussion_parts.append(sibling.get_text(separator=' ', strip=True))
+                if sibling.name == 'dl':
+                    dds = sibling.find_all('dd')
+                    if dds:
+                        for dd in dds[:-1]:
+                            discussion_parts.append(dd.get_text(separator=' ', strip=True))
+                        last_dd = dds[-1]
+            discussion_text = ' '.join(discussion_parts) if discussion_parts else 'No discussion found'
+            outcome = extract_outcome_from_dd(last_dd) if last_dd else 'Outcome not found'
+            entity_url = url + '#' + title
+            discussions.append({
+                "title": title,
+                "discussion": discussion_text,
+                "outcome": outcome,
+                "url": entity_url,
+                'date': url.split('Archive/')[-1]
+            })
+    return discussions
+
+def remove_first_sentence_if_q_number(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    if sentences and sentences[0].startswith('Q') and sentences[0][1:].isdigit():
+        return ' '.join(sentences[1:])
+    return text
+
+def process_discussions_by_url_list(url_list):
+    all_discussions = []
+    for url in url_list:
+        discussions = extract_discussions(url)
+        all_discussions.extend(discussions)
+    df = pd.DataFrame(all_discussions)
+    if not df.empty:
+        df['discussion'] = df['discussion'].apply(remove_first_sentence_if_q_number)
+    return df
+
+
+########################
+## Title based search ##
+########################
+
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import pysbd
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl', 'ul']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+    return text
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences)
+
+def clean_discussion_tag(tag):
+    for unwanted in tag.find_all(['span', 'img', 'a', 'div'], recursive=True):
+        unwanted.decompose()
+    return tag.get_text(separator=' ', strip=True)
+
+def extract_outcome_from_text_elements(elements):
+    consensus_keywords = [
+        'Deleted', 'Delete', 'delete', 'deleted', 
+        'kept', 'keep', 'Keep', 'Kept', 
+        'merge', 'Merge', 'Not done', 'No consensus', 'no consensus'
+    ]
+    for el in elements:
+        b_tags = el.find_all('b')
+        for b in b_tags:
+            if b.text.strip() in consensus_keywords:
+                return b.text.strip()
+    return ''
+
+def extract_discussion_section(soup, title):
+    h2_tag = soup.find('h2', id=title)
+    if not h2_tag:
+        print(f"No heading found with id={title}")
+        return '', '', ''
+
+    heading_div = h2_tag.find_parent('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    if not heading_div:
+        print(f"No heading div found for {title}")
+        return '', '', ''
+
+    next_heading_div = heading_div.find_next('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    discussion_nodes = []
+    for sibling in heading_div.next_siblings:
+        if sibling == next_heading_div:
+            break
+        discussion_nodes.append(sibling)
+
+    discussion_tags = []
+    for node in discussion_nodes:
+        if getattr(node, 'name', None) in ['p', 'ul', 'dl']:
+            if node.has_attr('class') and 'plainlinks' in node['class']:
+                continue
+            if node.get('style', '').lower() == 'visibility:hidden;display:none':
+                continue
+            if node.find('span', id=title):
+                continue
+            discussion_tags.append(node)
+
+    if not discussion_tags:
+        return '', '', ''
+    
+    label = extract_outcome_from_text_elements(discussion_tags)
+    discussion_html_parts = [str(tag) for tag in discussion_tags]
+    cleaned_parts = []
+    for tag in discussion_tags:
+        text = clean_discussion_tag(tag)
+        if text:
+            cleaned_parts.append(text)
+
+    cleaned_discussion = ' '.join(cleaned_parts)
+    discussion_html = '\n'.join(discussion_html_parts)
+    return discussion_html, label, cleaned_discussion
+
+def extract_div_from_title(title, url=''):
+    if url=='' or not url:
+        base_url = 'https://www.wikidata.org/wiki/Wikidata:Requests_for_deletions'
+        url = base_url + '#' + title
+        text_url = base_url
+        discussion_url = text_url + '#' + title
+
+
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Could not fetch {url}")
+        return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+    if title == '':
+        title = url.split('#')[-1]
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_html, label, cleaned_discussion = extract_discussion_section(soup, title)
+
+    text_url = 'https://www.wikidata.org/wiki/'+ url.split('#')[0]
+    discussion_url = url
+
+    df = pd.DataFrame([[title, text_url, discussion_url, cleaned_discussion, label]],
+                      columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+    if label:
+        df['label'] = df['label'].replace({
+            'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+            'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+            'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+            'No consensus':'no_consensus', 'no consensus':'no_consensus'
+        })
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+
+    return df
+
+
+########################
+## Collection function ##
+########################
+
+
+import pandas as pd
+
+def collect_wikidata_entity(mode='year', title='', url='', years=[]):
+    if mode not in ['title', 'year','url']:
+        raise ValueError("mode must be either 'title' or 'year'")
+
+    if mode == 'title':
+        if not title or years:
+            raise ValueError("For 'title' mode, 'title' must be provided and 'years' must be empty.")
+        df = extract_div_from_title(title)
+        df = df.rename(columns={'label':'outcome', 'discussion_cleaned':'discussion'})
+        return df
+    elif mode == 'url':
+        if 'Archive' in url:
+            archived_url = url.split('#')[0]
+            title = url.split('#')[-1]
+            disc_df = process_discussions_by_url_list([archived_url])
+            disc_df['title'] = disc_df['title'].str.strip()
+            title = title.strip()
+            df = disc_df[disc_df['title'] == title]
+            print(f"Found {len(df)} discussions for title {title}")
+            if df.empty:
+                return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+            df = df.rename(columns={'label':'outcome', 'discussion_cleaned':'discussion'})
+            return df
+        if title or years:
+            raise ValueError("For 'url' mode, 'url' must be provided and 'title' must be empty.")
+        df = extract_div_from_title('', url)
+        df = df.rename(columns={'label':'outcome', 'discussion_cleaned':'discussion'})
+        return df
+
+    elif mode == 'year':
+        if title or not years:
+            raise ValueError("For 'year' mode, 'years' must be provided and 'title' must be empty.")
+        if isinstance(years, list) and len(years) == 2:
+            start_year, end_year = years
+            years = list(range(start_year, end_year + 1))
+        elif isinstance(years, int):
+            years = [years]
+        df = pd.DataFrame()
+        for year in years:
+            print(f"Processing year: {year}")
+
+            year_urls = get_year_urls() 
+            if str(year) not in year_urls:
+                print(f"No URL found for year {year}")
+                continue
+            year_url = year_urls[str(year)]
+            month_day_urls = get_month_day_urls(year_url)
+            print(f"Found {len(month_day_urls)} month-day URLs for {year}")
+            discussions_df = process_discussions_by_url_list(month_day_urls)
+            
+            if discussions_df.empty:
+                continue
+
+            discussions_df.rename(columns={'url':'discussion_url', 'outcome':'label', 'discussion':'discussion_cleaned'}, inplace=True)
+            text_url = year_url
+            discussions_df['text_url'] = text_url
+            discussions_df['label'] = discussions_df['label'].replace({
+                'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+                'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+                'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+                'No consensus':'no_consensus', 'no consensus':'no_consensus'
+            })
+
+            desired_columns = ['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label']
+            for col in desired_columns:
+                if col not in discussions_df.columns:
+                    discussions_df[col] = ''
+            discussions_df = discussions_df[desired_columns]
+
+            df = pd.concat([df, discussions_df], ignore_index=True)
+            df = df.rename(columns={'label':'outcome', 'discussion_cleaned':'discussion'})
+        return df
+    
+

collect_data_wikidata_prop.py

@@ -0,0 +1,317 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import pysbd
+import re
+
+#####################
+# Utility functions #
+#####################
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl', 'ul']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+    return text
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences)
+
+def process_html_to_plaintext(df):
+    if df.empty:
+        return df
+    if 'discussion' in df.columns:
+        df['discussion_cleaned'] = df['discussion'].apply(html_to_plaintext)
+    return df
+
+def process_split_text_into_sentences(df):
+    if df.empty:
+        return df
+    if 'discussion_cleaned' in df.columns:
+        df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+    return df
+
+###########################
+# Year-based extraction   #
+###########################
+
+def extract_outcome_from_div(div):
+    try:
+        consensus_keywords = ['Deleted', 'Delete', 'delete', 'deleted', 'kept', 'keep', 'Keep', 'Kept', 'merge', 'Merge', 'Not done', 'No consensus', 'no consensus']
+        dd_tags = div.find_all('dd')
+        for dd in dd_tags:
+            b_tag = dd.find('b')
+            if b_tag and b_tag.text.strip() in consensus_keywords:
+                return b_tag.text.strip()
+            img_tag = dd.find('img')
+            if img_tag and 'X_mark.svg' in img_tag.get('src', ''):
+                next_b_tag = dd.find_next('b')
+                if next_b_tag and next_b_tag.text.strip() in consensus_keywords:
+                    return next_b_tag.text.strip()
+
+        return 'no consensus'
+    except Exception as e:
+        print(f"Error extracting outcome: {e}")
+        return 'unknown'
+
+
+def extract_cleaned_discussion(div):
+    discussion_parts = []
+    discussion_items = div.find_all(['li', 'dd'])
+    
+    for item in discussion_items:
+        for tag in item.find_all(['span', 'img', 'a']):
+            tag.decompose() 
+        cleaned_text = item.get_text(separator=' ', strip=True)
+        discussion_parts.append(cleaned_text)
+    return ' '.join(discussion_parts)
+
+def extract_div_contents_with_additional_columns(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'verdict', 'discussion'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    divs = soup.find_all('div', class_='boilerplate metadata discussion-archived mw-archivedtalk')
+    if len(divs) == 0:
+        print(f"No discussions found in {url}. Please check the structure.")
+    
+    data = []
+    for i, div in enumerate(divs):
+        try:
+            heading_div = div.find_previous('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+            if heading_div:
+                h2_tag = heading_div.find('h2')
+                if h2_tag:
+                    id = h2_tag.get('id', 'Unknown ID')
+                    if id:
+                        text_url = url+'#' + id 
+                        title = id.replace('(page does not exist)', '').strip()
+                    else:
+                        title = "Unknown Title"
+                        text_url = "Unknown URL"
+                else:
+                    title = "Unknown Title"
+                    text_url = "Unknown URL"
+            else:
+                # fallback for rare cases
+                title = "Unknown Title"
+                text_url = "Unknown URL"
+
+            deletion_discussion = div.prettify()
+            label = extract_outcome_from_div(div)
+            cleaned_discussion = extract_cleaned_discussion(div)
+            parts = deletion_discussion.split('<div class="mw-heading mw-heading3">')
+            discussion = parts[0] if len(parts) > 0 else ''
+            verdict = '<div class="mw-heading mw-heading3">' + parts[1] if len(parts) > 1 else ''
+
+            data.append([title, text_url, deletion_discussion, label, '', cleaned_discussion, verdict])
+        except Exception as e:
+            print(f"Error processing div #{i} in {url}: {e}")
+            continue
+
+    df = pd.DataFrame(data, columns=['title', 'text_url', 'deletion_discussion', 'label', 'confirmation', 'discussion', 'verdict'])
+    return df
+
+def scrape_wikidata_deletions(wikidata_url):
+    months_data = []
+    month_found = False
+    for month in range(1, 13):
+        month_url = f"{wikidata_url}/{month}"
+        print(f"Processing month: {month}")
+        response = requests.get(month_url)
+        if response.status_code == 200:
+            df = extract_div_contents_with_additional_columns(month_url)
+            if not df.empty:
+                df = process_html_to_plaintext(df)
+                df['discussion_cleaned'] = df['discussion_cleaned'].apply(lambda x: ' '.join(pysbd.Segmenter(language="en", clean=False).segment(x)[1:]) if x else x)
+                months_data.append(df)
+                month_found = True
+        else:
+            print(f"No month-specific page found for {month_url}.")
+
+    if month_found and months_data:
+        all_data = pd.concat(months_data, ignore_index=True)
+        return all_data
+    
+    print(f"Attempting year-based extraction for base URL: {wikidata_url}")
+    df = extract_div_contents_with_additional_columns(wikidata_url)
+    if not df.empty:
+        df = process_html_to_plaintext(df)
+        df['discussion_cleaned'] = df['discussion_cleaned'].apply(lambda x: ' '.join(pysbd.Segmenter(language="en", clean=False).segment(x)[1:]) if x else x)
+        return df
+
+    print("No data found using month-specific or year-based extraction.")
+    return pd.DataFrame()
+
+############################
+# Title-based extraction   #
+############################
+
+def extract_outcome_from_text_elements(elements):
+    consensus_keywords = [
+        'Deleted', 'Delete', 'delete', 'deleted', 
+        'kept', 'keep', 'Keep', 'Kept', 
+        'merge', 'Merge', 'Not done', 'No consensus', 'no consensus'
+    ]
+    for el in elements:
+        b_tags = el.find_all('b')
+        for b in b_tags:
+            if b.text.strip() in consensus_keywords:
+                return b.text.strip()
+    return ''
+
+def clean_discussion_tag(tag):
+    for unwanted in tag.find_all(['span', 'img', 'a', 'div'], recursive=True):
+        unwanted.decompose()
+    return tag.get_text(separator=' ', strip=True)
+
+def extract_discussion_section(soup, title):
+    h2_tag = soup.find('h2', id=title)
+    if not h2_tag:
+        print(f"No heading found with id={title}")
+        return '', '', ''
+    heading_div = h2_tag.find_parent('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    if not heading_div:
+        print(f"No heading div found for {title}")
+        return '', '', ''
+
+    next_heading_div = heading_div.find_next('div', class_='mw-heading mw-heading2 ext-discussiontools-init-section')
+    discussion_nodes = []
+    for sibling in heading_div.next_siblings:
+        if sibling == next_heading_div:
+            break
+        discussion_nodes.append(sibling)
+
+    discussion_tags = []
+    for node in discussion_nodes:
+        if getattr(node, 'name', None) in ['p', 'ul', 'dl']:
+            if node.find('span', id=title) or node.get('style', '').lower() == 'visibility:hidden;display:none':
+                continue
+            discussion_tags.append(node)
+
+    if not discussion_tags:
+        return '', '', ''
+
+    label = extract_outcome_from_text_elements(discussion_tags)
+    discussion_html_parts = [str(tag) for tag in discussion_tags]
+    cleaned_parts = []
+    for tag in discussion_tags:
+        text = clean_discussion_tag(tag)
+        if text:
+            cleaned_parts.append(text)
+
+    cleaned_discussion = ' '.join(cleaned_parts)
+    discussion_html = '\n'.join(discussion_html_parts)
+    return discussion_html, label, cleaned_discussion
+
+def extract_div_from_title(url, title):
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Could not fetch {url}")
+        return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_html, label, cleaned_discussion = extract_discussion_section(soup, title)
+
+    text_url = 'https://www.wikidata.org/wiki/Wikidata:Properties_for_deletion'
+    discussion_url = text_url + '#' + title
+
+    data = [[title, text_url, discussion_url, cleaned_discussion, label]]
+    df = pd.DataFrame(data, columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+    return df
+
+############################
+# Unified collect function #
+############################
+
+def collect_wikidata(mode='year', title='', url='', years=[]):
+    if mode not in ['title', 'year','url']:
+        raise ValueError("mode must be either 'title' or 'year' or 'url'.")
+
+    if mode == 'title':
+
+        if not title or years:
+            raise ValueError("For 'title' mode, 'title' must be provided and 'years' must be empty.")
+        url = 'https://www.wikidata.org/wiki/Wikidata:Properties_for_deletion#' + title
+        df = extract_div_from_title(url, title)
+        if not df.empty and 'label' in df.columns and df['label'].notnull().any():
+            df['label'] = df['label'].replace({
+                'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+                'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+                'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+                'No consensus':'no_consensus', 'no consensus':'no_consensus'
+            })
+        df = df.rename(columns={'discussion_cleaned':'discussion'})
+        return df
+    
+    elif mode == 'url':
+        if title or years:
+            raise ValueError("For 'url' mode, 'url' must be provided and 'title' must be empty.")
+        df = extract_div_contents_with_additional_columns(url)
+        if not df.empty and 'label' in df.columns and df['label'].notnull().any():
+            df['label'] = df['label'].replace({
+                'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+                'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+                'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+                'No consensus':'no_consensus', 'no consensus':'no_consensus'
+            })
+        else:
+            return ValueError("No data found for the provided URL.")
+        df = df.rename(columns={'discussion_cleaned':'discussion'})
+        return df
+
+    elif mode == 'year':
+        if title or not years:
+            raise ValueError("For 'year' mode, 'years' must be provided and 'title' must be empty.")
+
+        if isinstance(years, list) and len(years) == 2:
+            start_year, end_year = years
+            years = list(range(start_year, end_year + 1))
+        elif isinstance(years, int): 
+            years = [years]
+
+        df = pd.DataFrame()
+        for year in years:
+            wikidata_url = f'https://www.wikidata.org/wiki/Wikidata:Properties_for_deletion/Archive/{year}'  
+            deletions_df = scrape_wikidata_deletions(wikidata_url)
+            if deletions_df.empty:
+                continue
+
+            columns_to_drop = ['confirmation', 'discussion', 'verdict', 'deletion_discussion']
+            deletions_df = deletions_df.drop(columns=[col for col in columns_to_drop if col in deletions_df.columns], errors='ignore')
+
+            if 'label' in deletions_df.columns:
+                deletions_df.rename(columns={'label':'label'}, inplace=True)
+                deletions_df['label'] = deletions_df['label'].replace({
+                    'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+                    'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+                    'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+                    'No consensus':'no_consensus', 'no consensus':'no_consensus'
+                })
+
+
+            if 'text_url' in deletions_df.columns:
+                deletions_df.rename(columns={'text_url':'discussion_url'}, inplace=True)
+            deletions_df['text_url'] = wikidata_url
+            if 'label' not in deletions_df.columns:
+                deletions_df['label'] = ''
+
+            for col in ['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label']:
+                if col not in deletions_df.columns:
+                    deletions_df[col] = ''
+
+            deletions_df = deletions_df[['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label']]
+
+            deletions_df['year'] = year
+            df = pd.concat([df, deletions_df], ignore_index=True)
+            df = df.rename(columns={'discussion_cleaned':'discussion'})
+        return df

collect_data_wikinews.py

@@ -0,0 +1,435 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import pysbd
+import re
+
+
+################################
+# Year based data collection ###
+################################
+
+def get_soup(url):
+    response = requests.get(url)
+    response.raise_for_status()  
+    return BeautifulSoup(response.text, 'html.parser')
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+    return text
+
+def extract_fallback_discussion(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    discussion_parts = []
+    for element in soup.find_all(['p', 'li', 'dd', 'ol'], recursive=False):
+        discussion_parts.append(element.get_text(separator=' ', strip=True))
+    return ' '.join(discussion_parts).strip()
+
+def process_html_to_plaintext(df):
+    if df.empty:
+        return df
+    df['discussion_cleaned'] = df['discussion'].apply(html_to_plaintext)
+    for index, row in df.iterrows():
+        if not row['discussion_cleaned'].strip():
+            df.at[index, 'discussion_cleaned'] = extract_fallback_discussion(row['discussion_uncleaned']) 
+    return df
+
+def extract_outcome_from_div(div):
+    try:
+        result_phrase = div.find(text=re.compile(r'The result was to'))
+        if result_phrase:
+            result = result_phrase.find_next('b')
+            if result:
+                outcome_text = result.text.strip()
+                if outcome_text.lower() == "please do not modify it":
+                    return extract_following_sentence(div) or 'unknown'
+                elif validate_outcome(outcome_text):
+                    return outcome_text  
+        li_outcome = div.find('li')
+        if li_outcome and li_outcome.find('b'):
+            outcome_text = li_outcome.find('b').text.strip()
+            if outcome_text.lower() == "please do not modify it":
+                return extract_following_sentence(div) or 'unknown'
+            elif validate_outcome(outcome_text):
+                return outcome_text
+
+        dl_outcome = div.find('dl')
+        if dl_outcome and dl_outcome.find('b'):
+            outcome_text = dl_outcome.find('b').text.strip()
+            if outcome_text.lower() == "please do not modify it":
+                return extract_following_sentence(div) or 'unknown'
+            elif validate_outcome(outcome_text):
+                return outcome_text
+
+        outcome_italic = div.find('dd')
+        if outcome_italic and outcome_italic.find('i'):
+            outcome_text = outcome_italic.find('i').get_text(strip=True)
+            if outcome_text.lower() == "please do not modify it":
+                return extract_following_sentence(div) or 'unknown'
+            elif validate_outcome(outcome_text):
+                return outcome_text
+        return extract_following_sentence(div) or 'unknown'
+
+    except Exception as e:
+        print(f"Error extracting outcome: {e}")
+        return 'unknown'
+
+
+def extract_following_sentence(div):
+    try:
+        phrases = [
+            "No further edits should be made to this discussion",
+            "Please do not add any more comments and votes to this request",
+            "No further edits should be made to this discussion."
+        ]
+        
+        for phrase in phrases:
+            phrase_location = div.find(text=re.compile(phrase))
+            if phrase_location:
+                following_text = ""
+                for sibling in phrase_location.find_all_next(string=True):
+                    if "Please do not modify it" in sibling:
+                        continue
+                    following_text += sibling.strip() + " "
+                    if "." in sibling:
+                        break
+                sentence = following_text.split('.')[0].strip()
+                if validate_outcome(sentence):
+                    return sentence
+        
+        return None 
+
+    except Exception as e:
+        print(f"Error extracting following sentence: {e}")
+        return None
+
+def validate_outcome(outcome_text):
+    label_mapping = {
+        'delete': [
+            'delete', 'delete ... unanimous', 'deleted', 'deleted as abandoned',
+            'speedy delete', 'Delete', 'delete as redundant to existing template',
+            'delete as unlikely to be used', 'delete but no prejudice against recreation when needed',
+            'delete after Ottawahitech chose not to provide a rationale',
+            'Delete, with no objection to recreation when needed.', 'Deleted', 
+            'delete the Cigarette redirect and keep the NHS redirect.', 'Delete all articles', 'Tentatively sending through the usual abandonment process',
+            'Delete all articles','This was completed already.'
+        ],
+'speedy delete': [ 
+
+            'speedy delete', 'speedy deleted', 'speedy deleted test page', 'Speedy-deleted', 'Speedy deleted', 'Speedy-deleted, no meaningful content',
+            'Speeded as "old prep"', 'Speedied as "old prep"  -- Pi zero ( talk ) 23:42, 10 February 2020 (UTC)   [  reply  ]  __DTELLIPSISBUTTON__{"threadItem":{"timestamp":"2020-02-10T23:42:00'
+],
+
+        'keep': [
+             'keep',
+            'Do not undelete. The content should be kept by the author off-wiki, and can be included as a part of another story that is current',
+            'Personal details have been redacted and hidden from public view together with a NOINDEX flag',
+
+            ],
+        'redirect': [
+            'soft redirect'
+        ],
+        'merge': [
+            'convert near-clone of mainspace article to use {{topic cat}}; apply {{correction}} to mainspace article'
+        ],
+        'no_consensus': [
+            'No consensus to delete. However, there clearly is a consensus that if we are to have this template, we aren\'t to use it in its present form.',
+            'no consensus', 'No consensus',
+            "At this time, it's unclear if there's a consensus to keep but abundantly clear there isn't one to delete."
+        ],
+        'comment': [
+            'Remove', 'SVT', 'withdraw the deletion request', 'On consideration, speedied as unused and lacking fair-use rationale',
+            'Moved to userspace', 'Withdrawn to allow interview re-focus','More userspace drafts       This is the second batch of a large number of draft articles in userspace',
+            'This was completed already ', 'Do not undelete. The content should be kept by the author off-wiki, and can be included as a part of another story that is current',
+
+        ],
+        'withdrawn': ['Withdrawn to allow interview re-focus', 
+        ]
+    }
+
+    
+
+    outcome_to_label = {outcome.lower(): label for label, outcomes in label_mapping.items() for outcome in outcomes}
+    return outcome_to_label.get(outcome_text.lower(), 'unknown')
+
+
+def update_unknown_outcomes(df):
+    base_url = "https://en.wikinews.org/w/index.php?title="
+
+    for i in df.index:
+        if df.at[i, 'outcome'] == 'unknown':
+            title = df.at[i, 'title'].replace(" ", "_") 
+            url = f"{base_url}{title}&action=edit&redlink=1"
+            print(f"Checking page: {url}")
+
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    page_soup = BeautifulSoup(response.content, 'html.parser')
+                    
+                    # Look for the specific warning div
+                    warning_div = page_soup.find('div', class_='cdx-message cdx-message--block cdx-message--warning mw-warning-with-logexcerpt')
+                    if warning_div:
+                        df.at[i, 'outcome'] = 'delete'
+                    else:
+                        df.at[i, 'outcome'] = 'keep'
+                else:
+                    print(f"Failed to retrieve page: {url}")
+            
+            except Exception as e:
+                print(f"Error accessing {url}: {e}")
+
+    return df
+
+    
+def collect_wikinews_deletions(years=None):
+    base_url = 'https://en.wikinews.org/wiki/Wikinews:Deletion_requests/Archives'
+    response = requests.get(base_url)
+    if response.status_code != 200:
+        print("Failed to retrieve the archive page.")
+        return None
+    
+    soup = get_soup(base_url)
+    titles = []
+    text_urls = []
+    outcomes = []
+    deletion_discussions = []  
+    discussion_uncleaned = [] 
+    year_links = []
+    for a in soup.select('a[href^="/wiki/Wikinews:Deletion_requests/Archives/"]'):
+        year_text = re.findall(r'\d{4}', a.get_text()) 
+        if year_text:
+            year_links.append((year_text[0], a['href']))
+    if years:
+        if len(years) == 1:
+            start_year = end_year = years[0]
+        elif len(years) == 2:
+            start_year, end_year = min(years), max(years)
+        else:
+            print("Invalid years input. Provide one or two years.")
+            return None
+        year_links = [(year, link) for year, link in year_links if start_year <= int(year) <= end_year]
+    for year, year_link in year_links:
+        year_url = 'https://en.wikinews.org' + year_link
+        print(f"Processing year: {year_url}")
+        year_soup = get_soup(year_url)
+        discussion_divs = year_soup.find_all('div', class_=lambda x: x and 'boilerplate metadata' in x)
+        
+        for div in discussion_divs:
+            title_tag = div.find(['h2', 'h3'])
+            if title_tag:
+                link_tag = title_tag.find('a', title=True)
+                if link_tag:
+                    title = link_tag.get_text(strip=True)
+                    titles.append(title)
+                    text_url = year_url + '#' + link_tag['title'].replace(' ', '_')
+                    text_urls.append(text_url)
+                else:
+                    titles.append(title_tag.get_text(strip=True))
+                    text_urls.append(year_url)
+            else:
+                dl_tag = div.find('dl')
+                if dl_tag and dl_tag.find('b'):
+                    titles.append(dl_tag.find('b').get_text(strip=True))
+                else:
+                    titles.append('No title found')
+                text_urls.append(year_url)
+            deletion_discussions.append(div.prettify())
+            discussion_uncleaned.append(div.prettify())
+            outcome = extract_outcome_from_div(div)
+            outcomes.append(outcome)
+
+    df = pd.DataFrame({
+        'title': titles,
+        'url': text_urls,
+        'outcome': outcomes,
+        'discussion': deletion_discussions,
+        'discussion_uncleaned': discussion_uncleaned  
+    })
+
+
+    df = process_html_to_plaintext(df)
+    for i in df.index:
+        if df.at[i,'outcome'] == 'Please do not modify it' or df.at[i,'outcome'] == 'Please do not modify it.':
+            df.at[i,'outcome'] = extract_following_sentence(BeautifulSoup(df.at[i,'discussion_uncleaned'], 'html.parser')) or 'unknown'
+    df['outcome'] = df['outcome'].apply(lambda x: validate_outcome(x) if x else 'unknown')
+    df = update_unknown_outcomes(df)
+    return df
+
+def collect_wikinews(years=None):
+    df = collect_wikinews_deletions(years=years)
+    if df is None:
+        print('Error collecting Wikinews deletions.')
+        return None
+    return df
+
+
+##################################
+## Ttitle based data collection ##
+##################################
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl', 'ul']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+    return text
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    return ' '.join(sentences)
+
+def clean_discussion_tag(tag):
+    for unwanted in tag.find_all(['span', 'img', 'a', 'div'], recursive=True):
+        unwanted.decompose()
+    return tag.get_text(separator=' ', strip=True)
+
+def extract_outcome_from_text_elements(elements):
+    consensus_keywords = [
+        'Deleted', 'Delete', 'delete', 'deleted', 
+        'kept', 'keep', 'Keep', 'Kept', 
+        'merge', 'Merge', 'Not done', 'No consensus', 'no consensus', 'Done'
+    ]
+    for el in elements:
+        b_tags = el.find_all('b')
+        for b in b_tags:
+            if b.text.strip() in consensus_keywords:
+                return b.text.strip()
+    return ''
+
+def extract_discussion_section(soup, title):
+    """Extracts discussion section, label, and cleaned text."""
+    try:
+        h3_id = title.replace(" ", "_") 
+        h3_tag = soup.find('h3', {'id': h3_id})
+
+        if not h3_tag:
+            print(f"h3 tag with id '{h3_id}' not found.")
+            return '', '', ''
+
+        heading_div = h3_tag.parent
+
+        if not heading_div:
+            print("Parent div not found.")
+            return '', '', ''
+
+        next_heading_div = heading_div.find_next_sibling('div', class_='mw-heading mw-heading3')
+        discussion_nodes = []
+        for sibling in heading_div.next_siblings:
+            if sibling == next_heading_div:
+                break
+            discussion_nodes.append(sibling)
+
+        discussion_tags = []
+        for node in discussion_nodes:
+            if getattr(node, 'name', None) in ['p', 'ul', 'dl']:
+                discussion_tags.append(node)
+
+        if not discussion_tags:
+            return '', '', ''
+
+        label = extract_outcome_from_text_elements(discussion_tags)
+
+        discussion_html_parts = [str(tag) for tag in discussion_tags]
+        cleaned_parts = []
+        for tag in discussion_tags:
+            text = clean_discussion_tag(tag)
+            if text:
+                cleaned_parts.append(text)
+
+        cleaned_discussion = ' '.join(cleaned_parts)
+        discussion_html = '\n'.join(discussion_html_parts)
+        return discussion_html, label, cleaned_discussion
+
+    except Exception as e:
+        print(f"Error processing title '{title}': {e}")
+        import traceback
+        traceback.print_exc()  
+        return '', '', ''
+
+def extract_div_from_title(title):
+    base_url = 'https://en.wikinews.org/wiki/Wikinews:Deletion_requests'
+    t = title.replace(' ', '_')
+    url = base_url + '#' + t
+
+    response = requests.get(url)
+    if response.status_code != 200:
+        return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    discussion_html, label, cleaned_discussion = extract_discussion_section(soup, title)
+ 
+    text_url = base_url
+    discussion_url = text_url + '#' + title.replace(' ', '_')
+
+    df = pd.DataFrame([[title, text_url, discussion_url, cleaned_discussion, label]],
+                      columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+
+    if label:
+        df['label'] = df['label'].replace({
+            'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 
+            'kept':'keep', 'keep':'keep', 'Keep':'keep', 'Kept':'keep', 
+            'merge':'merge', 'Merge':'merge', 'Not done':'no_consensus', 
+            'No consensus':'no_consensus', 'no consensus':'no_consensus', 'Done':'delete'
+        })
+
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+    df = df.rename(columns={'discussion_cleaned':'discussion'})
+    return df
+
+########################
+## Umbrella function  ##
+########################
+
+def collect_wikinews(mode, title=None, url ='', year=None):
+
+    if mode == 'title':
+        if not title:
+            raise ValueError("Title is required for 'title' mode.")
+        return extract_div_from_title(title)
+    
+    elif mode == 'url':
+        if 'Archives' in url.split('/')[-2]:
+            year = int(url.split('/')[-1].split('#')[0])
+            print(f"Year extracted from URL: {year}")
+            df = collect_wikinews_deletions(years=[year])
+            #keep the row with the title only
+            df = df[df['title'] == url.split('#')[-1].replace('_', ' ')]
+            if df.empty:
+                return pd.DataFrame(columns=['title', 'text_url', 'discussion_url', 'discussion_cleaned', 'label'])
+            df = df[['title','url','discussion_cleaned','outcome']]
+            df = df.rename(columns={'discussion_cleaned':'discussion'})   
+            return df
+
+        if not url:
+            raise ValueError("URL is required for 'url' mode.")
+        title = url.split('#')[-1].replace('_', ' ')
+        print(f"Title extracted from URL: {title}")
+        return extract_div_from_title(title)
+
+    elif mode == 'year':
+        if not year:
+            raise ValueError("Year or year range is required for 'year' mode.")
+        return collect_wikinews_deletions(years=year)
+
+    else:
+        raise ValueError("Invalid mode. Please specify 'title' or 'year' or 'url'.")
+
+# year_df = collect_wikinews(mode='year', year=[2023]) 
+# title_df = collect_wikinews(mode='title', title="NurMi spam") 
+
+# print(year_df)
+# print(title_df)

collect_data_wikiquote.py

@@ -0,0 +1,151 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import pysbd
+import re
+
+def extract_outcome_from_div(div):
+    try:
+        # Extracting the decision from <b> tag that contains result like 'no consensus', 'deleted', etc.
+        result = div.find(text=re.compile(r'The result was:')).find_next('b')
+        if result:
+            return result.text.strip()
+        return 'no consensus'
+    except Exception as e:
+        print(f"Error extracting outcome: {e}")
+        return 'unknown'
+
+def html_to_plaintext(html_content):
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for tag in soup.find_all(['p', 'li', 'dd', 'dl']):
+        tag.insert_before('\n')
+        tag.insert_after('\n')
+    for br in soup.find_all('br'):
+        br.replace_with('\n')
+
+    text = soup.get_text(separator=' ', strip=True)
+    text = '\n'.join([line.strip() for line in text.splitlines() if line.strip() != ''])
+    return text
+
+def process_html_to_plaintext(df):
+    if df.empty:
+        return df
+    df['discussion_cleaned'] = df['discussion'].apply(html_to_plaintext)
+    return df
+
+def split_text_into_sentences(text):
+    seg = pysbd.Segmenter(language="en", clean=False)
+    sentences = seg.segment(text)
+    for i, sentence in enumerate(sentences):
+        if 'The result was:' in sentence:
+            return ' '.join(sentences[i+1:])
+    return ' '.join(sentences[1:])
+
+
+def process_split_text_into_sentences(df):
+    if df.empty:
+        return df
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(split_text_into_sentences)
+    df['discussion_cleaned'] = df['discussion_cleaned'].apply(lambda x: x.replace("The above discussion is preserved as an archive of the debate. Please do not modify it. Subsequent comments should be made on the appropriate discussion page (such as the article's talk page or in a deletion review ). No further edits should be made to this page.", ''))
+    #df['discussion_cleaned'] = df['discussion_cleaned'].apply(cleanup_initial_sentences)
+    return df
+
+def collect_wikiquote_title(title='all', base_url='https://en.wikiquote.org/wiki/Wikiquote:Votes_for_deletion_archive'):
+    titles = []
+    text_urls = []
+    labels = []
+    deletion_discussions = []
+    if title == 'all':
+        url = base_url
+    else:
+        url = base_url + '#' + title.replace(' ', '_')
+    
+    response = requests.get(url)
+    
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.text, 'html.parser')
+
+        if title == 'all':
+            divs = soup.find_all('div', class_='boilerplate metadata vfd')
+        else:
+            # For specific title, find a div that matches the title
+            divs = soup.find_all('div', class_='boilerplate metadata vfd')
+            divs = [div for div in divs if div.find('div', class_="mw-heading mw-heading2 ext-discussiontools-init-section") and title in div.find('div', class_="mw-heading mw-heading2 ext-discussiontools-init-section").text]
+
+        no_divs = len(divs)
+        print(f"Found {no_divs} div(s) with the expected classes.")
+
+        if no_divs >= 1:
+            for div in divs:
+                heading_div = div.find('div', class_="mw-heading mw-heading2 ext-discussiontools-init-section")
+                if heading_div:
+                    found_title = heading_div.text.strip()
+                    titles.append(found_title.replace('[edit]', ''))
+                    text_url = base_url + '#' + found_title.replace(' ', '_') 
+                    text_urls.append(text_url)
+                    label = extract_outcome_from_div(div)
+                    labels.append(label)
+                    deletion_discussions.append(div.prettify())
+                else:
+                    print("No heading div found with the expected classes.")
+
+            df = pd.DataFrame({'title': titles, 'text_url': text_urls, 'label': labels, 'discussion': deletion_discussions})
+            df = process_html_to_plaintext(df)
+            df = process_split_text_into_sentences(df)
+            df['label'] = df['label'].replace({
+                'Deleted':'delete', 'Delete':'delete', 'delete':'delete', 'deleted':'delete', 'deleted.':'delete', 'speedy deleted test page':'delete', 'Deleted and protected with a message':'delete',
+                'delete both':'delete', 'delete everything':'delete', 'Deleted due to copyvio':'delete', 'delete after various merges':'delete', 'delete 3 quoteless, keep 1 redirect':'delete',
+                'Consensus to remove from Wikiquote, but only if it is not merged into another article':'delete', 'Consensus to remove from Wikiquote, but not how':'delete', 'delete, pending technical fix':'delete', 'delete all':'delete', 'delete Portal:portal, no consensus/keep Template:Wikimedia':'delete',
+                'Speedy-deleted':'delete', 'Speedy deleted':'delete', 'Speedy-deleted, no meaningful content':'delete',
+                'kept':'keep', 'Kept.':'keep', 'Keep':'keep', 'keep':'keep', 'Kept':'keep', 'No consensus/keep':'keep', 'kept/no consensus':'keep', 'Kept; lack of consensus':'keep', 'kept after copyvio removal':'keep',
+                'Speedy-kept':'keep', 'Speedy kept':'keep',
+                'merge':'merge', 'Merge':'merge', 'merged':'merge', 'Merged':'merge', 'merged into Azerbaijani proverbs':'merge', 'Merge with Stephen Covey':'merge', 'Merge with Lyrics':'merge',
+                'merge and redirect':'merge', 'merge with Crusade (TV series)':'merge', 'Merged to Health':'merge', 'merge with 3rd Rock from the Sun':'merge',
+                'redirect to List of proverbs':'redirect', 'keep as redirect':'redirect', 'Redirect to Inuyasha':'redirect', 'Redirected to Humor':'redirect', 'Redirected to Doctor Who':'redirect',
+                'Redirect without text':'redirect', 'Proverbs turned to redirect to List of proverbs':'redirect', 'redirect to Drugs':'redirect', 'redirect to Advertising slogans':'redirect',
+                'redirect to Jalal al-Din Muhammad Rumi':'redirect', 'redirect':'redirect', 'Redirected':'redirect', 'move to Category:United States Marines':'redirect', 'move to Die Hard: With a Vengeance':'redirect',
+                'move to Star Wars Jedi Knight: Jedi Academy':'redirect', 'move to Lucien Lévy-Bruhl':'redirect', 'move to Dave Finlay':'redirect', 'move to User:Quenzer':'redirect', 'moved':'redirect', 
+                'moved to Monument inscriptions':'redirect', 'transwiki to Wikipedia, then delete':'redirect', 'Transwiki to Wikipedia':'redirect', 'Transwiki to Wikipedia':'redirect',
+                'delete His Holiness the Dalai Lama, redirect Dalai Lama to Tenzin Gyatso, 14th Dalai Lama':'redirect',
+                'move':'redirect', 'keep Just war theory, redirect Just war, delete Just War Theory':'no_consensus','move to Wikisource':'redirect',\
+                'kept.':'keep', 'Keep as Redirect':'redirect', 'Deleted.':'delete', '1 delete, 1 redirect':'redirect', 'moved to User:Quenzer':'redirect',\
+                'transwiki, then delete':'delete', 'merge with Lyrics':'redirect','Deleted all three images':'delete',\
+                'No consensus':'no_consensus', 'no consensus':'no_consensus', 'inconclusive; no action taken.':'no_consensus', 'UNIDENTIFIED':'no_consensus'
+            })
+            return df
+        else:
+            print("No divs found with the expected classes.")
+            return None
+    else:
+        print("Failed to retrieve the page.")
+        return None
+
+def collect_wikiquote(mode ='title',title = 'all', url = ''):
+    if mode not in ['title', 'url']:
+        raise ValueError("mode must be either 'title' or 'url'.")
+    if mode == 'title' and title == 'all':
+        base_url = 'https://en.wikiquote.org/wiki/Wikiquote:Votes_for_deletion_archive'
+        df = collect_wikiquote_title(title, base_url)
+        if df is not None:
+            if 'discussion_cleaned' in df.columns:
+                df = df[['title', 'text_url', 'label', 'discussion_cleaned']]
+                df = df.rename(columns={'discussion_cleaned': 'discussion'})
+            return df
+    elif mode == 'url':
+        df = collect_wikiquote_title('all', url)
+        title = url.split('#')[-1].replace('_', ' ')
+        df = df[df['title'].str.lower() == title.lower()].reset_index(drop=True)
+        if not df.empty:
+            if 'discussion_cleaned' in df.columns:
+                df = df[['title', 'text_url', 'label', 'discussion_cleaned']]
+                df = df.rename(columns={'discussion_cleaned': 'discussion'})
+            return df
+        else:
+            raise ValueError(f"No data found for the url: {url}")
+    else:
+        base_url = 'https://en.wikiquote.org/wiki/Wikiquote:Votes_for_deletion'
+        df = collect_wikiquote_title(title, base_url)
+        if 'discussion_cleaned' in df.columns:
+                df = df[['title', 'text_url', 'label', 'discussion_cleaned']]
+                df = df.rename(columns={'discussion_cleaned': 'discussion'})
+        return df

data_collect.py

@@ -0,0 +1,180 @@
+
+from datetime import datetime
+from wide_analysis.data.process_data import prepare_dataset
+from datasets import load_dataset
+
+from collect_data_wikidata_ent import collect_wikidata_entity
+from collect_data_wikidata_prop import collect_wikidata
+from collect_data_wikinews import collect_wikinews
+from collect_data_wikiquote import collect_wikiquote
+from collect_data_es import collect_es
+from collect_data_gr import collect_gr
+
+def normalize_outcome(o):
+    lowered = o.lower()
+    if 'διαγρ' in lowered:
+        return 'Διαγραφή'
+    elif 'διατήρη' in lowered or 'παραμονή' in lowered:
+        return 'Διατήρηση'
+    elif 'συγχών' in lowered:
+        return 'συγχώνευση'
+    else:
+        return 'Δεν υπάρχει συναίνεση'
+
+def collect(mode, start_date=None, end_date=None, url=None, title=None, output_path=None,
+            platform=None, lang=None, date=None, years=None):
+    if mode not in ['date_range', 'date', 'title','url','wide_2023']:
+        raise ValueError("Invalid mode. Choose from ['date_range', 'date', 'title','url','wide_2023']")
+
+    if mode == 'wide_2023':
+        dataset = load_dataset('hsuvaskakoty/wide_analysis')
+        print('Dataset loaded successfully as huggingface dataset')
+        print('The dataset has the following columns:', dataset.column_names)
+        return dataset
+    underlying_mode = mode
+    if mode in ['date', 'date_range']:
+        underlying_mode = 'year'
+    if mode == 'url':
+        underlying_mode = 'url'
+    if (platform is None and lang is None) or (platform=='wikipedia' and lang=='en'):
+        if mode in ['date_range', 'date', 'title']:
+            return prepare_dataset(
+                mode=mode,
+                start_date=start_date,
+                end_date=end_date,
+                url=url,
+                title=title,
+                output_path=output_path
+            )
+        else:
+            print("Invalid input. Choose from ['date_range', 'date', 'title','wide_2023']")
+            return None
+
+    if platform == 'wikidata_entity':
+            if underlying_mode == 'title':
+                if not title or (years and len(years)>0):
+                    raise ValueError("For 'title' mode in wikidata entity, 'title' must be provided and 'years' must be empty.")
+                return collect_wikidata_entity(mode='title', title=title, years=[])
+
+            elif underlying_mode == 'year':
+                if start_date and end_date:
+                    start_year = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    end_year = int(datetime.strptime(end_date, "%Y-%m-%d").year)
+                    return collect_wikidata_entity(mode='year', years=[start_year, end_year])
+                elif start_date:
+                    single_year = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    return collect_wikidata_entity(mode='year', years=single_year)
+                else:
+                    raise ValueError("For 'year' mode in wikidata entity, start_date (and optionally end_date) is required.")
+            elif underlying_mode == 'url':
+                if not url:
+                    raise ValueError("For 'url' mode in wikidata entity, 'url' must be provided.")
+                return collect_wikidata_entity(mode='url', url=url)
+            else:
+                raise ValueError("Invalid mode for wikidata entity. Use 'title' or 'year'.")
+            
+
+    elif platform == 'wikidata_property':
+            if underlying_mode == 'title':
+                if not title or (years and len(years)>0):
+                    raise ValueError("For 'title' mode in wikidata property, 'title' must be provided and 'years' must be empty.")
+                return collect_wikidata(mode='title', title=title, years=[])
+            elif underlying_mode == 'url':
+                if not url:
+                    raise ValueError("For 'url' mode in wikidata property, 'url' must be provided.")
+                return collect_wikidata(mode='url', title='', url=url, years=[])
+            
+            elif underlying_mode == 'year':
+                if start_date and end_date:
+                    start_year = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    end_year = int(datetime.strptime(end_date, "%Y-%m-%d").year)
+                    return collect_wikidata(mode='year', years=[start_year, end_year])
+                elif start_date:
+                    single_year = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    return collect_wikidata(mode='year', years=single_year)
+                else:
+                    raise ValueError("For 'year' mode in wikidata property, start_date (and optionally end_date) is required.")
+            else:
+                raise ValueError("Invalid mode for wikidata property. Use 'title' or 'year'.")
+
+        # else:
+        #     raise ValueError("Invalid lang for wikidata. Use 'entity' or 'property'.")
+
+    elif platform == 'wikinews':
+        if underlying_mode == 'title':
+            if not title:
+                raise ValueError("For 'title' mode in wikinews, 'title' is required.")
+            return collect_wikinews(mode='title', title=title)
+        elif underlying_mode == 'url':
+            if not url:
+                raise ValueError("For 'url' mode in wikinews, 'url' is required.")
+            return collect_wikinews(mode='url', url=url)
+        elif underlying_mode == 'year':
+            if start_date and end_date:
+                start_y = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                end_y = int(datetime.strptime(end_date, "%Y-%m-%d").year)
+                return collect_wikinews(mode='year', year=[start_y, end_y])
+            elif start_date:
+                single_y = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                return collect_wikinews(mode='year', year=single_y)
+            else:
+                raise ValueError("For 'year' mode in wikinews, start_date (and optionally end_date) is required.")
+        else:
+            raise ValueError("Invalid mode for wikinews. Use 'title' or 'year' or  'url'.")
+
+    # elif platform == 'wikiquote':
+    #     if underlying_mode != 'title':
+    #         raise ValueError("Wikiquote collection currently only supports 'title' mode.")
+    #     if not title:
+    #         title = 'all'
+    #     return collect_wikiquote(mode='title', title=title)
+    elif platform == 'wikiquote':
+        if underlying_mode not in ['title', 'url']:
+            raise ValueError("Wikiquote collection currently only supports 'title' or 'url' mode.")
+
+        if underlying_mode == 'title':
+            if not title:
+                title = 'all'
+            return collect_wikiquote(mode='title', title=title)
+        elif underlying_mode == 'url':
+            if not url:
+                raise ValueError("For 'url' mode in wikiquote, 'url' must be provided.")
+            return collect_wikiquote(mode='url', url=url)
+
+
+    elif platform == 'wikipedia':
+        if lang == 'es':
+            if underlying_mode == 'title':
+                if not title or date:
+                    raise ValueError("For 'title' mode in spanish wikipedia, 'title' must be provided and 'date' must be empty.")
+                return collect_es(mode='title', title=title, date='')
+            elif underlying_mode == 'year':
+                if not date:
+                    raise ValueError("For 'year' mode in spanish wikipedia, 'date' parameter (dd/mm/yyyy) is required.")
+                return collect_es(mode='year', title='', date=date)
+            else:
+                raise ValueError("Invalid mode for spanish wikipedia. Use 'title' or 'year'.")
+
+        elif lang == 'gr':
+            if underlying_mode == 'title':
+                if not title or not years or len(years) != 1:
+                    raise ValueError("For 'title' mode in greek wikipedia, 'title' and a single-element list years=['mm/yyyy'] are required.")
+                return collect_gr(mode='title', title=title, years=years)
+            elif underlying_mode == 'year':
+                if start_date and end_date:
+                    start_y = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    end_y = int(datetime.strptime(end_date, "%Y-%m-%d").year)
+                    return collect_gr(mode='year', title='', years=[start_y,end_y])
+                elif start_date:
+                    single_y = int(datetime.strptime(start_date, "%Y-%m-%d").year)
+                    return collect_gr(mode='year', title='', years=[single_y])
+                else:
+                    raise ValueError("For 'year' mode in greek wikipedia, start_date (and optionally end_date) is required.")
+            else:
+                raise ValueError("Invalid mode for greek wikipedia. Use 'title' or 'year'.")
+
+        else:
+            raise ValueError("Invalid lang for wikipedia. Use 'en', 'es', or 'gr'.")
+
+    else:
+        raise ValueError("Invalid platform. Use 'wikipedia', 'wikidata_entity', Wikidata_property', 'wikinews', or 'wikiquote'.")

model_predict.py

@@ -1,25 +1,290 @@
-#using pipeline to predict the input text
+# #using pipeline to predict the input text
+# import pandas as pd
+# from transformers import pipeline, AutoTokenizer
+# import pysbd
+
+# #-----------------Outcome Prediction-----------------
+# def outcome(text):
+#     label_mapping = {
+#         'delete': [0, 'LABEL_0'],
+#         'keep': [1, 'LABEL_1'],
+#         'merge': [2, 'LABEL_2'],
+#         'no consensus': [3, 'LABEL_3'],
+#         'speedy keep': [4, 'LABEL_4'],
+#         'speedy delete': [5, 'LABEL_5'],
+#         'redirect': [6, 'LABEL_6'],
+#         'withdrawn': [7, 'LABEL_7']
+#     }
+#     model_name = "research-dump/roberta-large_deletion_multiclass_complete_final"
+#     tokenizer = AutoTokenizer.from_pretrained(model_name)
+#     model = pipeline("text-classification", model=model_name, return_all_scores=True)
+    
+#     # Tokenize and truncate the text
+#     tokens = tokenizer(text, truncation=True, max_length=512)
+#     truncated_text = tokenizer.decode(tokens['input_ids'], skip_special_tokens=True)
+    
+#     results = model(truncated_text)
+    
+#     res_list = []
+#     for result in results[0]:
+#         for key, value in label_mapping.items():
+#             if result['label'] == value[1]:
+#                 res_list.append({'sentence': truncated_text, 'outcome': key, 'score': result['score']})
+#                 break
+    
+#     return res_list
+
+
+# #-----------------Stance Prediction-----------------
+
+# def extract_response(text, model_name, label_mapping):
+#     tokenizer = AutoTokenizer.from_pretrained(model_name)
+#     pipe = pipeline("text-classification", model=model_name, tokenizer=tokenizer, top_k=None)
+
+#     tokens = tokenizer(text, truncation=True, max_length=512)
+#     truncated_text = tokenizer.decode(tokens['input_ids'], skip_special_tokens=True)
+    
+#     results = pipe(truncated_text)
+    
+#     final_scores = {key: 0.0 for key in label_mapping}
+#     for result in results[0]:
+#         for key, value in label_mapping.items():
+#             if result['label'] == f'LABEL_{value}':
+#                 final_scores[key] = result['score']
+#                 break
+    
+#     return final_scores
+
+
+# def get_stance(text):
+#     label_mapping = {
+#             'delete': 0,
+#             'keep': 1,
+#             'merge': 2,
+#             'comment': 3
+#         }
+#     seg = pysbd.Segmenter(language="en", clean=False)
+#     text_list = seg.segment(text)
+#     model = 'research-dump/bert-large-uncased_wikistance_v1'
+#     res_list = []
+#     for t in text_list:
+#         res = extract_response(t, model,label_mapping) #, access_token)
+#         highest_key = max(res, key=res.get)
+#         highest_score = res[highest_key]
+#         result = {'sentence':t,'stance': highest_key, 'score': highest_score}
+#         res_list.append(result)
+    
+#     return res_list
+
+
+# #-----------------Policy Prediction-----------------
+# def get_policy(text):
+#     label_mapping = {'Wikipedia:Notability': 0,
+#             'Wikipedia:What Wikipedia is not': 1,
+#             'Wikipedia:Neutral point of view': 2,
+#             'Wikipedia:Verifiability': 3,
+#             'Wikipedia:Wikipedia is not a dictionary': 4,
+#             'Wikipedia:Wikipedia is not for things made up one day': 5,
+#             'Wikipedia:Criteria for speedy deletion': 6,
+#             'Wikipedia:Deletion policy': 7,
+#             'Wikipedia:No original research': 8,
+#             'Wikipedia:Biographies of living persons': 9,
+#             'Wikipedia:Arguments to avoid in deletion discussions': 10,
+#             'Wikipedia:Conflict of interest': 11,
+#             'Wikipedia:Articles for deletion': 12
+#             }
+    
+
+#     seg = pysbd.Segmenter(language="en", clean=False)
+#     text_list = seg.segment(text)
+#     model = 'research-dump/bert-large-uncased_wikistance_policy_v1'
+#     res_list = []
+    
+#     for t in text_list:
+#         res = extract_response(t, model,label_mapping)
+#         highest_key = max(res, key=res.get)
+#         highest_score = res[highest_key]
+#         result = {'sentence': t, 'policy': highest_key, 'score': highest_score}
+#         res_list.append(result)
+    
+#     return res_list
+
+
+
+# #-----------------Sentiment Analysis-----------------
+
+# def extract_highest_score_label(res):
+#     flat_res = [item for sublist in res for item in sublist]
+#     highest_score_item = max(flat_res, key=lambda x: x['score'])
+#     highest_score_label = highest_score_item['label']
+#     highest_score_value = highest_score_item['score']    
+#     return highest_score_label, highest_score_value
+
+
+# def get_sentiment(text):
+#     #sentiment analysis
+#     model_name = "cardiffnlp/twitter-roberta-base-sentiment-latest"
+#     tokenizer = AutoTokenizer.from_pretrained(model_name)
+#     model = pipeline("text-classification", model=model_name, top_k= None)
+
+#     #sentence tokenize the text using pysbd
+#     seg = pysbd.Segmenter(language="en", clean=False)
+#     text_list = seg.segment(text)
+
+#     res = []
+#     for t in text_list:
+#         results = model(t)
+#         highest_label, highest_score = extract_highest_score_label(results)
+#         result = {'sentence': t,'sentiment': highest_label, 'score': highest_score}
+#         res.append(result)
+#     return res
+
+
+# #-----------------Toxicity Prediction-----------------
+
+# def get_offensive_label(text):
+#     #offensive language detection model
+#     model_name = "cardiffnlp/twitter-roberta-base-offensive"
+#     tokenizer = AutoTokenizer.from_pretrained(model_name)
+#     model = pipeline("text-classification", model=model_name, top_k= None)
+
+#     #sentence tokenize the text using pysbd
+#     seg = pysbd.Segmenter(language="en", clean=False)
+#     text_list = seg.segment(text)
+
+#     res = []
+#     for t in text_list:
+#         results = model(t)
+#         highest_label, highest_score = extract_highest_score_label(results)
+#         result = {'sentence': t,'offensive_label': highest_label, 'score': highest_score}
+#         res.append(result)
+#     return res
+
+
+# #create the anchor function
+# def predict_text(text, model_name):
+#     if model_name == 'outcome':
+#         return outcome(text)
+#     elif model_name == 'stance':
+#         return get_stance(text)
+#     elif model_name == 'policy':
+#         return get_policy(text)
+#     elif model_name == 'sentiment':
+#         return get_sentiment(text)
+#     elif model_name == 'offensive':
+#         return get_offensive_label(text)
+#     else:
+#         return "Invalid model name"
+
+
 import pandas as pd
 from transformers import pipeline, AutoTokenizer
 import pysbd
+import torch
+
+
+label_mapping_wikipedia_en = {
+    'delete': [0, 'LABEL_0'],
+    'keep': [1, 'LABEL_1'],
+    'merge': [2, 'LABEL_2'],
+    'no consensus': [3, 'LABEL_3'],
+    'speedy keep': [4, 'LABEL_4'],
+    'speedy delete': [5, 'LABEL_5'],
+    'redirect': [6, 'LABEL_6'],
+    'withdrawn': [7, 'LABEL_7']
+}
+
+label_mapping_es = {
+    'Borrar': [0, 'LABEL_0'],
+    'Mantener': [1, 'LABEL_1'],
+    'Fusionar': [2, 'LABEL_2'],
+    'Otros': [3, 'LABEL_3']
+}
+
+label_mapping_gr = {
+    'Διαγραφή': [0, 'LABEL_0'],
+    'Δεν υπάρχει συναίνεση': [1, 'LABEL_1'],
+    'Διατήρηση': [2, 'LABEL_2'],
+    'συγχώνευση': [3, 'LABEL_3']
+}
+
+label_mapping_wikidata_ent = {
+    'delete': [0, 'LABEL_0'],
+    'no_consensus': [1, 'LABEL_1'],
+    'merge': [2, 'LABEL_2'],
+    'keep': [3, 'LABEL_3'],
+    'comment': [4, 'LABEL_4'],
+    'redirect': [5, 'LABEL_5']
+}
+
+label_mapping_wikidata_prop = {
+    'deleted': [0, 'LABEL_0'],
+    'keep': [1, 'LABEL_1'],
+    'no_consensus': [2, 'LABEL_2']
+}
+
+label_mapping_wikinews = {
+    'delete': [0, 'LABEL_0'],
+    'no_consensus': [1, 'LABEL_1'],
+    'speedy delete': [2, 'LABEL_2'],
+    'keep': [3, 'LABEL_3'],
+    'redirect': [4, 'LABEL_4'],
+    'comment': [5, 'LABEL_5'],
+    'merge': [6, 'LABEL_6'],
+    'withdrawn': [7, 'LABEL_7']
+}
+
+label_mapping_wikiquote = {
+    'merge': [0, 'LABEL_0'],
+    'keep': [1, 'LABEL_1'],
+    'no_consensus': [2, 'LABEL_2'],
+    'redirect': [3, 'LABEL_3'],
+    'delete': [4, 'LABEL_4']
+}
+
+best_models_tasks = {
+    'wikipedia': 'research-dump/roberta-large_deletion_multiclass_complete_final_v2',
+    'wikidata_entity': 'research-dump/roberta-large_wikidata_ent_outcome_prediction_v1',
+    'wikidata_property': 'research-dump/roberta-large_wikidata_prop_outcome_prediction_v1',
+    'wikinews': 'research-dump/all-roberta-large-v1_wikinews_outcome_prediction_v1',
+    'wikiquote': 'research-dump/roberta-large_wikiquote_outcome_prediction_v1'
+}
+
+best_models_langs = {
+    'en': 'research-dump/roberta-large_deletion_multiclass_complete_final_v2',
+    'es': 'research-dump/xlm-roberta-large_deletion_multiclass_es',
+    'gr': 'research-dump/xlm-roberta-large_deletion_multiclass_gr'
+}
 
 #-----------------Outcome Prediction-----------------
-def outcome(text):
-    label_mapping = {
-        'delete': [0, 'LABEL_0'],
-        'keep': [1, 'LABEL_1'],
-        'merge': [2, 'LABEL_2'],
-        'no consensus': [3, 'LABEL_3'],
-        'speedy keep': [4, 'LABEL_4'],
-        'speedy delete': [5, 'LABEL_5'],
-        'redirect': [6, 'LABEL_6'],
-        'withdrawn': [7, 'LABEL_7']
-    }
-    model_name = "research-dump/roberta-large_deletion_multiclass_complete_final"
+
+def outcome(text, lang='en', platform='wikipedia', date='', years=None):
+    if lang == 'en':
+        if platform not in best_models_tasks:
+            raise ValueError(f"For lang='en', platform must be one of {list(best_models_tasks.keys())}")
+        model_name = best_models_tasks[platform]
+        if platform == 'wikipedia':
+            label_mapping = label_mapping_wikipedia_en
+        elif platform == 'wikidata_entity':
+            label_mapping = label_mapping_wikidata_ent
+        elif platform == 'wikidata_property':
+            label_mapping = label_mapping_wikidata_prop
+        elif platform == 'wikinews':
+            label_mapping = label_mapping_wikinews
+        elif platform == 'wikiquote':
+            label_mapping = label_mapping_wikiquote
+    elif lang in ['es', 'gr']:
+        if platform != 'wikipedia':
+            raise ValueError(f"For lang='{lang}', only platform='wikipedia' is supported.")
+        model_name = best_models_langs[lang]
+        label_mapping = label_mapping_es if lang == 'es' else label_mapping_gr
+    else:
+        raise ValueError("Invalid lang. Use 'en', 'es', or 'gr'.")
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = pipeline("text-classification", model=model_name, return_all_scores=True)
-    
-    # Tokenize and truncate the text
+    model = pipeline("text-classification", model=model_name, return_all_scores=True, device=device)
+
     tokens = tokenizer(text, truncation=True, max_length=512)
     truncated_text = tokenizer.decode(tokens['input_ids'], skip_special_tokens=True)
     
@@ -31,10 +296,8 @@ def outcome(text):
             if result['label'] == value[1]:
                 res_list.append({'sentence': truncated_text, 'outcome': key, 'score': result['score']})
                 break
-    
     return res_list
 
-#-----------------Stance Prediction-----------------
 
 def extract_response(text, model_name, label_mapping):
     tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -54,7 +317,7 @@ def extract_response(text, model_name, label_mapping):
     
     return final_scores
 
-
+#-----------------Stance Detection-----------------
 def get_stance(text):
     label_mapping = {
             'delete': 0,
@@ -160,10 +423,9 @@ def get_offensive_label(text):
     return res
 
 
-#create the anchor function
-def predict_text(text, model_name):
+def predict_text(text, model_name, lang='en', platform='wikipedia', date='', years=None):
     if model_name == 'outcome':
-        return outcome(text)
+        return outcome(text, lang=lang, platform=platform, date=date, years=years)
     elif model_name == 'stance':
         return get_stance(text)
     elif model_name == 'policy':
@@ -173,4 +435,4 @@ def predict_text(text, model_name):
     elif model_name == 'offensive':
         return get_offensive_label(text)
     else:
-        return "Invalid Task name"
+        return "Invalid model name"