import re
import requests
from dataclasses import dataclass
from typing import List, Optional
import time
import os
import pickle
from datetime import date

@dataclass
class ArxivPaper:
    arxiv_id: str
    title: str
    author: List[str]
    subject: List[str]
    comment: List[str]
    abstract: Optional[str]



def arxiv_parser(txt):
    """
        parse arxiv format, e.g., https://export.arxiv.org/abs/2110.14416?fmt=txt
    """
    title, authors, abstract = re.findall('Title: (.*?)\nAuthors: (.*?)\nCategories.*\\\\\n(.*?)\n\\\\', txt, re.DOTALL)[0]
    title = title.replace('\n', ' ')
    authors = authors.replace('\n', ' ')
    abstract = abstract.replace('\n', ' ')
    return title, authors, abstract

def decode_date(arxiv_raw):
    delim = r'(?:.*?)'
    pat1 = r'#item(\d*?)">'
    pat2 = r'([A-Za-z]*, \d* [A-Za-z]* \d*)<'
    pat = delim.join([pat1,pat2])
    return re.findall(pat, arxiv_raw, re.DOTALL)

def decode_arxiv_webpage(arxiv_raw):
    """
        convert webpage (html) or recent publications into a list of arxiv entries.
    """
    pat_coarse = r'<dt><a name="item(\d*?)">\[(?:\d*?)\]</a>(.*?)</dd>'
    delim = r'(?:.*?)'
    #pat_arxiv_id = r'<a href="https://arxiv.org/abs/(\d\d\d\d\.\d\d\d\d\d)"'  ## if we manually download
    pat_arxiv_id = r'<a href="/abs/(\d\d\d\d\.\d\d\d\d\d)"' ## if we crawl from web
    pat_title = r'Title:</span>(.*?)</div>'
    pat_author_raw = r'<span class="descriptor">Authors:</span>(.*?)</div>'
    pat_subject_raw = r'<span class="descriptor">Subjects:</span>(.*?)</div>'
    pat_comment = r'<span class="descriptor">Comments:</span>(.*?)</div>'
    pat_comment_ = re.compile(pat_comment, re.DOTALL)
    pat_author = r'<a href="(?:.*?)">(.*?)</a>'
    pat_author_ = re.compile(pat_author, re.DOTALL)
    pat_subject = r'([\sA-Za-z]*?\([A-Za-z]*?\.[A-Za-z]*?\))'
    pat_subject_ = re.compile(pat_subject, re.DOTALL)
    pat = delim.join([pat_arxiv_id,pat_title,pat_author_raw,pat_subject_raw])
    pat_ = re.compile(pat, re.DOTALL)
    res = re.findall(pat_coarse, arxiv_raw, re.DOTALL)
    for indx, raw in res:
        ret = pat_.findall(raw)
        assert len(ret)==1, raw
        arxiv_id, title, author_raw, subject_raw = ret[0]
        author = pat_author_.findall(author_raw)
        comment = pat_comment_.findall(raw)
        subject = pat_subject_.findall(subject_raw)
        paper_obj = ArxivPaper(arxiv_id, title, author, subject, comment, None)
        yield (indx, paper_obj)

def get_abstract(arxiv_obj : ArxivPaper):
    ptext = requests.get(f'https://export.arxiv.org/abs/{arxiv_obj.arxiv_id}?fmt=txt').text
    title, authors, abstract = arxiv_parser(ptext)
    arxiv_obj.abstract= abstract
    return arxiv_obj

def get_today_paper(cate_list=['cs.LG', 'cs.CL', 'cs.AI'], is_save=False, verbose=True):
    if is_save:
        today=date.today()
        if os.path.exists(f'papers_{today}.pkl'):
            papers_dict = pickle.load(open(f'papers_{today}.pkl', 'rb'))
            return papers_dict

    paper_dict = {}
    for cate in cate_list:
        url = f"https://arxiv.org/list/{cate}/pastweek?skip=0&show=2000"
        print(f"fetching {url}")

        retry_count = 0
        while retry_count <= 5:
            try:
                arxiv_raw = requests.get(url).text
                time.sleep(20) # TODO
                break
            except:
                retry_count += 1

        papers = list(decode_arxiv_webpage(arxiv_raw))
        today_max = int(decode_date(arxiv_raw)[0][0]) - 1 ## decode_date returns a list of (start index, date) for previous dates.
        papers_today = papers[:today_max]
        for _, obj in papers_today:
            paper_dict[obj.arxiv_id] = obj

    for index, arxiv_obj in paper_dict.items():
        retry_count = 0
        while retry_count <= 5:
            try:
                ret = get_abstract(arxiv_obj)
                time.sleep(1)
                break
            except:
                retry_count += 1
        if verbose:
            print(index, ret)
    if is_save:
        today=date.today()
        with open(f'papers_{today}.pkl', 'wb') as f:
            pickle.dump(paper_dict, f)
    return paper_dict

def get_url(arxivObj):
    return 'https://www.arxiv.org/abs/'+arxivObj.arxiv_id

arxiv_util.py
Kaffaljidhmah2/Arxiv-Recommender
MY_AUTHOR_LIST=[ 
    'Zico Kolter'
]
filter_util.py
Kaffaljidhmah2/Arxiv-Recommender
from langchain.embeddings import OpenAIEmbeddings
from langchain.chat_models import ChatOpenAI
from langchain.schema import AIMessage, HumanMessage, SystemMessage  
import json
import time 
import os
import pickle
from datetime import date
from research_topic import RESEARCH_TOPIC_DICT

class OpenAIChat:
    model = ChatOpenAI(openai_api_key=open('.openai_api_key').read().strip()) # ' model_name = 'gpt-4'gpt-3.5-turbo-0613'

    @classmethod
    def chat(cls, request):
        response = cls.model([HumanMessage(content=request)]).content
        json_response = json.loads(response)
        return json_response

def get_summary_score(arxiv_obj, topic):
    meta_prompt = """
    You are a professional research assistant. Please read the title and the abstract of the paper I provide you below and summarize it into one sentence as concise as possible. Next, please judge if the paper is relevant to my current research topic.

    ** Score Scale
    Please use the following score scale:
    - 0 (most irrelevant, completely different directions)
    - 1 (somewhat relevant, there is only one keyword match)
    - 2 (medium, there are merely keyword matches, but the topic deviates a bit from the my research topic)
    - 3 (a must read, the paper is exactly working on the similar direction as my research topic). 

    ** My current research topic: 
    {topic}

    ** Arxiv paper title: 
    {title}

    ** Arxiv paper abstract: 
    {abstract}

    ** Response
    Please respond in the following format, and make sure it is parsable by JSON.
    {{
        "thoughts": "", # put your thinking step-by-step here.
        "summarization": "", # summarization of the contribution of the paper in one sentence.
        "score": "" # 0, 1, 2, 3.
    }}
    """

    result = OpenAIChat.chat(meta_prompt.format(topic=topic, title=arxiv_obj.title, abstract=arxiv_obj.abstract)) 
    return result

# Note: as the candidate papers are all about artificial intelligence, the papers may look relevant to my research topic at a first glance. However, you should try to discriminate if indeed there will be a significant overlapping with my research topic, and use high scores sparingly. 

def get_chatGPT_response(papers_dict, is_save=False, verbose=True):
    if is_save:
        today=date.today()
        if os.path.exists(f'response_{today}.pkl'): 
            response_dict = pickle.load(open(f'response_{today}.pkl', 'rb'))
            return response_dict


    response_dict={}
    for key, research_topic in RESEARCH_TOPIC_DICT.items():
        response_list = {}
        for index, arxiv_obj in papers_dict.items():
            try:
                chatGPT_response = get_summary_score(arxiv_obj, research_topic)
                if verbose:
                    print(chatGPT_response)
                response_list[index] = chatGPT_response
                time.sleep(1)
            except Exception as ex:
                print("Error occured", ex)
                response_list[index] = None
        assert len(papers_dict) == len(response_list)
        response_dict[key] = response_list

    if is_save:
        today=date.today()
        with open(f'response_{today}.pkl', 'wb') as f:
            pickle.dump(response_dict, f)
    return response_dict

llm.py
Kaffaljidhmah2/Arxiv-Recommender
from email.mime.text import MIMEText
import smtplib
from mail_config import *  

def SendEmail(t): 
	mailserver=smtplib.SMTP(smtpserver,587);
	mailserver.login(sender,passcode);
	mes=MIMEText(t,'plain','utf-8');
	mes['Subject']='Arxiv Paper';
	mes['From']=sender;
	mes['To']=receiver;
	mailserver.sendmail(sender,[receiver],mes.as_string());
	mailserver.quit();

if __name__ == "__main__":
	SendEmail("Test")
mail.py
Kaffaljidhmah2/Arxiv-Recommender
sender=''
passcode=''
smtpserver=''
receiver=''
mail_config.py
Kaffaljidhmah2/Arxiv-Recommender
from arxiv_util import get_today_paper, get_url
from llm import get_chatGPT_response
import time
import pickle
from research_topic import RESEARCH_TOPIC_DICT
from filter_util import MY_AUTHOR_LIST
from datetime import date
from collections import defaultdict
from mail import SendEmail

buffer=[]
def my_print(s):
    buffer.append(s)

def my_format(arxiv_obj, summary): 
    return '\t'.join([get_url(arxiv_obj), arxiv_obj.title, '(' + ( ', '.join(arxiv_obj.author)) + ')', '\n' + summary])

CATEGORY_LIST = ['cs.LG']
IS_SAVE = True
today=date.today()

papers_dict = get_today_paper(cate_list=CATEGORY_LIST, is_save=IS_SAVE)
my_print(f"we have {len(papers_dict)} papers today")
response_dict = get_chatGPT_response(papers_dict, is_save=IS_SAVE)

MY_AUTHOR_LIST_lower = [_.lower() for _ in MY_AUTHOR_LIST]
default_key=list(response_dict.keys())[0]
for idx, arxiv_obj in papers_dict.items():
    contains_author = False
    for aut in arxiv_obj.author:
        if aut.lower() in MY_AUTHOR_LIST_lower:
            contains_author= True
            break
    if contains_author:
        summary = response_dict[default_key][idx]['summarization']
        my_print(my_format(arxiv_obj, summary))
        my_print('')

my_print("==========================")
            


recommended_papers = defaultdict(list)
for key in RESEARCH_TOPIC_DICT.keys():
    for idx in papers_dict.keys():
        arxiv_obj = papers_dict[idx]
        response = response_dict[key][idx]
        if response is not None and  response['score'] in ['3', 3]:
            recommended_papers[key].append([arxiv_obj, response['summarization']])

for key in recommended_papers.keys():
    my_print(key)
    for p in recommended_papers[key]:
        my_print(my_format(*p))
        my_print('')
    my_print("----------------")
    

#SendEmail('\n'.join(buffer))
print('\n'.join(buffer))
main.py
Kaffaljidhmah2/Arxiv-Recommender
RESEARCH_TOPIC_DICT = {
    'vlm-attack': """The adversarial attack and defense of large language models: modern language models are RLHF-tuned so are aligned, but users can adversarially attack the model by injecting adversarially trained prompt inside to make models output unintentional content.""",
}
research_topic.py
Kaffaljidhmah2/Arxiv-Recommender