Some useful script for automatic survey!
# THIS FILE IS PART OF ScholarHelper (https://github.com/sdycodes/ScholarHelper)
# chase_arxiv.py - used for fetching papers from arxiv.org
# THIS PROGRAM IS LICENSED GPL-v3.0
# Copyright (c) 2023-2033 Dingyuan Shi (sdycodes@Github)


import urllib.request
import pickle
from lxml import etree
from collections import defaultdict
import datetime

        
            
subareas = ["AI", "LG", "RO"]
key_words = ["plan", "decision", 
            ("generat", "plan"), ("decision", "transformer"), ("plan", "diffusion")]
HEADER = "Your own header"

def get_html(url: str):
    headers = HEADER
    request = urllib.request.Request(url=url, headers=headers)
    res = urllib.request.urlopen(request, timeout=60)
    html = res.read().decode('utf-8')
    return html

def get_day(past_offset: int):
    today = datetime.date.today()
    return today + datetime.timedelta(days=-past_offset)

def get_paper_list(subareas, key_words, is_dump=False):
    paper_list = defaultdict(list)
    for area in subareas:
        url = f"https://arxiv.org/list/cs.{area}/pastweek?show=1000"
        print(url)
        try:
            html = get_html(url)
        except Exception as e:
            print(e)
            print(area)
        if is_dump:
            pickle.dump(html, open(f"./rec_{area}.html", "wb"))
        # html = pickle.load(open(f"./rec_{area}.html", "rb"))
        html = etree.HTML(html.encode())
        for day in range(1, 6):
            date = get_day(day - 1)
            print(date)
            k = 0
            while True:
                k += 1
                title_x_path = f"//*[@id=\"dlpage\"]/dl[{day}]/dd[{k}]/div/div[1]/text()"
                link_x_path = f"//*[@id=\"dlpage\"]/dl[{day}]/dt[{k}]/span/a[1]/text()"
                
                title = html.xpath(title_x_path)
                if len(title) == 0:
                    break
                title = title[1].strip().lower()
                
                link = html.xpath(link_x_path)[0].split(":")[1]
                link = f"https://arxiv.org/abs/{link}"
                
                # select
                for word in key_words:
                    if type(word) is str and word in title:
                        paper_list[date].append((title, link))
                        break
                    elif type(word) is tuple and all([w in title for w in word]):
                        paper_list[date].append((title, link))
                        break
            paper_list[date] = list(set(paper_list[date]))
    return paper_list




paper_list = get_paper_list(subareas, key_words, is_dump=False)
# out put
lines = []
for date in paper_list:
    lines.append(f"## {date}\n")
    for paper in paper_list[date]:
        lines.append(f"- [{paper[0]}]({paper[1]})\n")        
with open("README.md", "w") as f:
    f.writelines(lines)
chase_arxiv.py
sdycodes/ScholarHelper
# THIS FILE IS PART OF ScholarHelper (https://github.com/sdycodes/ScholarHelper)
# fetch_from_dblp.py - used for fetching paper titles from dblp
# THIS PROGRAM IS LICENSED GPL-v3.0
# Copyright (c) 2023-2033 Dingyuan Shi (sdycodes@Github)

from utils import get_header
import urllib.request
from lxml import etree


TARGET_LINK = "https://dblp.uni-trier.de/db/conf/icml/icml2015.html"
HEADER = get_header()

def get_html(url: str):
    headers = HEADER
    request = urllib.request.Request(url=url, headers=headers)
    res = urllib.request.urlopen(request, timeout=100)
    html = res.read().decode('utf-8')
    return html
    
html = get_html(TARGET_LINK)
html = etree.HTML(html.encode())
for each in html.xpath('//*[@class="title"]/text()'):
    print(each)
fetch_from_dblp.py
sdycodes/ScholarHelper
# THIS FILE IS PART OF ScholarHelper (https://github.com/sdycodes/ScholarHelper)
# find_citation.py - used for get citation of a paper
# THIS PROGRAM IS LICENSED GPL-v3.0
# Copyright (c) 2023-2033 Dingyuan Shi (sdycodes@Github)


import urllib.request
import re
import json
from tqdm import tqdm
from selenium import webdriver
import pickle
import os
from utils import get_header

LINK_PREFIX = "https://scholar.google.com/scholar?q="
# PATTERN = re.compile(r'Cited by \d+')
PATTERN = re.compile(r'被引用次数：\d+')
HEADER = get_header()
USE_DRIVER = True

if USE_DRIVER:
    driver = webdriver.Chrome()

def parse_json(file_path: str):
    titles = []
    with open(file_path, 'r') as json_file:
        all = json.load(json_file)
        contents = all["result"]["hits"]["hit"]
        for item in contents:
            title = item["info"]["title"]
            titles.append(title)
    return titles
    

def get_html(url: str):
    if USE_DRIVER:
        driver.get(url)
        html = driver.execute_script("return document.documentElement.outerHTML")
    else:
        headers = HEADER
        request = urllib.request.Request(url=url, headers=headers)
        res = urllib.request.urlopen(request, timeout=100)
        html = res.read().decode('utf-8')
    return html
    

def get_citation(titles: list[str]):
    if os.path.exists("cache.pkl"):
        name_citation_dict = pickle.load(open("cache.pkl", "rb"))
    else:
        name_citation_dict = dict()
    try:
        for k, title in tqdm(enumerate(titles), total=len(titles)):
            if title in name_citation_dict:
                continue
            name = title.replace(' ', '+')
            url = LINK_PREFIX + name
            try:
                html = get_html(url)
                m = PATTERN.search(html)
                if m:
                    to_be_parse = m.group(0)
                    k = len(to_be_parse) - 1
                    while k >= 0 and to_be_parse[k] in "0123456789":
                        k -= 1
                    citation = int(to_be_parse[k + 1:])
                else:
                    citation = 0
            except Exception as e:
                print(e)
                try:
                    print("some error happens!")
                    input()
                    html = get_html(url)
                    m = PATTERN.search(html)
                    if m:
                        to_be_parse = m.group(0)
                        k = len(to_be_parse) - 1
                        while k >= 0 and to_be_parse[k] in "0123456789":
                            k -= 1
                        citation = int(to_be_parse[k + 1:])
                    else:
                        citation = 0
                except Exception:
                    citation = -1
            name_citation_dict[title] = citation
    except KeyboardInterrupt:
        print(k)
        pickle.dump(name_citation_dict, open("cache.pkl", "wb"))
    pickle.dump(name_citation_dict, open("cache.pkl", "wb"))
    name_citation_pair = [(name, name_citation_dict[name]) for name in name_citation_dict]
    return name_citation_pair

    

titles_all = []
# for file in ["iclr13.json", "iclr14.json", "iclr15.json", "iclr16.json", "iclr17.json", "iclr18.json", "iclr19.json"]:
#     titles = parse_json("jsons/" + file)
#     titles_all.extend(titles)

for file in ["icml15.txt", "icml16.txt", "icml17.txt", "icml18.txt", "icml19.txt", "icml20.txt"]:
    with open("lists/" + file, "r") as f:
        lines = f.readlines()
        while lines:
            titles_all.extend([line.strip() for line in lines])
            lines = f.readlines()
for title in titles_all:
    print(title)
title_citations = get_citation(titles_all)
title_citations.sort(key=lambda x: -x[1])
print("+++++++++++++")
for title, citation in title_citations:
    print(title, citation)
find_citation.py
sdycodes/ScholarHelper

def get_header():
    with open("myheader.txt", "r") as f:
        line = f.readline()
        return eval(line.strip())
utils.py
sdycodes/ScholarHelper