import sys
from utils import *
import subprocess
from tqdm import tqdm

def main():
    info_fp = './info.json'
    aug_out_fp = './info_aug.json'

    augment_info(info_fp, aug_out_fp)

def augment_info(info_fp, aug_out_fp, hparam=None):
    data = read_json(info_fp)
    lady_cnt = 0
    dur_util = DurUtil(hparam)
    pbar = tqdm(data)
    for id in pbar:
        pbar.set_description('ID: {}'.format(id))
        entry = data[id]
        
        t = entry['description'].strip().lower().replace('\n', '')

        if 'female' in t or 'lady' in t or 'ladies' in t or 'fenale' in t:
            data[id]['sexist'] = True
            lady_cnt += 1
        else:
            data[id]['sexist'] = False

        if 'master room' in t:
            data[id]['is_master'] = True
        else:
            data[id]['is_master'] = False

        if 'no aircon' in t:
            data[id]['no_aircon'] = True
        else:
            data[id]['no_aircon'] = False

        address = entry['location']
        
        res = dur_util.compute_dur(address, hparam['place_work_1'])
        time1 = res['routes'][0]['localizedValues']['duration']['text'] # Transit time for work

        res = dur_util.compute_dur(address, hparam['place_work_2'])
        time2 = res['routes'][0]['localizedValues']['duration']['text'] # Transit time for XX
            
        data[id]['time_to_work'] = time1
        data[id]['time_to_xx'] = time2

        
    print('{}/{} for lady only'.format(lady_cnt, len(data)))
    dur_util.save_alias()
    save_json(data, aug_out_fp)

class DurUtil:
    def __init__(self, hparam):
        with open(jpath(os.path.dirname(__file__),'google_map_dur_.sh')) as f:
            cmd = f.read()
        self.cmd = cmd.replace('[MAP_KEY]', hparam['google_map_key'])

        self.alias = {}
        self.alias_fp = hparam['data_util_fp']
        if os.path.exists(self.alias_fp):
            self.alias = read_json(self.alias_fp)

    def save_alias(self):
        save_json(self.alias, self.alias_fp)

    def compute_dur(self, loc1, loc2):
        
        dur_gotten = False
        while dur_gotten == False:
            try:
                cmd = self.cmd.replace('[ORIG]', loc1).replace('[DEST]', loc2)
                completed_process = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=True)
                res = completed_process.stdout
                assert 'routes' in res
                dur_gotten = True
            except:
                # print(loc1, loc2)
                if loc1 in self.alias:
                    loc1 = self.alias[loc1]
                else:
                    print('What is the postcode of \n{} ? \nAnswer in format XXXXXX'.format(loc1))
                    t = input('> ')
                    t = '{}, Singapore'.format(t)
                    self.alias[loc1] = t
                    loc1 = t
        res = json.loads(res)
        return res
        

def parse_time(time_str):
    t = time_str.strip().split(' ')
    if len(t) <=2:
        h = 0
        m = t[0]
    else:
        h = t[0]
        m = t[2]
    return int(h), int(m)

if __name__ == '__main__':
    main()

selenium_guru/augment_property.py
Sonata165/SimpleGuru
import os
import sys
import time
from tqdm import tqdm
from selenium import webdriver
from selenium.webdriver.common.by import By
# from LyricSpider.src.utils import read_json, save_json
import random

# Set up the driver
options = webdriver.SafariOptions()
options.add_argument('--disable-blink-features=AutomationControlled')  # Disabling the flag that identifies automation control
options.add_argument("--start-maximized")  # Start browser maximized

jpath = os.path.join

def _main():
    url = 'https://www.propertyguru.com.sg/property-for-rent?market=residential&beds[]=-1&listing_type=rent&minprice=800&maxprice=1200&MRT_STATIONS[]=EW24&MRT_STATIONS[]=JE5&MRT_STATIONS[]=NS1&MRT_STATIONS[]=EW23&MRT_STATIONS[]=EW22&MRT_STATIONS[]=CC22&MRT_STATIONS[]=EW21&MRT_STATIONS[]=EW20&MRT_STATIONS[]=EW19&MRT_STATIONS[]=EW18&MRT_STATIONS[]=EW17&MRT_STATIONS[]=EW16&MRT_STATIONS[]=NE3&MRT_STATIONS[]=TE17&MRT_STATIONS[]=EW15&MRT_STATIONS[]=CC21&MRT_STATIONS[]=CC20&MRT_STATIONS[]=CC19&MRT_STATIONS[]=DT9&MRT_STATIONS[]=CC23&MRT_STATIONS[]=CC24&MRT_STATIONS[]=CC25&freetext=EW24/NS1/JE5+Jurong+East+MRT,+EW23+Clementi+MRT,+EW22+Dover+MRT,+CC22/EW21+Buona+Vista+MRT,+EW20+Commonwealth+MRT,+EW19+Queenstown+MRT,+EW18+Redhill+MRT,+EW17+Tiong+Bahru+MRT,+EW16/NE3/TE17+Outram+Park+MRT,+EW15+Tanjong+Pagar+MRT,+CC21+Holland+Village+MRT,+CC20+Farrer+Road+MRT,+CC19/DT9+Botanic+Gardens+MRT,+CC23+One-North+MRT,+CC24+Kent+Ridge+MRT,+CC25+Haw+Par+Villa+MRT&search=true'
    out_dir = './download'
    crawl_index(url, out_dir)

def crawl_index(url, out_dir):
    '''
    Craw index according to url, save to indiate path
    '''
    page_cnt = 0

    # Prepare path
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    base_link = url
    link = base_link
    while link != None:
        page_cnt += 1
        print('Trying to crawl index page {} ...'.format(page_cnt))
        driver = webdriver.Safari()
        driver.get(link)
        
        sleep_s = random.uniform(1,2)
        time.sleep(sleep_s)
        driver.execute_script("window.scrollTo(0, 1000)")
        with open(jpath(out_dir, '{}.html'.format(page_cnt)), 'w') as f:
            f.write(driver.page_source)

        try:
            next_page_button =  driver.find_element(By.CSS_SELECTOR, '#search-results-container > div.container.search-result-inner-container > div.contents > div.columned-content > div.columned-content-row.main-content-wrapper > section > div.listing-pagination > ul > li.pagination-next > a')
            link = next_page_button.get_property('href')
        except:
            link = None
        driver.quit()
    print('Index crawling finished.')

if __name__ == '__main__':
    _main()

selenium_guru/craw_index.py
Sonata165/SimpleGuru
import os
import sys
import time
from tqdm import tqdm
from selenium import webdriver
from selenium.webdriver.common.by import By
# from LyricSpider.src.utils import read_json, save_json
import random
from tqdm import tqdm

from utils import *

# Set up the driver
options = webdriver.SafariOptions()
options.add_argument('--disable-blink-features=AutomationControlled')  # Disabling the flag that identifies automation control
options.add_argument("--start-maximized")  # Start browser maximized

def _main():
    index_fp = './download/index.json'
    out_dir = './download/info'

    craw_property_info(index_fp, out_dir)

def craw_property_info(index_fp, out_dir):
    data = read_json(index_fp)
    data_l = list(data.items())
    data_l.sort()
    data = dict(data_l)

    # Prepare dir
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    pbar = tqdm(data)
    for id in pbar:
        entry = data[id]
        title, url = entry
        id = url.strip().split('-')[-1]
        pbar.set_description('ID: {}'.format(id))
        out_fp = jpath(out_dir, '{}.html'.format(id))

        if not os.path.exists(out_fp):
            driver = webdriver.Safari()
            driver.get(url)
            time.sleep(1)
            with open(out_fp, 'w') as f:
                f.write(driver.page_source)
            driver.quit()
        else:
            pass
        
    print('Info crawling finished.')


if __name__ == '__main__':
    _main()

selenium_guru/craw_info.py
Sonata165/SimpleGuru
# from html.parser import HTMLParser
import scrapy
from scrapy.selector import Selector
from utils import print_json, read_json, save_json, ls, jpath

def main():
    data_dir = './download/index/'
    out_path = './download/index.json'

    parse_index(data_dir, out_path)

def parse_index(data_dir, out_path):
    fns = ls(data_dir)
    fns.sort()

    ret = []
    ret = {}

    for fn in fns:
        fpath = jpath(data_dir, fn)
        data_path = fpath
        with open(data_path) as f:
            html = f.read()
        s = 'div.col-xs-12.col-sm-7.listing-description'
        s = s + ' h3 > a'
        s_href = s + '::attr(href)'
        s_title = s + '::attr(title)'
        res_href = Selector(text=html).css(s_href).getall()
        res_title = Selector(text=html).css(s_title).getall()
        res = list(zip(res_title, res_href))

        for title, url in res:
            id = url.strip().split('-')[-1]
            ret[id] = [title, url]

    print('Total No. Properties:', len(ret))
    save_json(ret, out_path)

if __name__ == '__main__':
    main()

selenium_guru/index_parser.py
Sonata165/SimpleGuru
# from html.parser import HTMLParser
import scrapy
from scrapy.selector import Selector
from utils import print_json, read_json, save_json, ls, jpath
from tqdm import tqdm

def main():
    data_dir = './download/info'
    out_path = './info.json'
    index_path = None

    parse_info(data_dir, out_path, index_path)

def parse_info(data_dir, out_path, index_path):
    index_data = read_json(index_path)

    fns = ls(data_dir)
    fns.sort()

    res = {}

    for fn in tqdm(fns):
        # Skip inactive properties
        id = fn.strip().split('.')[0]
        if id not in index_data:
            print(fn, 'not active')
            continue

        fpath = jpath(data_dir, fn)
        data_path = fpath
        with open(data_path) as f:
            html = f.read()
        
        # url
        s = 'meta[property="og:url"]::attr(content)'
        url = Selector(text=html).css(s).getall()[0]

        # title
        s = 'div.col-lg-8.col-md-12 div.property-overview-section.row h1::text'
        title = Selector(text=html).css(s).getall()[0]
        
        # location
        s = 'div.col-lg-8.col-md-12 > div > div > div.property-overview-section.row > div > div.location-info > div.full-address > p::text'
        location = Selector(text=html).css(s).getall()[0]

        # price
        s = 'div.col-lg-8.col-md-12 > div > div > div.property-overview-section.row > div > div.property-info > div.price-summary > div > h2::text'
        price = Selector(text=html).css(s).getall()[-1].strip().split(' ')[1].replace(',','')

        # abstract
        s = 'div:nth-child(5) > div > div.col-lg-8.col-md-12 > div > div > section.about-section > div > h3::text'
        abstract = Selector(text=html).css(s).getall()[0]

        # description
        s = 'meta[property="og:description"]::attr(content)'
        description = abstract + '\n' + Selector(text=html).css(s).getall()[0]

        # size
        s = 'div.col-lg-8.col-md-12 > div > div > section.details-section > section > div.meta-table-root > table > tbody > tr:nth-child(1) > td:nth-child(2) > div > div > div.meta-table__item__value.col-md-12.col-7 > div::text'
        size = Selector(text=html).css(s).getall()[0].strip().split(' ')[0]

        # furnish
        s = 'div.col-lg-8.col-md-12 > div > div > section.details-section > section > div.meta-table-root > table > tbody > tr:nth-child(2) > td:nth-child(1) > div > div > div.meta-table__item__value.col-md-12.col-7 > div::text'
        furnish = Selector(text=html).css(s).getall()[0]

        # top
        s = 'div.col-lg-8.col-md-12 > div > div > section.details-section > section > div.meta-table-root > table > tbody > tr:nth-child(3) > td:nth-child(2) > div > div > div.meta-table__item__value.col-md-12.col-7 > div::text'
        top = Selector(text=html).css(s).getall()[0]

        # psf
        s = 'div.col-lg-8.col-md-12 > div > div > section.details-section > section > div.meta-table-root > table > tbody > tr:nth-child(2) > td:nth-child(2) > div > div > div.meta-table__item__value.col-md-12.col-7 > div::text'
        psf = Selector(text=html).css(s).getall()[0].strip().split(' ')[1]

        # availability
        try:
            s = 'div.col-lg-8.col-md-12 > div > div > section.details-section > section > div.meta-table-root > table > tbody > tr:nth-child(5) > td:nth-child(2) > div > div > div.meta-table__item__value.col-md-12.col-7 > div::text'
            available = Selector(text=html).css(s).getall()[0]
        except:
            available = None

        # id
        id = url.split('-')[-1]

        # is_active

        res[id] = {
            'title': title,
            'url': url,
            'location': location,
            'price': price,
            'size': size,
            'availability': available,
            'description': description,
            'top_year': top,
            'furnish': furnish,
            'psf': psf,
        }

    save_json(res, out_path) 

if __name__ == '__main__':
    main()

selenium_guru/info_parser.py
Sonata165/SimpleGuru
from utils import *
import pandas as pd


def main():
    info_fp = './info_aug.json'
    out_fp = 'data.csv'

    rank(info_fp, out_fp)

def rank(info_fp, out_fp, hparam=None):
    data = read_json(info_fp)

    # Data type convert
    for id in data:
        entry = data[id]
        data[id]['price'] = int(entry['price'])
        data[id]['size'] = min(max(int(entry['size']), hparam['size']['worst']), hparam['size']['best'])
        try:
            year = int(entry['top_year'].strip().split(' ')[-1])
            data[id]['top_year'] = min(max(year, hparam['year']['worst']), hparam['year']['best'])
        except:
            data[id]['top_year'] = 2000
        try:
            data[id]['psf'] = float(entry['psf'])
        except:
            data[id]['psf'] = 5
        
        data[id]['time_to_work'] = norm_time_to_min(entry['time_to_work'])
        data[id]['time_to_xx'] = norm_time_to_min(entry['time_to_xx'])

    # Normalize by range: 
    df = pd.DataFrame.from_dict(data, orient='index')
    for k in ['size', 'top_year']:
        k_n = k + '_n'
        df[k_n] = normalize_column(df[k])
    data_n = df.to_dict(orient='index')

    # Compute score for each entry
    for id in data:
        entry = data[id]
        entry_n = data_n[id]
        score = compute_score(entry_n, hparam)
        data[id]['score'] = score

    # Sort by score
    data = dict(sorted(data.items(), key=lambda item: item[1]['score'], reverse=True))

    # Put score to front
    data = {k: {'score': v['score'], **{kk: vv for kk, vv in v.items() if kk != 'score'}} for k, v in data.items()}
    
    # Convert to CSV
    df = pd.DataFrame.from_dict(data, orient='index')
    df.to_csv(out_fp, index=False)  

def norm_time_to_min(time_str):
    work_time = time_str.split(' ')
    if len(work_time) <= 2:
        work_h = 0
        work_m = int(work_time[0])
    else:
        work_h = int(work_time[0])
        work_m = int(work_time[2])
    work_min = work_h * 60 + work_m
    return work_min

def compute_score(entry, hparam):
    '''
    Compute score, considering price, size, top_year, sexist, is_master, time_to_work, time_to_XX
    psf not included
    '''
    if hparam['i_am_girl']:
        s_sexist = 1
    else:
        s_sexist = 0 if entry['sexist'] == True else 1
    if not hparam['require_aircon']:
        s_aircon = 1
    else:
        s_aircon = 0 if entry['no_aircon'] else 1

    s_price = number_to_score(entry['price'], hparam['price']['best'], hparam['price']['worst'], False)
    s_size = 1 - entry['size_n']
    s_year = entry['top_year_n']
    s_master = 1 if entry['is_master'] else 0
    s_time_work = number_to_score(entry['time_to_work'], hparam['work_1_time']['best'], hparam['work_1_time']['worst'], False)
    s_time_xx = number_to_score(entry['time_to_xx'], hparam['work_2_time']['best'], hparam['work_2_time']['worst'], False)
    

    weights = hparam['weights']
    tot = sum([weights[i] for i in weights])
    score = (s_price*weights['price'] + s_size*weights['size'] + s_year*weights['year'] 
            + s_master*weights['master'] + s_time_work*weights['time_work'] + s_time_xx*weights['time_xx']) / tot
    return score * s_sexist * s_aircon

def normalize_column(column):
    min_value = column.min()
    max_value = min(column.max(), 2020)
    normalized_column = (column - min_value) / (max_value - min_value)
    return normalized_column

def number_to_score(n, best, worst, higher_better):
    if higher_better == True:
        assert best > worst
        score = max((n - worst), 0) / (best - worst)
    else:
        assert worst > best
        score = 1 - min(max((n - best), 0) / (worst - best), 1)
    return score

if __name__ == '__main__':
    main()
selenium_guru/rank.py
Sonata165/SimpleGuru
import os
import json
ls = os.listdir
jpath = os.path.join

def read_json(path):
    with open(path, 'r', encoding='utf8') as f:
        data = f.read()
        data = json.loads(data)
    return data

def save_json(data, path, sort=False):
    with open(path, 'w', encoding='utf8') as f:
        f.write(json.dumps(data, indent=4, sort_keys=sort, ensure_ascii=False).replace('\r', ''))

def print_json(data):
    print(json.dumps(data,indent=4,ensure_ascii=False))
selenium_guru/utils.py
Sonata165/SimpleGuru
import os
import time
import platform
import mlconfig

from selenium_guru.craw_index import crawl_index
from selenium_guru.index_parser import parse_index
from selenium_guru.craw_info import craw_property_info
from selenium_guru.info_parser import parse_info
from selenium_guru.augment_property import augment_info
from selenium_guru.rank import rank

jpath = os.path.join

def main(hparam):

    # Crawl index
    if hparam['do_crawl_index']:
        print('Start crawling ...')
        crawl_index(hparam['index_url'], out_dir=hparam['out_index_dir'])
    else:
        print('Index crawling skipped.')

    # Parse index
    if hparam['do_parse_index']:
        print('Parse index ...')
        parse_index(data_dir=hparam['out_index_dir'], out_path=hparam['out_index_fp'])
    else:
        print('Index parsing skipped.')

    # Crawl property info
    if hparam['do_crawl_properties']:
        print('Crawl property info ...')
        craw_property_info(index_fp=hparam['out_index_fp'], out_dir=hparam['out_info_dir'])
    else:
        print('Info crawling skipped')

    # Parse property info
    if hparam['do_parse_properties']:
        print('Parse property info ...')
        parse_info(hparam['out_info_dir'], hparam['out_info_fp'], hparam['out_index_fp'])
    else:
        print('Info parsing skipped')

    # Augment property info
    if hparam['do_augment_info']:
        print('Augment property info ...')
        augment_info(hparam['out_info_fp'], hparam['aug_out_fp'], hparam)
    else:
        print('Info augmentation skipped')

    # Rank properties
    if hparam['do_rank']:
        print('Start ranking ...')
        rank(info_fp=hparam['aug_out_fp'], out_fp=hparam['rank_out_fp'], hparam=hparam)
    else:
        print('Ranking skipped.')
    
    print('Finish. Good luck. ')


if __name__ == '__main__':
    machine = platform.platform().split('-')[0]  # macOS | Linux
    if machine == 'macOS':  # Local
        arg_path = './private/my_conf.yaml'
        hparam = mlconfig.load(arg_path)
    else:
        raise NotImplementedError()


    # Prepare dir
    print('Debug mode:', hparam['debug'])
    if hparam['debug']:
        hparam['output_dir'] = hparam['output_dir']+'_debug'
    hparam['log_fn'] = jpath(hparam['output_dir'], 'log.txt')
    if not os.path.exists(hparam['output_dir']):
        os.makedirs(hparam['output_dir'])

    main(hparam)



simpleguru.py
Sonata165/SimpleGuru
import os
import json
ls = os.listdir
jpath = os.path.join

def read_json(path):
    with open(path, 'r', encoding='utf8') as f:
        data = f.read()
        data = json.loads(data)
    return data

def save_json(data, path, sort=False):
    with open(path, 'w', encoding='utf8') as f:
        f.write(json.dumps(data, indent=4, sort_keys=sort, ensure_ascii=False).replace('\r', ''))

def print_json(data):
    print(json.dumps(data,indent=4,ensure_ascii=False))
utils.py
Sonata165/SimpleGuru