(self)
update(self) - Fill Queue with new Pastebin IDs
update(self) - Fill Queue with new Pastebin IDs
update ( self ) - Fill Queue with new Pastebin IDs
def update(self):
        '''update(self) - Fill Queue with new Pastebin IDs'''
        logging.info('Retrieving Pastebin ID\'s')
        new_pastes = []
        raw = None
        while not raw:
            try:
                raw = urllib.request.urlopen("https://scrape.pastebin.com/api_scraping.php?limit=" + str(SCRAPE_LIMIT))
            except:
                logging.info('Error with pastebin')
                raw = None
                sleep(5)
        # import API result as JSON
        decoded = raw.read().decode('utf-8')
        raw_json = json.loads(decoded)
        #parse json to get keys
        results = []
        #populate results list with paste_ids
        for paste_bulk in raw_json:
            results.append(paste_bulk['key'])
        if not self.ref_id:
            #up to 100 new pastes
            results = results[:100]
        for entry in results:
            paste = PastebinPaste(entry)
            # Check to see if we found our last checked paste_id
            if paste.id == self.ref_id:
                #if paste_id matches last checked id, no more new stuff
                break
            new_pastes.append(paste)
        for entry in new_pastes[::-1]:
            logging.info('Adding URL: ' + entry.url)
            self.put(entry)
def update ( self ) : logging . info ( 'Retrieving Pastebin ID\'s' ) new_pastes = [ ] raw = None while not raw : try : raw = urllib . request . urlopen ( "https://scrape.pastebin.com/api_scraping.php?limit=" + str ( SCRAPE_LIMIT ) ) except : logging . info ( 'Error with pastebin' ) raw = None sleep ( 5 ) # import API result as JSON decoded = raw . read ( ) . decode ( 'utf-8' ) raw_json = json . loads ( decoded ) #parse json to get keys results = [ ] #populate results list with paste_ids for paste_bulk in raw_json : results . append ( paste_bulk [ 'key' ] ) if not self . ref_id : #up to 100 new pastes results = results [ : 100 ] for entry in results : paste = PastebinPaste ( entry ) # Check to see if we found our last checked paste_id if paste . id == self . ref_id : #if paste_id matches last checked id, no more new stuff break new_pastes . append ( paste ) for entry in new_pastes [ : : - 1 ] : logging . info ( 'Adding URL: ' + entry . url ) self . put ( entry )