code: def filter_duplicate(self, url):
          
          if self.filterDuplicate:
              if url in self.historys: 
                  raise Exception('duplicate excepiton: %s is duplicate' % url)
              else:
                  self.historys.add(url)
          else:
              pass
docs: url去重