(self, domain, domain_dict)
return similar
Compare a domain to a list of domains and return similar domains.

        This function takes a domain and a dict object that contains
        as key domain names and value the calculated MinHash value for that
        domain as well as the domains depth (mbl.is is 2, foobar.evil.com would
        be 3). It will then strip www. if needed from the domain, and compare
        the Jaccard distance between all domains in the dict and the supplied
        domain (removing the TLD extension from all domains).

        If the Jaccard distance between the supplied domain and one or more of
        the domains in the domain dict is higher than the configured threshold
        the domain is further tested to see if there are overlapping substrings
        between the two domains. If there is a common substring that is longer
        than half the domain name and the Jaccard distance is above the
        threshold the domain is considered to be similar.

        Args:
            domain: string with a full domain, eg. www.google.com
            domain_dict: dict with domain names (keys) and MinHash objects
                (values) for all domains to compare against.

        Returns:
            a list of tuples (score, similar_domain_name) with the names of
            the similar domains as well as the Jaccard distance between
            the supplied domain and the matching one.
Compare a domain to a list of domains and return similar domains.
Compare a domain to a list of domains and return similar domains .
def _get_similar_domains(self, domain, domain_dict):
        """Compare a domain to a list of domains and return similar domains.

        This function takes a domain and a dict object that contains
        as key domain names and value the calculated MinHash value for that
        domain as well as the domains depth (mbl.is is 2, foobar.evil.com would
        be 3). It will then strip www. if needed from the domain, and compare
        the Jaccard distance between all domains in the dict and the supplied
        domain (removing the TLD extension from all domains).

        If the Jaccard distance between the supplied domain and one or more of
        the domains in the domain dict is higher than the configured threshold
        the domain is further tested to see if there are overlapping substrings
        between the two domains. If there is a common substring that is longer
        than half the domain name and the Jaccard distance is above the
        threshold the domain is considered to be similar.

        Args:
            domain: string with a full domain, eg. www.google.com
            domain_dict: dict with domain names (keys) and MinHash objects
                (values) for all domains to compare against.

        Returns:
            a list of tuples (score, similar_domain_name) with the names of
            the similar domains as well as the Jaccard distance between
            the supplied domain and the matching one.
        """
        domain = utils.strip_www_from_domain(domain)

        similar = []
        if '.' not in domain:
            return similar

        if domain in domain_dict:
            return similar

        if any(domain.endswith('.{0:s}'.format(x)) for x in domain_dict):
            return similar

        # We want to get rid of the TLD extension of the domain.
        # This is only used in the substring match in case the Jaccard
        # distance is above the threshold.
        domain_items = domain.split('.')
        domain_depth = len(domain_items)
        domain_part = '.'.join(domain_items[:-1])

        minhashes = {}
        for index in range(0, domain_depth - 1):
            minhashes[domain_depth - index] = self._get_minhash_from_domain(
                '.'.join(domain_items[index:]))

        for watched_domain, watched_item in iter(domain_dict.items()):
            watched_hash = watched_item.get('hash')
            watched_depth = watched_item.get('depth')

            minhash = minhashes.get(watched_depth)
            if not minhash:
                # The supplied domains length does not match this watched
                # domain.
                continue
            score = watched_hash.jaccard(minhash)
            if score < self.domain_scoring_threshold:
                continue

            watched_domain_items = watched_domain.split('.')
            watched_domain_part = '.'.join(watched_domain_items[:-1])

            # Check if there are also any overlapping strings.
            sequence = difflib.SequenceMatcher(
                None, domain_part, watched_domain_part)
            match = sequence.find_longest_match(
                0, len(domain_part), 0, len(watched_domain_part))

            # We want to have at least half of the domain matching.
            # TODO: This can be improved, this is a value and part that
            # needs or can be tweaked. Perhaps move this to a config option
            # that is the min length of strings.
            match_size = min(
                int(len(domain_part)/2), int(len(watched_domain_part)/2))
            if match.size < match_size:
                continue
            similar.append((watched_domain, score))

        return similar
def _get_similar_domains ( self , domain , domain_dict ) : domain = utils . strip_www_from_domain ( domain ) similar = [ ] if '.' not in domain : return similar if domain in domain_dict : return similar if any ( domain . endswith ( '.{0:s}' . format ( x ) ) for x in domain_dict ) : return similar # We want to get rid of the TLD extension of the domain. # This is only used in the substring match in case the Jaccard # distance is above the threshold. domain_items = domain . split ( '.' ) domain_depth = len ( domain_items ) domain_part = '.' . join ( domain_items [ : - 1 ] ) minhashes = { } for index in range ( 0 , domain_depth - 1 ) : minhashes [ domain_depth - index ] = self . _get_minhash_from_domain ( '.' . join ( domain_items [ index : ] ) ) for watched_domain , watched_item in iter ( domain_dict . items ( ) ) : watched_hash = watched_item . get ( 'hash' ) watched_depth = watched_item . get ( 'depth' ) minhash = minhashes . get ( watched_depth ) if not minhash : # The supplied domains length does not match this watched # domain. continue score = watched_hash . jaccard ( minhash ) if score < self . domain_scoring_threshold : continue watched_domain_items = watched_domain . split ( '.' ) watched_domain_part = '.' . join ( watched_domain_items [ : - 1 ] ) # Check if there are also any overlapping strings. sequence = difflib . SequenceMatcher ( None , domain_part , watched_domain_part ) match = sequence . find_longest_match ( 0 , len ( domain_part ) , 0 , len ( watched_domain_part ) ) # We want to have at least half of the domain matching. # TODO: This can be improved, this is a value and part that # needs or can be tweaked. Perhaps move this to a config option # that is the min length of strings. match_size = min ( int ( len ( domain_part ) / 2 ) , int ( len ( watched_domain_part ) / 2 ) ) if match . size < match_size : continue similar . append ( ( watched_domain , score ) ) return similar