(self)
return (
            '{0:d} potentially phishy domains discovered.').format(
                similar_domain_counter)
Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result
Entry point for the analyzer.
Entry point for the analyzer .
def run(self):
        """Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result
        """
        query = (
            '{"query": { "bool": { "should": [ '
            '{ "exists" : { "field" : "url" }}, '
            '{ "exists" : { "field" : "domain" }} ] } } }')

        return_fields = ['domain', 'url', 'message', 'human_readable']

        events = self.event_stream(
            '', query_dsl=query, return_fields=return_fields)

        domains = {}
        domain_counter = collections.Counter()
        tld_counter = collections.Counter()

        for event in events:
            domain = event.source.get('domain')

            if not domain:
                continue

            domain_counter[domain] += 1
            domains.setdefault(domain, [])
            domains[domain].append(event)

            tld = utils.get_tld_from_domain(domain)
            tld_counter[tld] += 1

        if not domain_counter:
            return 'No domains discovered, so no phishy domains.'

        watched_domains_list = current_app.config.get(
            'DOMAIN_ANALYZER_WATCHED_DOMAINS', [])
        domain_threshold = current_app.config.get(
            'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD', 10)
        watched_domains_list.extend([
            utils.strip_www_from_domain(x)
            for x, _ in domain_counter.most_common(domain_threshold)])
        watched_domains_list.extend([
            x for x, _ in tld_counter.most_common(domain_threshold)])
        watched_domains_list.extend(self.WATCHED_DOMAINS_BASE_LIST)
        watched_domains_list_temp = set(watched_domains_list)
        watched_domains_list = []
        for domain in watched_domains_list_temp:
            if domain in self.domain_scoring_exclude_domains:
                continue
            if any(domain.endswith(x)
                   for x in self.domain_scoring_exclude_domains):
                continue

            if '.' not in domain:
                continue
            watched_domains_list.append(domain)

        watched_domains = {}
        for domain in watched_domains_list:
            minhash = self._get_minhash_from_domain(domain)
            watched_domains[domain] = {
                'hash': minhash,
                'depth': len(domain.split('.'))
            }

        similar_domain_counter = 0
        allowlist_encountered = False
        evil_emoji = emojis.get_emoji('SKULL_CROSSBONE')
        phishing_emoji = emojis.get_emoji('FISHING_POLE')
        for domain, _ in iter(domain_counter.items()):
            emojis_to_add = []
            tags_to_add = []
            text = None

            similar_domains = self._get_similar_domains(
                domain, watched_domains)

            if similar_domains:
                similar_domain_counter += 1
                emojis_to_add.append(evil_emoji)
                emojis_to_add.append(phishing_emoji)
                tags_to_add.append('phishy-domain')
                similar_text_list = ['{0:s} [score: {1:.2f}]'.format(
                    phishy_domain,
                    score) for phishy_domain, score in similar_domains]
                text = 'Domain {0:s} is similar to {1:s}'.format(
                    domain, ', '.join(similar_text_list))
                if any(domain.endswith(
                        x) for x in self.domain_scoring_exclude_domains):
                    tags_to_add.append('known-domain')
                    allowlist_encountered = True

            for event in domains.get(domain, []):
                event.add_emojis(emojis_to_add)
                event.add_tags(tags_to_add)
                if text:
                    event.add_human_readable(text, self.NAME, append=False)

                # Commit the event to the datastore.
                event.commit()

        if similar_domain_counter:
            self.sketch.add_view(
                view_name='Phishy Domains', analyzer_name=self.NAME,
                query_string='tag:"phishy-domain"')

            if allowlist_encountered:
                self.sketch.add_view(
                    view_name='Phishy Domains, excl. known domains',
                    analyzer_name=self.NAME,
                    query_string=(
                        'tag:"phishy-domain" AND NOT tag:"known-domain"'))

        return (
            '{0:d} potentially phishy domains discovered.').format(
                similar_domain_counter)
def run ( self ) : query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }}, ' '{ "exists" : { "field" : "domain" }} ] } } }' ) return_fields = [ 'domain' , 'url' , 'message' , 'human_readable' ] events = self . event_stream ( '' , query_dsl = query , return_fields = return_fields ) domains = { } domain_counter = collections . Counter ( ) tld_counter = collections . Counter ( ) for event in events : domain = event . source . get ( 'domain' ) if not domain : continue domain_counter [ domain ] += 1 domains . setdefault ( domain , [ ] ) domains [ domain ] . append ( event ) tld = utils . get_tld_from_domain ( domain ) tld_counter [ tld ] += 1 if not domain_counter : return 'No domains discovered, so no phishy domains.' watched_domains_list = current_app . config . get ( 'DOMAIN_ANALYZER_WATCHED_DOMAINS' , [ ] ) domain_threshold = current_app . config . get ( 'DOMAIN_ANALYZER_WATCHED_DOMAINS_THRESHOLD' , 10 ) watched_domains_list . extend ( [ utils . strip_www_from_domain ( x ) for x , _ in domain_counter . most_common ( domain_threshold ) ] ) watched_domains_list . extend ( [ x for x , _ in tld_counter . most_common ( domain_threshold ) ] ) watched_domains_list . extend ( self . WATCHED_DOMAINS_BASE_LIST ) watched_domains_list_temp = set ( watched_domains_list ) watched_domains_list = [ ] for domain in watched_domains_list_temp : if domain in self . domain_scoring_exclude_domains : continue if any ( domain . endswith ( x ) for x in self . domain_scoring_exclude_domains ) : continue if '.' not in domain : continue watched_domains_list . append ( domain ) watched_domains = { } for domain in watched_domains_list : minhash = self . _get_minhash_from_domain ( domain ) watched_domains [ domain ] = { 'hash' : minhash , 'depth' : len ( domain . split ( '.' ) ) } similar_domain_counter = 0 allowlist_encountered = False evil_emoji = emojis . get_emoji ( 'SKULL_CROSSBONE' ) phishing_emoji = emojis . get_emoji ( 'FISHING_POLE' ) for domain , _ in iter ( domain_counter . items ( ) ) : emojis_to_add = [ ] tags_to_add = [ ] text = None similar_domains = self . _get_similar_domains ( domain , watched_domains ) if similar_domains : similar_domain_counter += 1 emojis_to_add . append ( evil_emoji ) emojis_to_add . append ( phishing_emoji ) tags_to_add . append ( 'phishy-domain' ) similar_text_list = [ '{0:s} [score: {1:.2f}]' . format ( phishy_domain , score ) for phishy_domain , score in similar_domains ] text = 'Domain {0:s} is similar to {1:s}' . format ( domain , ', ' . join ( similar_text_list ) ) if any ( domain . endswith ( x ) for x in self . domain_scoring_exclude_domains ) : tags_to_add . append ( 'known-domain' ) allowlist_encountered = True for event in domains . get ( domain , [ ] ) : event . add_emojis ( emojis_to_add ) event . add_tags ( tags_to_add ) if text : event . add_human_readable ( text , self . NAME , append = False ) # Commit the event to the datastore. event . commit ( ) if similar_domain_counter : self . sketch . add_view ( view_name = 'Phishy Domains' , analyzer_name = self . NAME , query_string = 'tag:"phishy-domain"' ) if allowlist_encountered : self . sketch . add_view ( view_name = 'Phishy Domains, excl. known domains' , analyzer_name = self . NAME , query_string = ( 'tag:"phishy-domain" AND NOT tag:"known-domain"' ) ) return ( '{0:d} potentially phishy domains discovered.' ) . format ( similar_domain_counter )