(self)
return (
            '{0:d} Safe Browsing result(s) on {1:d} URL(s), '
            '{2:d} on the allow list.').format(
                len(safebrowsing_results),
                len(urls),
                url_allowlisted,
            )
Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result
Entry point for the analyzer.
Entry point for the analyzer .
def run(self):
        """Entry point for the analyzer.

        Returns:
            String with summary of the analyzer result
        """
        # Exit ASAP if the API key is missing.
        if not self._safebrowsing_api_key:
            return 'Safe Browsing API requires an API key!'

        query = (
            '{"query": { "bool": { "should": [ '
            '{ "exists" : { "field" : "url" }} ] } } }')

        return_fields = ['url']

        events = self.event_stream(
            query_dsl=query,
            return_fields=return_fields,
        )

        urls = {}

        for event in events:
            url = self._sanitize_url(event.source.get('url'))

            if not url:
                continue

            urls.setdefault(url, []).append(event)

        # Exit early if there are no URLs in the data set to analyze.
        if not urls:
            return 'No URLs to analyze.'

        url_allowlisted = 0

        url_allowlist = set(
            interface.get_yaml_config(
                self._URL_ALLOW_LIST_CONFIG,
            ),
        )

        if not url_allowlist:
            domain_analyzer_allowlisted = current_app.config.get(
                'DOMAIN_ANALYZER_EXCLUDE_DOMAINS',
                [],
            )
            for domain in domain_analyzer_allowlisted:
                url_allowlist.add('*.%s/*' % domain)

        logger.info(
            '{0:d} entries on the allowlist.'.format(len(url_allowlist)),
        )

        safebrowsing_platforms = current_app.config.get(
            'SAFEBROWSING_PLATFORMS',
            ['ANY_PLATFORM'],
        )

        safebrowsing_types = current_app.config.get(
            'SAFEBROWSING_THREATTYPES',
            ['MALWARE'],
        )

        lookup_urls = []

        for url in urls:
            if self._is_url_allowlisted(url, url_allowlist):
                url_allowlisted += 1
                continue

            lookup_urls.append(url)

        try:
            safebrowsing_results = self._do_safebrowsing_lookup(
                lookup_urls,
                safebrowsing_platforms,
                safebrowsing_types,
            )
        except requests.HTTPError:
            return 'Couldn\'t reach the Safe Browsing API.'

        for url in lookup_urls:
            safebrowsing_result = safebrowsing_results.get(url)

            if not safebrowsing_result:
                continue

            for event in urls[url]:
                tags = ['google-safebrowsing-url']

                threat_type = safebrowsing_result.get('threatType')

                if threat_type:
                    tags.append(
                        'google-safebrowsing-%s' % threat_type.lower(),
                    )

                event.add_tags(tags)

                threat_attributes = []
                for item in safebrowsing_result.items():
                    threat_attributes.append('%s: %s' % item)

                event.add_attributes(
                    {
                        'google-safebrowsing-threat': ', '.join(
                            threat_attributes),
                    },
                )
                event.commit()

        return (
            '{0:d} Safe Browsing result(s) on {1:d} URL(s), '
            '{2:d} on the allow list.').format(
                len(safebrowsing_results),
                len(urls),
                url_allowlisted,
            )
def run ( self ) : # Exit ASAP if the API key is missing. if not self . _safebrowsing_api_key : return 'Safe Browsing API requires an API key!' query = ( '{"query": { "bool": { "should": [ ' '{ "exists" : { "field" : "url" }} ] } } }' ) return_fields = [ 'url' ] events = self . event_stream ( query_dsl = query , return_fields = return_fields , ) urls = { } for event in events : url = self . _sanitize_url ( event . source . get ( 'url' ) ) if not url : continue urls . setdefault ( url , [ ] ) . append ( event ) # Exit early if there are no URLs in the data set to analyze. if not urls : return 'No URLs to analyze.' url_allowlisted = 0 url_allowlist = set ( interface . get_yaml_config ( self . _URL_ALLOW_LIST_CONFIG , ) , ) if not url_allowlist : domain_analyzer_allowlisted = current_app . config . get ( 'DOMAIN_ANALYZER_EXCLUDE_DOMAINS' , [ ] , ) for domain in domain_analyzer_allowlisted : url_allowlist . add ( '*.%s/*' % domain ) logger . info ( '{0:d} entries on the allowlist.' . format ( len ( url_allowlist ) ) , ) safebrowsing_platforms = current_app . config . get ( 'SAFEBROWSING_PLATFORMS' , [ 'ANY_PLATFORM' ] , ) safebrowsing_types = current_app . config . get ( 'SAFEBROWSING_THREATTYPES' , [ 'MALWARE' ] , ) lookup_urls = [ ] for url in urls : if self . _is_url_allowlisted ( url , url_allowlist ) : url_allowlisted += 1 continue lookup_urls . append ( url ) try : safebrowsing_results = self . _do_safebrowsing_lookup ( lookup_urls , safebrowsing_platforms , safebrowsing_types , ) except requests . HTTPError : return 'Couldn\'t reach the Safe Browsing API.' for url in lookup_urls : safebrowsing_result = safebrowsing_results . get ( url ) if not safebrowsing_result : continue for event in urls [ url ] : tags = [ 'google-safebrowsing-url' ] threat_type = safebrowsing_result . get ( 'threatType' ) if threat_type : tags . append ( 'google-safebrowsing-%s' % threat_type . lower ( ) , ) event . add_tags ( tags ) threat_attributes = [ ] for item in safebrowsing_result . items ( ) : threat_attributes . append ( '%s: %s' % item ) event . add_attributes ( { 'google-safebrowsing-threat' : ', ' . join ( threat_attributes ) , } , ) event . commit ( ) return ( '{0:d} Safe Browsing result(s) on {1:d} URL(s), ' '{2:d} on the allow list.' ) . format ( len ( safebrowsing_results ) , len ( urls ) , url_allowlisted , )