(self, name, config)
return 'Feature extraction [{0:s}] extracted {1:d} features.'.format(
            name, event_counter)
Extract features from events.

        Args:
            name: String with the name describing the feature to be extracted.
            config: A dict that contains the configuration for the feature
                extraction. See data/features.yaml for fields and further
                documentation of what needs to be defined.

        Returns:
            String with summary of the analyzer result.
Extract features from events.
Extract features from events .
def extract_feature(self, name, config):
        """Extract features from events.

        Args:
            name: String with the name describing the feature to be extracted.
            config: A dict that contains the configuration for the feature
                extraction. See data/features.yaml for fields and further
                documentation of what needs to be defined.

        Returns:
            String with summary of the analyzer result.
        """
        query = config.get('query_string')
        query_dsl = config.get('query_dsl')
        attribute = config.get('attribute')
        store_type_list = config.get('store_type_list', False)
        keep_multimatch = config.get('keep_multimatch', False)
        overwrite_store_as = config.get('overwrite_store_as', True)
        overwrite_and_merge_store_as = config.get(
            'overwrite_and_merge_store_as', False)

        if not attribute:
            logger.warning('No attribute defined.')
            return ''

        store_as = config.get('store_as')
        if not store_as:
            logger.warning('No attribute defined to store results in.')
            return ''

        tags = config.get('tags', [])

        expression_string = config.get('re')
        if not expression_string:
            logger.warning('No regular expression defined.')
            return ''

        expression = utils.compile_regular_expression(
            expression_string=expression_string,
            expression_flags=config.get('re_flags'))

        emoji_names = config.get('emojis', [])
        emojis_to_add = [emojis.get_emoji(x) for x in emoji_names]

        return_fields = [attribute, store_as]

        events = self.event_stream(
            query_string=query, query_dsl=query_dsl,
            return_fields=return_fields)

        event_counter = 0
        for event in events:
            attribute_field = event.source.get(attribute)
            if isinstance(attribute_field, six.text_type):
                attribute_value = attribute_field
            elif isinstance(attribute_field, (list, tuple)):
                attribute_value = ','.join(attribute_field)
            elif isinstance(attribute_field, (int, float)):
                attribute_value = attribute_field
            else:
                attribute_value = None

            if not attribute_value:
                continue

            result = expression.findall(attribute_value)
            if not result:
                continue
            result = list(set(result))

            event_counter += 1
            store_as_current_val = event.source.get(store_as)
            if store_as_current_val and not overwrite_store_as:
                continue
            if isinstance(store_as_current_val, six.text_type):
                store_type_list = False
            elif isinstance(store_as_current_val, (list, tuple)):
                store_type_list = True
            new_value = self._get_attribute_value(
                store_as_current_val,
                result,
                keep_multimatch,
                overwrite_and_merge_store_as,
                store_type_list
                )
            if not new_value:
                continue
            event.add_attributes({store_as: new_value})
            event.add_emojis(emojis_to_add)
            event.add_tags(tags)

            # Commit the event to the datastore.
            event.commit()

        aggregate_results = config.get('aggregate', False)
        create_view = config.get('create_view', False)

        # If aggregation is turned on, we automatically create an aggregation.
        if aggregate_results:
            create_view = True

        if create_view and event_counter:
            view = self.sketch.add_view(
                name, self.NAME, query_string=query, query_dsl=query_dsl)

            if aggregate_results:
                params = {
                    'field': store_as,
                    'limit': 20,
                    'index': [self.timeline_id],
                }
                self.sketch.add_aggregation(
                    name='Top 20 for: {0:s} [{1:s}]'.format(store_as, name),
                    agg_name='field_bucket', agg_params=params,
                    description='Created by the feature extraction analyzer',
                    view_id=view.id, chart_type='hbarchart')

        return 'Feature extraction [{0:s}] extracted {1:d} features.'.format(
            name, event_counter)
def extract_feature ( self , name , config ) : query = config . get ( 'query_string' ) query_dsl = config . get ( 'query_dsl' ) attribute = config . get ( 'attribute' ) store_type_list = config . get ( 'store_type_list' , False ) keep_multimatch = config . get ( 'keep_multimatch' , False ) overwrite_store_as = config . get ( 'overwrite_store_as' , True ) overwrite_and_merge_store_as = config . get ( 'overwrite_and_merge_store_as' , False ) if not attribute : logger . warning ( 'No attribute defined.' ) return '' store_as = config . get ( 'store_as' ) if not store_as : logger . warning ( 'No attribute defined to store results in.' ) return '' tags = config . get ( 'tags' , [ ] ) expression_string = config . get ( 're' ) if not expression_string : logger . warning ( 'No regular expression defined.' ) return '' expression = utils . compile_regular_expression ( expression_string = expression_string , expression_flags = config . get ( 're_flags' ) ) emoji_names = config . get ( 'emojis' , [ ] ) emojis_to_add = [ emojis . get_emoji ( x ) for x in emoji_names ] return_fields = [ attribute , store_as ] events = self . event_stream ( query_string = query , query_dsl = query_dsl , return_fields = return_fields ) event_counter = 0 for event in events : attribute_field = event . source . get ( attribute ) if isinstance ( attribute_field , six . text_type ) : attribute_value = attribute_field elif isinstance ( attribute_field , ( list , tuple ) ) : attribute_value = ',' . join ( attribute_field ) elif isinstance ( attribute_field , ( int , float ) ) : attribute_value = attribute_field else : attribute_value = None if not attribute_value : continue result = expression . findall ( attribute_value ) if not result : continue result = list ( set ( result ) ) event_counter += 1 store_as_current_val = event . source . get ( store_as ) if store_as_current_val and not overwrite_store_as : continue if isinstance ( store_as_current_val , six . text_type ) : store_type_list = False elif isinstance ( store_as_current_val , ( list , tuple ) ) : store_type_list = True new_value = self . _get_attribute_value ( store_as_current_val , result , keep_multimatch , overwrite_and_merge_store_as , store_type_list ) if not new_value : continue event . add_attributes ( { store_as : new_value } ) event . add_emojis ( emojis_to_add ) event . add_tags ( tags ) # Commit the event to the datastore. event . commit ( ) aggregate_results = config . get ( 'aggregate' , False ) create_view = config . get ( 'create_view' , False ) # If aggregation is turned on, we automatically create an aggregation. if aggregate_results : create_view = True if create_view and event_counter : view = self . sketch . add_view ( name , self . NAME , query_string = query , query_dsl = query_dsl ) if aggregate_results : params = { 'field' : store_as , 'limit' : 20 , 'index' : [ self . timeline_id ] , } self . sketch . add_aggregation ( name = 'Top 20 for: {0:s} [{1:s}]' . format ( store_as , name ) , agg_name = 'field_bucket' , agg_params = params , description = 'Created by the feature extraction analyzer' , view_id = view . id , chart_type = 'hbarchart' ) return 'Feature extraction [{0:s}] extracted {1:d} features.' . format ( name , event_counter )