(self, sketch_id)
return jsonify(schema)
Handles POST request to the resource.
        Handler for /api/v1/sketches/:sketch_id/explore/

        Args:
            sketch_id: Integer primary key for a sketch database model

        Returns:
            JSON with list of matched events
Handles POST request to the resource.
        Handler for /api/v1/sketches/:sketch_id/explore/
Handles POST request to the resource . Handler for / api / v1 / sketches / : sketch_id / explore /
def post(self, sketch_id):
        """Handles POST request to the resource.
        Handler for /api/v1/sketches/:sketch_id/explore/

        Args:
            sketch_id: Integer primary key for a sketch database model

        Returns:
            JSON with list of matched events
        """
        sketch = Sketch.query.get_with_acl(sketch_id)
        if not sketch:
            abort(
                HTTP_STATUS_CODE_NOT_FOUND, 'No sketch found with this ID.')

        if not sketch.has_permission(current_user, 'read'):
            abort(HTTP_STATUS_CODE_FORBIDDEN,
                  'User does not have read access controls on sketch.')

        if sketch.get_status.status == 'archived':
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'Unable to query on an archived sketch.')

        form = forms.ExploreForm.build(request)

        if not form.validate_on_submit():
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'Unable to explore data, unable to validate form data')

        # TODO: Remove form and use json instead.
        query_dsl = form.dsl.data
        enable_scroll = form.enable_scroll.data
        scroll_id = form.scroll_id.data
        file_name = form.file_name.data
        count = bool(form.count.data)

        query_filter = request.json.get('filter', {})
        parent = request.json.get('parent', None)
        incognito = request.json.get('incognito', False)

        return_field_string = form.fields.data
        if return_field_string:
            return_fields = [x.strip() for x in return_field_string.split(',')]
        else:
            return_fields = query_filter.get('fields', [])
            return_fields = [field['field'] for field in return_fields]
            return_fields.extend(DEFAULT_SOURCE_FIELDS)

        if not query_filter:
            query_filter = {}

        all_indices = list({
            t.searchindex.index_name for t in sketch.timelines})
        indices = query_filter.get('indices', all_indices)

        # If _all in indices then execute the query on all indices
        if '_all' in indices:
            indices = all_indices

        # Make sure that the indices in the filter are part of the sketch.
        # This will also remove any deleted timeline from the search result.
        indices, timeline_ids = get_validated_indices(indices, sketch)

        # Remove indices that don't exist from search.
        indices = utils.validate_indices(indices, self.datastore)

        if not indices:
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'No valid search indices were found to perform the search on.')

        # Make sure we have a query string or star filter
        if not (form.query.data, query_filter.get('star'),
                query_filter.get('events'), query_dsl):
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'The request needs a query string/DSL and or a star filter.')

        # Aggregate hit count per index.
        index_stats_agg = {
            'indices': {
                'terms': {
                    'field': '_index',
                    'min_doc_count': 0,
                    'size': len(sketch.timelines)
                }
            },
            'timelines': {
                'terms': {
                    'field': '__ts_timeline_id',
                    'min_doc_count': 0,
                    'size': len(sketch.timelines)
                }
            },
            'count_over_time': {
                'auto_date_histogram': {
                    'field': 'datetime',
                    'buckets': 50,
                }
            }
        }
        if count:
            # Count operations do not support size parameters.
            if 'size' in query_filter:
                _ = query_filter.pop('size')
            if 'terminate_after' in query_filter:
                _ = query_filter.pop('terminate_after')

            try:
                result = self.datastore.search(
                    sketch_id=sketch_id,
                    query_string=form.query.data,
                    query_filter=query_filter,
                    query_dsl=query_dsl,
                    indices=indices,
                    timeline_ids=timeline_ids,
                    count=True)
            except ValueError as e:
                abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))

            # Get number of matching documents per index.
            schema = {'meta': {'total_count': result}, 'objects': []}
            return jsonify(schema)

        if file_name:
            file_object = io.BytesIO()

            form_data = {
                'created_at': datetime.datetime.utcnow().isoformat(),
                'created_by': current_user.username,
                'sketch': sketch_id,
                'query': form.query.data,
                'query_dsl': query_dsl,
                'query_filter': query_filter,
                'return_fields': return_fields,
            }
            with zipfile.ZipFile(file_object, mode='w') as zip_file:
                zip_file.writestr('METADATA', data=json.dumps(form_data))
                fh = export.query_to_filehandle(
                    query_string=form.query.data,
                    query_dsl=query_dsl,
                    query_filter=query_filter,
                    indices=indices,
                    sketch=sketch,
                    datastore=self.datastore,
                    return_fields=return_fields,
                    timeline_ids=timeline_ids)
                fh.seek(0)
                zip_file.writestr('query_results.csv', fh.read())
            file_object.seek(0)
            return send_file(
                file_object, mimetype='zip',
                attachment_filename=file_name)

        if scroll_id:
            # pylint: disable=unexpected-keyword-arg
            result = self.datastore.client.scroll(
                scroll_id=scroll_id, scroll='1m')
        else:
            try:
                result = self.datastore.search(
                    sketch_id=sketch_id,
                    query_string=form.query.data,
                    query_filter=query_filter,
                    query_dsl=query_dsl,
                    indices=indices,
                    aggregations=index_stats_agg,
                    return_fields=return_fields,
                    enable_scroll=enable_scroll,
                    timeline_ids=timeline_ids)
            except ValueError as e:
                abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))

        # Get number of matching documents over time.
        histogram_interval = result.get(
            'aggregations', {}).get('count_over_time', {}).get('interval', '')
        count_over_time = {
            'data': {},
            'interval': histogram_interval
        }
        try:
            for bucket in result['aggregations']['count_over_time']['buckets']:
                key = bucket.get('key')
                if key:
                    count_over_time['data'][key] = bucket.get('doc_count')
        except KeyError:
            pass

        # Get number of matching documents per index.
        count_per_index = {}
        try:
            for bucket in result['aggregations']['indices']['buckets']:
                key = bucket.get('key')
                if key:
                    count_per_index[key] = bucket.get('doc_count')
        except KeyError:
            pass

        # Get number of matching documents per timeline.
        count_per_timeline = {}
        try:
            for bucket in result['aggregations']['timelines']['buckets']:
                key = bucket.get('key')
                if key:
                    count_per_timeline[key] = bucket.get('doc_count')
        except KeyError:
            pass

        # Total count for query regardless of returned results.
        count_total_complete = sum(count_per_index.values())

        comments = {}
        if 'comment' in return_fields:
            events = Event.query.filter_by(
                sketch=sketch).all()
            for event in events:
                for comment in event.comments:
                    comments.setdefault(event.document_id, [])
                    comments[event.document_id].append(comment.comment)

        # Get labels for each event that matches the sketch.
        # Remove all other labels.
        for event in result['hits']['hits']:
            event['selected'] = False
            event['_source']['label'] = []
            try:
                for label in event['_source']['timesketch_label']:
                    if sketch.id != label['sketch_id']:
                        continue
                    event['_source']['label'].append(label['name'])
                del event['_source']['timesketch_label']
            except KeyError:
                pass

            if 'comment' in return_fields:
                event['_source']['comment'] = comments.get(event['_id'], [])

        # Update or create user state view. This is used in the UI to let
        # the user get back to the last state in the explore view.
        # TODO: Deprecate this and change how last activity is determined, e.g
        # use the new Search History feature instead.
        view = View.get_or_create(
            user=current_user, sketch=sketch, name='')
        view.update_modification_time()
        view.query_string = form.query.data
        view.query_filter = json.dumps(query_filter, ensure_ascii=False)
        view.query_dsl = json.dumps(query_dsl, ensure_ascii=False)
        db_session.add(view)
        db_session.commit()

        # Search History
        search_node = None
        new_search = SearchHistory(user=current_user, sketch=sketch)

        if parent:
            previous_search = SearchHistory.query.get(parent)
        else:
            previous_search = SearchHistory.query.filter_by(
                user=current_user, sketch=sketch).order_by(
                    SearchHistory.id.desc()).first()

        if not incognito:
            is_same_query = False
            is_same_filter = False

            new_search.query_string = form.query.data
            new_search.query_filter = json.dumps(
                query_filter, ensure_ascii=False)

            new_search.query_result_count = count_total_complete
            new_search.query_time = result['took']

            if previous_search:
                new_search.parent = previous_search

                new_query = new_search.query_string
                new_filter = new_search.query_filter
                previous_query = previous_search.query_string
                previous_filter = previous_search.query_filter

                is_same_query = previous_query == new_query
                is_same_filter = previous_filter == new_filter

            if not all([is_same_query, is_same_filter]):
                db_session.add(new_search)
                db_session.commit()
                # Create metric if user creates a new branch.
                if new_search.parent:
                    if len(new_search.parent.children) > 1:
                        METRICS['searchhistory'].labels(
                            action='branch').inc()
            else:
                METRICS['searchhistory'].labels(
                    action='ignore_same_query').inc()
        else:
            METRICS['searchhistory'].labels(action='incognito').inc()

        search_node = new_search if new_search.id else previous_search

        if not search_node:
            abort(HTTP_STATUS_CODE_BAD_REQUEST, 'Unable to save search')

        search_node = search_node.build_tree(search_node, {}, recurse=False)

        # Add metadata for the query result. This is used by the UI to
        # render the event correctly and to display timing and hit count
        # information.
        tl_colors = {}
        tl_names = {}
        for timeline in sketch.timelines:
            tl_colors[timeline.searchindex.index_name] = timeline.color
            tl_names[timeline.searchindex.index_name] = timeline.name

        meta = {
            'es_time': result['took'],
            'es_total_count': result['hits']['total'],
            'es_total_count_complete': count_total_complete,
            'timeline_colors': tl_colors,
            'timeline_names': tl_names,
            'count_per_index': count_per_index,
            'count_per_timeline': count_per_timeline,
            'count_over_time': count_over_time,
            'scroll_id': result.get('_scroll_id', ''),
            'search_node': search_node
        }

        # Elasticsearch version 7.x returns total hits as a dictionary.
        # TODO: Refactor when version 6.x has been deprecated.
        if isinstance(meta['es_total_count'], dict):
            meta['es_total_count'] = meta['es_total_count'].get('value', 0)

        schema = {'meta': meta, 'objects': result['hits']['hits']}
        return jsonify(schema)
def post ( self , sketch_id ) : sketch = Sketch . query . get_with_acl ( sketch_id ) if not sketch : abort ( HTTP_STATUS_CODE_NOT_FOUND , 'No sketch found with this ID.' ) if not sketch . has_permission ( current_user , 'read' ) : abort ( HTTP_STATUS_CODE_FORBIDDEN , 'User does not have read access controls on sketch.' ) if sketch . get_status . status == 'archived' : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'Unable to query on an archived sketch.' ) form = forms . ExploreForm . build ( request ) if not form . validate_on_submit ( ) : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'Unable to explore data, unable to validate form data' ) # TODO: Remove form and use json instead. query_dsl = form . dsl . data enable_scroll = form . enable_scroll . data scroll_id = form . scroll_id . data file_name = form . file_name . data count = bool ( form . count . data ) query_filter = request . json . get ( 'filter' , { } ) parent = request . json . get ( 'parent' , None ) incognito = request . json . get ( 'incognito' , False ) return_field_string = form . fields . data if return_field_string : return_fields = [ x . strip ( ) for x in return_field_string . split ( ',' ) ] else : return_fields = query_filter . get ( 'fields' , [ ] ) return_fields = [ field [ 'field' ] for field in return_fields ] return_fields . extend ( DEFAULT_SOURCE_FIELDS ) if not query_filter : query_filter = { } all_indices = list ( { t . searchindex . index_name for t in sketch . timelines } ) indices = query_filter . get ( 'indices' , all_indices ) # If _all in indices then execute the query on all indices if '_all' in indices : indices = all_indices # Make sure that the indices in the filter are part of the sketch. # This will also remove any deleted timeline from the search result. indices , timeline_ids = get_validated_indices ( indices , sketch ) # Remove indices that don't exist from search. indices = utils . validate_indices ( indices , self . datastore ) if not indices : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'No valid search indices were found to perform the search on.' ) # Make sure we have a query string or star filter if not ( form . query . data , query_filter . get ( 'star' ) , query_filter . get ( 'events' ) , query_dsl ) : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'The request needs a query string/DSL and or a star filter.' ) # Aggregate hit count per index. index_stats_agg = { 'indices' : { 'terms' : { 'field' : '_index' , 'min_doc_count' : 0 , 'size' : len ( sketch . timelines ) } } , 'timelines' : { 'terms' : { 'field' : '__ts_timeline_id' , 'min_doc_count' : 0 , 'size' : len ( sketch . timelines ) } } , 'count_over_time' : { 'auto_date_histogram' : { 'field' : 'datetime' , 'buckets' : 50 , } } } if count : # Count operations do not support size parameters. if 'size' in query_filter : _ = query_filter . pop ( 'size' ) if 'terminate_after' in query_filter : _ = query_filter . pop ( 'terminate_after' ) try : result = self . datastore . search ( sketch_id = sketch_id , query_string = form . query . data , query_filter = query_filter , query_dsl = query_dsl , indices = indices , timeline_ids = timeline_ids , count = True ) except ValueError as e : abort ( HTTP_STATUS_CODE_BAD_REQUEST , str ( e ) ) # Get number of matching documents per index. schema = { 'meta' : { 'total_count' : result } , 'objects' : [ ] } return jsonify ( schema ) if file_name : file_object = io . BytesIO ( ) form_data = { 'created_at' : datetime . datetime . utcnow ( ) . isoformat ( ) , 'created_by' : current_user . username , 'sketch' : sketch_id , 'query' : form . query . data , 'query_dsl' : query_dsl , 'query_filter' : query_filter , 'return_fields' : return_fields , } with zipfile . ZipFile ( file_object , mode = 'w' ) as zip_file : zip_file . writestr ( 'METADATA' , data = json . dumps ( form_data ) ) fh = export . query_to_filehandle ( query_string = form . query . data , query_dsl = query_dsl , query_filter = query_filter , indices = indices , sketch = sketch , datastore = self . datastore , return_fields = return_fields , timeline_ids = timeline_ids ) fh . seek ( 0 ) zip_file . writestr ( 'query_results.csv' , fh . read ( ) ) file_object . seek ( 0 ) return send_file ( file_object , mimetype = 'zip' , attachment_filename = file_name ) if scroll_id : # pylint: disable=unexpected-keyword-arg result = self . datastore . client . scroll ( scroll_id = scroll_id , scroll = '1m' ) else : try : result = self . datastore . search ( sketch_id = sketch_id , query_string = form . query . data , query_filter = query_filter , query_dsl = query_dsl , indices = indices , aggregations = index_stats_agg , return_fields = return_fields , enable_scroll = enable_scroll , timeline_ids = timeline_ids ) except ValueError as e : abort ( HTTP_STATUS_CODE_BAD_REQUEST , str ( e ) ) # Get number of matching documents over time. histogram_interval = result . get ( 'aggregations' , { } ) . get ( 'count_over_time' , { } ) . get ( 'interval' , '' ) count_over_time = { 'data' : { } , 'interval' : histogram_interval } try : for bucket in result [ 'aggregations' ] [ 'count_over_time' ] [ 'buckets' ] : key = bucket . get ( 'key' ) if key : count_over_time [ 'data' ] [ key ] = bucket . get ( 'doc_count' ) except KeyError : pass # Get number of matching documents per index. count_per_index = { } try : for bucket in result [ 'aggregations' ] [ 'indices' ] [ 'buckets' ] : key = bucket . get ( 'key' ) if key : count_per_index [ key ] = bucket . get ( 'doc_count' ) except KeyError : pass # Get number of matching documents per timeline. count_per_timeline = { } try : for bucket in result [ 'aggregations' ] [ 'timelines' ] [ 'buckets' ] : key = bucket . get ( 'key' ) if key : count_per_timeline [ key ] = bucket . get ( 'doc_count' ) except KeyError : pass # Total count for query regardless of returned results. count_total_complete = sum ( count_per_index . values ( ) ) comments = { } if 'comment' in return_fields : events = Event . query . filter_by ( sketch = sketch ) . all ( ) for event in events : for comment in event . comments : comments . setdefault ( event . document_id , [ ] ) comments [ event . document_id ] . append ( comment . comment ) # Get labels for each event that matches the sketch. # Remove all other labels. for event in result [ 'hits' ] [ 'hits' ] : event [ 'selected' ] = False event [ '_source' ] [ 'label' ] = [ ] try : for label in event [ '_source' ] [ 'timesketch_label' ] : if sketch . id != label [ 'sketch_id' ] : continue event [ '_source' ] [ 'label' ] . append ( label [ 'name' ] ) del event [ '_source' ] [ 'timesketch_label' ] except KeyError : pass if 'comment' in return_fields : event [ '_source' ] [ 'comment' ] = comments . get ( event [ '_id' ] , [ ] ) # Update or create user state view. This is used in the UI to let # the user get back to the last state in the explore view. # TODO: Deprecate this and change how last activity is determined, e.g # use the new Search History feature instead. view = View . get_or_create ( user = current_user , sketch = sketch , name = '' ) view . update_modification_time ( ) view . query_string = form . query . data view . query_filter = json . dumps ( query_filter , ensure_ascii = False ) view . query_dsl = json . dumps ( query_dsl , ensure_ascii = False ) db_session . add ( view ) db_session . commit ( ) # Search History search_node = None new_search = SearchHistory ( user = current_user , sketch = sketch ) if parent : previous_search = SearchHistory . query . get ( parent ) else : previous_search = SearchHistory . query . filter_by ( user = current_user , sketch = sketch ) . order_by ( SearchHistory . id . desc ( ) ) . first ( ) if not incognito : is_same_query = False is_same_filter = False new_search . query_string = form . query . data new_search . query_filter = json . dumps ( query_filter , ensure_ascii = False ) new_search . query_result_count = count_total_complete new_search . query_time = result [ 'took' ] if previous_search : new_search . parent = previous_search new_query = new_search . query_string new_filter = new_search . query_filter previous_query = previous_search . query_string previous_filter = previous_search . query_filter is_same_query = previous_query == new_query is_same_filter = previous_filter == new_filter if not all ( [ is_same_query , is_same_filter ] ) : db_session . add ( new_search ) db_session . commit ( ) # Create metric if user creates a new branch. if new_search . parent : if len ( new_search . parent . children ) > 1 : METRICS [ 'searchhistory' ] . labels ( action = 'branch' ) . inc ( ) else : METRICS [ 'searchhistory' ] . labels ( action = 'ignore_same_query' ) . inc ( ) else : METRICS [ 'searchhistory' ] . labels ( action = 'incognito' ) . inc ( ) search_node = new_search if new_search . id else previous_search if not search_node : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'Unable to save search' ) search_node = search_node . build_tree ( search_node , { } , recurse = False ) # Add metadata for the query result. This is used by the UI to # render the event correctly and to display timing and hit count # information. tl_colors = { } tl_names = { } for timeline in sketch . timelines : tl_colors [ timeline . searchindex . index_name ] = timeline . color tl_names [ timeline . searchindex . index_name ] = timeline . name meta = { 'es_time' : result [ 'took' ] , 'es_total_count' : result [ 'hits' ] [ 'total' ] , 'es_total_count_complete' : count_total_complete , 'timeline_colors' : tl_colors , 'timeline_names' : tl_names , 'count_per_index' : count_per_index , 'count_per_timeline' : count_per_timeline , 'count_over_time' : count_over_time , 'scroll_id' : result . get ( '_scroll_id' , '' ) , 'search_node' : search_node } # Elasticsearch version 7.x returns total hits as a dictionary. # TODO: Refactor when version 6.x has been deprecated. if isinstance ( meta [ 'es_total_count' ] , dict ) : meta [ 'es_total_count' ] = meta [ 'es_total_count' ] . get ( 'value' , 0 ) schema = { 'meta' : meta , 'objects' : result [ 'hits' ] [ 'hits' ] } return jsonify ( schema )