(
            self, file_extension, timeline_name, index_name, sketch, form,
            enable_stream, original_filename='', data_label='', file_path='',
            events='', meta=None)
return self.to_json(
            timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
Creates a full pipeline for an uploaded file and returns the results.

        Args:
            file_extension: the extension of the uploaded file.
            timeline_name: name the timeline will be stored under in the
                           datastore.
            index_name: the OpenSearch index name for the timeline.
            sketch: Instance of timesketch.models.sketch.Sketch
            form: a dict with the configuration for the upload.
            enable_stream: boolean indicating whether this is file is part of a
                           stream or not.
            original_filename: Original filename from the upload.
            data_label: Optional string with a data label for the search index.
            file_path: the path to the file to be uploaded (optional).
            events: a string with events to upload (optional).
            meta: optional dict with additional meta fields that will be
                  included in the return.

        Returns:
            A timeline if created otherwise a search index in JSON (instance
            of flask.wrappers.Response)
Creates a full pipeline for an uploaded file and returns the results.
Creates a full pipeline for an uploaded file and returns the results .
def _upload_and_index(
            self, file_extension, timeline_name, index_name, sketch, form,
            enable_stream, original_filename='', data_label='', file_path='',
            events='', meta=None):
        """Creates a full pipeline for an uploaded file and returns the results.

        Args:
            file_extension: the extension of the uploaded file.
            timeline_name: name the timeline will be stored under in the
                           datastore.
            index_name: the OpenSearch index name for the timeline.
            sketch: Instance of timesketch.models.sketch.Sketch
            form: a dict with the configuration for the upload.
            enable_stream: boolean indicating whether this is file is part of a
                           stream or not.
            original_filename: Original filename from the upload.
            data_label: Optional string with a data label for the search index.
            file_path: the path to the file to be uploaded (optional).
            events: a string with events to upload (optional).
            meta: optional dict with additional meta fields that will be
                  included in the return.

        Returns:
            A timeline if created otherwise a search index in JSON (instance
            of flask.wrappers.Response)
        """
        searchindex = self._get_index(
            name=timeline_name,
            description=timeline_name,
            sketch=sketch,
            index_name=index_name,
            data_label=data_label,
            extension=file_extension)

        if not searchindex:
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'We were unable to acquire a searchindex and therefore not '
                'able to upload data, please try again. If this error persist '
                'please create an issue on Github: https://github.com/'
                'google/timesketch/issues/new/choose')

        timelines = Timeline.query.filter_by(
            name=timeline_name, sketch=sketch).all()

        timeline = None
        for timeline_ in timelines:
            if timeline_.searchindex.index_name == searchindex.index_name:
                timeline = timeline_
                break

            logger.error(
                'There is a timeline in the sketch that has the same name '
                'but is stored in a different index: name {0:s} attempting '
                'index: {1:s} but found index {2:s} - retrying with a '
                'different timeline name.'.format(
                    timeline_name, searchindex.index_name,
                    timeline_.searchindex.index_name))

            timeline_name = '{0:s}_{1:s}'.format(
                timeline_name, uuid.uuid4().hex[-5:])
            return self._upload_and_index(
                file_extension=file_extension, timeline_name=timeline_name,
                index_name=searchindex.index_name, sketch=sketch, form=form,
                enable_stream=enable_stream,
                original_filename=original_filename, data_label=data_label,
                file_path=file_path, events=events, meta=meta)

        searchindex.set_status('processing')

        if not timeline:
            timeline = Timeline.get_or_create(
                name=timeline_name,
                description=timeline_name,
                sketch=sketch,
                user=current_user,
                searchindex=searchindex)

        if not timeline:
            abort(
                HTTP_STATUS_CODE_BAD_REQUEST,
                'Unable to get or create a new Timeline object.')

        # If the timeline already existed and has associated data sources
        # then we don't want to set the status to processing.
        if not timeline.datasources:
            timeline.set_status('processing')

        sketch.timelines.append(timeline)

        labels_to_prevent_deletion = current_app.config.get(
            'LABELS_TO_PREVENT_DELETION', [])
        for sketch_label in sketch.get_labels:
            if sketch_label not in labels_to_prevent_deletion:
                continue
            timeline.add_label(sketch_label)
            searchindex.add_label(sketch_label)

        file_size = form.get('total_file_size', 0)
        datasource = DataSource(
            timeline=timeline,
            user=current_user,
            provider=form.get('provider', 'N/A'),
            context=form.get('context', 'N/A'),
            file_on_disk=file_path,
            file_size=int(file_size),
            original_filename=original_filename,
            data_label=data_label
        )

        timeline.datasources.append(datasource)
        db_session.add(datasource)
        db_session.add(timeline)
        db_session.commit()

        sketch_id = sketch.id
        # Start Celery pipeline for indexing and analysis.
        # Import here to avoid circular imports.
        # pylint: disable=import-outside-toplevel
        from timesketch.lib import tasks
        pipeline = tasks.build_index_pipeline(
            file_path=file_path, events=events, timeline_name=timeline_name,
            index_name=searchindex.index_name, file_extension=file_extension,
            sketch_id=sketch_id, only_index=enable_stream,
            timeline_id=timeline.id)
        task_id = uuid.uuid4().hex
        pipeline.apply_async(task_id=task_id)

        if meta is None:
            meta = {}

        meta['task_id'] = task_id
        return self.to_json(
            timeline, status_code=HTTP_STATUS_CODE_CREATED, meta=meta)
def _upload_and_index ( self , file_extension , timeline_name , index_name , sketch , form , enable_stream , original_filename = '' , data_label = '' , file_path = '' , events = '' , meta = None ) : searchindex = self . _get_index ( name = timeline_name , description = timeline_name , sketch = sketch , index_name = index_name , data_label = data_label , extension = file_extension ) if not searchindex : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'We were unable to acquire a searchindex and therefore not ' 'able to upload data, please try again. If this error persist ' 'please create an issue on Github: https://github.com/' 'google/timesketch/issues/new/choose' ) timelines = Timeline . query . filter_by ( name = timeline_name , sketch = sketch ) . all ( ) timeline = None for timeline_ in timelines : if timeline_ . searchindex . index_name == searchindex . index_name : timeline = timeline_ break logger . error ( 'There is a timeline in the sketch that has the same name ' 'but is stored in a different index: name {0:s} attempting ' 'index: {1:s} but found index {2:s} - retrying with a ' 'different timeline name.' . format ( timeline_name , searchindex . index_name , timeline_ . searchindex . index_name ) ) timeline_name = '{0:s}_{1:s}' . format ( timeline_name , uuid . uuid4 ( ) . hex [ - 5 : ] ) return self . _upload_and_index ( file_extension = file_extension , timeline_name = timeline_name , index_name = searchindex . index_name , sketch = sketch , form = form , enable_stream = enable_stream , original_filename = original_filename , data_label = data_label , file_path = file_path , events = events , meta = meta ) searchindex . set_status ( 'processing' ) if not timeline : timeline = Timeline . get_or_create ( name = timeline_name , description = timeline_name , sketch = sketch , user = current_user , searchindex = searchindex ) if not timeline : abort ( HTTP_STATUS_CODE_BAD_REQUEST , 'Unable to get or create a new Timeline object.' ) # If the timeline already existed and has associated data sources # then we don't want to set the status to processing. if not timeline . datasources : timeline . set_status ( 'processing' ) sketch . timelines . append ( timeline ) labels_to_prevent_deletion = current_app . config . get ( 'LABELS_TO_PREVENT_DELETION' , [ ] ) for sketch_label in sketch . get_labels : if sketch_label not in labels_to_prevent_deletion : continue timeline . add_label ( sketch_label ) searchindex . add_label ( sketch_label ) file_size = form . get ( 'total_file_size' , 0 ) datasource = DataSource ( timeline = timeline , user = current_user , provider = form . get ( 'provider' , 'N/A' ) , context = form . get ( 'context' , 'N/A' ) , file_on_disk = file_path , file_size = int ( file_size ) , original_filename = original_filename , data_label = data_label ) timeline . datasources . append ( datasource ) db_session . add ( datasource ) db_session . add ( timeline ) db_session . commit ( ) sketch_id = sketch . id # Start Celery pipeline for indexing and analysis. # Import here to avoid circular imports. # pylint: disable=import-outside-toplevel from timesketch . lib import tasks pipeline = tasks . build_index_pipeline ( file_path = file_path , events = events , timeline_name = timeline_name , index_name = searchindex . index_name , file_extension = file_extension , sketch_id = sketch_id , only_index = enable_stream , timeline_id = timeline . id ) task_id = uuid . uuid4 ( ) . hex pipeline . apply_async ( task_id = task_id ) if meta is None : meta = { } meta [ 'task_id' ] = task_id return self . to_json ( timeline , status_code = HTTP_STATUS_CODE_CREATED , meta = meta )