(writer, export_instance, documents, progress_tracker=None)
Write rows to the given open _Writer.
    Rows will be written to each table in the export instance for each of
    the given documents.
    :param writer: An open _Writer
    :param export_instance: An ExportInstance
    :param documents: An iterable yielding documents
    :param progress_tracker: A task for soil to track progress against
    :return: None
Write rows to the given open _Writer.
    Rows will be written to each table in the export instance for each of
    the given documents.
    :param writer: An open _Writer
    :param export_instance: An ExportInstance
    :param documents: An iterable yielding documents
    :param progress_tracker: A task for soil to track progress against
    :return: None
Write rows to the given open _Writer . Rows will be written to each table in the export instance for each of the given documents . : param writer : An open _Writer : param export_instance : An ExportInstance : param documents : An iterable yielding documents : param progress_tracker : A task for soil to track progress against : return : None
def write_export_instance(writer, export_instance, documents, progress_tracker=None):
    """
    Write rows to the given open _Writer.
    Rows will be written to each table in the export instance for each of
    the given documents.
    :param writer: An open _Writer
    :param export_instance: An ExportInstance
    :param documents: An iterable yielding documents
    :param progress_tracker: A task for soil to track progress against
    :return: None
    """
    with TaskProgressManager(progress_tracker, src="export") as progress_manager:
        if progress_tracker:
            progress_manager.set_progress(0, documents.count)

        start = _time_in_milliseconds()
        total_bytes = 0
        total_rows = 0
        track_load = load_counter(export_instance.type, "export", export_instance.domain)

        for row_number, doc in enumerate(documents):
            total_bytes += sys.getsizeof(doc)
            for table in export_instance.selected_tables:
                try:
                    rows = table.get_rows(
                        doc,
                        row_number,
                        split_columns=export_instance.split_multiselects,
                        transform_dates=export_instance.transform_dates,
                    )
                except Exception as e:
                    notify_exception(None, "Error exporting doc", details={
                        'domain': export_instance.domain,
                        'export_instance_id': export_instance.get_id,
                        'export_table': table.label,
                        'doc_id': doc.get('_id'),
                    })
                    e.sentry_capture = False
                    raise

                for row in rows:
                    # It might be bad to write one row at a time from a performance perspective.
                    # Regardless, we should handle the batching of rows in the _Writer class, not here.
                    writer.write(table, row)

                total_rows += len(rows)

            track_load()
            if progress_tracker:
                progress_manager.set_progress(row_number + 1, documents.count)

    end = _time_in_milliseconds()
    tags = {'format': writer.format}
    _record_datadog_export_duration(end - start, total_bytes, total_rows, tags)
    _record_export_duration(end - start, export_instance)
def write_export_instance ( writer , export_instance , documents , progress_tracker = None ) : with TaskProgressManager ( progress_tracker , src = "export" ) as progress_manager : if progress_tracker : progress_manager . set_progress ( 0 , documents . count ) start = _time_in_milliseconds ( ) total_bytes = 0 total_rows = 0 track_load = load_counter ( export_instance . type , "export" , export_instance . domain ) for row_number , doc in enumerate ( documents ) : total_bytes += sys . getsizeof ( doc ) for table in export_instance . selected_tables : try : rows = table . get_rows ( doc , row_number , split_columns = export_instance . split_multiselects , transform_dates = export_instance . transform_dates , ) except Exception as e : notify_exception ( None , "Error exporting doc" , details = { 'domain' : export_instance . domain , 'export_instance_id' : export_instance . get_id , 'export_table' : table . label , 'doc_id' : doc . get ( '_id' ) , } ) e . sentry_capture = False raise for row in rows : # It might be bad to write one row at a time from a performance perspective. # Regardless, we should handle the batching of rows in the _Writer class, not here. writer . write ( table , row ) total_rows += len ( rows ) track_load ( ) if progress_tracker : progress_manager . set_progress ( row_number + 1 , documents . count ) end = _time_in_milliseconds ( ) tags = { 'format' : writer . format } _record_datadog_export_duration ( end - start , total_bytes , total_rows , tags ) _record_export_duration ( end - start , export_instance )