()
Sync data that is neither event or page based with hubspot/Kissmetrics
    :return:
Sync data that is neither event or page based with hubspot/Kissmetrics
    :return:
Sync data that is neither event or page based with hubspot / Kissmetrics : return :
def track_periodic_data():
    """
    Sync data that is neither event or page based with hubspot/Kissmetrics
    :return:
    """
    # Start by getting a list of web users mapped to their domains

    if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED:
        return

    time_started = datetime.utcnow()

    three_months_ago = date.today() - timedelta(days=90)

    user_query = (UserES()
                  .web_users()
                  .last_logged_in(gte=three_months_ago)
                  .sort('date_joined', desc=True)
                  .source(['domains', 'email', 'date_joined', 'username'])
                  .analytics_enabled())

    total_users = user_query.count()
    chunk_size = 100
    num_chunks = int(math.ceil(float(total_users) / float(chunk_size)))

    # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD
    hubspot_number_of_users_processed = 0
    hubspot_number_of_domains_with_forms_gt_threshold = 0
    hubspot_number_of_users_blocked = 0

    blocked_domains = get_blocked_hubspot_domains()
    blocked_users = emails_that_accepted_invitations_to_blocked_hubspot_domains()

    for chunk in range(num_chunks):
        users_to_domains = (user_query
                            .size(chunk_size)
                            .start(chunk * chunk_size)
                            .run()
                            .hits)

        # users_to_domains is a list of dicts
        domains_to_forms = (FormES()
                            .terms_aggregation('domain.exact', 'domain')
                            .size(0)
                            .run()
                            .aggregations.domain.counts_by_bucket())
        domains_to_mobile_users = (UserES()
                                   .mobile_users()
                                   .terms_aggregation('domain.exact', 'domain')
                                   .size(0)
                                   .run()
                                   .aggregations
                                   .domain
                                   .counts_by_bucket())

        # Keep track of india and www data seperately
        env = get_instance_string()

        for num_forms in domains_to_forms.values():
            if num_forms > HUBSPOT_THRESHOLD:
                hubspot_number_of_domains_with_forms_gt_threshold += 1

        # For each web user, iterate through their domains and select the max number of form submissions and
        # max number of mobile workers
        submit = []
        for user in users_to_domains:
            email = user.get('email') or user.get('username')
            if not _email_is_valid(email):
                continue

            if (user.get('email') in blocked_users
                    or user.get('username') in blocked_users):
                # User had accepted an invitation to a project space whose
                # Billing Account has blocked HubSpot analytics, so we
                # should not send any data about them going forward
                metrics_counter(
                    'commcare.hubspot_data.rejected.periodic_task.invitation',
                )
                hubspot_number_of_users_blocked += 1
                continue

            date_created = user.get('date_joined')
            max_forms = 0
            max_workers = 0
            max_export = 0
            max_report = 0

            is_member_of_blocked_domain = False
            for domain in user['domains']:
                if domain in blocked_domains:
                    metrics_counter(
                        'commcare.hubspot_data.rejected.periodic_task.domain',
                        tags={
                            'domain': domain,
                        }
                    )
                    is_member_of_blocked_domain = True
                    break
                if domain in domains_to_forms and domains_to_forms[domain] > max_forms:
                    max_forms = domains_to_forms[domain]
                if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers:
                    max_workers = domains_to_mobile_users[domain]
                if _get_export_count(domain) > max_export:
                    max_export = _get_export_count(domain)
                if _get_report_count(domain) > max_report:
                    max_report = _get_report_count(domain)

            if is_member_of_blocked_domain:
                # user is a member of a project space whose Billing Account
                # has blocked HubSpot analytics, so we must not send any data
                # about them.
                hubspot_number_of_users_blocked += 1
                continue

            hubspot_number_of_users_processed += 1

            project_spaces_created = ", ".join(get_domains_created_by_user(email))

            user_json = {
                'email': email,
                'properties': [
                    {
                        'property': '{}max_form_submissions_in_a_domain'.format(env),
                        'value': max_forms
                    },
                    {
                        'property': '{}max_mobile_workers_in_a_domain'.format(env),
                        'value': max_workers
                    },
                    {
                        'property': '{}project_spaces_created_by_user'.format(env),
                        'value': project_spaces_created,
                    },
                    {
                        'property': '{}over_300_form_submissions'.format(env),
                        'value': max_forms > HUBSPOT_THRESHOLD
                    },
                    {
                        'property': '{}date_created'.format(env),
                        'value': date_created
                    },
                    {
                        'property': '{}max_exports_in_a_domain'.format(env),
                        'value': max_export
                    },
                    {
                        'property': '{}max_custom_reports_in_a_domain'.format(env),
                        'value': max_report
                    }
                ]
            }
            submit.append(user_json)

        submit_json = json.dumps(submit)
        submit_data_to_hub_and_kiss(submit_json)

    metrics_gauge('commcare.hubspot.web_users_processed', hubspot_number_of_users_processed,
        multiprocess_mode=MPM_LIVESUM)
    metrics_gauge('commcare.hubspot.web_users_blocked', hubspot_number_of_users_blocked,
        multiprocess_mode=MPM_LIVESUM)
    metrics_gauge(
        'commcare.hubspot.domains_with_forms_gt_threshold', hubspot_number_of_domains_with_forms_gt_threshold,
        multiprocess_mode=MPM_MAX
    )

    task_time = datetime.utcnow() - time_started
    metrics_gauge(
        'commcare.hubspot.runtimes.track_periodic_data',
        task_time.seconds,
        multiprocess_mode=MPM_LIVESUM
    )
def track_periodic_data ( ) : # Start by getting a list of web users mapped to their domains if not KISSMETRICS_ENABLED and not HUBSPOT_ENABLED : return time_started = datetime . utcnow ( ) three_months_ago = date . today ( ) - timedelta ( days = 90 ) user_query = ( UserES ( ) . web_users ( ) . last_logged_in ( gte = three_months_ago ) . sort ( 'date_joined' , desc = True ) . source ( [ 'domains' , 'email' , 'date_joined' , 'username' ] ) . analytics_enabled ( ) ) total_users = user_query . count ( ) chunk_size = 100 num_chunks = int ( math . ceil ( float ( total_users ) / float ( chunk_size ) ) ) # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD hubspot_number_of_users_processed = 0 hubspot_number_of_domains_with_forms_gt_threshold = 0 hubspot_number_of_users_blocked = 0 blocked_domains = get_blocked_hubspot_domains ( ) blocked_users = emails_that_accepted_invitations_to_blocked_hubspot_domains ( ) for chunk in range ( num_chunks ) : users_to_domains = ( user_query . size ( chunk_size ) . start ( chunk * chunk_size ) . run ( ) . hits ) # users_to_domains is a list of dicts domains_to_forms = ( FormES ( ) . terms_aggregation ( 'domain.exact' , 'domain' ) . size ( 0 ) . run ( ) . aggregations . domain . counts_by_bucket ( ) ) domains_to_mobile_users = ( UserES ( ) . mobile_users ( ) . terms_aggregation ( 'domain.exact' , 'domain' ) . size ( 0 ) . run ( ) . aggregations . domain . counts_by_bucket ( ) ) # Keep track of india and www data seperately env = get_instance_string ( ) for num_forms in domains_to_forms . values ( ) : if num_forms > HUBSPOT_THRESHOLD : hubspot_number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [ ] for user in users_to_domains : email = user . get ( 'email' ) or user . get ( 'username' ) if not _email_is_valid ( email ) : continue if ( user . get ( 'email' ) in blocked_users or user . get ( 'username' ) in blocked_users ) : # User had accepted an invitation to a project space whose # Billing Account has blocked HubSpot analytics, so we # should not send any data about them going forward metrics_counter ( 'commcare.hubspot_data.rejected.periodic_task.invitation' , ) hubspot_number_of_users_blocked += 1 continue date_created = user . get ( 'date_joined' ) max_forms = 0 max_workers = 0 max_export = 0 max_report = 0 is_member_of_blocked_domain = False for domain in user [ 'domains' ] : if domain in blocked_domains : metrics_counter ( 'commcare.hubspot_data.rejected.periodic_task.domain' , tags = { 'domain' : domain , } ) is_member_of_blocked_domain = True break if domain in domains_to_forms and domains_to_forms [ domain ] > max_forms : max_forms = domains_to_forms [ domain ] if domain in domains_to_mobile_users and domains_to_mobile_users [ domain ] > max_workers : max_workers = domains_to_mobile_users [ domain ] if _get_export_count ( domain ) > max_export : max_export = _get_export_count ( domain ) if _get_report_count ( domain ) > max_report : max_report = _get_report_count ( domain ) if is_member_of_blocked_domain : # user is a member of a project space whose Billing Account # has blocked HubSpot analytics, so we must not send any data # about them. hubspot_number_of_users_blocked += 1 continue hubspot_number_of_users_processed += 1 project_spaces_created = ", " . join ( get_domains_created_by_user ( email ) ) user_json = { 'email' : email , 'properties' : [ { 'property' : '{}max_form_submissions_in_a_domain' . format ( env ) , 'value' : max_forms } , { 'property' : '{}max_mobile_workers_in_a_domain' . format ( env ) , 'value' : max_workers } , { 'property' : '{}project_spaces_created_by_user' . format ( env ) , 'value' : project_spaces_created , } , { 'property' : '{}over_300_form_submissions' . format ( env ) , 'value' : max_forms > HUBSPOT_THRESHOLD } , { 'property' : '{}date_created' . format ( env ) , 'value' : date_created } , { 'property' : '{}max_exports_in_a_domain' . format ( env ) , 'value' : max_export } , { 'property' : '{}max_custom_reports_in_a_domain' . format ( env ) , 'value' : max_report } ] } submit . append ( user_json ) submit_json = json . dumps ( submit ) submit_data_to_hub_and_kiss ( submit_json ) metrics_gauge ( 'commcare.hubspot.web_users_processed' , hubspot_number_of_users_processed , multiprocess_mode = MPM_LIVESUM ) metrics_gauge ( 'commcare.hubspot.web_users_blocked' , hubspot_number_of_users_blocked , multiprocess_mode = MPM_LIVESUM ) metrics_gauge ( 'commcare.hubspot.domains_with_forms_gt_threshold' , hubspot_number_of_domains_with_forms_gt_threshold , multiprocess_mode = MPM_MAX ) task_time = datetime . utcnow ( ) - time_started metrics_gauge ( 'commcare.hubspot.runtimes.track_periodic_data' , task_time . seconds , multiprocess_mode = MPM_LIVESUM )