Open Source log & event analytics platform
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin

from accounts.forms import CustomUserCreationForm, CustomUserChangeForm
from accounts.models import Org, User, Account, AccountApiToken


@admin.register(Org)
class OrgAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "name",
        "created_at",
    ]


@admin.register(Account)
class AccountAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "created_at",
        "owner",
    ]


@admin.register(User)
class CustomUserAdmin(UserAdmin):
    add_form = CustomUserCreationForm
    form = CustomUserChangeForm
    model = User
    list_display = ("email", "org", "account", "is_staff", "is_active",)
    list_filter = ("email", "org", "account", "is_staff", "is_active",)
    fieldsets = (
        (None, {"fields": ("email", "password")}),
        ("Permissions", {"fields": ("is_staff", "is_active", "groups", "user_permissions")}),
        ("org", {"fields": ("org", "account",)}),
    )
    add_fieldsets = (
        (None, {
            "classes": ("wide",),
            "fields": (
                "email", "password1", "password2", "is_staff",
                "is_active", "groups", "user_permissions", "org", "account",
            )}
         ),
    )
    search_fields = ("email",)
    ordering = ("email",)


@admin.register(AccountApiToken)
class AccountApiTokenAdmin(admin.ModelAdmin):
    list_display = [
        "key",
        "account",
        "created_by",
        "created_at",
    ]
    search_fields = ("account",)
    list_filter = ("created_by", "account",)
    fieldsets = (
        (None, {"fields": ("account", "created_by")}),
    )

accounts/admin.py
DrDroidLab/kenobi
from allauth.account.adapter import DefaultAccountAdapter
from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.template.loader import render_to_string
from rest_framework import exceptions, HTTP_HEADER_ENCODING
from rest_framework.authentication import TokenAuthentication

from accounts.cache import GLOBAL_ACCOUNT_API_TOKEN_CACHE
from accounts.models import AccountApiToken, AccountApiTokenUser
from prototype.utils.headers import X_REQUEST_AWS_FIREHOSE
from prototype.utils.utils import build_absolute_uri


class AccountApiTokenAuthentication(TokenAuthentication):
    model = AccountApiToken
    keyword = 'Bearer'

    def authenticate_credentials(self, key):
        token = GLOBAL_ACCOUNT_API_TOKEN_CACHE.get(api_key=key)
        return AccountApiTokenUser(token.account_id), token


class AwsKinesisApiTokenAuthentication(AccountApiTokenAuthentication):
    def authenticate(self, request):
        request_firehose_access_header = request.headers[X_REQUEST_AWS_FIREHOSE]

        if not request_firehose_access_header:
            raise exceptions.AuthenticationFailed(f'Missing {X_REQUEST_AWS_FIREHOSE} header')

        auth = request_firehose_access_header
        if isinstance(request_firehose_access_header, str):
            # Work around django test client oddness
            auth = request_firehose_access_header.encode(HTTP_HEADER_ENCODING)

        auth = auth.split()
        if not auth or auth[0].lower() != self.keyword.lower().encode():
            return None

        if len(auth) == 1:
            raise exceptions.AuthenticationFailed('Invalid token header. No credentials provided.')
        elif len(auth) > 2:
            raise exceptions.AuthenticationFailed('Invalid token header. Token string should not contain spaces.')

        try:
            token = auth[1].decode()
        except UnicodeError:
            raise exceptions.AuthenticationFailed(
                'Invalid token header. Token string should not contain invalid characters.')

        return self.authenticate_credentials(token)


def user_display(user):
    return user.email


class AccountAdapter(DefaultAccountAdapter):
    def get_email_confirmation_url(self, request, emailconfirmation):
        location = settings.ACCOUNT_EMAIL_VERIFICATION_URL.format(emailconfirmation.key)
        protocol = settings.ACCOUNT_DEFAULT_HTTP_PROTOCOL
        enabled = settings.EMAIL_USE_SITES
        if enabled:
            uri = build_absolute_uri(None, location, protocol, enabled)
        else:
            uri = build_absolute_uri(request, location, protocol, enabled)
        return uri

    def render_mail(self, template_prefix, email, context, headers=None):
        if not template_prefix == "account/email/email_confirmation_signup" or \
                template_prefix == "account/email/email_confirmation":
            return super().render_mail(template_prefix, email, context, headers)

        to = [email] if isinstance(email, str) else email
        subject = render_to_string("email_confirmation_subject.txt", context)
        # remove superfluous line breaks
        subject = " ".join(subject.splitlines()).strip()
        subject = self.format_email_subject(subject)

        from_email = self.get_from_email()

        context["custom_user_name"] = context["user"].first_name + " " + context["user"].last_name

        text_body = render_to_string(
            "email_confirmation_message.txt",
            context,
            self.request,
        ).strip()

        html_body = render_to_string(
            "email_confirmation_message.html",
            context,
            self.request,
        ).strip()

        msg = EmailMultiAlternatives(subject, text_body, from_email, to, headers=headers)
        msg.attach_alternative(html_body, "text/html")
        return msg

accounts/authentication.py
DrDroidLab/kenobi
from django.conf import settings
from django.core.cache import caches
from rest_framework import exceptions

from accounts.models import AccountApiToken, Account


class AccountApiTokenCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled
        # Set the timeout to 30 seconds for api keys.
        self._timeout = 30

    def _get(self, api_key: str):
        try:
            token = AccountApiToken.objects.get(key=api_key)
        except AccountApiToken.DoesNotExist:
            raise exceptions.AuthenticationFailed('Invalid token.')
        return token

    def get(self, api_key):
        if not self._enabled:
            return self._get(api_key)

        key = f'account_api_token:{api_key}'
        value = caches[self._cache_key].get(key)
        if value is not None:
            return value
        token = self._get(api_key)
        caches[self._cache_key].set(key, token, timeout=self._timeout)
        return token


class AccountCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def _is_whitelisted(self, account_id):
        return Account.objects.get(id=account_id).is_whitelisted

    def is_whitelisted(self, account_id):
        if not self._enabled:
            return self._is_whitelisted(account_id)
        key = f'account:{account_id}:is_whitelisted'
        value = caches[self._cache_key].get(key)
        if value is not None:
            return value
        enabled = self._is_whitelisted(account_id)
        caches[self._cache_key].set(key, enabled, timeout=30)
        return enabled

    def daily_account_quota(self, date, account_id, quota):
        key = f'account:{account_id}:{quota}:{date}'
        value = caches[self._cache_key].get(key)
        if value is not None:
            return value
        caches[self._cache_key].set(key, 0, timeout=86400)
        return 0

    def incr_daily_account_quota(self, date, account_id, quota, delta):
        key = f'account:{account_id}:{quota}:{date}'
        try:
            caches[self._cache_key].incr(key, delta)
        except ValueError:
            caches[self._cache_key].set(key, delta, timeout=86400)


GLOBAL_ACCOUNT_API_TOKEN_CACHE = AccountApiTokenCache(**settings.GLOBAL_ACCOUNT_API_TOKEN_CACHE)
GLOBAL_ACCOUNT_CACHE = AccountCache(**settings.GLOBAL_ACCOUNT_CACHE)

accounts/cache.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-14 03:34

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='Org',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=64, unique=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
            ],
        ),
        migrations.AddField(
            model_name='user',
            name='org',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='accounts.org'),
        ),
        migrations.AlterUniqueTogether(
            name='user',
            unique_together={('org', 'email')},
        ),
    ]

accounts/migrations/0002_org_user_org_alter_user_unique_together.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-17 06:26

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0002_org_user_org_alter_user_unique_together'),
    ]

    operations = [
        migrations.CreateModel(
            name='ApiToken',
            fields=[
                ('key', models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name='Key')),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='api_token', to=settings.AUTH_USER_MODEL, verbose_name='User')),
                ('org', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='api_token', to='accounts.org')),
            ],
            options={
                'verbose_name': 'ApiToken',
                'verbose_name_plural': 'ApiTokens',
            },
        ),
    ]

accounts/migrations/0003_apitoken.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-25 05:10

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0003_apitoken'),
    ]

    operations = [
        migrations.CreateModel(
            name='Account',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='owned_account', to=settings.AUTH_USER_MODEL, verbose_name='Owner')),
            ],
            options={
                'verbose_name': 'account',
                'verbose_name_plural': 'accounts',
                'unique_together': {('owner',)},
            },
        ),
        migrations.CreateModel(
            name='AccountApiToken',
            fields=[
                ('key', models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name='Key')),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='account_api_token', to='accounts.account')),
                ('created_by', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='account_api_token', to=settings.AUTH_USER_MODEL, verbose_name='User')),
            ],
            options={
                'verbose_name': 'AccountApiToken',
                'verbose_name_plural': 'AccountApiTokens',
            },
        ),
        migrations.AlterUniqueTogether(
            name='user',
            unique_together=set(),
        ),
        migrations.AddField(
            model_name='user',
            name='account',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='accounts.account'),
        ),
        migrations.AlterUniqueTogether(
            name='user',
            unique_together={('account', 'org', 'email')},
        ),
    ]

accounts/migrations/0004_account_accountapitoken_alter_user_unique_together_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-27 18:08

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0004_account_accountapitoken_alter_user_unique_together_and_more'),
    ]

    operations = [
        migrations.AlterField(
            model_name='account',
            name='owner',
            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='owned_account', to=settings.AUTH_USER_MODEL, verbose_name='Owner'),
        ),
    ]

accounts/migrations/0005_alter_account_owner.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-06-16 10:04

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
    ]

    operations = [
        migrations.AddField(
            model_name='account',
            name='is_whitelisted',
            field=models.BooleanField(default=False),
        ),
    ]

accounts/migrations/0006_account_is_whitelisted.py
DrDroidLab/kenobi
import os

import binascii
from allauth.account.utils import has_verified_email
from django.conf import settings
from django.contrib.auth.base_user import BaseUserManager
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.utils.functional import cached_property
from google.protobuf.timestamp_pb2 import Timestamp
from google.protobuf.wrappers_pb2 import BoolValue

from protos.event.account_pb2 import AccountApiToken as AccountApiTokenProto, User as UserProto, UserFlags
from prototype.threadlocal import get_current_request_org, get_current_request, set_current_request_org, \
    get_current_request_account, set_current_request_account, get_current_request_user, set_current_request_user
from prototype.utils.headers import X_REQUEST_ORG


class Account(models.Model):
    created_at = models.DateTimeField(auto_now_add=True)
    owner = models.ForeignKey(
        settings.AUTH_USER_MODEL, related_name='owned_account',
        on_delete=models.SET_NULL, verbose_name="Owner", null=True
    )
    is_whitelisted = models.BooleanField(default=False)

    class Meta:
        verbose_name = "account"
        verbose_name_plural = "accounts"
        unique_together = [['owner'], ]

    def __str__(self):
        return f'{self.id}'


class MissingAccountException(ValueError):
    pass


def get_request_account():
    account: Account = get_current_request_account()
    if account:
        return account

    user: User = get_request_user()

    if user:
        account: Account = user.account
        set_current_request_account(account)
    else:
        raise MissingAccountException(f'Missing user info')

    return account


def get_request_user():
    user: User = get_current_request_user()
    if user:
        return user

    request = get_current_request()

    if request.user.is_authenticated:
        user = request.user
        set_current_request_user(user)
    else:
        raise MissingAccountException(f'User is unanuthenticated. Missing user info')

    return user


class MissingOrgHeaderException(ValueError):
    pass


class Org(models.Model):
    name = models.CharField(max_length=64, unique=True, db_index=True)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return f'{self.name}'


def get_request_org():
    org: Org = get_current_request_org()
    if org:
        return org

    request = get_current_request()

    if request.user.is_authenticated:
        user = request.user
        org: Org = user.org
    else:
        # Workflow definitions for the workflow
        request_org_name = request.headers[X_REQUEST_ORG]
        if not request_org_name:
            raise MissingOrgHeaderException(f'{X_REQUEST_ORG} missing from request')
        org: Org = Org.objects.get(name=request_org_name)
    set_current_request_org(org)
    return org


class UserManager(BaseUserManager):
    """
    Custom user model manager where email is the unique identifiers
    for authentication instead of usernames.
    """

    def create_user(self, email, password, **extra_fields):
        """
        Create and save a user with the given email and password.
        """
        if not email:
            raise ValueError("The Email must be set")
        email = self.normalize_email(email)
        user = self.model(email=email, **extra_fields)
        user.set_password(password)
        user.save()
        return user

    def create_superuser(self, email, password, **extra_fields):
        """
        Create and save a SuperUser with the given email and password.
        """
        extra_fields.setdefault("is_staff", True)
        extra_fields.setdefault("is_superuser", True)
        extra_fields.setdefault("is_active", True)

        if extra_fields.get("is_staff") is not True:
            raise ValueError("Superuser must have is_staff=True.")
        if extra_fields.get("is_superuser") is not True:
            raise ValueError("Superuser must have is_superuser=True.")
        return self.create_user(email, password, **extra_fields)


class User(AbstractUser):
    username = None
    email = models.EmailField(
        verbose_name="email address",
        max_length=255,
        unique=True,
    )
    org = models.ForeignKey(Org, on_delete=models.CASCADE, null=True, blank=True)
    account = models.ForeignKey(Account, on_delete=models.CASCADE, null=True, blank=True)
    EMAIL_FIELD = "email"
    USERNAME_FIELD = "email"
    REQUIRED_FIELDS = []

    objects = UserManager()

    class Meta:
        verbose_name = "user"
        verbose_name_plural = "users"
        abstract = False
        unique_together = [['account', 'org', 'email'], ]

    def __str__(self):
        if self.account:
            return f'{self.account}:{self.email}'
        elif self.org:
            return f'{self.org}:{self.email}'
        else:
            return self.email

    @property
    def user_flags_proto(self):
        return UserFlags(
            is_email_verified=BoolValue(value=has_verified_email(self)),
            is_account_owner=BoolValue(value=(self.account.owner == self))
        )

    @property
    def proto(self):
        return UserProto(id=self.id, email=self.email, first_name=self.first_name, last_name=self.last_name,
                         user_flags=self.user_flags_proto)


class ApiToken(models.Model):
    key = models.CharField("Key", max_length=40, primary_key=True)
    org = models.ForeignKey(Org, related_name='api_token', on_delete=models.CASCADE)
    created_by = models.ForeignKey(
        settings.AUTH_USER_MODEL, related_name='api_token',
        on_delete=models.CASCADE, verbose_name="User"
    )
    created_at = models.DateTimeField(auto_now_add=True)

    class Meta:
        verbose_name = 'ApiToken'
        verbose_name_plural = 'ApiTokens'

    def __str__(self):
        return f'{self.org_id}:{self.key}'

    @classmethod
    def generate_key(cls):
        return binascii.hexlify(os.urandom(20)).decode()

    def save(self, *args, **kwargs):
        if not self.key:
            self.key = self.generate_key()
        return super().save(*args, **kwargs)

    @cached_property
    def created_at_timestamp(self):
        timestamp = Timestamp()
        timestamp.FromDatetime(self.created_at)
        return timestamp


class ApiTokenUser:
    is_active = True

    def __init__(self, org_id):
        self._org_id = org_id

    def __str__(self):
        return f'OrgUser {self._org_id}'

    @cached_property
    def org(self):
        return Org(id=self._org_id)

    @cached_property
    def org_id(self):
        return self._org_id

    def __eq__(self, other):
        return self.org_id == other.org_id

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.org_id)

    @property
    def is_anonymous(self):
        return False

    @property
    def is_authenticated(self):
        return True

    def save(self):
        raise NotImplementedError("Token users have no DB representation")

    def delete(self):
        raise NotImplementedError("Token users have no DB representation")

    def set_password(self, raw_password):
        raise NotImplementedError("Token users have no DB representation")

    def check_password(self, raw_password):
        raise NotImplementedError("Token users have no DB representation")


class AccountApiToken(models.Model):
    key = models.CharField("Key", max_length=40, primary_key=True)
    account = models.ForeignKey(Account, related_name='account_api_token', on_delete=models.CASCADE)
    created_by = models.ForeignKey(
        settings.AUTH_USER_MODEL, related_name='account_api_token',
        on_delete=models.CASCADE, verbose_name="User"
    )
    created_at = models.DateTimeField(auto_now_add=True)

    class Meta:
        verbose_name = 'AccountApiToken'
        verbose_name_plural = 'AccountApiTokens'

    def __str__(self):
        return f'{self.account_id}:{self.key}'

    @classmethod
    def generate_key(cls):
        return binascii.hexlify(os.urandom(20)).decode()

    def save(self, *args, **kwargs):
        if not self.key:
            self.key = self.generate_key()
        return super().save(*args, **kwargs)

    @cached_property
    def created_at_timestamp(self):
        timestamp = Timestamp()
        timestamp.FromDatetime(self.created_at)
        return timestamp

    @cached_property
    def proto(self):
        return AccountApiTokenProto(
            key=self.key,
            created_by=self.created_by.email,
            created_at=self.created_at_timestamp,
            account_id=self.account_id
        )


class AccountApiTokenUser:
    is_active = True

    def __init__(self, account_id):
        self._account_id = account_id

    def __str__(self):
        return f'AccountUser {self._account_id}'

    @cached_property
    def account(self):
        return Account(id=self._account_id)

    @cached_property
    def account_id(self):
        return self._account_id

    def __eq__(self, other):
        return self.account_id == other.account_id

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.account_id)

    @property
    def is_anonymous(self):
        return False

    @property
    def is_authenticated(self):
        return True

    def save(self):
        raise NotImplementedError("Token users have no DB representation")

    def delete(self):
        raise NotImplementedError("Token users have no DB representation")

    def set_password(self, raw_password):
        raise NotImplementedError("Token users have no DB representation")

    def check_password(self, raw_password):
        raise NotImplementedError("Token users have no DB representation")

accounts/models.py
DrDroidLab/kenobi
import requests
from dj_rest_auth.registration.serializers import RegisterSerializer
from dj_rest_auth.serializers import LoginSerializer
from django.conf import settings
from rest_framework import serializers


class AccountLoginSerializer(LoginSerializer):
    username = None
    email = serializers.EmailField(required=True)
    password = serializers.CharField(style={'input_type': 'password'})


class AccountRegisterSerializer(RegisterSerializer):
    username = None
    password1 = None
    password2 = None

    email = serializers.EmailField(required=True)
    password = serializers.CharField(required=True, write_only=True)
    first_name = serializers.CharField(required=True)
    last_name = serializers.CharField(required=True)

    def validate_password(self, password):
        return self.validate_password1(password)

    def validate(self, data):
        return data

    def custom_signup(self, request, user):
        pass

    def get_cleaned_data(self):
        return {
            'password1': self.validated_data.get('password', ''),
            'email': self.validated_data.get('email', ''),
            'first_name': self.validated_data.get('first_name', ''),
            'last_name': self.validated_data.get('last_name', ''),
        }

accounts/serializer.py
DrDroidLab/kenobi
from allauth.account.signals import email_confirmed
from django.contrib.auth import get_user_model
from django.db.models.signals import post_save
from django.dispatch import receiver

from accounts import tasks
from accounts.models import User, Account, AccountApiToken
from prototype.utils.decorators import skip_signal


@receiver(post_save, sender=User)
@skip_signal()
def create_user_account(sender, instance, created, **kwargs):
    instance.skip_signal = True
    if not created:
        return
    if instance.account:
        return
    if instance.is_staff:
        return
    account = Account(owner=instance)
    account.save()
    print(f'Created account for user: {instance}')

    instance.account = account
    instance.save()
    print(f'Associated account for user: {instance}')
    tasks.setup_account.delay(account.id)
    print(f'Created account setup task for: {instance}')

    log_dict = {"msg": "Signup_New_User", "user_email": instance.email, "user_id": instance.id}
    print(log_dict)


@receiver(email_confirmed)
def generate_account_token_on_email_confirmed(request, email_address, **kwargs):
    user_model = get_user_model()
    try:
        user = user_model.objects.get(email=email_address)
        account = Account.objects.get(owner=user)
        token = AccountApiToken(account=account, created_by=user)
        token.save()
        print(f'Created account api token for account: {account}')
    except Exception as e:
        pass

accounts/signals.py
DrDroidLab/kenobi
from celery import shared_task

from accounts.models import Account
from event.utils.account_data import setup_account_event_data
from utils.error_utils import error_dict


@shared_task
def setup_account(account_id: int):
    try:
        account = Account.objects.get(id=account_id)
        setup_account_event_data(account)
        return {'account_id': account_id, 'setup_status': 'SUCCESS'}
    except Account.DoesNotExist:
        return {'account_id': account_id, 'setup_status': 'FAILED', 'message': 'account setup failed',
                'error': f"account id:{account_id} doesn't exist"}
    except Exception as e:
        return {'account_id': account_id, 'setup_status': 'FAILED', **error_dict('account setup failed', e)}

accounts/tasks.py
DrDroidLab/kenobi
from dj_rest_auth.registration.views import VerifyEmailView, ResendEmailVerificationView, RegisterView
from dj_rest_auth.views import LoginView, LogoutView
from django.conf import settings
from django.urls import path, re_path
from django.views.generic import TemplateView

from accounts import views

accounturlpatterns = [
    # URLs that do not require a session or valid token
    path('login/', LoginView.as_view(), name='rest_login'),
    # URLs that require a user to be logged in with a valid session / token.
    path('logout/', LogoutView.as_view(), name='rest_logout'),
    path('user/', views.get_user, name='rest_user_details'),
    path('signup/', RegisterView.as_view(), name='rest_signup'),
    path('verify-email/', VerifyEmailView.as_view(), name='rest_verify_email'),
    path('resend-email/', ResendEmailVerificationView.as_view(), name="rest_resend_email"),

    re_path(
        r'^confirm-email-link/(?P<key>[-:\w]+)/$', views.confirm_email_link,
        name='account_confirm_email',
    ),
    path(
        'account-email-verification-sent/', TemplateView.as_view(),
        name='account_email_verification_sent',
    ),
]

if getattr(settings, 'REST_USE_JWT', False):
    from rest_framework_simplejwt.views import TokenVerifyView

    from dj_rest_auth.jwt_auth import get_refresh_view

    accounturlpatterns += [
        path('token/verify/', TokenVerifyView.as_view(), name='token_verify'),
        path('token/refresh/', get_refresh_view().as_view(), name='token_refresh'),
    ]

urlpatterns = [
    *accounturlpatterns,
    path('account_api_tokens', views.get_account_api_tokens),
    path('account_api_tokens/create', views.create_account_api_token),
    path('account_api_tokens/delete', views.delete_account_api_token),

    # path('registration/', include('dj_rest_auth.registration.urls')),
    # path('auth/', include('allauth.urls')),
]

accounts/urls.py
DrDroidLab/kenobi
from allauth.account.utils import has_verified_email
from django.conf import settings

from accounts.cache import GLOBAL_ACCOUNT_CACHE
from prototype.threadlocal import get_current_request
from utils.time import get_current_day_epoch

_event_count_quota_key = 'event_count'


class AccountDailyEventQuotaReached(Exception):
    pass


def get_account_quota_error_message():
    return f'Account reached daily event quota. Please reach out to enable more events for account.'


def check_account_event_quota(account_id, event_count):
    date = get_current_day_epoch()
    GLOBAL_ACCOUNT_CACHE.incr_daily_account_quota(date, account_id, _event_count_quota_key, event_count)
    return GLOBAL_ACCOUNT_CACHE.daily_account_quota(
        date, account_id,
        _event_count_quota_key
    ) <= settings.ACCOUNT_DAILY_EVENT_QUOTA


def do_process_events(account_id, event_count):
    if GLOBAL_ACCOUNT_CACHE.is_whitelisted(account_id):
        return True
    if check_account_event_quota(account_id, event_count):
        return True
    raise AccountDailyEventQuotaReached()


def is_request_user_email_verified():
    user = get_current_request().user
    return has_verified_email(user)

accounts/utils.py
DrDroidLab/kenobi
from typing import Union

from allauth.account.models import EmailConfirmationHMAC, EmailConfirmation
from django.http import HttpResponse, HttpRequest, Http404
from django.views.decorators.csrf import csrf_exempt
from google.protobuf.wrappers_pb2 import BoolValue

from accounts.models import get_request_account, Account, AccountApiToken
from accounts.utils import is_request_user_email_verified
from protos.event.api_pb2 import GetAccountApiTokensRequest, GetAccountApiTokensResponse, \
    CreateAccountApiTokenRequest, CreateAccountApiTokenResponse, DeleteAccountApiTokenRequest, \
    DeleteAccountApiTokenResponse, GetUserRequest, GetUserResponse
from prototype.threadlocal import get_current_request
from prototype.utils.decorators import web_api
from prototype.utils.meta import get_meta
from prototype.utils.queryset import filter_page


@web_api(GetAccountApiTokensRequest)
def get_account_api_tokens(request_message: GetAccountApiTokensRequest) -> Union[
    GetAccountApiTokensResponse, HttpResponse]:
    if not is_request_user_email_verified():
        return GetAccountApiTokensResponse(meta=get_meta(), account_api_tokens=[])

    account: Account = get_request_account()
    qs = account.account_api_token.all()

    total_count = qs.count()
    page = request_message.meta.page
    account_api_tokens = [account_api_token.proto for account_api_token in
                          filter_page(qs.order_by("-created_at"), page)]

    return GetAccountApiTokensResponse(
        meta=get_meta(page=page, total_count=total_count),
        account_api_tokens=account_api_tokens)


@web_api(CreateAccountApiTokenRequest)
def create_account_api_token(request_message: CreateAccountApiTokenRequest) -> Union[
    CreateAccountApiTokenResponse, HttpResponse]:
    if not is_request_user_email_verified():
        return CreateAccountApiTokenResponse(success=BoolValue(value=False))

    user = get_current_request().user
    account: Account = get_request_account()

    api_token = AccountApiToken(account=account, created_by=user)
    api_token.save()
    return CreateAccountApiTokenResponse(success=BoolValue(value=True), account_api_token=api_token.proto)


@web_api(DeleteAccountApiTokenRequest)
def delete_account_api_token(request_message: DeleteAccountApiTokenRequest) -> Union[
    DeleteAccountApiTokenResponse, HttpResponse]:
    if not is_request_user_email_verified():
        return DeleteAccountApiTokenResponse(success=BoolValue(value=False))

    account: Account = get_request_account()
    key: str = request_message.account_api_token_key
    if key == "":
        return DeleteAccountApiTokenResponse(success=BoolValue(value=False))
    try:
        api_token = AccountApiToken.objects.get(key=key, account=account)
    except AccountApiToken.DoesNotExist:
        return DeleteAccountApiTokenResponse(success=BoolValue(value=False))

    api_token.delete()
    return DeleteAccountApiTokenResponse(success=BoolValue(value=True))


@csrf_exempt
def confirm_email_link(request: HttpRequest, key):
    email_confirmation = EmailConfirmationHMAC.from_key(key)
    if not email_confirmation:
        queryset = EmailConfirmation.objects.all_valid()
        queryset = queryset.select_related("email_address__user")
        try:
            email_confirmation = queryset.get(key=key.lower())
        except EmailConfirmation.DoesNotExist:
            raise Http404()
    email_confirmation.confirm(request)
    return HttpResponse({'success': True})


@web_api(GetUserRequest)
def get_user(request_message: GetUserRequest) -> Union[GetUserResponse, HttpResponse]:
    request = get_current_request()
    user = request.user
    return GetUserResponse(user=user.proto)

accounts/views.py
DrDroidLab/kenobi
from django.contrib import admin

from connectors.models import ConnectorRequest, TransformerMapping, Connector, ConnectorKey


@admin.register(ConnectorRequest)
class ConnectorRequestAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "created_at",
        "connector_type",
    ]
    list_filter = ("account", "connector_type",)


@admin.register(ConnectorKey)
class ConnectorKeyAdmin(admin.ModelAdmin):
    list_display = [
        "account",
        "connector",
        "key_type",
        "created_at",
        "is_active",
    ]


@admin.register(Connector)
class ConnectorAdmin(admin.ModelAdmin):
    list_display = [
        "account",
        "connector_type",
        "created_at",
        "is_active",
    ]

@admin.register(TransformerMapping)
class TransformerMappingAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "decoder_type",
        "transformer_type",
        "is_active",
    ]
    list_filter = ("account", "transformer_type", "decoder_type", "is_active")

connectors/admin.py
DrDroidLab/kenobi
from django.apps import AppConfig


class ConnectorsConfig(AppConfig):
    default_auto_field = 'django.db.models.BigAutoField'
    name = 'connectors'

connectors/apps.py
DrDroidLab/kenobi
import requests
import datetime
import math

from connectors.models import ConnectorKey
from event.processors.process_events_payload import process_account_events_payload
from protos.event.connectors_pb2 import Connector as ConnectorProto, ConnectorKey as ConnectorKeyProto, PeriodicRunStatus, ConnectorType

from protos.event.schema_pb2 import IngestionEventPayload
from event.views import transform_event_json

def event_qualifies(timestamp, time_range):
    timestamp_time = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ").timestamp()
    start_time = datetime.datetime.strptime(time_range[0], "%Y-%m-%dT%H:%M:%S").timestamp()
    end_time = datetime.datetime.strptime(time_range[1], "%Y-%m-%dT%H:%M:%S").timestamp()

    if start_time <= timestamp_time < end_time:
        return True
    return False


def matches_user(ev, user_id, user_ip):
    if (user_id and ev.get('user', {}).get('id') == user_id) or (
            user_ip and ev.get('user', {}).get('ip_address') == user_ip):
        return True
    return False


class SentryPollingEngine():
    SENTRY_EVENT_TYPE_MAP = {'error': 'Sentry_Error', 'warning': 'Sentry_Warning'}
    TAGS_BLACKLIST = ['environment', 'handled', 'level', 'logger', 'mechanism', 'runtime', 'runtime.name']
    SENTRY_PROJECT_URL = 'https://sentry.io/api/0/projects/'
    SENTRY_EVENTS_URL = 'https://sentry.io/api/0/issues/{}/events/'

    def __init__(self, connector, chunk_offset=2):
        self.connector = connector

        try:
            sentry_key = ConnectorKey.objects.get(connector=self.connector,
                                                  key_type=ConnectorKeyProto.KeyType.SENTRY_API_KEY,
                                                  is_active=True).key

            self.SENTRY_BEARER_TOKEN = sentry_key
            self.HEADERS = {
                'Authorization': 'Bearer {}'.format(self.SENTRY_BEARER_TOKEN)
            }
            self.CHUNK_SIZE = int(self.connector.metadata.get('polling_frequency', 300))
            self.CHUNK_OFFSET = chunk_offset
            self.HOURS_OFFSET = math.ceil((self.CHUNK_OFFSET * self.CHUNK_SIZE) / 3600) + 1

            self.projects = self.get_sentry_projects()
        except ConnectorKey.DoesNotExist:
            raise Exception("Sentry API key not found for connector {}".format(self.connector.name))

    def get_sentry_projects(self):
        url = self.SENTRY_PROJECT_URL
        response = requests.request("GET", url, headers=self.HEADERS)

        projects = response.json()
        slugs = list((x['organization']['slug'], x['slug']) for x in projects)

        return slugs

    def get_issue_level_events(self, issue_id, time_range):

        events = []
        url = self.SENTRY_EVENTS_URL.format(issue_id)
        response = requests.request("GET", url, headers=self.HEADERS)
        event_data = response.json()

        for ev in event_data:
            if event_qualifies(ev['dateCreated'], time_range):
                events.append(ev)
        return events

    def get_error_events(self, error, sentry_data, time_range):
        result = {}
        id = error['id']
        events = self.get_issue_level_events(id, time_range)
        sentry_data += events

    def get_user_level_sentry_errors(self, project_slug, start_time_str, end_time_str):

        sentry_data = []

        url = self.SENTRY_PROJECT_URL + project_slug[0] + "/" + project_slug[
            1] + "/issues/?query={}".format("timestamp:>{} timestamp:<{}".format(start_time_str, end_time_str))
        response = requests.request("GET", url, headers=self.HEADERS)
        error_data = response.json()

        for e in error_data:
            self.get_error_events(e, sentry_data, (start_time_str, end_time_str))
        return sentry_data

    def create_error_timeline(self, sentry_errors, user_id, user_ip):
        result = []

        for se in sentry_errors:
            message = ""
            t_obj = {'transaction': list(filter(lambda x: x['key'] == 'transaction', se['tags']))[0]['value'],
                     'title': se['title'], 'type': se['event.type'], 'message': se['message'],
                     'timestamp': se['dateCreated'], 'epoch_timestamp': datetime.datetime.strptime(se['dateCreated'],
                                                                                                   "%Y-%m-%dT%H:%M:%SZ").timestamp()}

            if user_ip:
                message = "Received {} => [{}] in transaction {} from the user's IP address at {}".format(
                    t_obj['type'], t_obj['title'], t_obj['transaction'], t_obj['timestamp'])

            if user_id:
                message = "Received {} => [{}] in transaction {} for the user at {}".format(t_obj['type'],
                                                                                            t_obj['title'],
                                                                                            t_obj['transaction'],
                                                                                            t_obj['timestamp'])

            result.append({'message': message, 't_obj': t_obj})

        return result

    def get_polling_period_time_range(self, current_time):
        start_offset_time = (current_time - datetime.timedelta(hours=self.HOURS_OFFSET)).replace(minute=0, second=0)
        current_date_chunks = []

        end_time = start_offset_time

        while end_time <= current_time:
            period_start_time = end_time
            end_time = period_start_time + datetime.timedelta(seconds=self.CHUNK_SIZE)
            current_date_chunks.append((period_start_time, end_time))

        return current_date_chunks[-self.CHUNK_OFFSET]

    def create_events(self, errors_data):
        events = []

        if errors_data:
            for err in errors_data:
                ev_name = self.SENTRY_EVENT_TYPE_MAP.get(err['event.type'], 'Sentry_Error')
                ev_timestamp = datetime.datetime.strptime(err['dateCreated'],
                                                          "%Y-%m-%dT%H:%M:%SZ").timestamp() * 1000
                ev = {'message': err['message'], 'title': err['title']}

                if err.get('user'):
                    if err.get('user').get('id'):
                        ev['user_id'] = err.get('user').get('id')
                    if err.get('user').get('ip'):
                        ev['user_ip'] = err.get('user').get('ip')
                    if err.get('user').get('email'):
                        ev['user_email'] = err.get('user').get('email')

                for tag in err['tags']:
                    if tag['key'] not in self.TAGS_BLACKLIST:
                        ev[tag['key']] = tag['value']

                events.append({"name": ev_name, "timestamp": ev_timestamp, "payload": ev})

        transformed_events = list(transform_event_json(event) for event in events)
        process_account_events_payload(self.connector.account, IngestionEventPayload(events=transformed_events))

    def poll_sentry_data(self):

        current_time = datetime.datetime.utcnow()
        start_time, end_time = self.get_polling_period_time_range(current_time)

        start_time_str, end_time_str = datetime.datetime.strftime(start_time,
                                                                  '%Y-%m-%dT%H:%M:%S'), datetime.datetime.strftime(
            end_time, '%Y-%m-%dT%H:%M:%S')

        for proj in self.projects:
            sentry_errors = self.get_user_level_sentry_errors(proj, start_time_str, end_time_str)
            self.create_events(sentry_errors)
connectors/connector_engine.py
DrDroidLab/kenobi
import json
import math

import requests
from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v1.api.metrics_api import MetricsApi

from datadog_api_client.v2.api.metrics_api import MetricsApi as MetricsApi2
from datadog_api_client.v2.model.formula_limit import FormulaLimit
from datadog_api_client.v2.model.metrics_data_source import MetricsDataSource
from datadog_api_client.v2.model.metrics_timeseries_query import MetricsTimeseriesQuery
from datadog_api_client.v2.model.query_formula import QueryFormula
from datadog_api_client.v2.model.timeseries_formula_query_request import TimeseriesFormulaQueryRequest
from datadog_api_client.v2.model.timeseries_formula_request import TimeseriesFormulaRequest
from datadog_api_client.v2.model.timeseries_formula_request_attributes import TimeseriesFormulaRequestAttributes
from datadog_api_client.v2.model.timeseries_formula_request_queries import TimeseriesFormulaRequestQueries
from datadog_api_client.v2.model.timeseries_formula_request_type import TimeseriesFormulaRequestType

from connectors.models import Connector, ConnectorKey
from protos.event.connectors_pb2 import ConnectorType
from protos.event.connectors_pb2 import ConnectorKey as ConnectorKeyProto




class DatadogConnector(object):
    def __init__(self, tr, account_id):
        self.start_time = tr.time_geq
        self.end_time = tr.time_lt
        try:
            self.dd_connector = Connector.objects.get(account_id=account_id, connector_type=ConnectorType.DATADOG, is_active=True)
            self.dd_connector_keys = ConnectorKey.objects.filter(account_id=account_id, connector_id=self.dd_connector.id, is_active=True)
            self.configuration = Configuration()
            self.configuration.api_key["apiKeyAuth"] = self.dd_connector_keys.get(key_type=ConnectorKeyProto.DATADOG_API_KEY).key
            self.configuration.api_key["appKeyAuth"] = self.dd_connector_keys.get(key_type=ConnectorKeyProto.DATADOG_APP_KEY).key
        except Connector.DoesNotExist:
            raise Exception("Datadog connector not found")
        except ConnectorKey.DoesNotExist:
            raise Exception("Datadog connector keys not found")

    def fetch_metric(self, metric, tr=None):
        with ApiClient(self.configuration) as api_client:
            api_instance = MetricsApi(api_client)

            s_time, e_time = self.start_time, self.end_time
            if tr:
                s_time, e_time = tr.time_geq, tr.time_lt

            time_duration = e_time - s_time

            metric_type = metric.split(':')[0]
            metric = metric + '.rollup({}, {})'.format(metric_type, time_duration)

            response = api_instance.query_metrics(s_time, e_time, metric)

            series = response.get('series', [])
            if len(series) > 0:
                pointlist = series[0].get('pointlist', [])
                aggr = series[0].get('aggr', 'avg')
                unit = series[0].get('unit', [])[0]['short_name']
                if not unit:
                    unit = ''
                if aggr == 'sum':
                    return {"value": round(sum(list(x._data_store['value'][1] for x in pointlist))), "unit": unit}
                else:
                    return {"value": round(sum(list(x._data_store['value'][1] for x in pointlist)) / len(pointlist), 2), "unit": unit}
            return {}


    def fetch_metric_timeseries(self, metric, tr=None):
        s_time, e_time = self.start_time, self.end_time
        if tr:
            s_time, e_time = tr.time_geq, tr.time_lt

        resolution = math.ceil((e_time - s_time) / 15 / 60) * 60

        body = {
            "data": {
                "attributes": {
                    "from": s_time*1000,
                    "interval": resolution*1000,
                    "queries": [
                        {
                            "data_source": "metrics",
                            "query": metric
                        }
                    ],
                    "to": e_time*1000
                },
                "type": "timeseries_request"
            }
        }

        url = 'https://api.datadoghq.com/api/v2/query/timeseries'
        headers = {"DD-API-KEY": self.configuration.api_key["apiKeyAuth"], "DD-APPLICATION-KEY": self.configuration.api_key["appKeyAuth"], "Content-Type": "application/json"}
        response = requests.post(url, headers=headers, data=json.dumps(body))

        response = response.json()

        times = response.get('data', {}).get('attributes', {}).get('times', [])
        values = response.get('data', {}).get('attributes', {}).get('values', [])
        if len(times) > 0:
            pointlist = values[0]
            series = []
            for idx, x in enumerate(times):
                series.append({'timestamp': x, 'value': pointlist[idx]})
            return {"series": series, "unit": ''}
        return {"series": [], "unit": ''}
connectors/integrations/datadog.py
DrDroidLab/kenobi
from newrelic_api import Applications
import datetime

from connectors.models import Connector, ConnectorKey
from protos.event.connectors_pb2 import ConnectorType
from protos.event.connectors_pb2 import ConnectorKey as ConnectorKeyProto

def format_dt(ts):
    dt_object = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
    return dt_object

class NewRelicConnector(object):
    def __init__(self, tr, account_id):
        self.start_time = tr.time_geq
        self.end_time = tr.time_lt
        try:
            self.nr_connector = Connector.objects.get(account_id=account_id, connector_type=ConnectorType.NEW_RELIC, is_active=True)
            self.nr_connector_keys = ConnectorKey.objects.filter(account_id=account_id, connector_id=self.nr_connector.id, is_active=True)
            self.app = Applications(api_key=self.nr_connector_keys.get(key_type=ConnectorKeyProto.NEWRELIC_API_KEY).key)
            self.app_id = self.nr_connector_keys.get(key_type=ConnectorKeyProto.NEWRELIC_APP_ID).key
        except Connector.DoesNotExist:
            raise Exception("NewRelic connector not found")
        except ConnectorKey.DoesNotExist:
            raise Exception("NewRelic connector keys not found")

    def fetch_metric(self, metric, tr=None):
        metric_tokens = metric.split(":")
        metric_type = metric_tokens[0]
        metric_name = metric_tokens[1]

        s_time, e_time = self.start_time, self.end_time
        if tr:
            s_time, e_time = tr.time_geq, tr.time_lt

        metric_response = self.app.metric_data(self.app_id, [metric_name], values=[metric_type], from_dt=format_dt(s_time), to_dt=format_dt(e_time), summarize=True)

        series = metric_response.get('metric_data', {}).get('metrics', [])[0].get('timeslices', [])
        if len(series) > 0:
            metric_values = series[0].get('values', {})
            metric_value = metric_values.get(metric_type, None)
            unit = ''
            return {"value": round(metric_value, 2), "unit": unit}
        return {}

    def fetch_metric_timeseries(self, metric, tr=None):
        metric_tokens = metric.split(":")
        metric_type = metric_tokens[0]
        metric_name = metric_tokens[1]

        s_time, e_time = self.start_time, self.end_time
        if tr:
            s_time, e_time = tr.time_geq, tr.time_lt

        metric_response = self.app.metric_data(self.app_id, [metric_name], values=[metric_type], from_dt=format_dt(s_time), to_dt=format_dt(e_time), summarize=False)

        series = metric_response.get('metric_data', {}).get('metrics', [])[0].get('timeslices', [])
        if len(series) > 0:
            return {"series": series, "unit": '', "metric_type": metric_type}
        return {"series": [], "unit": '', "metric_type": ''}

connectors/integrations/newrelic.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-21 07:49

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
    ]

    operations = [
        migrations.CreateModel(
            name='Connector',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(max_length=255)),
                ('connector_type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SENTRY'), (2, 'SEGMENT'), (3, 'ELASTIC_SEARCH'), (4, 'AMPLITUDE')], default=0, null=True)),
                ('is_active', models.BooleanField(default=True)),
                ('metadata', models.JSONField()),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account', 'name', 'connector_type')},
            },
        ),
        migrations.CreateModel(
            name='ConnectorPeriodicRunMetadata',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('metadata', models.JSONField()),
                ('task_run_id', models.CharField(max_length=255)),
                ('status', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'STARTED'), (2, 'FINISHED'), (3, 'ERROR')], null=True)),
                ('started_at', models.DateTimeField(blank=True, db_index=True, null=True)),
                ('finished_at', models.DateTimeField(blank=True, db_index=True, null=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('connector', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='connectors.connector')),
            ],
        ),
        migrations.CreateModel(
            name='ConnectorRequest',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('connector_type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SENTRY'), (2, 'SEGMENT'), (3, 'ELASTIC_SEARCH'), (4, 'AMPLITUDE')], default=0, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account', 'connector_type')},
            },
        ),
        migrations.CreateModel(
            name='ConnectorKey',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('key_type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SENTRY_API_KEY')], null=True)),
                ('key', models.TextField()),
                ('is_active', models.BooleanField(default=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('connector', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='connectors.connector')),
            ],
            options={
                'unique_together': {('account', 'connector', 'key_type', 'key')},
            },
        ),
    ]

connectors/migrations/0001_initial.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-05-02 07:36

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('connectors', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='connector',
            name='connector_type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SENTRY'), (2, 'SEGMENT'), (3, 'ELASTIC_SEARCH'), (4, 'AMPLITUDE'), (5, 'AWS_KINESIS'), (6, 'CLOUDWATCH'), (7, 'CLEVERTAP'), (8, 'RUDDERSTACK'), (9, 'MOENGAGE'), (10, 'CRIBL'), (11, 'KAFKA'), (12, 'DATADOG'), (13, 'FILEBEAT'), (14, 'LOGSTASH'), (15, 'FLUENTD'), (16, 'FLUENTBIT'), (17, 'PAGER_DUTY'), (18, 'NEW_RELIC')], default=0, null=True),
        ),
        migrations.AlterField(
            model_name='connectorrequest',
            name='connector_type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SENTRY'), (2, 'SEGMENT'), (3, 'ELASTIC_SEARCH'), (4, 'AMPLITUDE'), (5, 'AWS_KINESIS'), (6, 'CLOUDWATCH'), (7, 'CLEVERTAP'), (8, 'RUDDERSTACK'), (9, 'MOENGAGE'), (10, 'CRIBL'), (11, 'KAFKA'), (12, 'DATADOG'), (13, 'FILEBEAT'), (14, 'LOGSTASH'), (15, 'FLUENTD'), (16, 'FLUENTBIT'), (17, 'PAGER_DUTY'), (18, 'NEW_RELIC')], default=0, null=True),
        ),
    ]

connectors/migrations/0002_alter_connector_connector_type_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-05-09 09:54

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
        ('connectors', '0002_alter_connector_connector_type_and_more'),
    ]

    operations = [
        migrations.CreateModel(
            name='TransformerMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('decoder_type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_DT'), (1, 'AWS_KINESIS_DECODER')], default=0, null=True)),
                ('transformer_type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_TT'), (2, 'SEGMENT_DFAULT_TRANSFORMER'), (3, 'AMPLITUDE_DEFAULT_TRANSFORMER'),], default=0, null=True)),
                ('is_active', models.BooleanField(default=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account', 'decoder_type', 'transformer_type')},
            },
        ),
    ]

connectors/migrations/0003_transformermapping.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-14 07:20

from django.db import migrations, models


class Migration(migrations.Migration):
    dependencies = [
        ('connectors', '0003_transformermapping'),
    ]

    operations = [
        migrations.AlterField(
            model_name='connectorkey',
            name='key_type',
            field=models.IntegerField(blank=True,
                                      choices=[(0, 'UNKNOWN'), (1, 'SENTRY_API_KEY'), (2, 'DATADOG_APP_KEY'),
                                               (3, 'DATADOG_API_KEY'), (4, 'NEWRELIC_API_KEY'), (5, 'NEWRELIC_APP_ID')],
                                      null=True),
        ),
        migrations.AlterField(
            model_name='transformermapping',
            name='decoder_type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_DT'), (1, 'AWS_KINESIS_DECODER'),
                                                           (2, 'AWS_CLOUDWATCH_KINESIS_DECODER')], default=0,
                                      null=True),
        ),
        migrations.AlterField(
            model_name='transformermapping',
            name='transformer_type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_TT'), (2, 'SEGMENT_DFAULT_TRANSFORMER'),
                                                           (3, 'AMPLITUDE_DEFAULT_TRANSFORMER'),
                                                           (5, 'CLOUDWATCH_JSON_LOG_TRANSFORMER')], default=0,
                                      null=True),
        ),
    ]

connectors/migrations/0004_alter_connectorkey_key_type_and_more.py
DrDroidLab/kenobi
from django.db import models
from google.protobuf.wrappers_pb2 import StringValue, BoolValue

from accounts.models import Account
from protos.event.connectors_pb2 import ConnectorKey as ConnectorKeyProto, PeriodicRunStatus, ConnectorType, \
    TransformerType, DecoderType

from utils.model_utils import generate_choices


class Connector(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE, db_index=True)
    name = models.CharField(max_length=255)
    connector_type = models.IntegerField(null=True, blank=True, choices=generate_choices(ConnectorType),
                                         default=ConnectorType.UNKNOWN)
    is_active = models.BooleanField(default=True)
    metadata = models.JSONField()
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'name', 'connector_type']]

    def __str__(self):
        return f'{self.account}:{self.connector_type}'


class ConnectorKey(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE, db_index=True)
    connector = models.ForeignKey(Connector, on_delete=models.CASCADE)
    key_type = models.IntegerField(null=True, blank=True,
                                   choices=generate_choices(ConnectorKeyProto.KeyType))
    key = models.TextField()
    is_active = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'connector', 'key_type', 'key']]


    def get_proto(self):
        return ConnectorKeyProto(key_type=self.key_type,
                                 key=StringValue(value=self.key),
                                 is_active=BoolValue(value=self.is_active))


class ConnectorPeriodicRunMetadata(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE, db_index=True)
    connector = models.ForeignKey(Connector, on_delete=models.CASCADE)
    metadata = models.JSONField()
    task_run_id = models.CharField(max_length=255)
    status = models.IntegerField(null=True, blank=True,
                                 choices=generate_choices(PeriodicRunStatus.StatusType))
    started_at = models.DateTimeField(blank=True, null=True, db_index=True)
    finished_at = models.DateTimeField(blank=True, null=True, db_index=True)


class ConnectorRequest(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE, db_index=True)
    connector_type = models.IntegerField(null=True, blank=True, choices=generate_choices(ConnectorType),
                                         default=ConnectorType.UNKNOWN)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'connector_type']]


class TransformerMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE, db_index=True)
    decoder_type = models.IntegerField(null=True, blank=True, choices=generate_choices(DecoderType),
                                       default=DecoderType.UNKNOWN_DT)
    transformer_type = models.IntegerField(null=True, blank=True, choices=generate_choices(TransformerType),
                                           default=TransformerType.UNKNOWN_TT)
    is_active = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'decoder_type', 'transformer_type']]

connectors/models.py
DrDroidLab/kenobi
from celery import shared_task

from connectors.models import Connector, ConnectorKey, ConnectorPeriodicRunMetadata
from protos.event.connectors_pb2 import Connector as ConnectorProto, ConnectorKey as ConnectorKeyProto, ConnectorType

from connectors.connector_engine import SentryPollingEngine
    
@shared_task
def sentry_connector_poll_events():
    sentry_connectors = Connector.objects.filter(is_active=True, connector_type=ConnectorType.SENTRY)
    for connector in sentry_connectors:
        sentry_per_connector_poll_events.delay(connector.id)

@shared_task
def sentry_per_connector_poll_events(connector_id):
    connector = Connector.objects.get(id=connector_id)
    polling_engine = SentryPollingEngine(connector)
    polling_engine.poll_sentry_data()


  
connectors/tasks.py
DrDroidLab/kenobi
import base64
import gzip
import json
from typing import Dict

from protos.event.connectors_pb2 import DecoderType
from protos.event.schema_pb2 import AWSKinesisEventPayload


class SourceDecoder:
    decoder_type: DecoderType = None

    @classmethod
    def decode(cls, message):
        pass


class AWSKinesisFirehoseRecordDecoder(SourceDecoder):

    def __init__(self):
        self.decoder = DecoderType.AWS_KINESIS_DECODER

    @classmethod
    def decode(cls, message: AWSKinesisEventPayload) -> Dict:
        data = base64.b64decode(message.data).decode('utf-8')
        data = json.loads(data)
        return data


class AWSCloudWatchKinesisFirehoseRecordDecoder(SourceDecoder):

    def __init__(self):
        self.decoder = DecoderType.AWS_CLOUDWATCH_KINESIS_DECODER

    @classmethod
    def decode(cls, message: AWSKinesisEventPayload) -> Dict:
        data = gzip.decompress(base64.b64decode(message.data))
        data = json.loads(data)
        return data


class SourceDecoderFacade:

    def __init__(self):
        self._map = {}

    def register(self, decoder_type: DecoderType, decoder: SourceDecoder):
        self._map[decoder_type] = decoder

    def decode(self, decoder_type: DecoderType, message):
        return self._map[decoder_type].decode(message)


decoder_facade = SourceDecoderFacade()
decoder_facade.register(DecoderType.AWS_KINESIS_DECODER, AWSKinesisFirehoseRecordDecoder())
decoder_facade.register(DecoderType.AWS_CLOUDWATCH_KINESIS_DECODER, AWSCloudWatchKinesisFirehoseRecordDecoder())

connectors/transformers/decoder.py
DrDroidLab/kenobi
import ast
import json
import logging
import re
from typing import List

from django.conf import settings

from connectors.models import TransformerMapping
from connectors.transformers.decoder import decoder_facade
from protos.event.connectors_pb2 import TransformerType, DecoderType
from prototype.utils.utils import current_milli_time


class TransformerFacadeException(ValueError):
    pass


logger = logging.getLogger(__name__)


class Transformer:
    type: TransformerType = None

    @classmethod
    def transform(cls, input_schema):
        pass


class SegmentDefaultTransformer(Transformer):

    def __init__(self):
        self.type = TransformerType.SEGMENT_DFAULT_TRANSFORMER

    @classmethod
    def transform(cls, input_schema) -> List:
        event = {}
        payload = {}

        if input_schema['type'] == 'track':
            event["name"] = input_schema.pop('event', None)
            payload['type'] = input_schema.pop('type', None)
            payload.update(input_schema['properties'])
            payload['userId'] = input_schema.pop('userId', None)
            event["payload"] = payload
        elif input_schema['type'] == 'page':
            event["name"] = input_schema.pop('name', None)
            payload['type'] = input_schema.pop('type', None)
            payload.update(input_schema['properties'])
            payload['userId'] = input_schema.pop('userId', None)
            event["payload"] = payload
        elif input_schema['type'] == 'screen':
            event["name"] = input_schema.pop('name', None)
            payload['type'] = input_schema.pop('type', None)
            payload.update(input_schema['properties'])
            event["payload"] = payload
        elif input_schema['type'] == 'group':
            event["name"] = input_schema.pop('type', None)
            payload['groupId'] = input_schema.pop('groupId', None)
            payload['userId'] = input_schema.pop('userId', None)
            traits = input_schema.pop('traits', None)
            if traits:
                payload.update(traits)
            event["payload"] = payload
        elif input_schema['type'] == 'identify':
            event["name"] = input_schema.pop('type', None)
            payload['userId'] = input_schema.pop('userId', None)
            traits = input_schema.pop('traits', None)
            if traits:
                payload.update(traits)
            event["payload"] = payload
        elif input_schema['type'] == 'alias':
            event["name"] = input_schema.pop('type', None)
            payload['previousId'] = input_schema.pop('previousId', None)
            payload['userId'] = input_schema.pop('userId', None)
            event["payload"] = payload

        if not event:
            raise TransformerFacadeException("Invalid Input Schema")

        return [event]


class AmplitudeDefaultTransformer(Transformer):

    def __init__(self):
        self.type = TransformerType.AMPLITUDE_DEFAULT_TRANSFORMER

    @classmethod
    def transform(cls, input_schema) -> List:
        event = {"name": input_schema.pop('event_type', None), "payload": input_schema}
        return [event]


class CloudwatchJsonLogTransformer(Transformer):

    def __init__(self):
        self.type = TransformerType.CLOUDWATCH_JSON_LOG_TRANSFORMER

    @classmethod
    def transform(cls, input_schema) -> List:
        events = []
        message_type = input_schema.get('messageType', None)
        if message_type and message_type == 'DATA_MESSAGE':
            log_events = input_schema.get('logEvents', None)
            if log_events:
                for log in log_events:
                    try:
                        log_message = log.get('message', None)
                        time_millis = log.get('timestamp', current_milli_time())
                        if log_message:
                            cleaned_log = json.loads(log_message)
                            if cleaned_log:
                                event_dict = cleaned_log
                                if event_dict and event_dict.get('msg', None) is not None:
                                    event_time = event_dict.pop('event_time', time_millis)
                                    event = {"name": event_dict.get('event_name', "").replace(" ", "_"), "payload": event_dict,
                                             "timestamp": event_time}
                                    events.append(event)

                    except Exception as e:
                        logger.error(
                            "Exception generating event for Cloudwatch JSON Log: {}, error: {}".format(log, e))
                        continue

        if not events:
            logger.error("No events generated for Cloudwatch JSON Logs: {}".format(input_schema))
        return events


class TransformerFacade:

    def __init__(self):
        self._map = {}

    def register(self, transformer_type: TransformerType, transformer: Transformer):
        self._map[transformer_type] = transformer

    def transform(self, transformer_mapping: TransformerMapping, message):

        decoder_type: DecoderType = transformer_mapping.decoder_type
        if not decoder_type:
            raise TransformerFacadeException(f'No decoder found with type {decoder_type}')

        transformer_type: TransformerType = transformer_mapping.transformer_type
        if not transformer_type:
            raise TransformerFacadeException(f'No transformer found with type {decoder_type}')

        if not transformer_mapping.is_active:
            raise TransformerFacadeException(f'In active transformer mapping found with id {transformer_mapping.id}')

        decoded_message = decoder_facade.decode(decoder_type, message)
        return self._map[transformer_type].transform(decoded_message)


transformer_facade = TransformerFacade()
transformer_facade.register(TransformerType.SEGMENT_DFAULT_TRANSFORMER, SegmentDefaultTransformer())
transformer_facade.register(TransformerType.AMPLITUDE_DEFAULT_TRANSFORMER, AmplitudeDefaultTransformer())
transformer_facade.register(TransformerType.CLOUDWATCH_JSON_LOG_TRANSFORMER, CloudwatchJsonLogTransformer())

connectors/transformers/transformer.py
DrDroidLab/kenobi
from django.urls import path

from . import views as connector_views

urlpatterns = [
    path('request', connector_views.connectors_request),
    path('create', connector_views.connectors_create),
    path('get_keys', connector_views.get_keys),
    path('delete_keys', connector_views.delete_keys),
    path('save_keys', connector_views.save_keys),
    path('transformer_mapping_create', connector_views.transformer_mapping_create),
]

connectors/urls.py
DrDroidLab/kenobi
from datetime import datetime
from typing import Union

import pytz
from django.core.mail import send_mail

from django.db import transaction
from django.http import HttpResponse
from django.template.loader import render_to_string

from django.db.utils import IntegrityError

from google.protobuf.wrappers_pb2 import BoolValue

from accounts.models import Account, get_request_account, get_request_user, User
from protos.event.api_pb2 import CreateConnectorRequest, CreateConnectorResponse, Message, RequestConnectorAPIRequest, \
    RequestConnectorAPIResponse, CreateTransformerMappingRequest, CreateTransformerMappingResponse, \
    GetConnectorKeysRequest, GetConnectorKeysResponse, SaveConnectorKeysResponse, SaveConnectorKeysRequest
from connectors.models import Connector, ConnectorKey, ConnectorRequest, TransformerMapping

from protos.event.connectors_pb2 import Connector as ConnectorProto, ConnectorKey as ConnectorKeyProto, ConnectorType, \
    DecoderType, TransformerType
from prototype.utils.decorators import web_api

from utils.proto_utils import proto_to_dict


@web_api(RequestConnectorAPIRequest)
def connectors_request(request_message: RequestConnectorAPIRequest) -> Union[
    RequestConnectorAPIResponse, HttpResponse]:
    account: Account = get_request_account()
    user: User = get_request_user()

    connector_type: ConnectorType = request_message.connector_type

    if not connector_type or connector_type == ConnectorType.UNKNOWN:
        return RequestConnectorAPIResponse(success=BoolValue(value=False),
                                           message=Message(title="Invalid Connector Request",
                                                           description="Input Connector not supported yet"))

    connectors_request, created = ConnectorRequest.objects.get_or_create(account=account,
                                                                         connector_type=connector_type,
                                                                         defaults={
                                                                             'account': account,
                                                                             'connector_type': connector_type,
                                                                         })

    if not created:
        return RequestConnectorAPIResponse(success=BoolValue(value=False), message=Message(title="Duplicate Request",
                                                                                           description="Connector already requested"))

    connector_name = ''
    for k, v in ConnectorType.items():
        if connector_type == v:
            connector_name = k
            break

    recipient_email_id = user.email
    timestamp = datetime.now(pytz.timezone('Asia/Kolkata')).strftime("%H:%M:%S %d-%m-%Y")
    msg_html = render_to_string('connector_reqeust_email_template.html',
                                {'connector_type': connector_name.capitalize(),
                                 'user_name': recipient_email_id,
                                 'timestamp': timestamp})
    send_mail(
        subject="Connector Request",
        message="test",
        from_email=None,
        recipient_list=[recipient_email_id, "dipesh@drdroid.io", "siddarth@drdroid.io"],
        html_message=msg_html
    )

    return RequestConnectorAPIResponse(success=BoolValue(value=True))


@web_api(CreateConnectorRequest)
def connectors_create(request_message: CreateConnectorRequest) -> Union[
    CreateConnectorResponse, HttpResponse]:
    account: Account = get_request_account()

    connector: ConnectorProto = request_message.connector
    connector_type = connector.type
    connector_keys = request_message.connector_keys

    if connector_type == ConnectorType.SENTRY:
        config = proto_to_dict(connector.sentry_config)

    try:
        with transaction.atomic():
            db_connector = Connector(account=account,
                                     name=connector.name.value,
                                     connector_type=connector_type,
                                     is_active=connector.is_active.value,
                                     metadata=config)
            db_connector.save()

            for c_key in connector_keys:
                connector_key: ConnectorKeyProto = c_key
                db_connector_key = ConnectorKey(account=account,
                                                connector=db_connector,
                                                key_type=connector_key.key_type,
                                                key=connector_key.key.value,
                                                is_active=connector_key.is_active.value)
                db_connector_key.save()
    except IntegrityError:
        return CreateConnectorResponse(success=BoolValue(value=False),
                                       message=Message(title='Failed to create connector'))

    return CreateConnectorResponse(success=BoolValue(value=True))


@web_api(CreateTransformerMappingRequest)
def transformer_mapping_create(request_message: CreateTransformerMappingRequest) -> Union[
    CreateTransformerMappingResponse, HttpResponse]:
    account: Account = get_request_account()

    if request_message.account_id.value:
        account = Account.objects.get(id=request_message.account_id.value)
    decoder_type: DecoderType = request_message.decoder_type
    transformer_type: TransformerType = request_message.transformer_type

    try:
        with transaction.atomic():
            db_transformer_mapping = TransformerMapping(account=account,
                                                        decoder_type=decoder_type,
                                                        transformer_type=transformer_type,
                                                        is_active=True)
            db_transformer_mapping.save()
    except IntegrityError:
        return CreateConnectorResponse(success=BoolValue(value=False),
                                       message=Message(title='Failed to create transformer mapping'))

    return CreateTransformerMappingResponse(success=BoolValue(value=True))


@web_api(GetConnectorKeysRequest)
def get_keys(request_message: GetConnectorKeysRequest) -> Union[
    GetConnectorKeysResponse, HttpResponse]:
    account: Account = get_request_account()

    connector_type: ConnectorType = request_message.connector_type

    if not connector_type or connector_type == ConnectorType.UNKNOWN:
        return GetConnectorKeysResponse(connector_keys=[])

    connector_keys = ConnectorKey.objects.filter(account=account, connector__connector_type=connector_type, is_active=True)
    connector_key_protos = list(x.get_proto() for x in connector_keys)

    return GetConnectorKeysResponse(connector_keys=connector_key_protos)


@web_api(GetConnectorKeysRequest)
def delete_keys(request_message: GetConnectorKeysRequest) -> Union[
    GetConnectorKeysResponse, HttpResponse]:
    account: Account = get_request_account()

    connector_type: ConnectorType = request_message.connector_type

    if not connector_type or connector_type == ConnectorType.UNKNOWN:
        return GetConnectorKeysResponse(message='Connector Type not found')

    ConnectorKey.objects.filter(account=account, connector__connector_type=connector_type, is_active=True).delete()
    return GetConnectorKeysResponse(message='Connector Keys Deleted')


@web_api(SaveConnectorKeysRequest)
def save_keys(request_message: SaveConnectorKeysRequest) -> Union[
    SaveConnectorKeysResponse, HttpResponse]:
    account: Account = get_request_account()

    connector_type: ConnectorType = request_message.connector_type
    connector_keys = request_message.connector_keys

    if not connector_type or connector_type == ConnectorType.UNKNOWN:
        return SaveConnectorKeysResponse(message='Connector Type not found')

    try:
        conn = Connector.objects.get(account=account, connector_type=connector_type, is_active=True)
    except Connector.DoesNotExist:
        conn = Connector.objects.create(account=account, connector_type=connector_type, is_active=True, metadata={"connector_type": connector_type})

    for conn_key in connector_keys:
        try:
            c_key = ConnectorKey.objects.get(account=account, connector=conn, is_active=True, key_type=conn_key.key_type)
            c_key.key = conn_key.key.value
            c_key.save()
        except ConnectorKey.DoesNotExist:
            c_key = ConnectorKey.objects.create(account=account, connector=conn, is_active=True, key_type=conn_key.key_type, key=conn_key.key.value)

    return SaveConnectorKeysResponse(message='Keys saved')
connectors/views.py
DrDroidLab/kenobi
from django.contrib import admin

from event.models import EventType, EventKey, Event, Monitor, MonitorTransaction, MonitorTransactionEventMapping, \
    Notification, Panel, Trigger, Alert, TriggerNotificationConfigMapping, AlertMonitorTransactionMapping, \
    MonitorTransactionStats, Entity, EntityInstance, EntityInstanceEventMapping, EntityEventKeyMapping, EntityTrigger, \
    EntityMonitorMapping


@admin.register(EventType)
class EventTypeAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "name",
        "account",
        "created_at",
        "event_sources",
    ]
    list_filter = ("account",)


@admin.register(EventKey)
class EventKeyAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "name",
        "event_type",
        "account",
        "created_at",
        "type",
    ]
    list_filter = ("account", "type")


@admin.register(Event)
class EventAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "event_type",
        "processed_kvs",
        "created_at",
        "timestamp",
        "event_source",
    ]
    list_filter = ("event_type", "account",)


@admin.register(Monitor)
class MonitorAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "name",
        "account",
        "primary_key",
        "secondary_key",
        "is_active",
        "created_at",
        "updated_at",
    ]
    list_filter = ("account", "primary_key", "secondary_key",)


class MonitorTransactionEventMappingAdminInline(admin.TabularInline):
    model = MonitorTransactionEventMapping
    raw_id_fields = ("monitor_transaction", "event", "account")


class MonitorTransactionStatsAdminInline(admin.TabularInline):
    model = MonitorTransactionStats
    raw_id_fields = ("monitor_transaction", "account")


@admin.register(EntityTrigger)
class EntityTriggerAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "name",
        "is_active",
        "entity",
        "type",
        "rule_type",
        "created_at"
    ]
    list_filter = ("entity", "type", "rule_type", "is_active")


@admin.register(MonitorTransaction)
class MonitorTransactionAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "monitor",
        "transaction",
        "created_at",
        "type",
    ]
    inlines = [MonitorTransactionEventMappingAdminInline, MonitorTransactionStatsAdminInline]
    list_filter = ("account", "monitor", "type",)
    list_per_page = 10


@admin.register(Notification)
class NotificationAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "channel",
        "config",
    ]
    list_filter = ("account", "channel")


class TriggerNotificationMappingAdminInline(admin.TabularInline):
    model = TriggerNotificationConfigMapping


@admin.register(TriggerNotificationConfigMapping)
class TriggerNotificationAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "trigger",
        "notification",
        "created_at",
    ]
    list_filter = ("account", "trigger", "notification",)


@admin.register(Panel)
class PanelAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "name",
        "created_at",
        "updated_at",
    ]
    list_filter = ("account",)


@admin.register(Trigger)
class TriggerAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "name",
        "monitor",
        "type",
        "priority",
        "is_active",
        "created_at",
        "updated_at",
    ]
    inlines = [TriggerNotificationMappingAdminInline]
    list_filter = ("account",)


class AlertMonitorTransactionMappingAdminInline(admin.TabularInline):
    model = AlertMonitorTransactionMapping


@admin.register(Alert)
class AlertAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "trigger",
        "triggered_at",
    ]
    inlines = [AlertMonitorTransactionMappingAdminInline]
    list_filter = ("account", 'trigger')


@admin.register(AlertMonitorTransactionMapping)
class AlertMonitorTransactionAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "alert",
        "monitor_transaction",
        "type",
        "created_at",
    ]
    list_filter = ("account", "alert", "monitor_transaction",)


class EntityEventKeyMappingAdminInline(admin.TabularInline):
    model = EntityEventKeyMapping
    raw_id_fields = ("entity", "event_key",)


class EntityMonitorMappingAdminInline(admin.TabularInline):
    model = EntityMonitorMapping
    raw_id_fields = ("entity", "monitor",)


@admin.register(Entity)
class EntityAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "name",
        "type",
        "is_active",
        "is_generated",
        "created_at",
    ]
    inlines = [EntityEventKeyMappingAdminInline, EntityMonitorMappingAdminInline]
    list_filter = ("account", "type", "is_active", "is_generated",)


@admin.register(EntityInstance)
class EntityInstanceAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "entity",
        "created_at",
    ]
    list_filter = ("account",)


@admin.register(EntityMonitorMapping)
class EntityMonitorMappingAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "account",
        "entity",
        "monitor",
        "is_active",
        "is_generated",
        "created_at"
    ]
    list_filter = ("account", "is_active", "is_generated", "entity", "monitor")

event/admin.py
DrDroidLab/kenobi
from collections import OrderedDict

from google.protobuf.wrappers_pb2 import BoolValue, StringValue

from protos.event.engine_options_pb2 import MetricOptions
from protos.event.metric_pb2 import AggregationFunction

metric_aggregation_function_options_dict = OrderedDict([
    (AggregationFunction.COUNT, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.COUNT,
        supports_empty_expression_selector=BoolValue(value=True),
        label=StringValue(value='Count'),
    )),
    (AggregationFunction.SUM, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.SUM,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Sum'),
    )),
    (AggregationFunction.AVG, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.AVG,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Avg'),
    )),
    (AggregationFunction.MIN, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.MIN,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Min'),
    )),
    (AggregationFunction.MAX, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.MAX,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Max'),
    )),
    (AggregationFunction.STD_DEV, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.STD_DEV,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Std Dev'),
    )),
    (AggregationFunction.VARIANCE, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.VARIANCE,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Variance'),
    )),
    (AggregationFunction.COUNT_DISTINCT, MetricOptions.AggregationFunctionOption(
        aggregation_function=AggregationFunction.COUNT_DISTINCT,
        supports_empty_expression_selector=BoolValue(value=False),
        label=StringValue(value='Count Distinct'),
    ))
])

metric_aggregation_function_options_list = list(metric_aggregation_function_options_dict.values())


def get_metric_aggregation_function_options():
    return metric_aggregation_function_options_list

event/base/aggregation_function.py
DrDroidLab/kenobi
import abc
from dataclasses import dataclass, field
from typing import Callable, Optional

from django.db.models import Subquery, OuterRef

from protos.event.literal_pb2 import LiteralType, IdLiteral
from protos.event.engine_options_pb2 import ColumnOption


class Options(abc.ABC):
    def get_options(self, account, obj):
        pass


@dataclass
class BaseColumn:
    name: str
    display_name: str
    is_filterable: bool

    def annotate(self):
        return {}

    def annotate_v2(self, paths: []):
        return {}


@dataclass
class Column(BaseColumn, Options):
    type: LiteralType
    is_groupable: bool
    is_orderable: bool
    aggregation_functions: Optional[list] = field(default_factory=list, repr=False)
    token_filter_op: Optional[object] = None
    id_options_cb: Optional[Callable] = None
    supported_ops: Optional[list] = field(default_factory=list, repr=False)
    id_label_resolver_cb: Optional[Callable] = None

    # TODO: Enable id literal resolution

    def get_options(self, account, obj):
        id_option = None
        if self.type == LiteralType.ID:
            id_option = ColumnOption.IdOption(
                type=IdLiteral.Type.LONG,
                long_options=self.id_options_cb(account, obj)
            )
        return ColumnOption(
            name=self.name,
            id_option=id_option,
            type=self.type,
            alias=self.display_name,
            is_groupable=self.is_groupable,
            aggregation_functions=self.aggregation_functions,
        )


@dataclass
class AnnotatedColumn(Column):
    annotation_relation: object = None

    def annotate(self):
        return {f'{self.name}': self.annotation_relation}


@dataclass
class AttributeColumn(BaseColumn, Options):
    annotation_relation: object
    attribute_options_cb: Callable
    attribute_field: str = None

    def annotate(self):
        return {f'{self.name}': self.annotation_relation}

    def get_options(self, account, obj):
        return self.attribute_options_cb(self.name, self.display_name, account, obj)


@dataclass
class AttributeColumnV2(BaseColumn, Options):
    general_annotation_relation: str
    associated_model_name: str
    associated_model: object
    attribute_options_cb_v2: Callable

    def annotate_v2(self, path):
        formatted_annotation_string = self.general_annotation_relation.format(path)
        subquery_object = None
        try:
            subquery_object = eval(formatted_annotation_string, {"Subquery": Subquery, "OuterRef": OuterRef,
                                                                 self.associated_model_name: self.associated_model})
        except Exception as e:
            print(f"Error occurred in generating annotation for AttributeColumnV2: {e}")
        if subquery_object:
            return {path: subquery_object}
        return None

    def get_options(self, account, obj):
        return self.attribute_options_cb_v2(account, obj)

event/base/column.py
DrDroidLab/kenobi
from typing import List, Dict

from django.db.models import Q

from event.base.filter_token_op import NumericColumnTokenFilterOp, StringColumnTokenFilterOp, \
    BooleanColumnTokenFilterOp, TimestampColumnTokenFilterOp, IDColumnTokenFilterOp, LiteralArrayColumnTokenFilterOp, \
    AttributeTokenFilterOp, TokenFilterOp, AttributeTokenV2FilterOp
from event.base.literal import is_scalar
from event.base.op import validate_query
from event.base.token import Token, OpToken, AttributeToken, ColumnToken, Filterable, Annotable, ExpressionTokenizer, \
    LiteralToken, AttributeTokenV2
from protos.event.literal_pb2 import LiteralType
from protos.event.query_base_pb2 import Op, Filter as FilterProto

CONNECTOR_OPS = [Op.AND, Op.OR, Op.NOT]


class FilterToken(Token):
    lhs: Token = None
    op: OpToken = None
    rhs: Token = None
    child_filters: List[Token] = None

    def __init__(self, lhs: Token, op: OpToken, rhs: Token, child_filters: List[Token] = None):
        self.lhs = lhs
        self.op = op
        self.rhs = rhs
        self.child_filters = child_filters or []

    def display(self):
        if self.child_filters:
            return f' {self.op.display()} '.join([f.display() for f in self.child_filters])
        else:
            return f'({self.lhs.display()} {self.op.display()} {self.rhs.display()})'


class FilterTokenProcessor:
    def __init__(self):
        self._default_scalar_column_literal_filter_op_dict = {
            LiteralType.LONG: NumericColumnTokenFilterOp(),
            LiteralType.DOUBLE: NumericColumnTokenFilterOp(),
            LiteralType.STRING: StringColumnTokenFilterOp(),
            LiteralType.BOOLEAN: BooleanColumnTokenFilterOp(),
            LiteralType.TIMESTAMP: TimestampColumnTokenFilterOp(),
            LiteralType.ID: IDColumnTokenFilterOp(),
        }

        self._default_array_column_literal_filter_op = LiteralArrayColumnTokenFilterOp()

        self._attribute_filter_op = AttributeTokenFilterOp()
        self._attribute_filter_op_v2 = AttributeTokenV2FilterOp()

    def _get_lhs_token_filter_op(self, lhs: Filterable) -> TokenFilterOp:
        if isinstance(lhs, AttributeToken):
            return self._attribute_filter_op
        elif isinstance(lhs, AttributeTokenV2):
            return self._attribute_filter_op_v2
        elif isinstance(lhs, ColumnToken):
            if lhs.column.token_filter_op and isinstance(lhs.column.token_filter_op, TokenFilterOp):
                return lhs.column.token_filter_op
            if is_scalar(lhs.column.type):
                return self._default_scalar_column_literal_filter_op_dict[lhs.column.type]
            else:
                return self._default_array_column_literal_filter_op
        else:
            raise ValueError(f'Unsupported LHS {lhs}')

    def process(self, f: Token) -> Q:
        if not isinstance(f, FilterToken):
            raise ValueError('FilterTokenProcessor can only process Filter')
        if f.op.op in CONNECTOR_OPS:
            if len(f.child_filters) == 0:
                return Q()
            op = f.op.op
            if op == Op.NOT and len(f.child_filters) > 1:
                raise ValueError('NOT expects a single child filter')
            processed = [self.process(child) for child in f.child_filters]
            base: Q = processed[0]
            if op == Op.NOT:
                return ~base
            elif op == Op.AND:
                for p in processed[1:]:
                    base = base & p
                return base
            elif op == Op.OR:
                for p in processed[1:]:
                    base = base | p
                return base

        return self._base_filter_processor(f)

    def _base_filter_processor(self, f: FilterToken) -> Q:
        if not isinstance(f.lhs, Filterable):
            raise ValueError(f'lhs {f.lhs} is not Filterable')
        token_filter_op = self._get_lhs_token_filter_op(f.lhs)
        return token_filter_op.process(f.lhs, f.op, f.rhs)


class FilterTokenAnnotator:
    def annotations(self, f: Token) -> Dict:
        if not isinstance(f, FilterToken):
            raise ValueError('FilterTokenAnnotator can only process Filter')
        annotations = {}
        if len(f.child_filters) > 0:
            for child in f.child_filters:
                annotations.update(self.annotations(child))

        if f.lhs and isinstance(f.lhs, Annotable):
            annotations.update(f.lhs.annotations())

        if f.rhs and isinstance(f.rhs, Annotable):
            annotations.update(f.rhs.annotations())

        return annotations


class FilterTokenizer:
    def __init__(self, columns):
        self._columns = columns
        self._expression_tokenizer = ExpressionTokenizer(columns)

    def tokenize(self, filter_proto: FilterProto) -> FilterToken:
        if filter_proto.ByteSize() == 0:
            return None
        child_filter_tokens = [self.tokenize(f) for f in filter_proto.filters]
        lhs_token = self._expression_tokenizer.tokenize(filter_proto.lhs)
        rhs_token = self._expression_tokenizer.tokenize(filter_proto.rhs)
        op_token = OpToken(op=filter_proto.op)

        return FilterToken(
            lhs=lhs_token,
            op=op_token,
            rhs=rhs_token,
            child_filters=child_filter_tokens
        )


class FilterTokenValidator:
    def validate(self, filter_token: FilterToken) -> (bool, str):
        if not filter_token or not filter_token.child_filters:
            return True, ''
        for token in filter_token.child_filters:
            if not isinstance(token, FilterToken):
                return False, 'Child filter must be Filter'
            is_filter_token_valid, err = self.validate(token)
            if not is_filter_token_valid:
                return False, err
        if filter_token.lhs is not None and filter_token.rhs is not None:
            lhs_token = filter_token.lhs
            rhs_token = filter_token.rhs
            op_token = filter_token.op

            if isinstance(lhs_token, ColumnToken) and isinstance(rhs_token, LiteralToken):
                is_filter_token_valid, err = validate_query(
                    lhs_token.column.type, op_token.op,
                    rhs_token.literal.literal_type
                )

                if not is_filter_token_valid:
                    return False, f'{err} not supported for column: {lhs_token.column.display_name}'

        return True, ''


class FilterTokenEvaluator:
    def __init__(self):
        self._filter_token_processor = FilterTokenProcessor()
        self._filter_token_annotator = FilterTokenAnnotator()

    def process(self, qs, filter_token: FilterToken):
        if not filter_token:
            return qs
        annotations = self._filter_token_annotator.annotations(filter_token)
        q = self._filter_token_processor.process(filter_token)
        filter_qs = qs.annotate(**annotations).filter(q)
        return filter_qs


class FilterEngine:
    def __init__(self, columns):
        self.columns = columns
        self._filter_tokenizer = FilterTokenizer(columns)
        self._filter_token_validator = FilterTokenValidator()
        self._filter_token_evaluator = FilterTokenEvaluator()

    def tokenize(self, filter_proto: FilterProto) -> FilterToken:
        return self._filter_tokenizer.tokenize(filter_proto)

    def validate(self, filter_token: FilterToken) -> (bool, str):
        return self._filter_token_validator.validate(filter_token)

    def evaluate(self, qs, filter_token: FilterToken):
        return self._filter_token_evaluator.process(qs, filter_token)

    def process(self, qs, filter_proto: FilterProto):
        filter_token: FilterToken = self.tokenize(filter_proto)
        if not filter_token:
            return qs
        query_validation_check, err = self.validate(filter_token)
        if not query_validation_check:
            raise ValueError(err)
        return self.evaluate(qs, filter_token)

event/base/filter_token.py
DrDroidLab/kenobi
import abc
from functools import reduce
from typing import Union

from django.db.models import F, Q, Value

from event.base.literal import is_scalar
from event.base.token import Filterable, ColumnToken, OpToken, LiteralToken, AttributeToken, AttributeTokenV2
from protos.event.query_base_pb2 import Op


class TokenFilterOp(abc.ABC):
    def process(self, lhs, op, rhs):
        pass


class ColumnTokenFilterOp(TokenFilterOp):
    supported_ops = None

    def process(self, lhs_token: ColumnToken, op_token: OpToken, rhs_token: Union[Filterable, LiteralToken]):
        lhs = lhs_token.filter_key()
        op = op_token.op

        rhs = self.rhs(rhs_token)

        if lhs_token.column.supported_ops:
            if op not in lhs_token.column.supported_ops:
                raise ValueError(f'Query {op} not supported for {lhs_token}')
        else:
            if self.supported_ops and op not in self.supported_ops:
                raise ValueError(f'Query {op} not supported for {lhs_token}')

        return self.q(lhs, op, rhs)

    def rhs(self, rhs_token):
        if isinstance(rhs_token, Filterable):
            return F(rhs_token.filter_key())
        elif isinstance(rhs_token, LiteralToken):
            return rhs_token.literal_value
        else:
            raise ValueError('RHS must be Filterable or Literal')

    def q(self, lhs, op, rhs):
        pass


class IDColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.IN]

    def q(self, lhs, op, rhs):
        if op == Op.EQ:
            return Q(**{f'{lhs}': rhs})
        elif op == Op.IN:
            return Q(**{f'{lhs}__in': rhs})


class TimestampColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.GTE, Op.GT, Op.LTE, Op.LT]

    def q(self, lhs, op, rhs) -> Q:
        if op == Op.EQ:
            return Q(**{f'{lhs}': rhs})
        elif op == Op.GTE:
            return Q(**{f'{lhs}__gte': rhs})
        elif op == Op.GT:
            return Q(**{f'{lhs}__gt': rhs})
        elif op == Op.LTE:
            return Q(**{f'{lhs}__lte': rhs})
        elif op == Op.LT:
            return Q(**{f'{lhs}__lt': rhs})


class StringColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.NEQ, Op.IN, Op.NOT_IN, Op.IS_NULL, Op.EXISTS]

    def q(self, lhs, op, rhs) -> Q:
        if op == Op.EQ:
            return Q(**{f'{lhs}': rhs})
        elif op == Op.NEQ:
            return ~Q(**{f'{lhs}': rhs})
        elif op == Op.IN:
            return Q(**{f'{lhs}__in': rhs})
        elif op == Op.NOT_IN:
            return ~Q(**{f'{lhs}__in': rhs})
        elif op == Op.IS_NULL:
            return Q(**{f'{lhs}__isnull': True})
        elif op == Op.EXISTS:
            return Q(**{f'{lhs}__isnull': False})


class BooleanColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.NEQ, Op.EXISTS, Op.IS_NULL]

    def q(self, lhs, op, rhs) -> Q:
        if op == Op.EQ:
            return Q(**{f'{lhs}': rhs})
        elif op == Op.NEQ:
            return ~Q(**{f'{lhs}': rhs})
        elif op == Op.IS_NULL:
            return Q(**{f'{lhs}__isnull': True})
        elif op == Op.EXISTS:
            return Q(**{f'{lhs}__isnull': False})


class NumericColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.NEQ, Op.GT, Op.GTE, Op.LT, Op.LTE, Op.IN, Op.NOT_IN, Op.IS_NULL, Op.EXISTS]

    def q(self, lhs, op, rhs) -> Q:
        if op == Op.EQ:
            return Q(**{f'{lhs}': rhs})
        elif op == Op.NEQ:
            return ~Q(**{f'{lhs}': rhs})
        elif op == Op.GT:
            return Q(**{f'{lhs}__gt': rhs})
        elif op == Op.LT:
            return Q(**{f'{lhs}__lt': rhs})
        elif op == Op.GTE:
            return Q(**{f'{lhs}__gte': rhs})
        elif op == Op.LTE:
            return Q(**{f'{lhs}__lte': rhs})
        elif op == Op.IN:
            return Q(**{f'{lhs}__in': rhs})
        elif op == Op.NOT_IN:
            return ~Q(**{f'{lhs}__in': rhs})
        elif op == Op.IS_NULL:
            return Q(**{f'{lhs}__isnull': True})
        elif op == Op.EXISTS:
            return Q(**{f'{lhs}__isnull': False})


class LiteralArrayColumnTokenFilterOp(ColumnTokenFilterOp):
    supported_ops = [Op.EQ, Op.NEQ, Op.IN, Op.NOT_IN, Op.EXISTS, Op.IS_NULL]

    def rhs(self, rhs_token):
        if not isinstance(rhs_token, LiteralToken):
            raise ValueError('RHS must be Literal')
        if is_scalar(rhs_token.literal.literal_type):
            return [rhs_token.literal_value]
        return rhs_token.literal_value

    def q(self, lhs, op, rhs) -> Q:
        if op == Op.EQ or op == Op.IN:
            return Q(**{f'{lhs}__contains': rhs})
        elif op == Op.NEQ or op == Op.NOT_IN:
            return ~Q(**{f'{lhs}__contains': rhs})
        elif op == Op.EXISTS:
            return ~Q(**{f'{lhs}': Value([None])})
        elif op == Op.IS_NULL:
            return Q(**{f'{lhs}': Value([None])})


class AttributeTokenFilterOp(TokenFilterOp):
    @staticmethod
    def typecheck(lhs_token, op, rhs):
        if op == Op.IN and type(rhs) is not list:
            raise ValueError(f'IN expects a list type for values, query op not supported for {lhs_token}')
        elif op == Op.NOT_IN and type(rhs) is not list:
            raise ValueError(f'NOT_IN expects a list type for values, query op not supported for {lhs_token}')
        return

    def process(self, lhs_token: AttributeToken, op_token: OpToken, rhs_token: Union[Filterable, LiteralToken]):
        lhs = lhs_token.filter_key()
        op = op_token.op

        if isinstance(rhs_token, Filterable):
            rhs = F(rhs_token.filter_key())
        elif isinstance(rhs_token, LiteralToken):
            rhs = rhs_token.literal_value
            self.typecheck(lhs_token, op, rhs)
        else:
            raise ValueError('RHS must be Filterable or Literal')

        if op == Op.EQ:
            return Q(**{lhs: rhs})
        elif op == Op.NEQ:
            return ~Q(**{lhs: rhs})
        elif op == Op.GT:
            return Q(**{f'{lhs}__gt': rhs})
        elif op == Op.LT:
            return Q(**{f'{lhs}__lt': rhs})
        elif op == Op.GTE:
            return Q(**{f'{lhs}__gte': rhs})
        elif op == Op.LTE:
            return Q(**{f'{lhs}__lte': rhs})
        elif op == Op.IN:
            return Q(**{f'{lhs}__in': rhs})
        elif op == Op.NOT_IN:
            return ~Q(**{f'{lhs}__in': rhs})
        elif op == Op.IS_NULL:
            return Q(**{f'{lhs}__isnull': True})
        elif op == Op.EXISTS:
            return Q(**{f'{lhs}__isnull': False})
        else:
            raise ValueError(f'Query op not supported for {lhs_token}')


class AttributeTokenV2FilterOp(TokenFilterOp):
    @staticmethod
    def typecheck(lhs_token, op, rhs):
        if op == Op.IN and type(rhs) is not list:
            raise ValueError(f'IN expects a list type for values, query op not supported for {lhs_token}')
        elif op == Op.NOT_IN and type(rhs) is not list:
            raise ValueError(f'NOT_IN expects a list type for values, query op not supported for {lhs_token}')
        return

    def process(self, lhs_token: AttributeTokenV2, op_token: OpToken, rhs_token: Union[Filterable, LiteralToken]):
        lhs_list = lhs_token.filter_key()
        op = op_token.op

        if isinstance(rhs_token, Filterable):
            rhs = F(rhs_token.filter_key())
        elif isinstance(rhs_token, LiteralToken):
            rhs = rhs_token.literal_value
            self.typecheck(lhs_token, op, rhs)
        else:
            raise ValueError('RHS must be Filterable or Literal')
        filter_list = []
        for lhs in lhs_list:
            if op == Op.EQ:
                filter_list.append(Q(**{lhs: rhs}))
            elif op == Op.NEQ:
                filter_list.append(~Q(**{lhs: rhs}))
            elif op == Op.GT:
                filter_list.append(Q(**{f'{lhs}__gt': rhs}))
            elif op == Op.LT:
                filter_list.append(Q(**{f'{lhs}__lt': rhs}))
            elif op == Op.GTE:
                filter_list.append(Q(**{f'{lhs}__gte': rhs}))
            elif op == Op.LTE:
                filter_list.append(Q(**{f'{lhs}__lte': rhs}))
            elif op == Op.IN:
                filter_list.append(Q(**{f'{lhs}__in': rhs}))
            elif op == Op.NOT_IN:
                filter_list.append(~Q(**{f'{lhs}__in': rhs}))
            elif op == Op.IS_NULL:
                filter_list.append(Q(**{f'{lhs}__isnull': True}))
            elif op == Op.EXISTS:
                filter_list.append(Q(**{f'{lhs}__isnull': False}))
            else:
                raise ValueError(f'Query op not supported for {lhs_token}')

        return reduce(lambda x, y: x | y, filter_list)

event/base/filter_token_op.py
DrDroidLab/kenobi
from collections import defaultdict
from datetime import datetime

from google.protobuf.wrappers_pb2 import UInt64Value, StringValue, DoubleValue, Int64Value, BoolValue

from protos.event.base_pb2 import EventKey
from protos.event.literal_pb2 import Literal, LiteralType, IdLiteral
from protos.event.metric_pb2 import AggregationFunction

_SCALAR_LITERALS = {
    LiteralType.STRING,
    LiteralType.LONG,
    LiteralType.DOUBLE,
    LiteralType.BOOLEAN,
    LiteralType.TIMESTAMP,
    LiteralType.ID
}


def is_scalar(lt: LiteralType):
    return lt in _SCALAR_LITERALS


def literal_to_obj(lt: Literal):
    if lt.literal_type == LiteralType.STRING:
        return lt.string.value
    elif lt.literal_type == LiteralType.LONG:
        return lt.long.value
    elif lt.literal_type == LiteralType.DOUBLE:
        return lt.double.value
    elif lt.literal_type == LiteralType.BOOLEAN:
        return lt.boolean.value
    elif lt.literal_type == LiteralType.TIMESTAMP:
        return datetime.utcfromtimestamp(lt.timestamp)
    elif lt.literal_type == LiteralType.ID:
        id_literal = lt.id
        if id_literal.type == IdLiteral.Type.LONG:
            return id_literal.long.value
        elif id_literal.type == IdLiteral.Type.STRING:
            return id_literal.string.value
    elif lt.literal_type == LiteralType.STRING_ARRAY:
        return list(lt.string_array)
    elif lt.literal_type == LiteralType.LONG_ARRAY:
        return list(lt.long_array)
    elif lt.literal_type == LiteralType.DOUBLE_ARRAY:
        return list(lt.double_array)
    elif lt.literal_type == LiteralType.BOOLEAN_ARRAY:
        return list(lt.boolean_array)
    elif lt.literal_type == LiteralType.ID_ARRAY:
        ids = []
        for id_literal in lt.id_array:
            if id_literal.type == IdLiteral.Type.LONG:
                ids.append(id_literal.long.value)
            elif id_literal.type == IdLiteral.Type.STRING:
                ids.append(id_literal.string.value)
        return ids
    elif lt.literal_type == LiteralType.NULL_STRING:
        return ''
    elif lt.literal_type == LiteralType.NULL_NUMBER:
        return 0

    return None


def display_literal(lt: Literal):
    if lt.literal_type == LiteralType.STRING:
        return f'"{lt.string.value}"'
    elif lt.literal_type == LiteralType.LONG:
        return lt.long.value
    elif lt.literal_type == LiteralType.DOUBLE:
        return lt.double.value
    elif lt.literal_type == LiteralType.BOOLEAN:
        return lt.boolean.value
    elif lt.literal_type == LiteralType.TIMESTAMP:
        return datetime.utcfromtimestamp(lt.timestamp)
    elif lt.literal_type == LiteralType.ID:
        id_literal = lt.id
        if id_literal.type == IdLiteral.Type.LONG:
            return id_literal.long.value
        elif id_literal.type == IdLiteral.Type.STRING:
            return f'"{id_literal.string.value}"'
    elif lt.literal_type == LiteralType.STRING_ARRAY:
        return [f'"{string}"' for string in lt.string_array]
    elif lt.literal_type == LiteralType.LONG_ARRAY:
        return list(lt.long_array)
    elif lt.literal_type == LiteralType.DOUBLE_ARRAY:
        return list(lt.double_array)
    elif lt.literal_type == LiteralType.BOOLEAN_ARRAY:
        return list(lt.boolean_array)
    elif lt.literal_type == LiteralType.ID_ARRAY:
        ids = []
        for id_literal in lt.id_array:
            if id_literal.type == IdLiteral.Type.LONG:
                ids.append(id_literal.long.value)
            elif id_literal.type == IdLiteral.Type.STRING:
                ids.append(f'"{id_literal.string.value}"')
        return ids
    elif lt.literal_type == LiteralType.NULL_STRING:
        return '""'
    elif lt.literal_type == LiteralType.NULL_NUMBER:
        return 0

    return None


def obj_to_literal(o, is_id=False):
    if o is None:
        return Literal(literal_type=LiteralType.NULL_NUMBER)
    elif isinstance(o, bool):
        return Literal(literal_type=LiteralType.BOOLEAN, boolean=BoolValue(value=o))
    elif isinstance(o, str):
        if is_id:
            return Literal(
                literal_type=LiteralType.ID,
                id=IdLiteral(type=IdLiteral.Type.STRING, string=StringValue(value=o))
            )
        return Literal(literal_type=LiteralType.STRING, string=StringValue(value=o))
    elif isinstance(o, int):
        if is_id:
            return Literal(
                literal_type=LiteralType.ID,
                id=IdLiteral(type=IdLiteral.Type.LONG, long=UInt64Value(value=o))
            )
        return Literal(literal_type=LiteralType.LONG, long=Int64Value(value=o))
    elif isinstance(o, float):
        return Literal(literal_type=LiteralType.DOUBLE, double=DoubleValue(value=o))
    elif isinstance(o, datetime):
        return Literal(literal_type=LiteralType.TIMESTAMP, timestamp=int(o.timestamp()))
    else:
        raise ValueError('Unexpected type')


_event_key_type_to_literal_type_dict = defaultdict(lambda: LiteralType.UNKNOWN_LT, {
    EventKey.KeyType.STRING: LiteralType.STRING,
    EventKey.KeyType.LONG: LiteralType.LONG,
    EventKey.KeyType.DOUBLE: LiteralType.DOUBLE,
    EventKey.KeyType.BOOLEAN: LiteralType.BOOLEAN,
})

_literal_type_is_groupable = defaultdict(lambda: False, {
    LiteralType.STRING: True,
    LiteralType.LONG: True,
    LiteralType.BOOLEAN: True,
    LiteralType.DOUBLE: False
})

_literal_type_aggregation_function = defaultdict(list, {
    LiteralType.STRING: [AggregationFunction.COUNT_DISTINCT],
    LiteralType.LONG: [
        AggregationFunction.COUNT_DISTINCT,
        AggregationFunction.SUM,
        AggregationFunction.AVG,
        AggregationFunction.MIN,
        AggregationFunction.MAX,
        AggregationFunction.STD_DEV,
        AggregationFunction.VARIANCE,
    ],
    LiteralType.BOOLEAN: [AggregationFunction.COUNT_DISTINCT],
    LiteralType.DOUBLE: [
        AggregationFunction.SUM,
        AggregationFunction.AVG,
        AggregationFunction.MIN,
        AggregationFunction.MAX,
        AggregationFunction.STD_DEV,
        AggregationFunction.VARIANCE,
    ]
})


def event_key_type_to_literal_type(key_type: EventKey.KeyType) -> LiteralType:
    return _event_key_type_to_literal_type_dict.get(key_type, LiteralType.UNKNOWN_LT)


def literal_type_is_groupable(literal_type: LiteralType) -> bool:
    return _literal_type_is_groupable.get(literal_type, False)


def literal_type_aggregation_functions(literal_type: LiteralType) -> [AggregationFunction]:
    return _literal_type_aggregation_function.get(literal_type, list())

event/base/literal.py
DrDroidLab/kenobi
from collections import OrderedDict
from functools import lru_cache
from typing import List, Dict
import re

from django.db import models
from django.db.models import QuerySet, Count, Sum, Avg, Max, Min, StdDev, Variance
from django.db.models.expressions import RawSQL, F
from django.db.models.fields.json import KeyTextTransform
from django.db.models.functions import Cast
from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value

from event.clickhouse.models import Events

from event.base.filter_token import FilterToken, FilterTokenizer, FilterTokenValidator, \
    FilterTokenEvaluator, FilterTokenAnnotator
from event.base.literal import obj_to_literal
from event.base.token import Token, Groupable, Filterable, ColumnToken, AttributeToken, ExpressionTokenizer, Annotable
from protos.event.literal_pb2 import LiteralType
from protos.event.metric_pb2 import AggregationFunction, LabeledData, MetricData, MetricDataMetadata, MetricExpression, \
    MetricSelector, TsDataPoint
from prototype.utils.timerange import DateTimeRange, filter_dtr

_FUNCTION_DICT = {
    AggregationFunction.SUM: Sum,
    AggregationFunction.AVG: Avg,
    AggregationFunction.MAX: Max,
    AggregationFunction.MIN: Min,
    AggregationFunction.STD_DEV: StdDev,
    AggregationFunction.VARIANCE: Variance,
    AggregationFunction.COUNT: Count,
}

_AGGREGATEABLE_TYPES = {
    LiteralType.LONG,
    LiteralType.DOUBLE,
    LiteralType.TIMESTAMP,
}


@lru_cache(maxsize=10)
def get_metric_for_idx(idx: int):
    return f'metric_{idx + 1}'


def get_float_metric(aggrfn, key, field):
    return aggrfn(
        Cast(
            KeyTextTransform(key, field), models.FloatField()
        )
    )


def get_label_group(label_group_keys, record):
    return '__'.join([str(record[key]) for key in label_group_keys])


class MetricSelectorToken(Token):
    function: AggregationFunction = None
    expression: Token = None
    alias: str = None
    metric_alias: str = None
    metric_selector: MetricSelector = None

    def display(self) -> str:
        return f'{AggregationFunction.Name(self.function)}({self.expression.display()}) as {self.alias}'


class MetricSelectorTokenizer:
    def __init__(self, columns):
        self._columns = columns
        self._expression_tokenizer = ExpressionTokenizer(columns)

    def tokenize(self, metric_selector: MetricSelector, idx) -> MetricSelectorToken:
        metric_selector_token = MetricSelectorToken()

        metric_selector_token.function = metric_selector.function
        metric_selector_token.expression = self._expression_tokenizer.tokenize(metric_selector.expression)
        metric_selector_token.alias = metric_selector.alias
        metric_selector_token.metric_alias = get_metric_for_idx(idx)
        metric_selector_token.metric_selector = metric_selector

        return metric_selector_token


class MetricSelectorTokenAnnotator:
    def annotations(self, selector: Token) -> Dict:
        if not isinstance(selector, MetricSelectorToken):
            raise ValueError('MetricSelectorTokenAnnotator can only process MetricSelectorToken')
        annotations = {}
        if isinstance(selector.expression, Annotable):
            annotations.update(selector.expression.annotations())
        return annotations


class MetricToken(Token):
    filter: FilterToken = None
    group_by: List[Token] = None
    selectors: List[MetricSelectorToken] = None
    selector_metric_idx_alias_dict: Dict = None
    is_timeseries: bool = False
    resolution: int = 30
    timestamp_field: str = None

    def display(self) -> str:
        return f'{" ,".join(selector.display() for selector in self.selectors)} WHERE {self.filter.display()} GROUP BY {", ".join([group_by.display() for group_by in self.group_by])}'


class MetricTokenizer:
    def __init__(self, columns, timestamp_field):
        self._columns = columns
        self._timestamp_field = timestamp_field
        self._filter_tokenizer = FilterTokenizer(columns)
        self._metric_selector_tokenizer = MetricSelectorTokenizer(columns)
        self._expression_tokenizer = ExpressionTokenizer(columns)

    def tokenize(self, metric_expression: MetricExpression) -> MetricToken:
        metric_token = MetricToken()
        metric_token.filter = self._filter_tokenizer.tokenize(metric_expression.filter)
        metric_token.group_by = [self._expression_tokenizer.tokenize(g) for g in metric_expression.group_by]
        metric_token.selectors = [self._metric_selector_tokenizer.tokenize(s, idx) for idx, s in
                                  enumerate(metric_expression.selectors)]

        metric_token.selector_metric_idx_alias_dict = {
            get_metric_for_idx(idx): s.alias for idx, s in enumerate(metric_token.selectors)
        }

        metric_token.is_timeseries = metric_expression.is_timeseries.value
        if metric_expression.resolution.value:
            metric_token.resolution = metric_expression.resolution.value
        metric_token.timestamp_field = self._timestamp_field
        return metric_token


class MetricTokenValidator:
    def __init__(self):
        self._filter_token_validator = FilterTokenValidator()

    def validate(self, metric_token: MetricToken) -> (bool, str):
        if metric_token.filter:
            is_filter_token_valid, err = self._filter_token_validator.validate(metric_token.filter)
            if not is_filter_token_valid:
                return False, err
        return True, ''

        # TODO: Add validation checks for group_by and selectors


class MetricTokenAnnotator:
    def __init__(self):
        self._metric_selector_token_annotator = MetricSelectorTokenAnnotator()
        self._filter_token_annotator = FilterTokenAnnotator()

    def annotations(self, token: Token) -> Dict:
        if not isinstance(token, MetricToken):
            raise ValueError('MetricTokenAnnotator can only process MetricToken')
        annotations = {}
        if token.filter:
            annotations.update(self._filter_token_annotator.annotations(token.filter))
        for g in token.group_by:
            if isinstance(g, Annotable):
                annotations.update(g.annotations())

        for selector in token.selectors:
            annotations.update(self._metric_selector_token_annotator.annotations(selector))

        return annotations


class MetricTokenProcessor:
    def __init__(self):
        self._filter_token_evaluator = FilterTokenEvaluator()
        self._metric_token_annotator = MetricTokenAnnotator()
        self._timestamp_bucket = 'timestamp_bucket'

    def _selector_annotations(self, selectors: List[MetricSelectorToken]) -> OrderedDict:
        selector_annotations = OrderedDict({})
        for idx, selector in enumerate(selectors):
            selector_function = selector.function
            expression: Filterable = selector.expression
            metric_alias = selector.metric_alias

            if selector_function is AggregationFunction.COUNT:
                if expression:
                    selector_annotations[metric_alias] = Count(F(expression.filter_key()))
                else:
                    selector_annotations[metric_alias] = Count('pk')
            elif selector_function is AggregationFunction.COUNT_DISTINCT:
                selector_annotations[metric_alias] = Count(F(expression.filter_key()), distinct=True)
            else:
                if isinstance(expression, ColumnToken):
                    column_type = expression.column.type
                    if column_type not in _AGGREGATEABLE_TYPES:
                        raise ValueError(
                            f'Function {AggregationFunction.Name(selector_function)} not valid for column {expression.column.display_name} of type {LiteralType.Name(column_type)}'
                        )
                    if selector_function not in expression.column.aggregation_functions:
                        raise ValueError(
                            f'Function {AggregationFunction.Name(selector_function)} not valid for column {expression.column.display_name}'
                        )

                    selector_annotations[metric_alias] = _FUNCTION_DICT[selector_function](F(expression.filter_key()))
                elif isinstance(expression, AttributeToken):
                    selector_annotations[metric_alias] = _FUNCTION_DICT[selector_function](
                        Cast(
                            KeyTextTransform(expression.attribute_identifier.name,
                                             expression.attribute_identifier.path), models.FloatField()
                        )
                    )
        return selector_annotations

    def process(self, qs: QuerySet, metric_token: MetricToken, dtr: DateTimeRange):
        is_timeseries = metric_token.is_timeseries
        ts_field = metric_token.timestamp_field
        resolution = metric_token.resolution

        # Filter timerange on the base qs
        qs: QuerySet = filter_dtr(qs, dtr, ts_field)

        annotations = self._metric_token_annotator.annotations(metric_token)
        qs = qs.annotate(**annotations)
        if metric_token.filter:
            # Apply filter on the base qs
            qs = self._filter_token_evaluator.process(qs, metric_token.filter)
        if is_timeseries:
            # Annotate each record with a timestamp bucket based on the resolution and the start timestamp
            qs = qs.annotate(timestamp_bucket=RawSQL(
                '{start_ts} + (((CAST(extract(epoch from "{table}"."{ts_field}") as BIGINT) - {start_ts}) /%s) * %s)'.format(
                    table=qs.model._meta.db_table,
                    ts_field=ts_field,
                    start_ts=int(dtr.time_geq.timestamp())
                )
                , [resolution, resolution])
            )

        # All group by labels from columns
        group_by: list = list()
        # All metric label metadata expressions
        metric_label_metadata = OrderedDict()
        for g in metric_token.group_by:
            if isinstance(g, Groupable):
                k = g.group_key()
                group_by.append(k)
                metric_label_metadata[k] = g

        if is_timeseries:
            group_by.insert(0, self._timestamp_bucket)

        selector_annotations = self._selector_annotations(metric_token.selectors)
        selector_annotation_aliases = list(selector_annotations.keys())

        if group_by:
            qs = qs.values(*(list(group_by)))
            qs = qs.annotate(**selector_annotations)
            if is_timeseries:
                qs = qs.order_by(self._timestamp_bucket)
        else:
            # In case there is no group by and no timeseries, we need to resolve the selections here only
            qs = qs.aggregate(**selector_annotations)

        resolved_qs = []
        if isinstance(qs, QuerySet):
            resolved_qs = list(qs)
        elif isinstance(qs, Dict):
            resolved_qs = [qs]

        label_key_ordering = metric_label_metadata.keys()

        label_group_data_map = {}
        for record in resolved_qs:
            label_group = get_label_group(label_key_ordering, record)
            if label_group not in label_group_data_map:
                label_group_data_map[label_group] = LabeledData(
                    label_group=label_group,
                    labels=[
                        metric_label_metadata[label_key].group_label(record[label_key])
                        for label_key in label_key_ordering
                    ],
                    alias_data_map={}
                )
            data = label_group_data_map[label_group]
            for metric_alias in selector_annotation_aliases:
                metric = obj_to_literal(record[metric_alias])
                if is_timeseries:
                    data.alias_data_map[metric_alias].timeseries_data.append(
                        TsDataPoint(
                            timestamp=int(record[self._timestamp_bucket]),
                            value=metric
                        )
                    )
                else:
                    data.alias_data_map[metric_alias].value.CopyFrom(metric)

        metric_data_metadata = MetricDataMetadata(
            labels_metadata=[
                metric_label_metadata[label_key].group_label_metadata()
                for label_key in label_key_ordering
            ],
            is_timeseries=BoolValue(value=is_timeseries),
            resolution=UInt64Value(value=resolution),
            time_range=dtr.to_tr(),
            metric_alias_selector_map={
                selector.metric_alias: selector.metric_selector
                for selector in metric_token.selectors
            }
        )

        return MetricData(
            metadata=metric_data_metadata,
            labeled_data=list(label_group_data_map.values())
        )


class EventsClickhouseMetricTokenProcessor(MetricTokenProcessor):

    def process(self, qs: QuerySet, metric_token: MetricToken, dtr: DateTimeRange):
        is_timeseries = metric_token.is_timeseries
        ts_field = metric_token.timestamp_field
        resolution = metric_token.resolution

        # Filter timerange on the base qs
        qs: QuerySet = filter_dtr(qs, dtr, ts_field)

        annotations = self._metric_token_annotator.annotations(metric_token)
        qs = qs.annotate(**annotations)
        if metric_token.filter:
            # Apply filter on the base qs
            qs = self._filter_token_evaluator.process(qs, metric_token.filter)
        if is_timeseries:
            # Annotate each record with a timestamp bucket based on the resolution and the start timestamp
            qs = qs.annotate(timestamp_bucket=RawSQL(
                '{start_ts} + (floor((toUInt64(toDateTime("{table}"."{ts_field}")) - {start_ts}) /%s) * %s)'.format(
                    table=qs.model._meta.db_table,
                    ts_field=ts_field,
                    start_ts=int(dtr.time_geq.timestamp())
                )
                , [resolution, resolution])
            )

        # All group by labels from columns
        group_by: list = list()
        # All metric label metadata expressions
        metric_label_metadata = OrderedDict()
        for g in metric_token.group_by:
            if isinstance(g, Groupable):
                k = g.group_key()
                group_by.append(k)
                metric_label_metadata[k] = g

        if is_timeseries:
            group_by.insert(0, self._timestamp_bucket)

        selector_annotations = self._selector_annotations(metric_token.selectors)
        selector_annotation_aliases = list(selector_annotations.keys())

        if group_by:
            qs = qs.values(*(list(group_by)))
            qs = qs.annotate(**selector_annotations)
            if is_timeseries:
                qs = qs.order_by(self._timestamp_bucket)
        else:
            # In case there is no group by and no timeseries, we need to resolve the selections here only
            qs = qs.aggregate(**selector_annotations)

        resolved_qs = []
        if isinstance(qs, QuerySet):
            resolved_qs = list(qs)
        elif isinstance(qs, Dict):
            resolved_qs = [qs]

        label_key_ordering = metric_label_metadata.keys()

        label_group_data_map = {}
        for record in resolved_qs:
            label_group = get_label_group(label_key_ordering, record)
            if label_group not in label_group_data_map:
                label_group_data_map[label_group] = LabeledData(
                    label_group=label_group,
                    labels=[
                        metric_label_metadata[label_key].group_label(record[label_key])
                        for label_key in label_key_ordering
                    ],
                    alias_data_map={}
                )
            data = label_group_data_map[label_group]
            for metric_alias in selector_annotation_aliases:
                metric = obj_to_literal(record[metric_alias])
                if is_timeseries:
                    data.alias_data_map[metric_alias].timeseries_data.append(
                        TsDataPoint(
                            timestamp=int(record[self._timestamp_bucket]),
                            value=metric
                        )
                    )
                else:
                    data.alias_data_map[metric_alias].value.CopyFrom(metric)

        metric_data_metadata = MetricDataMetadata(
            labels_metadata=[
                metric_label_metadata[label_key].group_label_metadata()
                for label_key in label_key_ordering
            ],
            is_timeseries=BoolValue(value=is_timeseries),
            resolution=UInt64Value(value=resolution),
            time_range=dtr.to_tr(),
            metric_alias_selector_map={
                selector.metric_alias: selector.metric_selector
                for selector in metric_token.selectors
            }
        )

        return MetricData(
            metadata=metric_data_metadata,
            labeled_data=list(label_group_data_map.values())
        )


class MetricExpressionEvaluator:
    def __init__(self, parent_model, columns, timestamp_field):
        self._metric_tokenizer = MetricTokenizer(columns, timestamp_field)
        self._metric_token_validator = MetricTokenValidator()

        if parent_model == Events:
            self._metric_token_processor = EventsClickhouseMetricTokenProcessor()
        else:
            self._metric_token_processor = MetricTokenProcessor()

    def process(self, qs, metric_expression: MetricExpression, dtr: DateTimeRange):
        metric_token: MetricToken = self._metric_tokenizer.tokenize(metric_expression)
        is_valid, err = self._metric_token_validator.validate(metric_token)
        if not is_valid:
            raise ValueError(err)
        return self._metric_token_processor.process(qs, metric_token, dtr)

    def process_batch(self, qs: QuerySet, metric_expressions: List[MetricExpression], dtr: DateTimeRange):
        metric_tokens: List[MetricToken] = [
            self._metric_tokenizer.tokenize(metric_expression) for metric_expression in metric_expressions
        ]
        for metric_token in metric_tokens:
            is_valid, err = self._metric_token_validator.validate(metric_token)
            if not is_valid:
                raise ValueError(err)
        return [
            self._metric_token_processor.process(qs, metric_token, dtr) for metric_token in metric_tokens
        ]

event/base/metric_token.py
DrDroidLab/kenobi
from protos.event.literal_pb2 import LiteralType, LiteralTypeDescription
from protos.event.query_base_pb2 import OpDescription, Op, OpRhs, OpMapping

OP_DESCRIPTIONS = [
    OpDescription(op=Op.EQ, label='='),
    OpDescription(op=Op.NEQ, label='!='),
    OpDescription(op=Op.GT, label='>'),
    OpDescription(op=Op.LT, label='<'),
    OpDescription(op=Op.GTE, label='>='),
    OpDescription(op=Op.LTE, label='<='),
    OpDescription(op=Op.IN, label='IN'),
    OpDescription(op=Op.NOT_IN, label='NOT IN'),
    OpDescription(op=Op.OR, label='OR', is_logical=True),
    OpDescription(op=Op.AND, label='AND', is_logical=True),
    OpDescription(op=Op.IS_NULL, label='IS NULL', is_unary=True),
    OpDescription(op=Op.EXISTS, label='EXISTS', is_unary=True),
    OpDescription(op=Op.NOT, label='NOT', is_unary=True, is_logical=True)
]
STRING_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.STRING),
    OpRhs(op=Op.NEQ, rhs=LiteralType.STRING),
    OpRhs(op=Op.IN, rhs=LiteralType.STRING_ARRAY),
    OpRhs(op=Op.NOT_IN, rhs=LiteralType.STRING_ARRAY),
    OpRhs(op=Op.IS_NULL, rhs=LiteralType.NULL_STRING),
    OpRhs(op=Op.EXISTS, rhs=LiteralType.NULL_STRING)
]
LONG_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.LONG),
    OpRhs(op=Op.NEQ, rhs=LiteralType.LONG),
    OpRhs(op=Op.GT, rhs=LiteralType.LONG),
    OpRhs(op=Op.LT, rhs=LiteralType.LONG),
    OpRhs(op=Op.GTE, rhs=LiteralType.LONG),
    OpRhs(op=Op.LTE, rhs=LiteralType.LONG),
    OpRhs(op=Op.IN, rhs=LiteralType.LONG_ARRAY),
    OpRhs(op=Op.NOT_IN, rhs=LiteralType.LONG_ARRAY),
    OpRhs(op=Op.IS_NULL, rhs=LiteralType.NULL_NUMBER),
    OpRhs(op=Op.EXISTS, rhs=LiteralType.NULL_NUMBER),
]
DOUBLE_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.NEQ, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.GT, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.LT, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.GTE, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.LTE, rhs=LiteralType.DOUBLE),
    OpRhs(op=Op.IN, rhs=LiteralType.DOUBLE_ARRAY),
    OpRhs(op=Op.NOT_IN, rhs=LiteralType.DOUBLE_ARRAY),
    OpRhs(op=Op.IS_NULL, rhs=LiteralType.NULL_NUMBER),
    OpRhs(op=Op.EXISTS, rhs=LiteralType.NULL_NUMBER),
]
BOOLEAN_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.BOOLEAN),
    OpRhs(op=Op.NEQ, rhs=LiteralType.BOOLEAN),
    OpRhs(op=Op.IS_NULL, rhs=LiteralType.NULL_NUMBER),
    OpRhs(op=Op.EXISTS, rhs=LiteralType.NULL_NUMBER),
]
TIMESTAMP_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.TIMESTAMP),
    OpRhs(op=Op.GT, rhs=LiteralType.TIMESTAMP),
    OpRhs(op=Op.LT, rhs=LiteralType.TIMESTAMP),
    OpRhs(op=Op.GTE, rhs=LiteralType.TIMESTAMP),
    OpRhs(op=Op.LTE, rhs=LiteralType.TIMESTAMP),
]
ID_LITERAL_OPS_RHS_MAP = [
    OpRhs(op=Op.EQ, rhs=LiteralType.ID),
    OpRhs(op=Op.IN, rhs=LiteralType.ID_ARRAY),
]
OP_MAPPINGS = [
    OpMapping(lhs=LiteralType.STRING, op_rhs=STRING_OPS_RHS_MAP),
    OpMapping(lhs=LiteralType.LONG, op_rhs=LONG_OPS_RHS_MAP),
    OpMapping(lhs=LiteralType.DOUBLE, op_rhs=DOUBLE_OPS_RHS_MAP),
    OpMapping(lhs=LiteralType.BOOLEAN, op_rhs=BOOLEAN_OPS_RHS_MAP),
    OpMapping(lhs=LiteralType.TIMESTAMP, op_rhs=TIMESTAMP_OPS_RHS_MAP),
    OpMapping(lhs=LiteralType.ID, op_rhs=ID_LITERAL_OPS_RHS_MAP),
]

LITERAL_TYPE_DESCRIPTIONS = [
    LiteralTypeDescription(literal_type=LiteralType.STRING, label='str'),
    LiteralTypeDescription(literal_type=LiteralType.LONG, label='long'),
    LiteralTypeDescription(literal_type=LiteralType.DOUBLE, label='double'),
    LiteralTypeDescription(literal_type=LiteralType.BOOLEAN, label='bool'),
    LiteralTypeDescription(literal_type=LiteralType.TIMESTAMP, label='timestamp'),
    LiteralTypeDescription(literal_type=LiteralType.ID, label='id'),
    LiteralTypeDescription(literal_type=LiteralType.STRING_ARRAY, label='str array'),
    LiteralTypeDescription(literal_type=LiteralType.LONG_ARRAY, label='long array'),
    LiteralTypeDescription(literal_type=LiteralType.DOUBLE_ARRAY, label='double array'),
    LiteralTypeDescription(literal_type=LiteralType.BOOLEAN_ARRAY, label='bool array'),
    LiteralTypeDescription(literal_type=LiteralType.ID_ARRAY, label='id array'),
]


def validate_query(lhs_type: LiteralType, op: Op, rhs_type: LiteralType) -> (bool, str):
    op_rhs: [OpRhs] = None
    for mapping in OP_MAPPINGS:
        if lhs_type == mapping.lhs:
            op_rhs = mapping.op_rhs
            break
    if not op_rhs:
        return False, op

    rhs: LiteralType = LiteralType.UNKNOWN_LT
    for op_rhs_mapping in op_rhs:
        if op == op_rhs_mapping.op:
            rhs = op_rhs_mapping.rhs
            break
    if rhs == LiteralType.UNKNOWN_LT or rhs_type != rhs:
        return False, "rhs type: {}".format(rhs_type)

    return True, ""

event/base/op.py
DrDroidLab/kenobi
from typing import Dict

from django.db.models import Q, F

from event.base.token import Token, ColumnToken, Annotable, ExpressionTokenizer
from protos.event.query_base_pb2 import Op, OrderByExpression, SortOrder


class OrderByToken(Token):
    order_by: Token = None
    sort_order: SortOrder = None
    allow_nulls = True
    nulls_last = True

    def __init__(self, order_by: Token, sort_order: SortOrder = SortOrder.DESC, allow_nulls=True, nulls_last=True):
        self.order_by = order_by
        self.sort_order = sort_order
        self.allow_nulls = allow_nulls
        self.nulls_last = nulls_last

    def display(self):
        return f'({self.order_by.display()}'


class OrderByTokenAnnotator:
    def annotations(self, t: Token) -> Dict:
        if not isinstance(t, OrderByToken):
            raise ValueError('FilterTokenAnnotator can only process valid OrderBy Expression')
        annotations = {}

        if t.order_by and isinstance(t.order_by, Annotable):
            annotations.update(t.order_by.annotations())

        return annotations


class OrderByTokenizer:
    def __init__(self, columns):
        self._columns = columns
        self._expression_tokenizer = ExpressionTokenizer(columns)

    def tokenize(self, order_by_expression: OrderByExpression) -> OrderByToken:
        if order_by_expression.ByteSize() == 0:
            return

        order_by_token = self._expression_tokenizer.tokenize(order_by_expression.expression)

        allow_nulls = True
        if order_by_expression.allow_nulls:
            allow_nulls = order_by_expression.allow_nulls.value

        nulls_last = True
        if order_by_expression.nulls_last:
            nulls_last = order_by_expression.nulls_last.value

        return OrderByToken(
            order_by=order_by_token,
            sort_order=order_by_expression.order,
            allow_nulls=allow_nulls,
            nulls_last=nulls_last
        )


class OrderByTokenValidator:
    def validate(self, order_by_token: OrderByToken) -> (bool, str):
        if not order_by_token:
            return True, ''
        if not isinstance(order_by_token.order_by, ColumnToken):
            return False, 'Can only order by on columns'

        if not order_by_token.order_by.column.is_orderable:
            return False, 'Selected column is not orderable'
        return True, ''


class OrderByTokenEvaluator:
    def __init__(self):
        self._order_by_token_annotator = OrderByTokenAnnotator()

    def process(self, qs, order_by_token: OrderByToken):
        if not order_by_token:
            return qs

        if isinstance(order_by_token.order_by, Annotable):
            annotations = self._order_by_token_annotator.annotations(order_by_token)
            qs = qs.annotate(**annotations)

        if not order_by_token.allow_nulls:
            qs = qs.filter(
                Q(**{f'{order_by_token.order_by.column_identifier.name}__isnull': order_by_token.allow_nulls}))

        if order_by_token.sort_order == SortOrder.DESC:
            if order_by_token.nulls_last:
                qs = qs.order_by(F(order_by_token.order_by.column_identifier.name).desc(nulls_last=True))
            else:
                qs = qs.order_by(F(order_by_token.order_by.column_identifier.name).desc(nulls_first=True))
        elif order_by_token.sort_order == SortOrder.ASC:
            if order_by_token.nulls_last:
                qs = qs.order_by(F(order_by_token.order_by.column_identifier.name).asc(nulls_last=True))
            else:
                qs = qs.order_by(F(order_by_token.order_by.column_identifier.name).asc(nulls_first=True))

        return qs


class OrderByEngine:
    def __init__(self, columns, default_order_expression: OrderByExpression):
        self.columns = columns
        self.default_order_expression = default_order_expression
        self._order_by_tokenizer = OrderByTokenizer(columns)
        self._order_by_token_validator = OrderByTokenValidator()
        self._order_by_token_evaluator = OrderByTokenEvaluator()

    def tokenize(self, order_by_expression: OrderByExpression) -> OrderByToken:
        return self._order_by_tokenizer.tokenize(order_by_expression)

    def validate(self, filter_token: OrderByToken) -> (bool, str):
        return self._order_by_token_validator.validate(filter_token)

    def evaluate(self, qs, filter_token: OrderByToken):
        return self._order_by_token_evaluator.process(qs, filter_token)

    def process(self, qs, order_by_expression: OrderByExpression):
        if order_by_expression.ByteSize() == 0:
            order_by_expression = self.default_order_expression
        order_by_token: OrderByToken = self.tokenize(order_by_expression)
        if not order_by_token:
            return qs
        order_by_validation_check, err = self.validate(order_by_token)
        if not order_by_validation_check:
            raise ValueError(err)
        return self.evaluate(qs, order_by_token)

event/base/order_by_token.py
DrDroidLab/kenobi
import abc
from abc import abstractmethod
from typing import Dict

from django.db.models import F

from event.base.column import Column, AttributeColumn, BaseColumn, AttributeColumnV2
from event.base.literal import literal_to_obj, display_literal, obj_to_literal
from protos.event.literal_pb2 import Literal, LiteralType
from protos.event.metric_pb2 import LabelMetadata, Label
from protos.event.query_base_pb2 import ColumnIdentifier, AttributeIdentifier, Op, AttributeIdentifierV2
from protos.event.query_base_pb2 import Expression as ExpressionProto


class Filterable(abc.ABC):
    @abstractmethod
    def filter_key(self):
        pass


class Annotable(abc.ABC):
    @abstractmethod
    def annotations(self):
        pass


class Groupable(abc.ABC):
    @abstractmethod
    def group_key(self):
        pass

    @abstractmethod
    def group_label_metadata(self):
        pass

    @abstractmethod
    def group_label(self, obj):
        pass


class Orderable(abc.ABC):
    @abstractmethod
    def order_key(self):
        pass


class Token(abc.ABC):
    @abstractmethod
    def display(self) -> str:
        pass

    def __str__(self):
        return self.display()


class ColumnToken(Token, Filterable, Annotable, Groupable, Orderable):
    column: Column = None
    column_identifier: ColumnIdentifier = None

    def __init__(self, column, column_identifier):
        self.column = column
        self.column_identifier = column_identifier
        self._is_id = self.column.type == LiteralType.ID

    def display(self) -> str:
        return f'"{self.column.display_name}"'

    def filter_key(self):
        return self.column_identifier.name

    def annotations(self):
        return self.column.annotate()

    def group_key(self):
        return self.column_identifier.name

    def group_label_metadata(self):
        return LabelMetadata(
            expression=ExpressionProto(
                column_identifier=self.column_identifier
            ),
            alias=self.column.display_name
        )

    def group_label(self, obj):
        return Label(
            value=obj_to_literal(obj, self._is_id),
            display_value=self.column.id_label_resolver_cb(obj) if self._is_id else ''
        )

    def order_key(self):
        return self.column_identifier.name


class AttributeToken(Token, Filterable, Annotable, Groupable):
    column: AttributeColumn = None
    attribute_identifier: AttributeIdentifier = None

    def __init__(self, column, attribute_identifier):
        self.column = column
        self.attribute_identifier = attribute_identifier
        self._full_path = f'{self.attribute_identifier.path}__{self.attribute_identifier.name}'
        # self._full_path = f'{self.attribute_identifier.path}__{self.attribute_identifier.name}'
        self._group_key_id = f'{self.attribute_identifier.path}_{self.attribute_identifier.name}'

    def display(self) -> str:
        return f'"{self.column.display_name}".{self.attribute_identifier.name}'

    def filter_key(self):
        return self._full_path

    def annotations(self):
        return {**self.column.annotate(), self._group_key_id: F(self._full_path)}

    def group_key(self):
        return self._group_key_id

    def group_label_metadata(self):
        return LabelMetadata(
            expression=ExpressionProto(
                attribute_identifier=self.attribute_identifier
            ),
            alias=self.display()
        )

    def group_label(self, obj):
        return Label(value=obj_to_literal(obj))


class AttributeTokenV2(Token, Filterable, Annotable):
    column: AttributeColumnV2 = None
    attribute_identifier_v2: AttributeIdentifierV2 = None

    def __init__(self, column, attribute_identifier_v2):
        self.column = column
        self.attribute_identifier_v2 = attribute_identifier_v2
        self._paths = self.attribute_identifier_v2.path

    def display(self) -> str:
        return f'"{self.column.display_name}".{self.attribute_identifier_v2.name}'

    def filter_key(self):
        filter_keys = []
        for path in self._paths:
            filter_keys.append(f'{path}__{self.attribute_identifier_v2.name}')
        return filter_keys

    def annotations(self):
        annotations = {}
        for path in self._paths:
            annotations.update(self.column.annotate_v2(path))
        return annotations


class LiteralToken(Token):
    literal: Literal = None
    literal_value = None

    def __init__(self, literal: Literal):
        self.literal = literal
        self.literal_value = literal_to_obj(literal)

    def display(self) -> str:
        return display_literal(self.literal)


class OpToken(Token):
    op: Op = None

    def __init__(self, op: Op):
        self.op = op

    def display(self) -> str:
        return Op.Name(self.op)


class ExpressionTokenizer:
    def __init__(self, columns: Dict[str, BaseColumn]):
        self._columns = columns

    def _column_tokenizer(self, identifier: ColumnIdentifier) -> Token:
        column = self._columns.get(identifier.name, None)
        if column is None:
            raise ValueError(f'Invalid column name {identifier.name}')
        return ColumnToken(column=column, column_identifier=identifier)

    def _attribute_tokenizer(self, identifier: AttributeIdentifier) -> Token:
        column = self._columns.get(identifier.path, None)
        if column is None:
            raise ValueError(f'Invalid column name {identifier.name}')
        if not isinstance(column, AttributeColumn):
            raise ValueError(f'Invalid column type {identifier.name}')
        return AttributeToken(column=column, attribute_identifier=identifier)

    def _attribute_tokenizer_v2(self, identifier: AttributeIdentifierV2) -> Token:
        column = self._columns.get('free_attribute_search_column', None)
        if column is None:
            raise ValueError(f'Free attribute search not supported for current model and {identifier.name}')
        return AttributeTokenV2(column=column, attribute_identifier_v2=identifier)

    def tokenize(self, expression: ExpressionProto) -> Token:
        if expression.HasField('column_identifier'):
            return self._column_tokenizer(expression.column_identifier)
        elif expression.HasField('attribute_identifier'):
            return self._attribute_tokenizer(expression.attribute_identifier)
        elif expression.HasField('literal'):
            return LiteralToken(literal=expression.literal)
        elif expression.HasField('attribute_identifier_v2'):
            return self._attribute_tokenizer_v2(expression.attribute_identifier_v2)
        return None

event/base/token.py
DrDroidLab/kenobi
import traceback

from django.conf import settings
from django.core.cache import caches

from protos.event.base_pb2 import Context


class PanelCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def get(self, account_id, name=None):
        try:
            if name:
                key = f'panel:{account_id}:{name}'
                value = caches[self._cache_key].get(key)
                return [value]
            c = caches[self._cache_key]
            values = [c.get(k) for k in c.keys(f'panel:{account_id}:*')]
            return values
        except Exception as ex:
            print(
                f"Error while getting panel config from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def create_or_update(self, account_id, name, panel, time_to_expire_s=None):
        try:
            key = f'panel:{account_id}:{name}'
            caches[self._cache_key].set(key, panel, timeout=time_to_expire_s)
            if not time_to_expire_s:
                caches[self._cache_key].persist(key)
            return panel, True
        except Exception as ex:
            print(
                f"Error while setting panel config from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def delete(self, account_id, name):
        try:
            if name:
                key = f'panel:{account_id}:{name}'
                value = caches[self._cache_key].delete(key)
                return [value]
        except Exception as ex:
            print(
                f"Error while deleting panel config from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")


class DashboardCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def get(self, account_id, name=None):
        try:
            if name:
                key = f'dashboard:{account_id}:{name}'
                value = caches[self._cache_key].get(key)
                return [value]
            c = caches[self._cache_key]
            values = [c.get(k) for k in c.keys(f'dashboard:{account_id}:*')]
            return values
        except Exception as ex:
            print(
                f"Error while getting/setting dashboard config from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def create_or_update(self, account_id, name, dashboard, time_to_expire_s=None):
        try:
            key = f'dashboard:{account_id}:{name}'
            caches[self._cache_key].set(key, dashboard, time_to_expire_s)
            if not time_to_expire_s:
                caches[self._cache_key].persist(key, dashboard, time_to_expire_s)
        except Exception as ex:
            print(
                f"Error while setting dashboard config in cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def delete(self, account_id, name):
        try:
            if name:
                key = f'dashboard:{account_id}:{name}'
                value = caches[self._cache_key].delete(key)
                return [value]
        except Exception as ex:
            print(
                f"Error while deleting dashboard config from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")


class EventSearchQueryRequestCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def get(self, account_id, search_context_id=None):
        try:
            if not search_context_id:
                raise Exception("search_context_id is required")

            key = f'event_search_query:{account_id}:{search_context_id}'
            return caches[self._cache_key].get(key)
        except Exception as ex:
            print(
                f"Error while getting event search query from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def create_or_update(self, account_id, search_context_id, event_search_query_request, time_to_expire_s=None):
        try:
            if not time_to_expire_s:
                time_to_expire_s = 60 * 60 * 24 * 2
            key = f'event_search_query:{account_id}:{search_context_id}'
            caches[self._cache_key].set(key, event_search_query_request, time_to_expire_s)
        except Exception as ex:
            print(
                f"Error while setting event search query in cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def delete(self, account_id, search_context_id):
        try:
            if not search_context_id:
                raise Exception("search_context_id is required")

            key = f'event_search_query:{account_id}:{search_context_id}'
            value = caches[self._cache_key].delete(key)
            return value
        except Exception as ex:
            print(
                f"Error while deleting event search querty  from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")


class MonitorTransactionSearchQueryRequestCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def get(self, account_id, search_context_id=None):
        try:
            if not search_context_id:
                raise Exception("search_context_id is required")

            key = f'monitor_transaction_search_query:{account_id}:{search_context_id}'
            return caches[self._cache_key].get(key)
        except Exception as ex:
            print(
                f"Error while getting monitor transaction search query from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def create_or_update(self, account_id, search_context_id, event_search_query_request, time_to_expire_s=None):
        try:
            if not time_to_expire_s:
                time_to_expire_s = 60 * 60 * 24 * 2
            key = f'monitor_transaction_search_query:{account_id}:{search_context_id}'
            caches[self._cache_key].set(key, event_search_query_request, time_to_expire_s)
        except Exception as ex:
            print(
                f"Error while setting monitor transaction search query in cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def delete(self, account_id, search_context_id):
        try:
            if not search_context_id:
                raise Exception("search_context_id is required")

            key = f'monitor_transaction_search_query:{account_id}:{search_context_id}'
            value = caches[self._cache_key].delete(key)
            return value
        except Exception as ex:
            print(
                f"Error while deleting monitor transaction search query from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")


class ExportRequestCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def get(self, account_id, export_context: Context, user_email):
        try:
            if not account_id or not export_context or not user_email:
                raise Exception("Missing required fields [account_id, export_context, user_email]")

            key = f'export_request:{account_id}:{export_context}:{user_email}:'
            return caches[self._cache_key].get(key)
        except Exception as ex:
            print(
                f"Error while getting export context from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def create_or_update(self, account_id, export_context: Context, user_email, export_id, time_to_expire_s=None):
        try:
            if not account_id or not export_context or not user_email:
                raise Exception("Missing required fields [account_id, export_context, user_email]")
            if not time_to_expire_s:
                time_to_expire_s = 60 * 60
            key = f'export_request:{account_id}:{export_context}:{user_email}:'
            caches[self._cache_key].set(key, export_id, time_to_expire_s)
        except Exception as ex:
            print(
                f"Error while setting export context in cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    def delete(self, account_id, export_context: Context, user_email):
        try:
            if not account_id or not export_context or not user_email:
                raise Exception("Missing required fields [account_id, export_context, user_email]")
            key = f'export_request:{account_id}:{export_context}:{user_email}:'
            value = caches[self._cache_key].delete(key)
            return value
        except Exception as ex:
            print(
                f"Error while deleting monitor transaction search query from cache:: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")


GLOBAL_PANEL_CACHE = PanelCache(**settings.GLOBAL_PANEL_CACHE)
GLOBAL_DASHBOARD_CACHE = DashboardCache(**settings.GLOBAL_DASHBOARD_CACHE)
GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE = EventSearchQueryRequestCache(
    **settings.GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE)
GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE = MonitorTransactionSearchQueryRequestCache(
    **settings.GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE)
GLOBAL_EXPORT_CONTEXT_CACHE = ExportRequestCache(**settings.GLOBAL_EXPORT_CONTEXT_CACHE)

event/cache.py
DrDroidLab/kenobi
import clickhouse_connect
from django.conf import settings

from utils.singleton import singleton_function


@singleton_function
def get_client():
    config = settings.CLICKHOUSE_CLIENT_CONFIG
    return clickhouse_connect.get_client(**config)

event/clickhouse/client.py
DrDroidLab/kenobi
"""
# events table schema
set allow_experimental_object_type = 1;
create table events (
    id UInt64,
    account_id UInt64,
    created_at DateTime64(3, 'UTC') default now(),
    timestamp DateTime64(3, 'UTC') default now(),
    event_type_id UInt64,
    event_type_name VARCHAR(256),
    event_source UInt16,
    processed_kvs JSON,
    ingested_event  JSON,
)
engine = MergeTree()
PARTITION BY toDate(timestamp)
primary key (account_id, event_type_id, timestamp)
order by (account_id, event_type_id, timestamp)
TTL toDateTime(created_at) + INTERVAL 1 WEEK DELETE;
"""
from datetime import datetime, timezone
from typing import List
import logging
import json

from event.clickhouse.client import get_client
from event.clickhouse.tableproxy import TableProxy
from event.models import is_filterable_key_type
from event.processors.utils import infer_key_type
from protos.event.schema_pb2 import KeyValue, IngestionEvent
from protos.kafka.base_pb2 import ProcessedIngestionEvent
from utils.proto_utils import get_value, proto_to_dict
from utils.singleton import singleton, singleton_function

logger = logging.getLogger(__name__)

def processed_kvs_to_kvs(processed_kvs: List[ProcessedIngestionEvent.ProcessedKeyValue]):
    return [
        KeyValue(key=kv.key.value, value=kv.value) for kv in processed_kvs
    ]


def transform_pkvs(pkvs: List[ProcessedIngestionEvent.ProcessedKeyValue]):
    processed_kvs = {}
    kvs = []
    for pkv in pkvs:
        k = pkv.key.value
        k_type = infer_key_type(pkv.value)

        value = get_value(pkv.value)
        if is_filterable_key_type(k_type):
            processed_kvs[k] = value
        kvs.append(KeyValue(key=k, value=pkv.value))
    return processed_kvs, kvs


@singleton
class EventsProxy(TableProxy):
    data_columns: List[str] = ['id', 'account_id', 'created_at', 'timestamp',
                               'event_type_id', 'event_type_name',
                               'event_source', 'processed_kvs', 'ingested_event']
    data_columns_types: List[str] = ['UInt64', 'UInt64', "DateTime64(3, 'UTC')",
                                     "DateTime64(3, 'UTC')", 'UInt64', 'String', 'UInt16',
                                     "Object('json')", "String"]
    table_name: str = 'events'

    def __init__(self):
        self.query_client = get_client()
        super().__init__(self.query_client)

    def _transform_processed_event(self, processed_event: ProcessedIngestionEvent):
        event_name = processed_event.event_type.name
        event_timestamp = processed_event.timestamp

        processed_kvs, kvs = transform_pkvs(processed_event.kvs)
        ingested_event = IngestionEvent(
            name=event_name,
            timestamp=event_timestamp,
            kvs=kvs
        )
        return [
            processed_event.id.value,
            processed_event.account_id.value,
            datetime.now(timezone.utc),
            datetime.utcfromtimestamp(event_timestamp / 1000).astimezone(timezone.utc),
            processed_event.event_type.id,
            event_name,
            processed_event.event_source,
            processed_kvs,
            json.dumps(proto_to_dict(ingested_event))
        ]

    def ingest_processed_events(self, processed_events: List[ProcessedIngestionEvent]):
        data = [self._transform_processed_event(processed_event) for processed_event in processed_events]
        self.insert_data(data)
        return

    def query_events(self, query: str):
        result = self.query_client.query(query)
        return result


@singleton_function
def get_events_proxy():
    return EventsProxy()

event/clickhouse/eventsproxy.py
DrDroidLab/kenobi
from datetime import datetime
import json
from datetime import timezone
from typing import List

from event.models import is_filterable_key_type
from event.clickhouse.models import Events, MonitorTransactions
from event.processors.utils import infer_key_type
from protos.event.schema_pb2 import KeyValue, IngestionEvent
from protos.kafka.base_pb2 import ProcessedIngestionEvent, ProcessedMonitorTransaction
from utils.proto_utils import get_value, proto_to_dict


def processed_kvs_to_kvs(processed_kvs: List[ProcessedIngestionEvent.ProcessedKeyValue]):
    return [KeyValue(key=kv.key.value, value=kv.value) for kv in processed_kvs]


def transform_pkvs(pkvs: List[ProcessedIngestionEvent.ProcessedKeyValue]):
    processed_kvs = {}
    kvs = []
    for pkv in pkvs:
        k = pkv.key.value
        k_type = infer_key_type(pkv.value)

        value = get_value(pkv.value)
        if is_filterable_key_type(k_type):
            processed_kvs[k] = value
        kvs.append(KeyValue(key=k, value=pkv.value))
    return processed_kvs, kvs


def ingest_processed_events(processed_events: List[ProcessedIngestionEvent]):
    events = []
    for processed_event in processed_events:
        event_name = processed_event.event_type.name
        event_timestamp = processed_event.timestamp

        processed_kvs, kvs = transform_pkvs(processed_event.kvs)
        ingested_event = IngestionEvent(
            name=event_name,
            timestamp=event_timestamp,
            kvs=kvs
        )
        events.append(
            Events(
                id=processed_event.id.value,
                account_id=processed_event.account_id.value,
                created_at=datetime.now(timezone.utc),
                timestamp=datetime.utcfromtimestamp(event_timestamp / 1000).astimezone(timezone.utc),
                event_type_id=processed_event.event_type.id,
                event_type_name=event_name,
                event_source=processed_event.event_source,
                processed_kvs=processed_kvs,
                ingested_event=json.dumps(proto_to_dict(ingested_event))
            )
        )

    if events:
        Events.objects.bulk_create(events, batch_size=1000)


def ingest_processed_monitor_transactions(processed_monitor_transactions: List[ProcessedMonitorTransaction]):
    monitor_transactions = []
    for processed_monitor_transaction in processed_monitor_transactions:
        monitor_transaction_id = processed_monitor_transaction.monitor_transaction.id.value
        transaction = processed_monitor_transaction.monitor_transaction.transaction.value
        monitor_id = processed_monitor_transaction.monitor_transaction.monitor.id
        status = processed_monitor_transaction.monitor_transaction_status

        ingestion_event: ProcessedIngestionEvent = processed_monitor_transaction.ingestion_event
        event_id = ingestion_event.id.value
        event_type_id = ingestion_event.event_type.id
        event_type_name = ingestion_event.event_type.name
        event_timestamp = ingestion_event.timestamp
        event_source = ingestion_event.event_source

        transaction_time = None
        if processed_monitor_transaction.transaction_time:
            transaction_time = processed_monitor_transaction.transaction_time.value

        processed_kvs, kvs = transform_pkvs(ingestion_event.kvs)
        ingested_event = IngestionEvent(
            name=event_type_name,
            timestamp=event_timestamp,
            kvs=kvs
        )

        monitor_transactions.append(
            MonitorTransactions(
                id=monitor_transaction_id,
                account_id=processed_monitor_transaction.account_id.value,
                monitor_id=monitor_id,
                transaction=transaction,
                status=status,
                transaction_time=transaction_time,
                event_id=event_id,
                monitor_transaction_event_type=processed_monitor_transaction.monitor_transaction_event_type,
                event_type_id=event_type_id,
                event_type_name=event_type_name,
                event_timestamp=datetime.utcfromtimestamp(event_timestamp / 1000).astimezone(timezone.utc),
                event_source=event_source,
                event_processed_kvs=processed_kvs,
                ingested_event=json.dumps(proto_to_dict(ingested_event)),
                created_at=datetime.now(timezone.utc),
            )
        )

    if monitor_transactions:
        MonitorTransactions.objects.bulk_create(monitor_transactions, batch_size=1000)

event/clickhouse/model_helper.py
DrDroidLab/kenobi
from django.utils.functional import cached_property

from clickhouse_backend import models as clickhouse_models
import json

from event.models import EventType
from protos.event.base_pb2 import Event as EventProto
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto, MonitorPartial


def clean_event_kvs(event_kvs: list):
    for kvs in event_kvs:
        if kvs.get('value').get('string_value'):
            kvs['value']['string_value'] = str(kvs.get('value')['string_value'])
        if kvs.get('value').get('int_value'):
            kvs['value']['int_value'] = int(kvs.get('value')['int_value'])
        if kvs.get('bool_value'):
            kvs['value']['bool_value'] = bool(kvs.get('value')['bool_value'])
        if kvs.get('double_value'):
            kvs['value']['double_value'] = float(kvs.get('value')['double_value'])
        if kvs.get('array_value'):
            kvs['value']['string_value'] = json.dumps(kvs.get('value')['array_value']['values'])
    return event_kvs


class Events(clickhouse_models.ClickhouseModel):
    id = clickhouse_models.UInt64Field(primary_key=True)
    account_id = clickhouse_models.UInt64Field()
    created_at = clickhouse_models.DateTime64Field()
    timestamp = clickhouse_models.DateTime64Field()
    event_type_id = clickhouse_models.UInt16Field()
    event_type_name = clickhouse_models.StringField()
    event_source = clickhouse_models.UInt16Field()
    processed_kvs = clickhouse_models.JSONField()
    ingested_event = clickhouse_models.StringField()

    class Meta:
        db_table = 'events'

    @cached_property
    def proto(self):
        try:
            kvs_json = json.loads(self.ingested_event)
        except:
            kvs_json = {}

        kvs_list = clean_event_kvs(kvs_json.get('kvs', []))
        proto = EventProto(
            id=int(self.id), event_type=EventType.proto_partial_from_sql(int(self.event_type_id), self.event_type_name),
            kvs=kvs_list,
            timestamp=int(self.timestamp.timestamp()),
            event_source=EventProto.EventSource.API
        )
        return proto


class MonitorTransactions(clickhouse_models.ClickhouseModel):
    id = clickhouse_models.UInt64Field(primary_key=True)
    account_id = clickhouse_models.UInt64Field()
    monitor_id = clickhouse_models.UInt64Field()
    transaction = clickhouse_models.StringField()
    status = clickhouse_models.UInt16Field()
    transaction_time = clickhouse_models.Float64Field()

    event_id = clickhouse_models.UInt64Field()
    monitor_transaction_event_type = clickhouse_models.UInt16Field()
    event_type_id = clickhouse_models.UInt64Field()
    event_type_name = clickhouse_models.StringField()
    event_timestamp = clickhouse_models.DateTime64Field()
    event_source = clickhouse_models.UInt16Field()
    event_processed_kvs = clickhouse_models.JSONField()
    ingested_event = clickhouse_models.StringField()

    created_at = clickhouse_models.DateTime64Field()

    class Meta:
        db_table = 'monitor_transactions'

event/clickhouse/models.py
DrDroidLab/kenobi
from typing import List

import clickhouse_connect.driver


class TableProxy:
    data_columns: List[str] = None
    data_columns_types: List[str] = None
    table_name: str = None

    def __init__(self, client: clickhouse_connect.driver.Client):
        if not self.data_columns or not self.data_columns_types or not self.table_name:
            raise NotImplementedError
        self.client = client

    def describe_table(self):
        result = self.client.query(f'DESCRIBE TABLE {self.table_name}')
        print(result.column_names[0:2])
        print(result.result_columns[0:2])

    def insert_data(self, data):
        self.client.insert(
            self.table_name,
            data,
            column_names=self.data_columns,
            column_type_names=self.data_columns_types
        )

event/clickhouse/tableproxy.py
DrDroidLab/kenobi
from abc import abstractmethod

from google.protobuf.wrappers_pb2 import UInt64Value, BoolValue

from event.clickhouse.models import Events
from accounts.models import Account
from event.base.aggregation_function import get_metric_aggregation_function_options
from event.base.filter_token import FilterEngine
from event.base.metric_token import MetricExpressionEvaluator
from event.base.order_by_token import OrderByEngine
from event.engine.model_columns import ColumnOptions, event_columns, monitor_transaction_columns, \
    entity_instance_columns, events_clickhouse_columns
from event.models import Monitor, EventType, Entity
from protos.event.base_pb2 import Context
from protos.event.engine_options_pb2 import MetricOptions, QueryOptions, QueryOptionsV2
from protos.event.literal_pb2 import IdLiteral, Literal, LiteralType
from protos.event.metric_pb2 import MetricExpression, MetricSelector, AggregationFunction
from protos.event.query_base_pb2 import QueryRequest, Filter, Expression, Op, ColumnIdentifier, OrderByExpression, \
    SortOrder


class ContextResolver:
    columns = None
    timestamp_field = None
    parent_model = None
    parent_column_name: str = None

    def __init__(self):
        self._filter_engine = FilterEngine(self.columns)
        self._order_by_engine = OrderByEngine(self.columns, default_order_expression=OrderByExpression(
            expression=Expression(column_identifier=ColumnIdentifier(name=self.timestamp_field)),
            order=SortOrder.DESC))
        self._metric_expression_evaluator = MetricExpressionEvaluator(
            self.parent_model, self.columns, self.timestamp_field
        )
        self._column_options = ColumnOptions(self.columns)

    @abstractmethod
    def qs(self, account):
        raise NotImplementedError()

    def filter_engine(self):
        return self._filter_engine

    def order_by_engine(self):
        return self._order_by_engine

    def metric_expression_evaluator(self) -> MetricExpressionEvaluator:
        return self._metric_expression_evaluator

    def get_parent_obj(self, account, id: IdLiteral = None):
        if id.type == IdLiteral.Type.LONG:
            return self.parent_model.objects.get(account=account, id=id.long.value)
        elif id.type == IdLiteral.Type.STRING:
            return id.string.value

    def get_search_options(self, account, parent_obj=None):
        column_options, attribute_options = self._column_options.options(account=account, obj=parent_obj)

        return QueryOptions(column_options=column_options, attribute_options=attribute_options)

    def get_search_options_v2(self, account, parent_obj=None):
        column_options, attribute_options_v2 = self._column_options.options_v2(account=account, obj=parent_obj)

        return QueryOptionsV2(column_options=column_options, attribute_options_v2=attribute_options_v2)

    def get_metric_options(self, account, parent_obj=None):
        column_options, attribute_options = self._column_options.options(account=account, obj=parent_obj)
        return MetricOptions(
            aggregation_function_options=get_metric_aggregation_function_options(),
            column_options=column_options,
            attribute_options=attribute_options
        )

    def get_parent_id_filter(self, id: IdLiteral = None):
        return Filter(
            lhs=Expression(column_identifier=ColumnIdentifier(name=self.parent_column_name, type=LiteralType.ID)),
            op=Op.EQ,
            rhs=Expression(
                literal=Literal(
                    literal_type=LiteralType.ID,
                    id=id
                )
            )
        )

    def get_default_metric_expression(self, account, id: IdLiteral = None):
        parent_id_filter = None
        if id is not None and id.ByteSize() != 0:
            parent_id_filter = self.get_parent_id_filter(id)

        return MetricExpression(
            filter=parent_id_filter,
            selectors=[
                MetricSelector(
                    function=AggregationFunction.COUNT,
                )
            ],
            is_timeseries=BoolValue(value=True),
            resolution=UInt64Value(value=10),
        )

    def get_default_query(self, account, obj=None):
        return QueryRequest()


class EventContextResolver(ContextResolver):
    columns = event_columns
    timestamp_field = 'timestamp'
    parent_model = EventType
    parent_column_name = 'event_type_id'

    def qs(self, account):
        return account.event_set.all()

    def get_default_query(self, account, obj=None):
        if type(account) is not Account:
            raise ValueError(f'{account} needs to be Account')

        return QueryRequest(filter=Filter(op=Op.AND, filters=[]))


class EventsClickhouseContextResolver(ContextResolver):
    columns = events_clickhouse_columns
    timestamp_field = 'timestamp'
    parent_model = Events
    parent_column_name = 'event_type_id'

    def qs(self, account):
        return Events.objects.filter(account_id=account.id)

    def get_default_query(self, account, obj=None):
        if type(account) is not Account:
            raise ValueError(f'{account} needs to be Account')

        return QueryRequest(filter=Filter(op=Op.AND, filters=[]))


default_monitor_transaction_search_filter = Filter(
    lhs=Expression(column_identifier=ColumnIdentifier(name="type", type=LiteralType.ID)),
    op=Op.IN,
    rhs=Expression(
        literal=Literal(
            literal_type=LiteralType.ID_ARRAY,
            id_array=[
                IdLiteral(
                    type=IdLiteral.Type.LONG,
                    long=UInt64Value(value=1)
                ),
                IdLiteral(
                    type=IdLiteral.Type.LONG,
                    long=UInt64Value(value=2)
                )
            ]
        )
    )
)


class MonitorTransactionContextResolver(ContextResolver):
    columns = monitor_transaction_columns
    timestamp_field = 'created_at'
    parent_model = Monitor
    parent_column_name = 'monitor_id'

    def qs(self, account):
        return account.monitortransaction_set.all()

    def get_default_query(self, account, obj=None):
        if type(account) is not Account:
            raise ValueError(f'{account} needs to be Account')

        # filters = [default_monitor_transaction_search_filter]
        filters = []

        if obj:
            filters.append(
                self.get_parent_id_filter(IdLiteral(
                    long=UInt64Value(value=obj.id),
                    type=IdLiteral.Type.LONG
                ))
            )

        return QueryRequest(filter=Filter(op=Op.AND, filters=filters))


class EntityInstanceContextResolver(ContextResolver):
    columns = entity_instance_columns
    timestamp_field = 'created_at'
    parent_model = Entity
    parent_column_name = 'entity_id'

    def qs(self, account):
        return account.entityinstance_set.all()

    def get_default_query(self, account, obj=None):
        if type(account) is not Account:
            raise ValueError(f'{account} needs to be Account')

        filters = []

        if obj:
            filters.append(
                self.get_parent_id_filter(IdLiteral(
                    long=UInt64Value(value=obj.id),
                    type=IdLiteral.Type.LONG
                ))
            )

        return QueryRequest(filter=Filter(op=Op.AND, filters=filters))


class EventTypeContextResolver(ContextResolver):
    def qs(self, account):
        return account.eventtype_set.all()


class MonitorContextResolver(ContextResolver):
    def qs(self, account):
        return account.monitor_set.all()


class EntityContextResolver(ContextResolver):
    def qs(self, account):
        return account.entity_set.all()


_context_to_resolver = {
    Context.EVENT: EventContextResolver(),
    Context.EVENTS_CLICKHOUSE: EventsClickhouseContextResolver(),
    Context.MONITOR_TRANSACTION: MonitorTransactionContextResolver(),
    Context.ENTITY_INSTANCE: EntityInstanceContextResolver(),
    Context.EVENT_TYPE: EventTypeContextResolver(),
    Context.MONITOR: MonitorContextResolver(),
    Context.ENTITY: EntityContextResolver(),
}


def get_context_resolver(context: Context) -> ContextResolver:
    return _context_to_resolver.get(context)

event/engine/context.py
DrDroidLab/kenobi
from django.contrib.postgres.search import TrigramWordSimilarity
from django.db.models import Q

from event.engine.context import ContextResolver, get_context_resolver
from event.models import EventType, Monitor, Entity
from protos.event.base_pb2 import Context


class FuzzyMatchEngineException(ValueError):
    pass


class FuzzyMatchEngine:

    def __init__(self, model, context_resolver: ContextResolver):
        self._model = model
        self._context_resolver = context_resolver

    def match_pattern(self, account, pattern: str, similarity_ratio_threshold=0.3):
        qs = self._context_resolver.qs(account)

        return qs.annotate(similarity=TrigramWordSimilarity(pattern, 'name')).filter(
            Q(name__trigram_word_similar=pattern) | Q(similarity__gte=similarity_ratio_threshold)).order_by(
            '-similarity')


event_type_fuzzy_match_engine = FuzzyMatchEngine(
    EventType,
    get_context_resolver(Context.EVENT_TYPE),
)

monitor_fuzzy_match_engine = FuzzyMatchEngine(
    Monitor,
    get_context_resolver(Context.MONITOR),
)

entity_fuzzy_match_engine = FuzzyMatchEngine(
    Entity,
    get_context_resolver(Context.ENTITY),
)

event/engine/fuzzy_match_engine.py
DrDroidLab/kenobi
import math
from typing import List

from event.clickhouse.models import Events
from event.engine.context import get_context_resolver
from event.models import EventType
from event.workflows.workflow_utils import get_metric_aggr_function_mapping
from protos.event.base_pb2 import Context
from protos.event.literal_pb2 import IdLiteral, LiteralType
from protos.event.metric_pb2 import MetricExpression
from prototype.utils.timerange import DateTimeRange
from utils.proto_utils import dict_to_proto


class MetricEngine:
    def process(self, account, context: Context, expressions: List[MetricExpression], dtr: DateTimeRange):
        if context == Context.EVENT:
            context = Context.EVENTS_CLICKHOUSE

        context_resolver = get_context_resolver(context)
        if context_resolver is None:
            raise Exception(f'invalid context: {context}')

        qs = context_resolver.qs(account)
        evaluator = context_resolver.metric_expression_evaluator()
        return evaluator.process_batch(qs, expressions, dtr)

    def get_options(self, account, context: Context, id: IdLiteral = None):
        context_resolver = get_context_resolver(context)
        if context_resolver is None:
            raise Exception(f'invalid context: {context}')
        parent_obj = context_resolver.get_parent_obj(account, id)
        return context_resolver.get_metric_options(account, parent_obj)

    def get_default_metric_expression(self, account, context: Context, id: IdLiteral = None):
        context_resolver = get_context_resolver(context)
        if context_resolver is None:
            raise Exception(f'invalid context: {context}')
        return context_resolver.get_default_metric_expression(account, id)


metric_engine = MetricEngine()


def process_workflow_monitor_metric_expression(account, metric_expr, dtr: DateTimeRange):
    start_time, end_time = dtr.to_tr_str()
    metric_timeseries_data = []

    tr = dtr.to_tr()
    time_duration = tr.time_lt - tr.time_geq
    account_id = account.id

    aggr_field = metric_expr.split(':')[1].split('{')[0]
    filters = list(
        {'lhs': x.split(':')[0], 'rhs': x.split(':')[1]} for x in metric_expr.split('{')[1].split('}')[0].split(','))

    resolution = math.ceil(time_duration / 15 / 60)

    aggr_type = metric_expr.split(':')[0]
    metric_type = aggr_field.split('.')[1]
    primary_event_name = list(filter(lambda x: x['lhs'] == 'primary_event_name', filters))[0]['rhs']
    secondary_event_name = list(filter(lambda x: x['lhs'] == 'secondary_event_name', filters))[0]['rhs']
    joining_key = list(filter(lambda x: x['lhs'] == 'joining_key', filters))[0]['rhs']

    filter_clause = ''
    event_filters = list(filter(
        lambda x: x['lhs'] != 'primary_event_name' and x['lhs'] != 'secondary_event_name' and x['lhs'] != 'joining_key',
        filters))
    for f in event_filters:
        filter_clause += ' and processed_kvs.{} = \'{}\''.format(f['lhs'], f['rhs'])

    query = None
    if aggr_type == 'percent':
        if metric_type == 'active':
            query = "SELECT toStartOfInterval(C.p_timestamp, INTERVAL {} minute) AS interval_start, (SUM(C.result) * 100.0 / COUNT(*)) AS id FROM (select IF(A.p_j_key = B.s_j_key, 0, IF(A.p_j_key != B.s_j_key, 1, 0)) AS result, A.p_timestamp from (select processed_kvs.{} as p_j_key, timestamp as p_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key) C GROUP BY interval_start ORDER BY interval_start".format(
                resolution, joining_key, account_id, primary_event_name, start_time, end_time, filter_clause, joining_key,
                account_id, secondary_event_name, start_time
            )
        if metric_type == 'finished':
            query = "SELECT toStartOfInterval(C.p_timestamp, INTERVAL {} minute) AS interval_start, (SUM(C.result) * 100.0 / COUNT(*)) AS id FROM (select IF(A.p_j_key = B.s_j_key, 1, IF(A.p_j_key != B.s_j_key, 0, 0)) AS result, A.p_timestamp from (select processed_kvs.{} as p_j_key, timestamp as p_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key) C GROUP BY interval_start ORDER BY interval_start".format(
                resolution, joining_key, account_id, primary_event_name, start_time, end_time, filter_clause,
                joining_key,
                account_id, secondary_event_name, start_time
            )

    if aggr_type == 'avg':
        if metric_type == 'time':
            query = "SELECT toStartOfInterval(C.p_timestamp, INTERVAL {} minute) AS interval_start, avg(id) as id FROM (select date_diff('ms', p_timestamp, s_timestamp) as id, p_timestamp  from (select processed_kvs.{} as p_j_key, timestamp as p_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key, timestamp as s_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key where A.p_j_key = B.s_j_key) C GROUP BY interval_start ORDER BY interval_start".format(
                resolution, joining_key, account_id, primary_event_name, start_time, end_time, filter_clause,
                joining_key,
                account_id, secondary_event_name, start_time
            )

    qs = Events.objects.raw(query)
    metric_data = list(qs)

    for d in metric_data:
        metric_timeseries_data.append({
            'timestamp': int(d.interval_start.timestamp()*1000),
            'value': d.id
        })
    return metric_timeseries_data



def process_workflow_metric_expression(account, metric_expr, dtr: DateTimeRange):

    tr = dtr.to_tr()
    time_duration = tr.time_lt - tr.time_geq

    aggregation_type = metric_expr.split(':')[0]
    aggregation_field = metric_expr.split(':')[1].split('{')[0]
    metric_context = metric_expr.split('{')[0].split(':')[1].split('.')[0].upper()
    aggregation_object = aggregation_field.split('.')[0]
    filters = []
    selectors = []
    resolution = math.ceil(time_duration / 15 / 60) * 60

    if aggregation_object == 'transaction':
        return process_workflow_monitor_metric_expression(account, metric_expr, dtr)

    for token in metric_expr.split('{')[1].split('}')[0].split(','):
        filter_lhs = token.split(':')[0]
        filter_rhs = token.split(':')[1]

        if filter_lhs == 'event_name':
            event_type = EventType.objects.filter(account=account, name=filter_rhs).first()
            filters.append({
                "lhs": {
                    "column_identifier": {
                        "name": "event_type_id"
                    }
                },
                "op": "EQ",
                "rhs": {
                    "literal": {
                        "literal_type": "ID",
                        "id": {
                            "type": "LONG",
                            "long": event_type.id
                        }
                    }
                }
            })
        else:
            filters.append({
                "lhs": {
                    {
                        "attribute_identifier": {
                            "name": filter_lhs,
                            "path": "event_attribute"
                        }
                    }
                },
                "op": "EQ",
                "rhs": {
                    "literal": {
                        "literal_type": "STRING",
                        "string": filter_rhs
                    }
                }
            })

    if '.' in aggregation_field and aggregation_field.split('.')[0] == 'event' and aggregation_field.split('.')[1]:
        selectors.append({
            "function": get_metric_aggr_function_mapping(aggregation_type),
            "expression": {
                "attribute_identifier": {
                    "name": aggregation_field.split('.')[1],
                    "path": "event_attribute"
                }
            }
        })
    else:
        selectors.append({
            "function": get_metric_aggr_function_mapping(aggregation_type),
            "expression": {
                "column_identifier": {
                    "name": "event_type_id"
                }
            }
        })

    expression = {
        "filter": {
            "op": "AND",
            "filters": filters
        },
        "selectors": selectors,
        "resolution": resolution,
        "is_timeseries": True,
    }

    expr = dict_to_proto(expression, MetricExpression)

    context = Context.EVENT
    if metric_context == 'monitor':
        context = Context.MONITOR_TRANSACTION

    metric_data = metric_engine.process(account, context, [expr], dtr)
    timeseries_data = metric_data[0].labeled_data[0].alias_data_map['metric_1'].timeseries_data

    metric_timeseries_data = []
    for ts_data in timeseries_data:
        metric_value = None
        if get_literal_value(ts_data.value):
            metric_value = float(get_literal_value(ts_data.value).value)

        metric_timeseries_data.append({
            'timestamp': ts_data.timestamp * 1000,
            'value': metric_value
        })

    return metric_timeseries_data


def get_literal_value(literal):
    literal_type = literal.literal_type
    if literal_type == LiteralType.LONG:
        return literal.long
    if literal_type == LiteralType.DOUBLE:
        return literal.double
    return None


def process_metric_expressions(account, context: Context, expressions: List[MetricExpression], dtr: DateTimeRange):
    return metric_engine.process(account, context, expressions, dtr)


def get_metric_options(account, context: Context, id: IdLiteral = None):
    return metric_engine.get_options(account, context, id)


def get_default_metric_expression(account, context: Context, id: IdLiteral = None):
    return metric_engine.get_default_metric_expression(account, context, id)

event/engine/metric_engine.py
DrDroidLab/kenobi
from functools import lru_cache
from typing import Dict

from django.contrib.postgres.aggregates import ArrayAgg
from django.db import models
from django.db.models import Value, F, Exists, OuterRef, Subquery
from django.db.models.fields.json import KeyTextTransform
from django.db.models.functions import TruncSecond, Cast
from google.protobuf.wrappers_pb2 import UInt64Value

from accounts.models import Account, get_request_account
from event.base.column import Column, AnnotatedColumn, AttributeColumn, AttributeColumnV2
from event.base.filter_token_op import LiteralArrayColumnTokenFilterOp
from event.base.literal import event_key_type_to_literal_type, literal_type_is_groupable, \
    literal_type_aggregation_functions
from event.models import AlertMonitorTransactionMapping, MonitorTransactionEventMapping, MonitorTransaction, \
    EntityInstanceEventMapping, AlertEntityInstanceMapping
from protos.event.engine_options_pb2 import AttributeOption, AttributeOptionV2
from protos.event.literal_pb2 import LiteralType, IdLiteral
from protos.event.metric_pb2 import AggregationFunction
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto

transaction_status_options = {
    MonitorTransactionProto.MonitorTransactionStatus.PRIMARY_RECEIVED: 'Active',
    MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED: 'Finished',
}


def transaction_status_options_cb(account, obj):
    return transaction_status_options


def get_enum_id_column_option_cb(proto_enum):
    def _enum_id_options(account, obj):
        return {v: k for k, v in proto_enum.items() if v != 0}

    return _enum_id_options


def get_id_option_cb(account_scope_cb):
    def _id_options(account, obj):
        if obj is not None:
            return {obj.id: obj.name}
        return {elem['id']: elem['name'] for elem in account_scope_cb(account).values('id', 'name')}

    return _id_options


def get_event_attribute_options(path, display_name, account, obj):
    if obj:
        qs = account.eventkey_set.filter(event_type=obj)
    else:
        qs = account.eventkey_set.all()

    qs = qs.select_related(
        'event_type'
    ).values('id', 'name', 'type', 'event_type__id', 'event_type__name')

    attribute_options: [AttributeOption] = []

    for q in qs:
        literal_type = event_key_type_to_literal_type(q['type'])
        attribute_options.append(
            AttributeOption(
                name=q['name'], path=path, type=literal_type,
                column_context=[
                    AttributeOption.ColumnContext(
                        alias='event_type_id',
                        id=IdLiteral(
                            long=UInt64Value(value=q['event_type__id']),
                            type=IdLiteral.Type.LONG
                        )
                    )
                ],
                is_groupable=literal_type_is_groupable(literal_type),
                aggregation_functions=literal_type_aggregation_functions(literal_type),
            )
        )

    return attribute_options


def get_monitor_attribute_options_cb(key='primary_key'):
    def get_monitor_attribute_options(path, display_name, account, obj):
        if obj:
            qs = account.monitor_set.filter(id=obj.id)
        else:
            qs = account.monitor_set.exclude(is_active=False)

        key_event_type_id = f'{key}__event_type__id'
        key_event_type_name = f'{key}__event_type__name'

        qs = qs.select_related(key, f'{key}__event_type').values(
            'id', 'name', key,
            key_event_type_id,
            key_event_type_name,
        )

        attribute_options: [AttributeOption] = []
        for monitor in qs:
            event_keys_qs = account.eventkey_set.filter(
                event_type=monitor[key_event_type_id]
            ).values(
                'id',
                'name',
                'type'
            )
            path_alias = f"{monitor[key_event_type_name]} ({display_name})"

            for ek in event_keys_qs:
                literal_type = event_key_type_to_literal_type(ek['type'])
                attribute_options.append(
                    AttributeOption(
                        name=ek['name'], path=path, type=literal_type,
                        path_alias=path_alias,
                        column_context=[
                            AttributeOption.ColumnContext(
                                alias='monitor_id',
                                id=IdLiteral(
                                    long=UInt64Value(value=monitor['id']),
                                    type=IdLiteral.Type.LONG
                                )
                            )
                        ],
                        is_groupable=literal_type_is_groupable(literal_type),
                        aggregation_functions=literal_type_aggregation_functions(literal_type),
                    )
                )
        return attribute_options

    return get_monitor_attribute_options


def get_monitor_attribute_options_v2(account, obj):
    if obj:
        qs = account.monitor_set.filter(id=obj.id)
    else:
        qs = account.monitor_set.exclude(is_active=False)

    attribute_options: [AttributeOptionV2] = []
    attribute_options_dict = {}
    for key in ['primary_key', 'secondary_key']:

        key_event_type_id = f'{key}__event_type__id'
        key_event_type_name = f'{key}__event_type__name'

        eqs = qs.select_related(key, f'{key}__event_type').values(
            'id', 'name', key,
            key_event_type_id,
            key_event_type_name,
        )

        for monitor in eqs:
            event_keys_qs = account.eventkey_set.filter(
                event_type=monitor[key_event_type_id]
            ).values(
                'id',
                'name',
                'type'
            )

            path = "monitor__{}__event_type".format(key)

            for ek in event_keys_qs:
                literal_type = event_key_type_to_literal_type(ek['type'])
                if f"{ek['name']}##{literal_type}" not in attribute_options_dict:
                    attribute_options_dict[f"{ek['name']}##{literal_type}"] = []
                attribute_options_dict[f"{ek['name']}##{literal_type}"].append(path)

    for key, value in attribute_options_dict.items():
        paths = list(set(value))
        name = key.split("##")[0]
        literal_type = int(key.split("##")[1])
        attribute_options.append(
            AttributeOptionV2(
                name=name,
                path=paths,
                type=literal_type,
            )
        )
    return attribute_options


def get_entity_attribute_options_v2(account, obj):
    if obj:
        qs = account.entityeventkeymapping_set.filter(entity_id=obj.id)
    else:
        qs = account.entityeventkeymapping_set.exclude(is_active=False)

    attribute_options: [AttributeOptionV2] = []
    attribute_options_dict = {}
    qs = qs.select_related('event_key__event_type').values('event_key__event_type__id')
    for q in qs:
        event_type_id = q["event_key__event_type__id"]

        event_keys_qs = account.eventkey_set.filter(event_type=event_type_id).values('id', 'name', 'type')

        path = str(event_type_id)
        for ek in event_keys_qs:
            literal_type = event_key_type_to_literal_type(ek['type'])
            if f"{ek['name']}##{literal_type}" not in attribute_options_dict:
                attribute_options_dict[f"{ek['name']}##{literal_type}"] = []
            attribute_options_dict[f"{ek['name']}##{literal_type}"].append(path)

    for key, value in attribute_options_dict.items():
        paths = list(set(value))
        name = key.split("##")[0]
        literal_type = int(key.split("##")[1])
        attribute_options.append(
            AttributeOptionV2(
                name=name,
                path=paths,
                type=literal_type,
            )
        )
    return attribute_options


@lru_cache(maxsize=256)
def get_event_type_name_from_id(account_id, event_type_id):
    return Account(id=account_id).eventtype_set.filter(id=event_type_id).values_list('name', flat=True)[0]


@lru_cache(maxsize=256)
def get_monitor_name_from_id(account_id, monitor_id):
    return Account(id=account_id).monitor_set.filter(id=monitor_id).values_list('name', flat=True)[0]


@lru_cache(maxsize=256)
def get_entity_name_from_id(account_id, entity_id):
    return Account(id=account_id).entity_set.filter(id=entity_id).values_list('name', flat=True)[0]


def resolve_transaction_status_options(transaction_status):
    return transaction_status_options[transaction_status]


def get_resolve_column_id_cb(get_obj_name_from_id_cb):
    def _resolve_column_id(column_id):
        account = get_request_account()
        return get_obj_name_from_id_cb(account.id, column_id)

    return _resolve_column_id


event_columns = {
    'event_type_id': Column(
        name='event_type_id',
        display_name='Event Type',
        type=LiteralType.ID,
        id_options_cb=get_id_option_cb(
            lambda account: account.eventtype_set
        ),
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.COUNT],
        id_label_resolver_cb=get_resolve_column_id_cb(get_event_type_name_from_id),
    ),
    'timestamp_in_seconds': AnnotatedColumn(
        name='timestamp_in_seconds',
        annotation_relation=TruncSecond('timestamp'),
        type=LiteralType.TIMESTAMP,
        display_name='Timestamp (Seconds)',
        is_groupable=False,
        is_filterable=True,
        is_orderable=True,
        aggregation_functions=[AggregationFunction.MIN, AggregationFunction.MAX],
    ),
    'timestamp': Column(
        name='timestamp',
        type=LiteralType.TIMESTAMP,
        display_name='Timestamp',
        is_groupable=False,
        is_orderable=True,
        is_filterable=False,
        aggregation_functions=[AggregationFunction.MIN, AggregationFunction.MAX],
    ),
    'monitor_transaction': AnnotatedColumn(
        name='monitor_transaction',
        display_name='Monitor Transaction',
        type=LiteralType.STRING,
        annotation_relation=ArrayAgg(
            'monitortransactioneventmapping__monitor_transaction__transaction',
            default=Value([])
        ),
        token_filter_op=LiteralArrayColumnTokenFilterOp(),
        is_groupable=False,
        is_filterable=True,
        is_orderable=False,
    ),
    'event_attribute': AttributeColumn(
        name='event_attribute',
        annotation_relation=F('processed_kvs'),
        display_name='Event Attribute',
        attribute_options_cb=get_event_attribute_options,
        attribute_field='processed_kvs',
        is_filterable=True
    )
}

events_clickhouse_columns = {
    'event_type_id': Column(
        name='event_type_id',
        display_name='Event Type',
        type=LiteralType.ID,
        id_options_cb=get_id_option_cb(
            lambda account: account.eventtype_set
        ),
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.COUNT],
        id_label_resolver_cb=get_resolve_column_id_cb(get_event_type_name_from_id),
    ),
    'timestamp_in_seconds': AnnotatedColumn(
        name='timestamp_in_seconds',
        annotation_relation=TruncSecond('timestamp'),
        type=LiteralType.TIMESTAMP,
        display_name='Timestamp (Seconds)',
        is_groupable=False,
        is_orderable=True,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.MIN, AggregationFunction.MAX],
    ),
    'timestamp': Column(
        name='timestamp',
        type=LiteralType.TIMESTAMP,
        display_name='Timestamp',
        is_groupable=False,
        is_orderable=True,
        is_filterable=False,
        aggregation_functions=[AggregationFunction.MIN, AggregationFunction.MAX],
    ),
    'event_attribute': AttributeColumn(
        name='event_attribute',
        annotation_relation=F('processed_kvs'),
        display_name='Event Attribute',
        attribute_options_cb=get_event_attribute_options,
        attribute_field='processed_kvs',
        is_filterable=True
    )
}

monitor_transaction_columns = {
    'monitor_id': Column(
        name='monitor_id',
        type=LiteralType.ID,
        display_name='Monitor',
        id_options_cb=get_id_option_cb(
            lambda account: account.monitor_set.exclude(is_active=False)
        ),
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.COUNT],
        id_label_resolver_cb=get_resolve_column_id_cb(get_monitor_name_from_id),
    ),
    'transaction': Column(
        name='transaction',
        type=LiteralType.STRING,
        display_name='Transaction Value',
        is_groupable=False,
        is_orderable=False,
        is_filterable=True,
    ),
    'type': Column(
        name='type',
        type=LiteralType.ID,
        display_name='Transaction Status',
        id_options_cb=transaction_status_options_cb,
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.COUNT, AggregationFunction.COUNT_DISTINCT],
        id_label_resolver_cb=resolve_transaction_status_options,
    ),
    'transaction_time': AnnotatedColumn(
        name='transaction_time',
        annotation_relation=Cast(
            KeyTextTransform("transaction_time", "monitortransactionstats__stats"), models.FloatField()
        ),
        type=LiteralType.DOUBLE,
        display_name='Transaction Time',
        is_groupable=False,
        is_orderable=True,
        is_filterable=True,
        aggregation_functions=literal_type_aggregation_functions(LiteralType.DOUBLE),
    ),
    'created_at': Column(
        name='created_at',
        type=LiteralType.TIMESTAMP,
        display_name='Created At',
        is_groupable=False,
        is_orderable=True,
        is_filterable=False,
        aggregation_functions=[AggregationFunction.MIN, AggregationFunction.MAX],
    ),
    'has_alerts': AnnotatedColumn(
        name='has_alerts',
        annotation_relation=Exists(
            AlertMonitorTransactionMapping.objects.filter(
                monitor_transaction=OuterRef('pk')).filter(
                created_at__gte=OuterRef('created_at'))
        ),
        type=LiteralType.BOOLEAN,
        display_name='Alerted',
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=literal_type_aggregation_functions(LiteralType.BOOLEAN),
    ),
    'primary_event_attribute': AttributeColumn(
        name='primary_event_attribute',
        annotation_relation=Subquery(
            MonitorTransactionEventMapping.objects.filter(
                monitor_transaction=OuterRef("pk")).filter(
                event__event_type=OuterRef(
                    "monitor__primary_key__event_type__id")).values(
                'event__processed_kvs').order_by(
                'event__timestamp')[:1]
        ),
        display_name='Primary Event',
        attribute_options_cb=get_monitor_attribute_options_cb(key='primary_key'),
        is_filterable=True
    ),
    'secondary_event_attribute': AttributeColumn(
        name='secondary_event_attribute',
        annotation_relation=Subquery(
            MonitorTransactionEventMapping.objects.filter(
                monitor_transaction=OuterRef("pk")).filter(
                event__event_type=OuterRef(
                    "monitor__secondary_key__event_type")).values(
                'event__processed_kvs').order_by(
                'event__timestamp')[:1]
        ),
        display_name='Secondary Event',
        attribute_options_cb=get_monitor_attribute_options_cb(key='secondary_key'),
        is_filterable=True
    ),
    'free_attribute_search_column': AttributeColumnV2(
        name='free_attribute_search_column',
        general_annotation_relation="Subquery(MonitorTransactionEventMapping.objects.filter(monitor_transaction=OuterRef('pk')).filter(event__event_type=OuterRef('{}')).values('event__processed_kvs').order_by('event__timestamp')[:1])",
        display_name='Attribute Search',
        associated_model_name='MonitorTransactionEventMapping',
        associated_model=MonitorTransactionEventMapping,
        attribute_options_cb_v2=get_monitor_attribute_options_v2,
        is_filterable=True
    )
}

entity_instance_columns = {
    'entity_id': Column(
        name='entity_id',
        type=LiteralType.ID,
        display_name='Entity',
        id_options_cb=get_id_option_cb(
            lambda account: account.entity_set.exclude(is_active=False)
        ),
        is_groupable=True,
        is_orderable=False,
        is_filterable=True,
        aggregation_functions=[AggregationFunction.COUNT],
        id_label_resolver_cb=get_resolve_column_id_cb(get_entity_name_from_id),
    ),
    'instance': Column(
        name='instance',
        type=LiteralType.STRING,
        display_name='Entity Instance Value',
        is_groupable=False,
        is_orderable=False,
        is_filterable=True,
    ),
    'has_alerts': AnnotatedColumn(
        name='has_alerts',
        annotation_relation=Exists(
            AlertEntityInstanceMapping.objects.filter(
                entity_instance=OuterRef('pk')).filter(
                created_at__gte=OuterRef('created_at'))
        ),
        type=LiteralType.BOOLEAN,
        display_name='Alerted',
        is_groupable=True,
        is_orderable=False,
        aggregation_functions=literal_type_aggregation_functions(LiteralType.BOOLEAN),
        is_filterable=True,
    ),
    'free_attribute_search_column': AttributeColumnV2(
        name='free_attribute_search_column',
        general_annotation_relation="Subquery(EntityInstanceEventMapping.objects.filter(entity_instance=OuterRef('pk')).filter(event__event_type_id={}).values('event_processed_kvs').order_by('event_timestamp')[:1])",
        display_name='Attribute Search',
        associated_model_name='EntityInstanceEventMapping',
        associated_model=EntityInstanceEventMapping,
        attribute_options_cb_v2=get_entity_attribute_options_v2,
        is_filterable=True
    )
}


class ColumnOptions:
    def __init__(self, columns: Dict):
        self._columns = columns

    def options(self, account, obj=None):
        column_options = []
        attribute_options = []
        for column_name, column in self._columns.items():
            if column.is_filterable:
                if isinstance(column, Column):
                    column_options.append(column.get_options(account, obj))
                elif isinstance(column, AttributeColumn):
                    attribute_options.extend(column.get_options(account, obj))

        return column_options, attribute_options

    def options_v2(self, account, obj=None):
        column_options = []
        attribute_options_v2 = []
        for column_name, column in self._columns.items():
            if column.is_filterable:
                if isinstance(column, Column):
                    column_options.append(column.get_options(account, obj))
                elif isinstance(column, AttributeColumnV2):
                    attribute_options_v2.extend(column.get_options(account, obj))

        return column_options, attribute_options_v2

event/engine/model_columns.py
DrDroidLab/kenobi
from django.db.models import QuerySet

from event.engine.context import ContextResolver, get_context_resolver
from event.models import Event, MonitorTransaction, EntityInstance
from event.clickhouse.models import Events
from protos.event.base_pb2 import Context
from protos.event.literal_pb2 import IdLiteral
from protos.event.query_base_pb2 import QueryRequest, OrderByExpression


class QueryEngineException(ValueError):
    pass


def get_default_query(account, obj) -> QueryRequest:
    return QueryRequest()


class QueryEngine:
    def __init__(self, model, context_resolver: ContextResolver):
        self._model = model
        self._context_resolver = context_resolver

    def process_query(self, qs: QuerySet, query_request: QueryRequest) -> QuerySet:
        if qs.model is not self._model:
            raise QueryEngineException(f'Query engine not supported for {qs.model.__name__}')
        return self._context_resolver.filter_engine().process(qs, query_request.filter)

    def process_query_with_filter(self, qs: QuerySet, query_filter) -> QuerySet:
        if qs.model is not self._model:
            raise QueryEngineException(f'Query engine not supported for {qs.model.__name__}')
        return self._context_resolver.filter_engine().process(qs, query_filter)

    def process_ordering(self, qs: QuerySet, query_request: QueryRequest) -> QuerySet:
        if qs.model is not self._model:
            raise QueryEngineException(f'Query engine not supported for {qs.model.__name__}')

        order_by_expression: OrderByExpression = query_request.order_by
        return self._context_resolver.order_by_engine().process(qs, order_by_expression)

    def get_obj(self, account, id: IdLiteral = None):
        return self._context_resolver.get_parent_obj(account, id)

    def get_search_options(self, account, parent_obj=None):
        return self._context_resolver.get_search_options(account, parent_obj)

    def get_search_options_v2(self, account, parent_obj=None):
        return self._context_resolver.get_search_options_v2(account, parent_obj)

    def get_default_query(self, account, obj=None):
        return self._context_resolver.get_default_query(account, obj)


global_event_search_clickhouse_engine = QueryEngine(
    Events,
    get_context_resolver(Context.EVENTS_CLICKHOUSE),
)

global_event_search_engine = QueryEngine(
    Event,
    get_context_resolver(Context.EVENT),
)

monitor_transaction_search_engine = QueryEngine(
    MonitorTransaction,
    get_context_resolver(Context.MONITOR_TRANSACTION),
)

entity_instance_search_engine = QueryEngine(
    EntityInstance,
    get_context_resolver(Context.ENTITY_INSTANCE),
)

event/engine/query_engine.py
DrDroidLab/kenobi
from django.db import transaction as dj_transaction
from django.db.models import Avg, Count, F
from google.protobuf.wrappers_pb2 import UInt64Value, StringValue

from accounts.models import Account
from event.cache import GLOBAL_PANEL_CACHE
from event.clickhouse.models import MonitorTransactions, Events
from event.models import Entity, EntityEventKeyMapping, EntityMonitorMapping
from event.monitors_crud import create_monitors
from event.workflows.funnel import Funnel
from protos.event.entity_pb2 import EntityPartial, Entity as EntityProto, WorkflowView
from protos.event.panel_pb2 import FunnelPanel, PanelV1, PanelData
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto
from prototype.utils.timerange import DateTimeRange, filter_dtr
from utils.proto_utils import proto_to_dict, dict_to_proto


class EntityFunnelCrudIncorrectPayloadException(ValueError):
    pass


class EntityFunnelCrudNotFoundException(ValueError):
    pass


def save_entity_funnel_panel_config(scope, entity_funnel_panel: PanelV1):
    try:
        GLOBAL_PANEL_CACHE.create_or_update(account_id=scope.id, name=entity_funnel_panel.meta_info.name,
                                            panel=proto_to_dict(entity_funnel_panel))
        return True
    except Exception as e:
        print(str(e))
    return False


def db_get_funnel_entity(scope: Account, funnel_entity: EntityPartial, is_active=None) -> Entity:
    filters = {}
    if funnel_entity.id.value:
        filters['id'] = funnel_entity.id.value
    if funnel_entity.name.value:
        filters['name'] = funnel_entity.name.value
    if is_active:
        filters['is_active'] = is_active
    try:
        if not filters:
            raise EntityFunnelCrudIncorrectPayloadException(f'Incorrect payload - Missing entity funnel details')
        return scope.entity_set.get(**filters)
    except Entity.DoesNotExist:
        raise EntityFunnelCrudNotFoundException(f'No Entity Funnel found with details {funnel_entity}')


def check_if_entity_funnel_exists(scope, entity_id=None, entity_name=None, is_active=None) -> bool:
    filters = {}
    if entity_id:
        filters['id'] = entity_id
    if entity_name:
        filters['name'] = entity_name
    if is_active:
        filters['is_active'] = is_active
    try:
        return scope.entity_set.filter(**filters).exists()
    except Exception as e:
        raise EntityFunnelCrudNotFoundException(f'Entity Check Failed with details {e}')


def db_save_entity_funnel(scope, name, event_key_ids, event_key_tuples):
    with dj_transaction.atomic():
        e = Entity(
            account=scope,
            name=name,
            type=EntityProto.Type.FUNNEL,
            is_active=True,
            is_generated=True
        )
        e.save()

        ems: [EntityMonitorMapping] = []
        for pair in event_key_tuples:
            primary_event_key_id = pair[0]
            secondary_event_key_id = pair[1]

            monitor_name = 'system_generated_monitor_' + str(primary_event_key_id) + '_' + str(secondary_event_key_id)
            db_monitor, created, error = create_monitors(scope, monitor_name, primary_event_key_id,
                                                         secondary_event_key_id,
                                                         is_generated=True)
            if not db_monitor and error:
                raise EntityFunnelCrudIncorrectPayloadException(f'Error Creating Entity Funnel: {error}')
            ems.append(
                EntityMonitorMapping(account=scope, entity=e, monitor=db_monitor, is_active=True, is_generated=True))

        if ems:
            EntityMonitorMapping.objects.bulk_create(ems, ignore_conflicts=True, batch_size=25)

        eekms: [EntityEventKeyMapping] = [EntityEventKeyMapping(account=scope, entity=e, event_key_id=event_key_id) for
                                          event_key_id in
                                          event_key_ids]

        if eekms:
            EntityEventKeyMapping.objects.bulk_create(
                eekms,
                ignore_conflicts=True,
                batch_size=25
            )
    return e


def entity_funnels_create(scope, entity_funnel_panel: PanelV1) -> (Entity, str):
    if not entity_funnel_panel or not entity_funnel_panel.meta_info or not entity_funnel_panel.meta_info.name:
        return None, 'Incorrect Payload: Missing Entity Funnel Name'

    entity_funnel_name: str = entity_funnel_panel.meta_info.name
    if check_if_entity_funnel_exists(scope, entity_name=entity_funnel_name):
        return None, f'Incorrect Payload: Entity Funnel with name {entity_funnel_name} already exists'

    if not save_entity_funnel_panel_config(scope, entity_funnel_panel):
        return None, f'Incorrect Payload: Unable to save Entity Funnel Panel in Cache'

    entity_funnel_panel_data: PanelData = entity_funnel_panel.data
    which_one_of = entity_funnel_panel_data.WhichOneof('data')
    if not entity_funnel_panel_data.type == PanelData.PanelDataType.FUNNEL or not which_one_of == 'funnel':
        return None, f'Incorrect Payload: Panel Data Type is not Funnel'

    entity_funnel_config: FunnelPanel = entity_funnel_panel_data.funnel
    event_key_name = entity_funnel_config.event_key_name
    event_type_ids = entity_funnel_config.event_type_ids
    event_id_tuples = [(event_type_ids[i], event_type_ids[i + 1]) for i in range(len(event_type_ids) - 1)]
    if not event_type_ids or not event_key_name:
        return None, 'Incorrect Payload: Missing Entity Funnel Event Type Ids/Event Key'

    event_keys = scope.eventkey_set.filter(event_type__in=event_type_ids, name=event_key_name).values('id',
                                                                                                      'event_type_id')
    if event_keys.count() != len(event_type_ids):
        return None, f'Incorrect Payload: Invalid Entity Funnel Event Keys'

    event_key_tuples = []
    for pair in event_id_tuples:
        primary_event_key_id = next((x['id'] for x in event_keys if x['event_type_id'] == pair[0]), None)
        secondary_event_key_id = next((x['id'] for x in event_keys if x['event_type_id'] == pair[1]), None)
        if not primary_event_key_id or not secondary_event_key_id:
            return None, f'Incorrect Payload: Missing Entity Funnel DB Event Keys'
        event_key_tuples.append((primary_event_key_id, secondary_event_key_id))

    try:
        db_entity_funnel = db_save_entity_funnel(scope, entity_funnel_name, event_keys.values_list('id', flat=True),
                                                 event_key_tuples)
        return db_entity_funnel, None
    except Exception as e:
        return None, f'Incorrect Payload: Unable to save Entity Funnel in DB'


def entity_funnels_get_clickhouse_events(account, dtr, entity_funnel_name, filter_key_name=None, filter_value=None):
    panels = GLOBAL_PANEL_CACHE.get(account_id=account.id, name=entity_funnel_name)
    panel_proto: PanelV1 = dict_to_proto(panels[0], PanelV1)
    panel_data = panel_proto.data
    if panel_data.type != PanelData.PanelDataType.FUNNEL or panel_data.funnel is None:
        raise EntityFunnelCrudIncorrectPayloadException(f'Incorrect Panel Payload: Panel Data Type is not Funnel')
    funnel = panel_data.funnel
    event_key_name = funnel.event_key_name
    funnel_event_type_ids = funnel.event_type_ids
    if not filter_key_name or not filter_value:
        filter_key_name = funnel.filter_key_name
        filter_value = funnel.filter_value
    funnel_event_type_ids_str = ','.join([str(e) for e in funnel_event_type_ids])
    dtr_geq = dtr.to_tr_str()[0]
    dtr_lt = dtr.to_tr_str()[1]
    events_group_query = f"select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, " \
                         f"groupArray(event_type_name) as event_type_name_group, " \
                         f"groupArray(timestamp) as timestamp_group, e_id from " \
                         f"(select id, event_type_id, event_type_name, timestamp, " \
                         f"processed_kvs.{event_key_name} as e_id from events where " \
                         f"event_type_id in ({funnel_event_type_ids_str}) and account_id = {account.id} and " \
                         f"timestamp between '{dtr_geq}' and '{dtr_lt}' and processed_kvs.{event_key_name} in " \
                         f"(select distinct processed_kvs.{event_key_name} from events where " \
                         f"event_type_id = {funnel_event_type_ids[0]} and account_id = {account.id} and " \
                         f"timestamp between '{dtr_geq}' and '{dtr_lt}') " \
                         f"order by timestamp asc, created_at asc) group by e_id;"

    if filter_key_name and filter_value:
        events_group_query = f"select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, " \
                             f"groupArray(event_type_name) as event_type_name_group, " \
                             f"groupArray(timestamp) as timestamp_group, groupArray(e_filter_value) as filter_group, " \
                             f"e_id from (select id, event_type_id, event_type_name, timestamp, " \
                             f"processed_kvs.{event_key_name} as e_id, " \
                             f"processed_kvs.{filter_key_name} as e_filter_value from events where " \
                             f"event_type_id in ({funnel_event_type_ids_str}) and account_id = {account.id} and " \
                             f"timestamp between '{dtr_geq}' and '{dtr_lt}' and processed_kvs.{event_key_name} in " \
                             f"(select distinct processed_kvs.{event_key_name} from events where " \
                             f"event_type_id = {funnel_event_type_ids[0]} and account_id = {account.id} and " \
                             f"timestamp between '{dtr_geq}' and '{dtr_lt}') " \
                             f"order by timestamp asc, created_at asc) group by e_id;"

    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    f = Funnel()

    for grouped_event in events_group_set:
        f.add_to_node_map(grouped_event, filter_key_name=filter_key_name, filter_value=filter_value)

    ordered_funnel_data = f.get_ordered_funnel_data(funnel_event_type_ids)
    return ordered_funnel_data


def entity_funnels_get(scope: Account, dtr: DateTimeRange, entity_funnel_name: str, filter_key_name=None,
                       filter_value=None):
    if not scope:
        raise EntityFunnelCrudIncorrectPayloadException(f'Incorrect Panel Payload: Missing Account')
    account_id = scope.id
    if not entity_funnel_name:
        raise EntityFunnelCrudIncorrectPayloadException(f'Incorrect Panel Payload: Missing Entity Funnel Name')

    try:
        db_entity_funnel = db_get_funnel_entity(scope,
                                                funnel_entity=EntityPartial(name=StringValue(value=entity_funnel_name)),
                                                is_active=True)
    except (EntityFunnelCrudIncorrectPayloadException, EntityFunnelCrudNotFoundException):
        raise EntityFunnelCrudNotFoundException(
            f'Incorrect Panel Payload: Active Entity with name {entity_funnel_name} does not exist')

    entity_monitor = scope.entitymonitormapping_set.filter(entity=db_entity_funnel, is_active=True, is_generated=True)
    entity_monitor = entity_monitor.prefetch_related('monitor').values('monitor_id',
                                                                       'monitor__primary_key__name',
                                                                       'monitor__primary_key__event_type_id',
                                                                       'monitor__primary_key__event_type__name',
                                                                       'monitor__secondary_key__event_type_id',
                                                                       'monitor__secondary_key__event_type__name')
    if not entity_monitor:
        raise EntityFunnelCrudIncorrectPayloadException(
            f'Incorrect Panel Payload: Entity Funnel Monitors not found for {entity_funnel_name}')
    entity_monitor_ids = [monitor['monitor_id'] for monitor in entity_monitor]

    head_monitor = entity_monitor[0]
    unique_transactions = MonitorTransactions.objects.filter(account_id=account_id).filter(
        monitor_id=head_monitor['monitor_id']).filter(event_type_id=head_monitor['monitor__primary_key__event_type_id'])
    unique_transactions = filter_dtr(unique_transactions, dtr, 'event_timestamp')
    if filter_key_name and filter_value:
        unique_transactions = unique_transactions.annotate(
            event_attribute=F('event_processed_kvs__{}'.format(filter_key_name)))
        unique_transactions = unique_transactions.filter(event_attribute=filter_value)
    unique_transactions = unique_transactions.values_list('transaction', flat=True).distinct()

    monitor_transactions = MonitorTransactions.objects.filter(account_id=account_id).filter(
        event_timestamp__gte=dtr.time_geq).filter(monitor_id__in=entity_monitor_ids).filter(
        monitor_transaction_event_type=MonitorTransactionProto.MonitorTransactionEventType.PRIMARY).filter(
        transaction__in=unique_transactions).values('monitor_id').annotate(
        transaction_count=Count('transaction', distinct=True))

    monitor_transactions_finished = MonitorTransactions.objects.filter(account_id=account_id).filter(
        event_timestamp__gte=dtr.time_geq).filter(monitor_id__in=entity_monitor_ids).filter(
        monitor_transaction_event_type=MonitorTransactionProto.MonitorTransactionEventType.SECONDARY).filter(
        transaction__in=unique_transactions).values('monitor_id').annotate(
        avg_duration=Avg('transaction_time')).annotate(completed_transaction_count=Count('transaction', distinct=True))

    monitor_transactions_count = {item['monitor_id']: item['transaction_count'] for item in monitor_transactions}
    monitor_transactions_finished_count = {
        item['monitor_id']: {'completed_transaction_count': item['completed_transaction_count'],
                             'avg_duration': item['avg_duration']} for item in monitor_transactions_finished}
    data = []
    links = []
    node_map = {}
    edge_map = {}
    start_conn_types = [WorkflowView.NodeConnectionType.OUT]
    mid_conn_types = [WorkflowView.NodeConnectionType.IN, WorkflowView.NodeConnectionType.OUT]
    end_conn_types = [WorkflowView.NodeConnectionType.IN]
    for idx, monitor in enumerate(entity_monitor):
        monitor_id = monitor['monitor_id']

        primary_node_transaction_count = monitor_transactions_count.get(monitor_id, 0)
        secondary_node_transaction_count = 0
        avg_duration = 0
        monitor_transactions_finished_stats = monitor_transactions_finished_count.get(monitor_id, {})
        if monitor_transactions_finished_stats:
            secondary_node_transaction_count = monitor_transactions_finished_stats.get('completed_transaction_count', 0)
            avg_duration = monitor_transactions_finished_stats.get('avg_duration', 0)

        primary_conn_types = mid_conn_types
        secondary_conn_types = mid_conn_types
        if len(data) == 0:
            primary_conn_types = start_conn_types
        elif idx >= len(entity_monitor) - 1:
            secondary_conn_types = end_conn_types

        primary_node = WorkflowView.Node(
            node_id=UInt64Value(value=monitor['monitor__primary_key__event_type_id']),
            label=StringValue(value=monitor['monitor__primary_key__event_type__name']),
            connection_types=primary_conn_types,
            metrics=[
                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                             value=StringValue(value=str(primary_node_transaction_count)),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ]
        )

        secondary_node = WorkflowView.Node(
            node_id=UInt64Value(value=monitor['monitor__secondary_key__event_type_id']),
            label=StringValue(value=monitor['monitor__secondary_key__event_type__name']),
            connection_types=secondary_conn_types,
            metrics=[
                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                             value=StringValue(value=str(secondary_node_transaction_count)),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ]
        )

        if primary_node.node_id.value not in node_map:
            node_map[primary_node.node_id.value] = primary_node
            data.append(primary_node)
        if secondary_node.node_id.value not in node_map:
            node_map[secondary_node.node_id.value] = secondary_node
            data.append(secondary_node)

        edge = "{}-{}".format(primary_node.node_id.value, secondary_node.node_id.value)

        drop_percent = 0
        if primary_node_transaction_count > 0 and secondary_node_transaction_count > 0:
            drop_percent = round(((primary_node_transaction_count - secondary_node_transaction_count) * 100 /
                                  primary_node_transaction_count), 2)
        if edge and edge not in edge_map:
            edge_map[edge] = True
        links.append(
            WorkflowView.Edge(
                start_node_id=UInt64Value(value=primary_node.node_id.value),
                end_node_id=UInt64Value(value=secondary_node.node_id.value),
                metrics=[
                    WorkflowView.Edge.EdgeMetric(name=StringValue(value="Drop"),
                                                 value=StringValue(value="{}%".format(str(drop_percent))),
                                                 metric_status=WorkflowView.MetricStatus.GREEN),
                    WorkflowView.Edge.EdgeMetric(name=StringValue(value="Avg Time"),
                                                 value=StringValue(value="{} sec".format(str(avg_duration))),
                                                 metric_status=WorkflowView.MetricStatus.GREEN)
                ],
            )
        )

    return WorkflowView(nodes=data, edges=links)

event/entity_funnels_crud.py
DrDroidLab/kenobi
csm_data = {
  "blittandgaines": {
    "source": 'SFTP',
    "customer": 'Blittandgaines',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '830'
  },
  "firstcreditservices": {
    "source": 'LV',
    "customer": 'FCS',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": '600'
  },
  "universalcredit": {
    "source": 'SFTP',
    "customer": 'Universal Credit ',
    "ingestion_start_utc": '1000',
    "ingestion_end_utc": '1200'
  },
  "trueaccord": {
    "source": 'SFTP',
    "customer": 'TrueAccord',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "resurgent": {
    "source": 'SFTP',
    "customer": 'Resurgent',
    "ingestion_start_utc": '600',
    "ingestion_end_utc": '800'
  },
  "psb": {
    "source": 'SFTP',
    "customer": 'PSB',
    "ingestion_start_utc": '600',
    "ingestion_end_utc": '800'
  },
  "mars": {
    "source": 'SFTP',
    "customer": 'Mars',
    "ingestion_start_utc": '600',
    "ingestion_end_utc": '900'
  },
  "annuity": {
    "source": 'TCN',
    "customer": 'Annuity',
    "ingestion_start_utc": '800',
    "ingestion_end_utc": '1200'
  },
  "ccmr3": {
    "source": 'TCN',
    "customer": 'CCMR3',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "creditsolutions3p": {
    "source": 'TCN',
    "customer": 'Creditsolutions3p',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "creditsolutionseo": {
    "source": 'TCN',
    "customer": 'CreditsolutionsEO',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "pac": {
    "source": 'TCN',
    "customer": 'PAC',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "pacfunding": {
    "source": 'TCN',
    "customer": 'PAC Funding',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "universalfidelity": {
    "source": 'TCN',
    "customer": 'Universal Fidelity',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "payplanuk": {
    "source": 'SFTP',
    "customer": 'Payplan UK',
    "ingestion_start_utc": '330',
    "ingestion_end_utc": '530'
  },
  "billgoslinguk": {
    "source": 'SFTP',
    "customer": 'BGO UK',
    "ingestion_start_utc": '600',
    "ingestion_end_utc": '800'
  },
  "billgoslingca": {
    "source": 'SFTP',
    "customer": 'BGO CA',
    "ingestion_start_utc": '730',
    "ingestion_end_utc": '930'
  },
  "gatestonecapitaloneca": {
    "source": 'TCN',
    "customer": 'gatestonecaponeca',
    "ingestion_start_utc": '1100',
    "ingestion_end_utc": '1300'
  },
  "dcsyncom": {
    "source": 'SFTP',
    "customer": 'DCSyncom',
    "ingestion_start_utc": '1200',
    "ingestion_end_utc": '1400'
  },
  "dcfirstcredit": {
    "source": 'SFTP',
    "customer": 'dcfirstcredit',
    "ingestion_start_utc": '830',
    "ingestion_end_utc": '1030'
  },
  "moorelaw": {
    "source": 'SFTP',
    "customer": 'MooreLaw',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '830'
  },
  "bca": {
    "source": 'SFTP',
    "customer": 'BCA',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '930'
  },
  "holliscobb": {
    "source": 'SFTP',
    "customer": 'Hollis Cobb',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '1030'
  },
  "cascade": {
    "source": 'TCN',
    "customer": 'Cascade',
    "ingestion_start_utc": '900',
    "ingestion_end_utc": '1100'
  },
  "Stillmanlaw": {
    "source": 'SFTP',
    "customer": 'SLO',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "timepayment": {
    "source": 'Five9, SFTP',
    "customer": 'TPT',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '830'
  },
  "pendrick": {
    "source": 'LV',
    "customer": 'Pendrick',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "harrisandharris": {
    "source": 'PCI SFTP',
    "customer": 'H&H',
    "ingestion_start_utc": '',
    "ingestion_end_utc": ''
  },
  "ffam": {
    "source": 'LV',
    "customer": 'FFAM',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "sherloq": {
    "source": 'SFTP',
    "customer": 'Sherloq ',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "zarzaurandschwartz": {
    "source": 'SFTP',
    "customer": 'ZZS(Zarzaur)',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '830'
  },
  "gatestoneus": {
    "source": 'LV',
    "customer": 'Gatestoneus',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "gatestoneusaa": {
    "source": 'LV',
    "customer": 'Gatestoneusaa',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "gatestonesci": {
    "source": 'LV',
    "customer": 'GatestoneSCI',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "gatestonecpe": {
    "source": 'LV',
    "customer": 'GatestoneCPE',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "revenuecycle": {
    "source": 'LV',
    "customer": 'Revenue Cycle',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "springoak": {
    "source": 'LV',
    "customer": 'SpringOaks ',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "everest": {
    "source": 'LV',
    "customer": 'Everest',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "halsted": {
    "source": 'SFTP',
    "customer": 'Halsted',
    "ingestion_start_utc": '530',
    "ingestion_end_utc": '930'
  },
  "rashcurtis": {
    "source": 'LV',
    "customer": 'RashCurtis',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "sequium": {
    "source": 'LV',
    "customer": 'Sequium',
    "ingestion_start_utc": '430',
    "ingestion_end_utc": '830'
  },
  "unifin": {
    "source": 'SFTP',
    "customer": 'Unifin',
    "ingestion_start_utc": '600',
    "ingestion_end_utc": '800'
  },
  "icca2": {
    "source": 'IC',
    "customer": 'Capital',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icgbs": {
    "source": 'IC',
    "customer": 'Global',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icpr": {
    "source": 'IC',
    "customer": 'PR',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icamfiman": {
    "source": 'IC',
    "customer": 'Amfiman',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icsra": {
    "source": 'IC',
    "customer": 'SRA',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icdatamax": {
    "source": 'IC',
    "customer": 'Datamax',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icmlc": {
    "source": 'IC',
    "customer": 'MLC',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icfcr": {
    "source": 'IC',
    "customer": 'FCR',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icprocom": {
    "source": 'IC',
    "customer": 'Procom',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icbh": {
    "source": 'IC',
    "customer": 'BH',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "iclcs": {
    "source": 'IC',
    "customer": 'LCS',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "icbw": {
    "source": 'IC',
    "customer": 'BW',
    "ingestion_start_utc": 'Streaming',
    "ingestion_end_utc": ''
  },
  "nes": {
    "source": 'SFTP',
    "customer": 'NES',
    "ingestion_start_utc": '1000',
    "ingestion_end_utc": '1300'
  },
  "billgosling": {
    "source": 'SFTP',
    "customer": 'BGO US',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '1030'
  },
  "zwicker": {
    "source": 'SFTP',
    "customer": 'Zwicker',
    "ingestion_start_utc": '1030',
    "ingestion_end_utc": '1330'
  },
  "prosper": {
    "source": 'FIVE9',
    "customer": 'Prosper',
    "ingestion_start_utc": '630',
    "ingestion_end_utc": '830'
  },
  "stenger": {
    "source": 'TCN',
    "customer": 'Stenger',
    "ingestion_start_utc": '825',
    "ingestion_end_utc": '1025'
  },
  "optio": {
    "source": 'SFTP',
    "customer": 'Optio',
    "ingestion_start_utc": '1330',
    "ingestion_end_utc": '1530'
  },
  "dccpc": {
    "source": 'SFTP',
    "customer": 'DCCPC',
    "ingestion_start_utc": '1200',
    "ingestion_end_utc": '1400'
  },
  "nra": {
    "source": 'LV',
    "customer": 'NRA',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  },
  "grc": {
    "source": 'LV',
    "customer": 'GRC',
    "ingestion_start_utc": '400',
    "ingestion_end_utc": '600'
  }
}
event/helpers/prodigal_csm_sla.py
DrDroidLab/kenobi
from datetime import datetime, timedelta, timezone

def append_metrics(metric_object, event_count_object):
    event_name_metric_name_map = {
        'p1-error': 'P1 Error',
        "p1-success": 'Metadata received',
        "p1-failed": 'Metadata received',
        'p2-audioReceived': 'Audio files received',
        'p2-success':   'Matched Records',
        'p3-filteredRecord':  'Filtered Records',
        'p6-success': 'Ingested Records',
        'publish-end': 'Published Records'
    }

    metric_array = metric_object['record_count_metrics']
    if event_count_object['event_type_name'] in event_name_metric_name_map:
        metric_name = event_name_metric_name_map[event_count_object['event_type_name']]
        for metric in metric_array:
            if metric['metric_name'] == metric_name:
                metric['metric_value'] = metric['metric_value'] + int(event_count_object['record_count'])
                metric['event_type_ids'].append(event_count_object['event_type_id'])

    for metric in metric_array:
        event_type_ids_str = '['
        event_type_ids_str_tokens = []
        for e_type_id in metric['event_type_ids']:
            event_type_ids_str_tokens.append('{"type": "LONG", "long": "' + str(e_type_id) + '"}')
        event_type_ids_str += ",".join(event_type_ids_str_tokens) + ']'
        metric['filter_query'] = '[{"op":"EQ","lhs":{"attribute_identifier":{"name":"tenantId","path":"event_attribute","type":"STRING"}},"rhs":{"literal":{"literal_type":"STRING","string":"' + metric_object['tenantId'] + '"}}},{"op":"EQ","lhs":{"attribute_identifier":{"name":"connectorId","path":"event_attribute","type":"STRING"}},"rhs":{"literal":{"literal_type":"STRING","string":"' + metric_object['connectorId'] + '"}}},{"op":"EQ","lhs":{"attribute_identifier":{"name":"integrationType","path":"event_attribute","type":"STRING"}},"rhs":{"literal":{"literal_type":"STRING","string":"' + metric_object['integrationType'] + '"}}},{"op":"IN","lhs":{"column_identifier":{"name":"event_type_id","type":"ID"}},"rhs":{"literal":{"literal_type":"ID_ARRAY","id_array":' + event_type_ids_str + '}}}]'


def get_timestamp_utc(timestamp_str):
    if not timestamp_str:
        return None

    if len(timestamp_str) == 3:
        hours = int(timestamp_str[0])
        minutes = int(timestamp_str[1:])
    elif len(timestamp_str) == 4:
        hours = int(timestamp_str[:2])
        minutes = int(timestamp_str[2:])
    else:
        return None

    now_utc = datetime.now(tz=timezone.utc)
    desired_time = now_utc.replace(hour=hours, minute=minutes, second=0, microsecond=0)

    return desired_time
event/helpers/prodigal_helper.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-25 05:10

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('accounts', '0004_account_accountapitoken_alter_user_unique_together_and_more'),
    ]

    operations = [
        migrations.CreateModel(
            name='EventKey',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=256)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('type', models.IntegerField(blank=True, choices=[(0, 'KEY_TYPE_UNKNOWN'), (1, 'KEY_TYPE_STRING'), (2, 'KEY_TYPE_LONG'), (3, 'KEY_TYPE_DOUBLE'), (4, 'KEY_TYPE_BOOLEAN'), (5, 'KEY_TYPE_BYTE'), (6, 'KEY_TYPE_ARRAY'), (7, 'KEY_TYPE_OBJECT')], db_index=True, default=0, null=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
        ),
        migrations.CreateModel(
            name='Monitor',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=256)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('primary_key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='primary_key', to='event.eventkey')),
                ('secondary_key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='secondary_key', to='event.eventkey')),
            ],
            options={
                'unique_together': {('account', 'name')},
            },
        ),
        migrations.CreateModel(
            name='MonitorTransaction',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('transaction', models.TextField(db_index=True)),
                ('type', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'PRIMARY_RECEIVED'), (2, 'SECONDARY_RECEIVED')], db_index=True, default=0, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('monitor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitor')),
            ],
            options={
                'unique_together': {('account', 'monitor', 'transaction')},
            },
        ),
        migrations.CreateModel(
            name='EventType',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=256)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account', 'name')},
            },
        ),
        migrations.AddField(
            model_name='eventkey',
            name='event_type',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.eventtype'),
        ),
        migrations.CreateModel(
            name='Event',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('event', models.JSONField()),
                ('processed_kvs', models.JSONField(null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('timestamp', models.DateTimeField(db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('event_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.eventtype')),
            ],
        ),
        migrations.CreateModel(
            name='Panel',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.TextField(blank=True, default='', null=True)),
                ('config', models.JSONField(blank=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(auto_now=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account', 'name')},
            },
        ),
        migrations.CreateModel(
            name='Notification',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('channel', models.CharField(max_length=256)),
                ('webhook_url', models.TextField()),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
            options={
                'unique_together': {('account',)},
            },
        ),
        migrations.CreateModel(
            name='MonitorTransactionEventMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.event')),
                ('monitor_transaction', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitortransaction')),
            ],
            options={
                'unique_together': {('account', 'event', 'monitor_transaction')},
            },
        ),
        migrations.AlterUniqueTogether(
            name='eventkey',
            unique_together={('account', 'event_type', 'name')},
        ),
    ]

event/migrations/0001_initial.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-02-28 11:28

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0004_account_accountapitoken_alter_user_unique_together_and_more'),
        ('event', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='monitortransaction',
            name='updated_at',
            field=models.DateTimeField(auto_now=True, db_index=True, null=True),
        ),
        migrations.CreateModel(
            name='Trigger',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=256)),
                ('type', models.IntegerField(choices=[(0, 'UNKNOWN_TT'), (1, 'MISSING_EVENT_TT'), (2, 'DELAYED_EVENT_TT')], db_index=True)),
                ('config', models.JSONField()),
                ('priority', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_P'), (1, 'TP_0'), (2, 'TP_1'), (3, 'TP_2'), (4, 'TP_3'), (5, 'TP_4')], db_index=True, default=5, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('monitor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='event.monitor')),
            ],
            options={
                'unique_together': {('account', 'name')},
            },
        ),
        migrations.CreateModel(
            name='Alert',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('triggered_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('monitor_transaction', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='event.monitortransaction')),
                ('trigger', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.trigger')),
            ],
            options={
                'unique_together': {('account', 'trigger', 'monitor_transaction')},
            },
        ),
    ]

event/migrations/0002_monitortransaction_updated_at_trigger_alert.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-02 07:12

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0002_monitortransaction_updated_at_trigger_alert'),
    ]

    operations = [
        migrations.AddField(
            model_name='event',
            name='event_source',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK')], db_index=True, default=0, null=True),
        ),
        migrations.AddField(
            model_name='event',
            name='is_generated',
            field=models.BooleanField(blank=True, default=False, null=True),
        ),
        migrations.AlterField(
            model_name='eventkey',
            name='type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'STRING'), (2, 'LONG'), (3, 'DOUBLE'), (4, 'BOOLEAN'), (5, 'BYTE'), (6, 'ARRAY'), (7, 'OBJECT')], db_index=True, default=0, null=True),
        ),
    ]

event/migrations/0003_event_event_source_event_is_generated_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-02 07:56

from django.db import migrations, models
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0003_event_event_source_event_is_generated_and_more'),
    ]

    operations = [
        migrations.AddField(
            model_name='eventtype',
            name='event_sources',
            field=prototype.fields.ChoiceArrayField(base_field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK')], null=True), default=list, size=None),
        ),
    ]

event/migrations/0004_eventtype_event_sources.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-02 11:09

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0004_eventtype_event_sources'),
    ]

    operations = [
        migrations.AlterField(
            model_name='trigger',
            name='type',
            field=models.IntegerField(choices=[(0, 'UNKNOWN'), (1, 'MISSING_EVENT'), (2, 'DELAYED_EVENT')], db_index=True),
        ),
    ]

event/migrations/0005_alter_trigger_type.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-02 15:46

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
        ('event', '0005_alter_trigger_type'),
    ]

    operations = [
        migrations.AlterUniqueTogether(
            name='notification',
            unique_together=set(),
        ),
        migrations.AddField(
            model_name='notification',
            name='config',
            field=models.JSONField(blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='notification',
            name='channel',
            field=models.IntegerField(choices=[(0, 'UNKNOWN'), (1, 'SLACK')], db_index=True),
        ),
        migrations.AlterUniqueTogether(
            name='notification',
            unique_together={('account', 'config')},
        ),
        migrations.CreateModel(
            name='TriggerNotificationConfigMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('notification', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.notification')),
                ('trigger', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.trigger')),
            ],
            options={
                'unique_together': {('account', 'trigger', 'notification')},
            },
        ),
        migrations.RemoveField(
            model_name='notification',
            name='webhook_url',
        ),
        migrations.AddField(
            model_name='notification',
            name='trigger',
            field=models.ManyToManyField(related_name='triggers', through='event.TriggerNotificationConfigMapping', to='event.trigger'),
        ),
    ]

event/migrations/0006_alter_notification_unique_together_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-07 11:27

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
        ('event', '0006_alter_notification_unique_together_and_more'),
    ]

    operations = [
        migrations.AlterUniqueTogether(
            name='alert',
            unique_together=set(),
        ),
        migrations.CreateModel(
            name='AlertMonitorTransactionMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('alert', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.alert')),
                ('monitor_transaction', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitortransaction')),
            ],
            options={
                'unique_together': {('account', 'alert', 'monitor_transaction')},
            },
        ),
        migrations.RemoveField(
            model_name='alert',
            name='monitor_transaction',
        ),
        migrations.AddField(
            model_name='monitortransaction',
            name='alert',
            field=models.ManyToManyField(related_name='alerts', through='event.AlertMonitorTransactionMapping', to='event.alert'),
        ),
    ]

event/migrations/0007_alter_alert_unique_together_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-07 15:11

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0007_alter_alert_unique_together_and_more'),
    ]

    operations = [
        migrations.AddField(
            model_name='alert',
            name='stats',
            field=models.JSONField(null=True),
        ),
    ]

event/migrations/0008_alert_stats.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-08 05:24

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0008_alert_stats'),
    ]

    operations = [
        migrations.AddField(
            model_name='alertmonitortransactionmapping',
            name='stats',
            field=models.JSONField(null=True),
        ),
        migrations.AddField(
            model_name='alertmonitortransactionmapping',
            name='type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'MISSED_TRANSACTION'), (2, 'DELAYED_TRANSACTION')], db_index=True, default=0, null=True),
        ),
    ]

event/migrations/0009_alertmonitortransactionmapping_stats_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-13 09:08

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0009_alertmonitortransactionmapping_stats_and_more'),
    ]

    operations = [
        migrations.AddField(
            model_name='monitor',
            name='is_active',
            field=models.BooleanField(blank=True, default=True, null=True),
        ),
        migrations.AlterField(
            model_name='notification',
            name='channel',
            field=models.IntegerField(choices=[(0, 'UNKNOWN'), (1, 'SLACK'), (2, 'EMAIL')], db_index=True),
        ),
    ]

event/migrations/0010_monitor_is_active_alter_notification_channel.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-13 09:37

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0010_monitor_is_active_alter_notification_channel'),
    ]

    operations = [
        migrations.AddField(
            model_name='trigger',
            name='is_active',
            field=models.BooleanField(blank=True, default=True, null=True),
        ),
    ]

event/migrations/0011_trigger_is_active.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-22 08:53

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0011_trigger_is_active'),
    ]

    operations = [
        migrations.AddField(
            model_name='trigger',
            name='generated_config',
            field=models.JSONField(blank=True, null=True),
        ),
    ]

event/migrations/0012_trigger_generated_config.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-22 10:39

from django.db import migrations

from event.models import Monitor


def set_trigger_generated_choices_status(apps, schema_editor):
    Trigger = apps.get_model("event", "Trigger")

    for trigger in Trigger.objects.all():  # event__event_type_id
        generated_config = {
            'primary_event_filters': {"event__event_type_id": trigger.monitor.primary_key.event_type.id},
            'secondary_event_filters': {"event__event_type_id": trigger.monitor.secondary_key.event_type.id}
        }
        trigger.generated_config = generated_config
        trigger.save(update_fields=['generated_config'])


class Migration(migrations.Migration):
    dependencies = [
        ('event', '0012_trigger_generated_config'),
    ]

    operations = [
        migrations.RunPython(set_trigger_generated_choices_status, migrations.RunPython.noop)
    ]

event/migrations/0013_populate_trigger_generated_config.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-03-28 07:27

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
        ('event', '0013_populate_trigger_generated_config'),
    ]

    operations = [
        migrations.CreateModel(
            name='MonitorTransactionStats',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('stats', models.JSONField(blank=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('monitor_transaction', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitortransaction')),
            ],
            options={
                'unique_together': {('account', 'monitor_transaction')},
            },
        ),
    ]

event/migrations/0014_monitortransactionstats.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-12 12:16

from django.db import migrations

from protos.event.trigger_pb2 import TriggerDefinition


def set_delayed_trigger_config_skip_unfinished_transactions(apps, schema_editor):
    Trigger = apps.get_model("event", "Trigger")

    for trigger in Trigger.objects.filter(type=TriggerDefinition.Type.DELAYED_EVENT):
        config = trigger.config
        if "skip_unfinished_transactions" not in config:
            config["skip_unfinished_transactions"] = True
            trigger.config = config
            trigger.save(update_fields=['config'])


class Migration(migrations.Migration):
    dependencies = [
        ('event', '0014_monitortransactionstats'),
    ]

    operations = [
        migrations.RunPython(set_delayed_trigger_config_skip_unfinished_transactions, migrations.RunPython.noop)
    ]

event/migrations/0015_populate_delayed_trigger_config.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-17 05:39

from django.db import migrations, models
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0015_populate_delayed_trigger_config'),
    ]

    operations = [
        migrations.AddField(
            model_name='monitortransaction',
            name='event_timestamp',
            field=models.DateTimeField(blank=True, db_index=True, null=True),
        ),
        migrations.AddField(
            model_name='monitortransactioneventmapping',
            name='type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN_MT_ET'), (1, 'PRIMARY'), (2, 'SECONDARY')], db_index=True, default=0, null=True),
        ),
    ]

event/migrations/0016_monitortransaction_event_timestamp_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-17 05:39

from django.db import migrations, models
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0016_monitortransaction_event_timestamp_and_more'),
    ]

    operations = [
        migrations.AlterField(
            model_name='event',
            name='event_source',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE')], db_index=True, default=0, null=True),
        ),
        migrations.AlterField(
            model_name='eventtype',
            name='event_sources',
            field=prototype.fields.ChoiceArrayField(base_field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE')], null=True), default=list, size=None),
        ),
    ]

event/migrations/0017_alter_event_event_source_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-17 08:03

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0017_alter_event_event_source_and_more'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='monitortransaction',
            name='event_timestamp',
        ),
        migrations.AddField(
            model_name='monitortransactioneventmapping',
            name='event_timestamp',
            field=models.DateTimeField(blank=True, db_index=True, null=True),
        ),
    ]

event/migrations/0018_remove_monitortransaction_event_timestamp_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-18 09:47

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_alter_account_owner'),
        ('event', '0018_remove_monitortransaction_event_timestamp_and_more'),
    ]

    operations = [
        migrations.CreateModel(
            name='Entity',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(db_index=True, max_length=256)),
                ('is_active', models.BooleanField(blank=True, default=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
            ],
        ),
        migrations.CreateModel(
            name='EntityInstance',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('instance', models.TextField(db_index=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entity')),
            ],
            options={
                'unique_together': {('account', 'entity', 'instance')},
            },
        ),
        migrations.CreateModel(
            name='EntityEventKeyMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('is_active', models.BooleanField(blank=True, db_index=True, default=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entity')),
                ('event_key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.eventkey')),
            ],
            options={
                'unique_together': {('account', 'entity', 'event_key')},
            },
        ),
        migrations.AddField(
            model_name='entity',
            name='event_key',
            field=models.ManyToManyField(related_name='event_keys', through='event.EntityEventKeyMapping', to='event.eventkey'),
        ),
        migrations.CreateModel(
            name='EntityInstanceMonitorTransactionMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('instance', models.TextField(db_index=True)),
                ('transaction', models.TextField(db_index=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entity')),
                ('entity_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entityinstance')),
                ('monitor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitor')),
                ('monitor_transaction', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitortransaction')),
            ],
            options={
                'unique_together': {('account', 'entity_instance', 'monitor_transaction')},
            },
        ),
        migrations.CreateModel(
            name='EntityInstanceEventMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('instance', models.TextField(db_index=True)),
                ('event_processed_kvs', models.JSONField(blank=True, null=True)),
                ('event_timestamp', models.DateTimeField(blank=True, db_index=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entity')),
                ('entity_instance', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entityinstance')),
                ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.event')),
                ('event_key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.eventkey')),
            ],
            options={
                'unique_together': {('account', 'entity_instance', 'event')},
            },
        ),
        migrations.AlterUniqueTogether(
            name='entity',
            unique_together={('account', 'name')},
        ),
    ]

event/migrations/0019_entity_entityinstance_entityeventkeymapping_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-25 09:26

from django.db import migrations

from protos.event.trigger_pb2 import TriggerDefinition


def set_delayed_trigger_config_type(apps, schema_editor):
    Trigger = apps.get_model("event", "Trigger")

    for trigger in Trigger.objects.filter(type=TriggerDefinition.Type.DELAYED_EVENT):
        config = trigger.config
        if "skip_unfinished_transactions" in config:
            config.pop("skip_unfinished_transactions")
        if "type" not in config:
            config["type"] = "DELAYED_EVENTS"
        trigger.config = config
        trigger.save(update_fields=['config'])


class Migration(migrations.Migration):
    dependencies = [
        ('event', '0019_entity_entityinstance_entityeventkeymapping_and_more'),
    ]

    operations = [
        migrations.RunPython(set_delayed_trigger_config_type, migrations.RunPython.noop)
    ]

event/migrations/0020_modify_delayed_event_trigger_config.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-04-25 09:30

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0020_modify_delayed_event_trigger_config'),
    ]

    operations = [
        migrations.AlterField(
            model_name='entity',
            name='event_key',
            field=models.ManyToManyField(related_name='entity_event_keys', through='event.EntityEventKeyMapping', to='event.eventkey'),
        ),
    ]

event/migrations/0021_alter_entity_event_key.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-05-09 09:54

from django.db import migrations, models
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ('event', '0021_alter_entity_event_key'),
    ]

    operations = [
        migrations.AlterField(
            model_name='event',
            name='event_source',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE'), (6, 'SNS')], db_index=True, default=0, null=True),
        ),
        migrations.AlterField(
            model_name='eventtype',
            name='event_sources',
            field=prototype.fields.ChoiceArrayField(base_field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE'), (6, 'SNS')], null=True), default=list, size=None),
        ),
    ]

event/migrations/0022_alter_event_event_source_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-06-12 11:30
from django.contrib.postgres.operations import TrigramExtension
from django.db import migrations


class Migration(migrations.Migration):
    dependencies = [
        ('event', '0022_alter_event_event_source_and_more'),
    ]

    operations = [
        TrigramExtension(),
    ]

event/migrations/0023_auto_20230612_1130.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-08 12:59

import clickhouse_backend.models
from django.db import migrations, models
import django.db.models.deletion
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ("accounts", "0006_account_is_whitelisted"),
        ("event", "0023_auto_20230612_1130"),
    ]

    operations = [
        migrations.CreateModel(
            name="EntityTrigger",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("name", models.CharField(db_index=True, max_length=256)),
                (
                    "type",
                    models.IntegerField(
                        choices=[
                            (0, "UNKNOWN"),
                            (1, "LAST_EVENT"),
                            (2, "EVENT_COUNT"),
                        ],
                        db_index=True,
                    ),
                ),
                ("config", models.JSONField()),
                ("generated_config", models.JSONField(blank=True, null=True)),
                ("is_active", models.BooleanField(blank=True, default=True, null=True)),
                ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                (
                    "updated_at",
                    models.DateTimeField(auto_now=True, db_index=True, null=True),
                ),
                (
                    "account",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="accounts.account",
                    ),
                ),
                (
                    "entity",
                    models.ForeignKey(
                        blank=True,
                        null=True,
                        on_delete=django.db.models.deletion.CASCADE,
                        to="event.entity",
                    ),
                ),
            ],
        ),
    ]

event/migrations/0024_events_alter_event_event_source_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-09 01:58

import clickhouse_backend.models
from django.db import migrations, models
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ("event", "0024_events_alter_event_event_source_and_more"),
    ]

    operations = [
        migrations.AddField(
            model_name="entitytrigger",
            name="rule_type",
            field=models.IntegerField(
                choices=[(0, "UNKNOWN"), (1, "LAST_EVENT"), (2, "EVENT_COUNT"), (3, "EVENT_OCCURS")],
                db_index=True,
                default=1,
            ),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name="entitytrigger",
            name="type",
            field=models.IntegerField(
                choices=[
                    (0, "UNKNOWN_TYPE"),
                    (1, "PER_EVENT"),
                    (2, "AGGREGATED_EVENTS"),
                ],
                db_index=True,
            ),
        )
    ]

event/migrations/0025_events_entitytrigger_rule_type_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-11 09:33

import clickhouse_backend.models
from django.db import migrations, models
import django.db.models.deletion
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ("event", "0025_events_entitytrigger_rule_type_and_more"),
    ]

    operations = [
        migrations.AddField(
            model_name="alert",
            name="entity_trigger",
            field=models.ForeignKey(
                null=True,
                on_delete=django.db.models.deletion.CASCADE,
                to="event.entitytrigger",
            ),
        ),
        migrations.AlterField(
            model_name="alert",
            name="trigger",
            field=models.ForeignKey(
                null=True,
                on_delete=django.db.models.deletion.CASCADE,
                to="event.trigger",
            ),
        ),
    ]

event/migrations/0026_events_alert_entity_trigger_alter_alert_trigger_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-11 14:58

import clickhouse_backend.models
from django.db import migrations, models
import django.db.models.deletion
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ("accounts", "0006_account_is_whitelisted"),
        ("event", "0026_events_alert_entity_trigger_alter_alert_trigger_and_more"),
    ]

    operations = [
        migrations.CreateModel(
            name="EntityTriggerNotificationConfigMapping",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                (
                    "account",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="accounts.account",
                    ),
                ),
                (
                    "entity_trigger",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="event.entitytrigger",
                    ),
                ),
                (
                    "notification",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="event.notification",
                    ),
                ),
            ],
            options={
                "unique_together": {("account", "entity_trigger", "notification")},
            },
        ),
        migrations.AddField(
            model_name="notification",
            name="entity_trigger",
            field=models.ManyToManyField(
                related_name="entity_triggers",
                through="event.EntityTriggerNotificationConfigMapping",
                to="event.entitytrigger",
            ),
        ),
    ]

event/migrations/0027_events_alter_event_event_source_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-09-18 09:22

import clickhouse_backend.models
from django.db import migrations, models
import django.db.models.deletion
import prototype.fields


class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0006_account_is_whitelisted'),
        ('event', '0027_events_alter_event_event_source_and_more'),
    ]

    operations = [
        migrations.AlterUniqueTogether(
            name='entity',
            unique_together=set(),
        ),
        migrations.AddField(
            model_name='entity',
            name='is_generated',
            field=models.BooleanField(blank=True, default=False, null=True),
        ),
        migrations.AddField(
            model_name='entity',
            name='type',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'DEFAULT'), (2, 'FUNNEL')], db_index=True, default=1, null=True),
        ),
        migrations.AddField(
            model_name='monitor',
            name='is_generated',
            field=models.BooleanField(blank=True, default=False, null=True),
        ),
        migrations.AlterField(
            model_name='event',
            name='event_source',
            field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE'), (6, 'SNS'), (7, 'CLOUDWATCH')], db_index=True, default=0, null=True),
        ),
        migrations.AlterField(
            model_name='eventtype',
            name='event_sources',
            field=prototype.fields.ChoiceArrayField(base_field=models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SAMPLE'), (2, 'API'), (3, 'SDK'), (4, 'SEGMENT'), (5, 'AMPLITUDE'), (6, 'SNS'), (7, 'CLOUDWATCH')], null=True), default=list, size=None),
        ),
        migrations.AlterUniqueTogether(
            name='entity',
            unique_together={('account', 'name', 'type')},
        ),
        migrations.CreateModel(
            name='EntityMonitorMapping',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('is_active', models.BooleanField(blank=True, default=True, null=True)),
                ('is_generated', models.BooleanField(blank=True, default=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='accounts.account')),
                ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.entity')),
                ('monitor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='event.monitor')),
            ],
            options={
                'unique_together': {('account', 'entity', 'monitor')},
            },
        ),
        migrations.AddField(
            model_name='entity',
            name='entity_monitors',
            field=models.ManyToManyField(related_name='entity_monitors', through='event.EntityMonitorMapping', to='event.monitor'),
        ),
    ]

event/migrations/0028_events_alter_entity_unique_together_and_more.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-10-06 03:43

import clickhouse_backend.models
from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ("accounts", "0006_account_is_whitelisted"),
        ("event", "0028_events_alter_entity_unique_together_and_more"),
    ]

    operations = [
        migrations.CreateModel(
            name="AlertEntityInstanceMapping",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                (
                    "type",
                    models.IntegerField(
                        blank=True,
                        choices=[
                            (0, "UNKNOWN"),
                            (1, "PER_EVENT"),
                            (2, "AGGREGATED_EVENTS"),
                        ],
                        db_index=True,
                        default=0,
                        null=True,
                    ),
                ),
                ("stats", models.JSONField(null=True)),
                ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
                (
                    "account",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="accounts.account",
                    ),
                ),
                (
                    "alert",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE, to="event.alert"
                    ),
                ),
                (
                    "entity_instance",
                    models.ForeignKey(
                        on_delete=django.db.models.deletion.CASCADE,
                        to="event.entityinstance",
                    ),
                ),
            ],
            options={
                "unique_together": {("account", "alert", "entity_instance")},
            },
        ),
        migrations.AddField(
            model_name="entityinstance",
            name="alert",
            field=models.ManyToManyField(
                related_name="entity_alerts",
                through="event.AlertEntityInstanceMapping",
                to="event.alert",
            ),
        ),
    ]

event/migrations/0029_events_monitortransactions_and_more.py
DrDroidLab/kenobi
import time
from datetime import timezone, timedelta

from django.db import models
from django.db.models import Q, F, Avg, FloatField
from django.db.models.functions import Cast
from django.utils.functional import cached_property
from google.protobuf.struct_pb2 import Struct
from google.protobuf.timestamp_pb2 import Timestamp
from google.protobuf.wrappers_pb2 import UInt64Value, BoolValue, DoubleValue, StringValue, Int64Value

from accounts.models import Account
from protos.event.alert_pb2 import AlertDetail as AlertDetailProto, AlertStats, AlertMonitorTransaction, \
    DelayedMonitorTransactionStats, AlertSummary as AlertSummaryProto, AlertEntityInstance
from protos.event.base_pb2 import EventKey as EventKeyProto, EventType as EventTypeProto, Event as EventProto, \
    EventTypePartial as EventTypePartialProto, EventTypeSummary, EventTypeStats, EventTypeDefinition, TimeRange
from protos.event.entity_pb2 import EntityPartial, EntityStats, EntitySummary, Entity as EntityProto, EntityDetail, \
    EntityEventKeyMapping as EntityEventKeyMappingProto, EntityInstancePartial, EntityInstanceStats, \
    EntityInstanceSummary, EntityInstanceDetail, EntityInstance as EntityInstanceProto, EntityTriggerDefinition, \
    EntityTrigger as EntityTriggerProto
from protos.event.literal_pb2 import Literal, LiteralType
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto, Monitor as MonitorProto, \
    MonitorPartial as MonitorPartialProto, MonitorDefinition, MonitorStats, MonitorEventTypeDetails, \
    MonitorTransactionStats as MonitorTransactionStatsProto, MonitorTransactionDetails
from protos.event.notification_pb2 import NotificationConfig, SlackConfiguration, EmailConfiguration
from protos.event.panel_pb2 import PanelV1
from protos.event.query_base_pb2 import Op, Filter
from protos.event.schema_pb2 import IngestionEvent
from protos.event.trigger_pb2 import TriggerPriority, Trigger as TriggerProto, TriggerDefinition, TriggerStats, \
    MissingEventTrigger, DelayedEventTrigger, TriggerSummary, BasicTriggerFilter
from prototype.aggregates import Percentile
from prototype.fields import ChoiceArrayField
from prototype.utils.queryset import filter_time_range
from utils.model_utils import generate_choices
from utils.proto_utils import dict_to_proto

TRANSACTIONAL_KEY_TYPES = {EventKeyProto.KeyType.STRING, EventKeyProto.KeyType.LONG, EventKeyProto.KeyType.DOUBLE}
FILTERABLE_KEY_TYPES = {EventKeyProto.KeyType.STRING, EventKeyProto.KeyType.LONG, EventKeyProto.KeyType.DOUBLE,
                        EventKeyProto.KeyType.BOOLEAN}


def is_transactional_key_type(event_key_type):
    return event_key_type in TRANSACTIONAL_KEY_TYPES


def is_filterable_key_type(event_key_type):
    return event_key_type in FILTERABLE_KEY_TYPES


def get_literal_for_event_key_type(event_key_type, value):
    if event_key_type == EventKeyProto.KeyType.LONG:
        return Literal(literal_type=LiteralType.LONG, long=Int64Value(value=value))
    elif event_key_type == EventKeyProto.KeyType.DOUBLE:
        return Literal(literal_type=LiteralType.DOUBLE, double=DoubleValue(value=value))
    elif event_key_type == EventKeyProto.KeyType.STRING:
        return Literal(literal_type=LiteralType.STRING, string=StringValue(value=value))
    elif event_key_type == EventKeyProto.KeyType.BOOLEAN:
        return Literal(literal_type=LiteralType.BOOLEAN, boolean=BoolValue(value=value))
    else:
        raise ValueError(f'Unknown event key type {event_key_type}')


def get_literal_array_for_event_key_type(event_key_type, value):
    if event_key_type == EventKeyProto.KeyType.LONG:
        return Literal(literal_type=LiteralType.LONG_ARRAY, long_array=value)
    elif event_key_type == EventKeyProto.KeyType.STRING:
        return Literal(literal_type=LiteralType.STRING_ARRAY, string_array=value)
    else:
        raise ValueError(f'Unknown event key type {event_key_type}')


def get_basic_monitor_trigger_filter_keys(scope, event_filters):
    result: [BasicTriggerFilter] = []
    event_type_id = event_filters.get('event__event_type_id', None)
    event_keys = scope.eventkey_set.filter(event_type_id=event_type_id).values('id', 'name', 'type')
    for k, v in event_filters.items():
        if k == 'event__event_type_id':
            continue

        filter_tokens = k.split('__')
        key_name = filter_tokens[2]

        suffix_op = None
        if len(filter_tokens) > 3:
            suffix_op = k.split('__')[3]

        for key in event_keys:
            if key_name == key['name']:
                if not suffix_op:
                    result.append(BasicTriggerFilter(event_key_id=key['id'], op=Op.EQ,
                                                 literal=get_literal_for_event_key_type(key['type'], v)))
                if suffix_op == 'in':
                    result.append(BasicTriggerFilter(event_key_id=key['id'], op=Op.IN,
                                                     literal=get_literal_array_for_event_key_type(key['type'], v)))
    return result


class EventType(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    event_sources = ChoiceArrayField(
        base_field=models.IntegerField(choices=generate_choices(EventProto.EventSource), null=True, blank=True),
        default=list, coerce=int
    )

    class Meta:
        unique_together = [['account', 'name']]

    def __str__(self):
        return f'{self.account_id}:{self.name}'

    @property
    def keys(self):
        return [key.proto_partial for key in self.eventkey_set.all()]

    @property
    def proto(self):
        return EventTypeProto(id=self.id, name=str(self.name), keys=self.keys, event_sources=list(self.event_sources))

    @property
    def proto_partial(self):
        return EventTypePartialProto(id=self.id, name=str(self.name))

    @staticmethod
    def proto_partial_from_sql(e_type_id, e_type_name):
        return EventTypePartialProto(id=e_type_id, name=e_type_name)

    def definition(self, tr: TimeRange = TimeRange()) -> EventTypeSummary:
        return EventTypeDefinition(event_type=self.proto)

    def summary(self, tr: TimeRange = TimeRange()) -> EventTypeSummary:
        return EventTypeSummary(event_type=self.proto_partial, stats=self.stats(tr))

    def monitor_and_keys_summary(self):
        return self.proto_partial, self.monitor_and_keys_stats()

    def stats(self, tr: TimeRange = TimeRange()):
        event_set = self.event_set
        if tr:
            event_set = filter_time_range(event_set, tr, 'created_at')

        event_keys = self.eventkey_set.values("id")
        keys = [k["id"] for k in event_keys]
        monitor_count = self.account.monitor_set.filter(
            Q(primary_key_id__in=event_keys) | Q(secondary_key_id__in=event_keys)).count()

        return EventTypeStats(keys_count=UInt64Value(value=len(keys)), event_count=UInt64Value(value=event_set.count()),
                              monitor_count=UInt64Value(value=monitor_count))

    def monitor_and_keys_stats(self):
        event_keys = self.eventkey_set.values("id")
        keys = [k["id"] for k in event_keys]
        monitor_count = self.account.monitor_set.filter(
            Q(primary_key_id__in=event_keys) | Q(secondary_key_id__in=event_keys)).count()

        return {'keys_count': len(keys), 'event_count': 0, 'monitor_count': monitor_count}


class EventKey(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    event_type = models.ForeignKey(EventType, on_delete=models.CASCADE, db_index=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    type = models.IntegerField(null=True, blank=True, choices=generate_choices(EventKeyProto.KeyType),
                               default=EventKeyProto.KeyType.UNKNOWN, db_index=True)

    # type, scope pending

    class Meta:
        unique_together = [['account', 'event_type', 'name']]

    def __str__(self):
        return f'{self.event_type}:{self.name}'

    @property
    def proto_partial(self):
        return EventKeyProto(id=self.id, key=str(self.name), key_type=self.type)

    @property
    def proto(self):
        return EventKeyProto(id=self.id, key=str(self.name), key_type=self.type,
                             event_type=self.event_type.proto_partial)

    @property
    def proto_event_type_partial(self) -> EventTypePartialProto:
        return self.event_type.proto_partial


class Event(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    event_type = models.ForeignKey(EventType, on_delete=models.CASCADE, db_index=True)
    event = models.JSONField()
    processed_kvs = models.JSONField(null=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    timestamp = models.DateTimeField(db_index=True)
    is_generated = models.BooleanField(default=False, null=True, blank=True)
    event_source = models.IntegerField(null=True, blank=True, choices=generate_choices(EventProto.EventSource),
                                       default=EventProto.EventSource.UNKNOWN, db_index=True)

    def __str__(self):
        return f'{self.event_type_id}:{self.id}'

    @cached_property
    def proto(self):
        ingested_event: IngestionEvent = dict_to_proto(self.event, IngestionEvent)
        proto = EventProto(
            id=self.id, event_type=self.event_type.proto_partial, kvs=ingested_event.kvs,
            timestamp=int(self.timestamp.replace(tzinfo=timezone.utc).timestamp()),
            event_source=self.event_source
        )
        return proto


class Monitor(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    primary_key = models.ForeignKey(EventKey, on_delete=models.CASCADE, db_index=True, related_name='primary_key')
    secondary_key = models.ForeignKey(EventKey, on_delete=models.CASCADE, db_index=True, related_name='secondary_key')
    is_active = models.BooleanField(default=True, null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True)
    is_generated = models.BooleanField(default=False, null=True, blank=True)

    class Meta:
        unique_together = [['account', 'name']]

    def __str__(self):
        return f'{self.account_id}:{self.name}'

    @property
    def proto_partial(self):
        return MonitorPartialProto(id=self.id, name=str(self.name))

    @cached_property
    def proto(self):
        return MonitorProto(id=self.id, name=str(self.name), primary_event_key=self.primary_key.proto,
                            secondary_event_key=self.secondary_key.proto, is_active=BoolValue(value=self.is_active))

    @property
    def proto_monitor_event_type_detail(self) -> MonitorEventTypeDetails:
        return MonitorEventTypeDetails(monitor=self.proto_partial,
                                       primary_event_type=self.primary_key.proto_event_type_partial,
                                       secondary_event_type=self.secondary_key.proto_event_type_partial)

    def definition(self, tr: TimeRange = TimeRange()):
        return MonitorDefinition(monitor=self.proto, stats=self.stats(tr))

    def stats(self, tr: TimeRange = TimeRange()):
        monitor_transactions = self.monitortransaction_set
        if tr:
            monitor_transactions = filter_time_range(monitor_transactions, tr, 'created_at')
        monitor_transactions_finished = monitor_transactions.filter(
            type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED
        )

        monitor_transactions_aggregates = monitor_transactions_finished.annotate(
            transaction_time=(Cast(F('monitortransactionstats__stats__transaction_time'), FloatField()))
        ).aggregate(
            p95=Percentile('transaction_time', percentile=0.95),
            p99=Percentile('transaction_time', percentile=0.99),
            avg_delay=Avg(F('transaction_time'))
        )

        avg_delay = monitor_transactions_aggregates.get('avg_delay')
        if not avg_delay:
            avg_delay = 0

        p95 = monitor_transactions_aggregates.get('p95', timedelta(seconds=0))
        if not p95:
            p95 = 0
        p99 = monitor_transactions_aggregates.get('p99', timedelta(seconds=0))
        if not p99:
            p99 = 0

        return MonitorStats(
            transaction_count=UInt64Value(value=monitor_transactions.count()),
            finished_transaction_count=UInt64Value(value=monitor_transactions_finished.count()),
            transaction_avg_delay=DoubleValue(value=round(avg_delay, 3)),
            percentiles={'p95': round(p95, 3), 'p99': round(p99, 3)}
        )


class Trigger(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    monitor = models.ForeignKey(Monitor, null=True, blank=True, on_delete=models.CASCADE, db_index=True)
    type = models.IntegerField(choices=generate_choices(TriggerDefinition.Type), db_index=True)
    config = models.JSONField()
    generated_config = models.JSONField(null=True, blank=True)
    priority = models.IntegerField(null=True, blank=True, choices=generate_choices(TriggerPriority),
                                   default=TriggerPriority.TP_4, db_index=True)
    is_active = models.BooleanField(default=True, null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'name']]

    def __str__(self):
        return f'{self.account}:{self.name}'

    @property
    def proto(self, definition=True, trigger_stats=False) -> TriggerProto:
        return TriggerProto(id=self.id, name=self.name, priority=self.priority,
                            monitor=self.monitor.proto if self.monitor else MonitorProto(),
                            definition=self.trigger_definition if definition else TriggerDefinition(),
                            stats=self.trigger_stats if trigger_stats else TriggerStats(),
                            is_active=BoolValue(value=self.is_active))

    @property
    def proto_summary(self) -> TriggerSummary:
        return TriggerSummary(id=self.id, name=self.name,
                              type=self.type,
                              monitor_details=self.monitor.proto_monitor_event_type_detail)

    @cached_property
    def trigger_definition(self) -> TriggerDefinition:
        basic_primary_monitor_trigger_filters = get_basic_monitor_trigger_filter_keys(self.account,
                                                                                      self.generated_config.get(
                                                                                          'primary_event_filters', {}))
        basic_secondary_monitor_trigger_filters = get_basic_monitor_trigger_filter_keys(self.account,
                                                                                        self.generated_config.get(
                                                                                            'secondary_event_filters',
                                                                                            {}))

        if self.type == TriggerDefinition.Type.MISSING_EVENT:
            return TriggerDefinition(type=self.type,
                                     missing_event_trigger=dict_to_proto(self.config, MissingEventTrigger),
                                     primary_event_filters=basic_primary_monitor_trigger_filters,
                                     secondary_event_filters=basic_secondary_monitor_trigger_filters)
        elif self.type == TriggerDefinition.Type.DELAYED_EVENT:
            return TriggerDefinition(type=self.type,
                                     delayed_event_trigger=dict_to_proto(self.config, DelayedEventTrigger),
                                     primary_event_filters=basic_primary_monitor_trigger_filters,
                                     secondary_event_filters=basic_secondary_monitor_trigger_filters)

    @cached_property
    def trigger_stats(self) -> TriggerStats:
        last_triggered_at = None
        if self.last_triggered_at:
            last_triggered_at = Timestamp()
            last_triggered_at.FromDatetime(self.last_triggered_at)

        ts = TriggerStats(
            last_triggered_at=last_triggered_at,
            alert_count=UInt64Value(value=self.alert_count),
        )
        return ts

    @cached_property
    def definition(self) -> TriggerDefinition:
        return dict_to_proto(self.trigger_rule, TriggerDefinition)


class Entity(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    is_active = models.BooleanField(default=True, null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True)
    event_key = models.ManyToManyField(
        EventKey,
        through='EntityEventKeyMapping',
        related_name='entity_event_keys'
    )

    entity_monitors = models.ManyToManyField(
        Monitor,
        through='EntityMonitorMapping',
        related_name='entity_monitors'
    )

    type = models.IntegerField(null=True, blank=True, choices=generate_choices(EntityProto.Type),
                               default=EntityProto.Type.DEFAULT, db_index=True)
    is_generated = models.BooleanField(default=False, null=True, blank=True)

    class Meta:
        unique_together = ('account', 'name', 'type')

    def __str__(self):
        return f'{self.name}'

    @property
    def proto_partial(self):
        return EntityPartial(id=UInt64Value(value=self.id), name=StringValue(value=self.name))

    @property
    def proto(self) -> EntityProto:
        #     TODO(mohit.agarwal) need to add created_at and updated at fields here
        return EntityProto(
            id=UInt64Value(value=self.id),
            name=StringValue(value=str(self.name)),
            is_active=BoolValue(value=bool(self.is_active)),
        )

    @property
    def stats(self) -> EntityStats:
        return EntityStats(
            new_instance_count=UInt64Value(value=getattr(self, 'new_instance_count', 0)),
            active_instance_count=UInt64Value(value=getattr(self, 'active_instance_count', 0)),
            event_count=UInt64Value(value=getattr(self, 'event_count', 0)),
            transaction_count=UInt64Value(value=getattr(self, 'transaction_count', 0)),
        )

    @property
    def summary(self) -> EntitySummary:
        return EntitySummary(
            entity=self.proto,
            stats=None
        )

    @property
    def detail(self) -> EntityDetail:
        return EntityDetail(
            entity=self.proto,
            stats=self.stats,
            entity_event_key_mappings=[ekm.proto for ekm in self.entityeventkeymapping_set.all()],
        )

    @property
    def detail_without_stats(self) -> EntityDetail:
        return EntityDetail(
            entity=self.proto,
            entity_event_key_mappings=[ekm.proto for ekm in self.entityeventkeymapping_set.all()],
        )


class EntityTrigger(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.CharField(max_length=256, db_index=True)
    entity = models.ForeignKey(Entity, null=True, blank=True, on_delete=models.CASCADE, db_index=True)
    type = models.IntegerField(choices=generate_choices(EntityTriggerDefinition.TriggerType), db_index=True)
    rule_type = models.IntegerField(choices=generate_choices(EntityTriggerDefinition.EntityTriggerRuleType),
                                    db_index=True)
    config = models.JSONField()
    generated_config = models.JSONField(null=True, blank=True)
    is_active = models.BooleanField(default=True, null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    def __str__(self):
        return f'{self.account}:{self.name}'

    @property
    def proto(self) -> EntityTriggerProto:
        return EntityTriggerProto(id=self.id, entity_id=self.entity.id, name=self.name,
                                  is_active=BoolValue(value=self.is_active), entity_name=self.entity.name,
                                  definition=EntityTriggerDefinition(type=self.type,
                                                                     trigger_rule_config=self.trigger_rule_config,
                                                                     rule_type=self.rule_type),
                                  filter=dict_to_proto(self.generated_config, Filter))

    @cached_property
    def trigger_rule_config(self) -> EntityTriggerDefinition.TriggerRuleConfig:
        return EntityTriggerDefinition.TriggerRuleConfig(event_id=int(self.config.get('event_id', -1)),
                                                         event_name=self.config.get('event_name', ''),
                                                         time_interval=int(self.config.get('time_interval', 0)),
                                                         threshold_count=int(self.config.get('threshold_count', 0)))


class EntityEventKeyMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity = models.ForeignKey(Entity, on_delete=models.CASCADE)
    event_key = models.ForeignKey(EventKey, on_delete=models.CASCADE)
    is_active = models.BooleanField(default=True, null=True, blank=True, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now=True)

    class Meta:
        unique_together = [['account', 'entity', 'event_key']]

    @property
    def proto(self):
        return EntityEventKeyMappingProto(
            id=UInt64Value(value=self.id),
            event_key=self.event_key.proto_partial,
            is_active=BoolValue(value=bool(self.is_active)),
        )


class Alert(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    trigger = models.ForeignKey(Trigger, on_delete=models.CASCADE, db_index=True, null=True)
    entity_trigger = models.ForeignKey(EntityTrigger, on_delete=models.CASCADE, db_index=True, null=True)
    triggered_at = models.DateTimeField(auto_now_add=True, db_index=True)
    stats = models.JSONField(null=True)

    def __str__(self):
        if self.trigger:
            return f'{self.trigger}:{self.triggered_at}'
        else:
            return f'{self.entity_trigger}:{self.triggered_at}'

    @cached_property
    def triggered_at_timestamp(self):
        timestamp = Timestamp()
        timestamp.FromDatetime(self.triggered_at)
        return timestamp

    @property
    def proto_details(self) -> AlertDetailProto:
        if self.trigger:
            return AlertDetailProto(id=self.id, trigger=self.trigger.proto,
                                    triggered_at=self.triggered_at_timestamp, stats=self.alert_stats)
        elif self.entity_trigger:
            return AlertDetailProto(id=self.id, entity_trigger=self.entity_trigger.proto,
                                    triggered_at=self.triggered_at_timestamp, stats=self.alert_stats)
        else:
            return AlertDetailProto(id=self.id)

    @property
    def proto_summary(self) -> AlertSummaryProto:
        if self.trigger:
            return AlertSummaryProto(id=self.id, trigger=self.trigger.proto_summary,
                                     triggered_at=int(self.triggered_at.replace(tzinfo=timezone.utc).timestamp()))
        elif self.entity_trigger:
            return AlertSummaryProto(id=self.id, entity_trigger=self.entity_trigger.proto,
                                     triggered_at=int(self.triggered_at.replace(tzinfo=timezone.utc).timestamp()))
        else:
            return AlertSummaryProto(id=self.id)

    @cached_property
    def alert_stats(self) -> AlertStats:
        if self.stats:
            return dict_to_proto(self.stats, AlertStats)
        return AlertStats()


class EntityInstance(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity = models.ForeignKey(Entity, on_delete=models.CASCADE)
    instance = models.TextField(db_index=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    alert = models.ManyToManyField(
        Alert,
        through='AlertEntityInstanceMapping',
        related_name='entity_alerts'
    )

    class Meta:
        unique_together = [['account', 'entity', 'instance']]

    @property
    def proto_partial(self) -> EntityInstancePartial:
        return EntityInstancePartial(
            id=UInt64Value(value=self.id),
            instance=StringValue(value=self.instance),
        )

    @property
    def proto(self) -> EntityInstanceProto:
        return EntityInstanceProto(
            id=UInt64Value(value=self.id),
            entity=self.entity.proto_partial,
            instance=StringValue(value=self.instance),
        )

    @property
    def stats(self) -> EntityInstanceStats:
        alert_id_trigger_name_mapping = AlertEntityInstanceMapping.objects.filter(
            entity_instance=self).values('alert_id', 'alert__entity_trigger__name')
        return EntityInstanceStats(
            event_count=UInt64Value(value=getattr(self, 'event_count', 0)),
            transaction_count=UInt64Value(value=getattr(self, 'transaction_count', 0)),
            alerts=list(EntityInstanceStats.EntityAlert(alert_id=x['alert_id'], entity_trigger_name=x['alert__entity_trigger__name']) for x in list(alert_id_trigger_name_mapping))
        )

    @property
    def summary(self) -> EntityInstanceSummary:
        return EntityInstanceSummary(
            entity_instance=self.proto_partial,
            stats=self.stats,
        )

    @property
    def detail(self) -> EntityInstanceDetail:
        return EntityInstanceDetail(
            entity_instance=self.proto,
            stats=self.stats,
        )


class AlertEntityInstanceMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    alert = models.ForeignKey(Alert, on_delete=models.CASCADE, db_index=True)
    entity_instance = models.ForeignKey(EntityInstance, on_delete=models.CASCADE, db_index=True)
    type = models.IntegerField(null=True, blank=True,
                               choices=generate_choices(AlertEntityInstance.Type),
                               default=AlertEntityInstance.Type.UNKNOWN,
                               db_index=True)
    stats = models.JSONField(null=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'alert', 'entity_instance']]

    @property
    def proto(self) -> AlertEntityInstance:
        if self.type == AlertEntityInstance.Type.PER_EVENT:
            return AlertEntityInstance(type=self.type,
                                           alert_id=self.alert_id,
                                           entity_instance=self.entity_instance.proto)
        elif self.type == AlertEntityInstance.Type.AGGREGATED_EVENTS:
            return AlertEntityInstance(type=self.type,
                                           alert_id=self.alert_id,
                                           entity_instance=self.entity_instance.proto)

    @property
    def miniproto(self) -> AlertEntityInstance:
        if self.type == AlertEntityInstance.Type.PER_EVENT:
            return AlertEntityInstance(type=self.type,
                                           alert_id=self.alert_id,
                                           trigger_name=self.alert.entity_trigger.name)
        return None



class EntityInstanceEventMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity_instance = models.ForeignKey(EntityInstance, on_delete=models.CASCADE)
    event = models.ForeignKey(Event, on_delete=models.CASCADE)

    event_key = models.ForeignKey(EventKey, on_delete=models.CASCADE)
    entity = models.ForeignKey(Entity, on_delete=models.CASCADE)
    instance = models.TextField(db_index=True)
    event_processed_kvs = models.JSONField(null=True, blank=True)
    event_timestamp = models.DateTimeField(null=True, blank=True, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'entity_instance', 'event']]


class MonitorTransaction(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    monitor = models.ForeignKey(Monitor, on_delete=models.CASCADE, db_index=True)
    transaction = models.TextField(db_index=True)
    type = models.IntegerField(null=True, blank=True,
                               choices=generate_choices(MonitorTransactionProto.MonitorTransactionStatus),
                               default=MonitorTransactionProto.MonitorTransactionStatus.UNKNOWN,
                               db_index=True)

    alert = models.ManyToManyField(
        Alert,
        through='AlertMonitorTransactionMapping',
        related_name='alerts'
    )

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'monitor', 'transaction']]

    @property
    def proto(self):
        return MonitorTransactionProto(
            id=self.id, transaction=str(self.transaction),
            monitor=self.monitor.proto_partial,
            created_at=int(self.created_at.replace(tzinfo=timezone.utc).timestamp()),
            status=self.type,
            has_alerts=BoolValue(value=getattr(self, 'has_alerts', False)),
            transaction_age=DoubleValue(
                value=round(time.time() - self.created_at.replace(tzinfo=timezone.utc).timestamp(), 3))
        )

    @property
    def proto_for_export(self):
        return MonitorTransactionProto(
            id=self.id, transaction=str(self.transaction),
            monitor=self.monitor.proto_partial,
            created_at=int(self.created_at.replace(tzinfo=timezone.utc).timestamp()),
            status=self.type,
            has_alerts=BoolValue(value=getattr(self, 'has_alerts', False)),
            transaction_time=DoubleValue(value=self.transaction_time),
            transaction_age=DoubleValue(
                value=round(time.time() - self.created_at.replace(tzinfo=timezone.utc).timestamp(), 3))
        )

    @property
    def details_proto(self):
        if self.stats:
            return MonitorTransactionDetails(
                monitor_transaction=self.proto,
                stats=dict_to_proto(self.stats, MonitorTransactionStatsProto)
            )
        else:
            return MonitorTransactionDetails(
                monitor_transaction=self.proto,
                stats=MonitorTransactionStatsProto()
            )


class MonitorTransactionStats(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    monitor_transaction = models.ForeignKey(MonitorTransaction, on_delete=models.CASCADE, db_index=True)
    stats = models.JSONField(null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    class Meta:
        unique_together = [['account', 'monitor_transaction']]

    @property
    def proto(self):
        return MonitorTransactionDetails(
            monitor_transaction=self.monitor_transaction.proto,
            stats=dict_to_proto(self.stats, MonitorTransactionStatsProto)
        )


class MonitorTransactionEventMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    monitor_transaction = models.ForeignKey(MonitorTransaction, on_delete=models.CASCADE, db_index=True)
    event = models.ForeignKey(Event, on_delete=models.CASCADE, db_index=True)
    type = models.IntegerField(null=True, blank=True,
                               choices=generate_choices(MonitorTransactionProto.MonitorTransactionEventType),
                               default=MonitorTransactionProto.MonitorTransactionEventType.UNKNOWN_MT_ET,
                               db_index=True)
    event_timestamp = models.DateTimeField(null=True, blank=True, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'event', 'monitor_transaction']]


class AlertMonitorTransactionMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    alert = models.ForeignKey(Alert, on_delete=models.CASCADE, db_index=True)
    monitor_transaction = models.ForeignKey(MonitorTransaction, on_delete=models.CASCADE, db_index=True)
    type = models.IntegerField(null=True, blank=True,
                               choices=generate_choices(AlertMonitorTransaction.Type),
                               default=AlertMonitorTransaction.Type.UNKNOWN,
                               db_index=True)
    stats = models.JSONField(null=True)
    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'alert', 'monitor_transaction']]

    @property
    def proto(self) -> AlertMonitorTransaction:
        if self.type == AlertMonitorTransaction.Type.MISSED_TRANSACTION:
            return AlertMonitorTransaction(type=self.type,
                                           alert_id=self.alert_id,
                                           monitor_transaction=self.monitor_transaction.proto)
        elif self.type == AlertMonitorTransaction.Type.DELAYED_TRANSACTION:
            return AlertMonitorTransaction(type=self.type,
                                           alert_id=self.alert_id,
                                           monitor_transaction=self.monitor_transaction.proto,
                                           delayed_monitor_transaction_stats=dict_to_proto(self.stats,
                                                                                           DelayedMonitorTransactionStats))

    @property
    def miniproto(self) -> AlertMonitorTransaction:
        if self.type == AlertMonitorTransaction.Type.MISSED_TRANSACTION:
            return AlertMonitorTransaction(type=self.type,
                                           alert_id=self.alert_id,
                                           trigger_name=self.alert.trigger.name)
        elif self.type == AlertMonitorTransaction.Type.DELAYED_TRANSACTION:
            return AlertMonitorTransaction(type=self.type,
                                           alert_id=self.alert_id,
                                           trigger_name=self.alert.trigger.name)


class Notification(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    channel = models.IntegerField(choices=generate_choices(NotificationConfig.Channel), db_index=True)
    config = models.JSONField(null=True, blank=True)
    trigger = models.ManyToManyField(
        Trigger,
        through='TriggerNotificationConfigMapping',
        related_name='triggers'
    )
    entity_trigger = models.ManyToManyField(
        EntityTrigger,
        through='EntityTriggerNotificationConfigMapping',
        related_name='entity_triggers'
    )

    class Meta:
        unique_together = [['account', 'config']]

    def __str__(self):
        return f'{self.account}:{self.config}'

    @property
    def proto(self) -> NotificationConfig:
        if self.channel == NotificationConfig.Channel.SLACK:
            return NotificationConfig(channel=self.channel,
                                      slack_configuration=dict_to_proto(self.config, SlackConfiguration))
        elif self.channel == NotificationConfig.Channel.EMAIL:
            return NotificationConfig(channel=self.channel,
                                      email_configuration=dict_to_proto(self.config, EmailConfiguration))


class TriggerNotificationConfigMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    trigger = models.ForeignKey(Trigger, on_delete=models.CASCADE, db_index=True)
    notification = models.ForeignKey(Notification, on_delete=models.CASCADE, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'trigger', 'notification']]


class EntityTriggerNotificationConfigMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity_trigger = models.ForeignKey(EntityTrigger, on_delete=models.CASCADE, db_index=True)
    notification = models.ForeignKey(Notification, on_delete=models.CASCADE, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'entity_trigger', 'notification']]


class Panel(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    name = models.TextField(null=True, blank=True, default="")
    config = models.JSONField(null=True, blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    class Meta:
        unique_together = [['account', 'name']]

    def __str__(self):
        return f'{self.account}:{self.name}'

    @property
    def proto(self):
        c: Struct = Struct()
        c.update(self.config)
        return PanelV1(id=self.id, name=self.name, config=c)


class EntityInstanceMonitorTransactionMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity_instance = models.ForeignKey(EntityInstance, on_delete=models.CASCADE)
    monitor_transaction = models.ForeignKey(MonitorTransaction, on_delete=models.CASCADE)

    entity = models.ForeignKey(Entity, on_delete=models.CASCADE)
    instance = models.TextField(db_index=True)
    monitor = models.ForeignKey(Monitor, on_delete=models.CASCADE)
    transaction = models.TextField(db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'entity_instance', 'monitor_transaction']]


class EntityMonitorMapping(models.Model):
    account = models.ForeignKey(Account, on_delete=models.CASCADE)
    entity = models.ForeignKey(Entity, on_delete=models.CASCADE)
    monitor = models.ForeignKey(Monitor, on_delete=models.CASCADE)
    is_active = models.BooleanField(default=True, null=True, blank=True)
    is_generated = models.BooleanField(default=True, null=True, blank=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(auto_now_add=True, db_index=True)

    class Meta:
        unique_together = [['account', 'entity', 'monitor']]

event/models.py
DrDroidLab/kenobi
from django.db.utils import IntegrityError
from google.protobuf.wrappers_pb2 import BoolValue, StringValue

from accounts.models import Account
from event.models import Monitor
from event.update_processor import monitor_update_processor
from protos.event.api_pb2 import CreateOrUpdateMonitorRequest, Message, CreateOrUpdateMonitorResponse
from protos.event.base_pb2 import EventKey
from protos.event.monitor_pb2 import UpdateMonitorOp


def get_db_monitor_dict(db_name=None, requested_name=None, primary_key=None, secondary_key=None, is_active=None,
                        is_generated=False):
    db_monitor_default = {'name': db_name}
    if requested_name:
        db_monitor_default['name'] = requested_name
    if primary_key:
        db_monitor_default['primary_key_id'] = primary_key['id']
    if secondary_key:
        db_monitor_default['secondary_key_id'] = secondary_key['id']
    if is_active is not None:
        db_monitor_default['is_active'] = is_active
    if is_generated:
        db_monitor_default['is_generated'] = is_generated
    return db_monitor_default


def get_db_monitor(account: Account, monitor_id=None, monitor_name=None, is_active=None, primary_key_id=None,
                   secondary_key_id=None):
    filters = {}
    if monitor_id:
        filters['id'] = monitor_id
    if is_active is not None:
        filters['is_active'] = is_active
    if monitor_name:
        filters['name'] = monitor_name
    if primary_key_id:
        filters['primary_key_id'] = primary_key_id
    if secondary_key_id:
        filters['secondary_key_id'] = secondary_key_id
    try:
        return account.monitor_set.filter(**filters)
    except Exception:
        return None


def create_monitors(account: Account, monitor_name: str, primary_event_key_id, secondary_event_key_id,
                    is_generated=False) -> (Monitor, bool, str):
    if not monitor_name or not primary_event_key_id or not secondary_event_key_id:
        return None, False, "Invalid Request: Missing monitor name/primary/secondary keys"

    keys = account.eventkey_set.filter(id__in=[primary_event_key_id, secondary_event_key_id]).values_list('id',
                                                                                                          flat=True)
    if not keys or len(keys) != 2:
        return None, False, "Invalid Request: Check Primary and Secondary Event keys"

    db_monitors = get_db_monitor(account, primary_key_id=primary_event_key_id, secondary_key_id=secondary_event_key_id)
    if not db_monitors.exists():
        try:
            db_monitor = Monitor(account=account,
                                 name=monitor_name,
                                 primary_key_id=primary_event_key_id,
                                 secondary_key_id=secondary_event_key_id,
                                 is_active=True,
                                 is_generated=is_generated)
            db_monitor.save()
            return db_monitor, True, None
        except IntegrityError:
            return None, False, f"Integrity Error: Monitor with {monitor_name} already exists"
    else:
        # Uniqueness constraint is on account and monitor name
        db_named_monitors = get_db_monitor(account, monitor_name=monitor_name)
        if db_named_monitors.exists():
            db_named_monitor = db_named_monitors.first()
            if db_named_monitor.primary_key_id != primary_event_key_id or db_named_monitor.secondary_key_id != secondary_event_key_id:
                return None, False, f"Duplicate Name: Monitor {monitor_name} with different keys already exists"
            if db_named_monitor.is_active:
                return db_named_monitor, False, f"Active Monitor Found: {monitor_name} : " \
                                                f"{primary_event_key_id} : {secondary_event_key_id} already exists"
            else:
                db_monitor = db_named_monitor
        else:
            active_db_monitor = db_monitors.filter(is_active=True)
            if active_db_monitor.exists():
                active_db_monitor = active_db_monitor.first()
                return active_db_monitor, False, f"Active Monitor Found: {active_db_monitor.name} " \
                                                 f"with {primary_event_key_id} and " \
                                                 f"{secondary_event_key_id} already exists"
            db_monitor = db_monitors.first()

        update_ops = [UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_STATUS,
                                      update_monitor_status=UpdateMonitorOp.UpdateMonitorStatus(
                                          is_active=BoolValue(value=True))),
                      UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_IS_GENERATED,
                                      update_monitor_is_generated=UpdateMonitorOp.UpdateMonitorIsGenerated(
                                          is_generated=BoolValue(value=is_generated))),
                      UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_NAME,
                                      update_monitor_name=UpdateMonitorOp.UpdateMonitorName(
                                          name=StringValue(value=monitor_name)))]
        monitor_update_processor.update(db_monitor, update_ops)
        return db_monitor, False, None


def update_monitors(account: Account, request: CreateOrUpdateMonitorRequest):
    monitor_id = request.id
    if not monitor_id:
        return CreateOrUpdateMonitorResponse(success=BoolValue(value=False),
                                             message=Message(title='Error updating monitor',
                                                             description='Missing monitor id'))
    db_monitor = get_db_monitor(account, monitor_id=monitor_id)
    db_monitor = db_monitor.first()
    if not db_monitor:
        return CreateOrUpdateMonitorResponse(success=BoolValue(value=False),
                                             message=Message(title='Error updating monitor',
                                                             description=f'Monitor with id {monitor_id} not found'))
    update_ops: [UpdateMonitorOp] = []
    if request.name:
        update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_NAME,
                                          update_monitor_name=UpdateMonitorOp.UpdateMonitorName(
                                              name=StringValue(value=request.name))))
    if request.primary_event_key_id:
        update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_PRIMARY_KEY,
                                          update_monitor_primary_key=UpdateMonitorOp.UpdateMonitorPrimaryKey(
                                              primary_event_key=EventKey(id=request.primary_event_key_id))))
    if request.secondary_event_key_id:
        update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_SECONDARY_KEY,
                                          update_monitor_secondary_key=UpdateMonitorOp.UpdateMonitorSecondaryKey(
                                              secondary_event_key=EventKey(id=request.secondary_event_key_id))))
    if request.is_active is not None:
        if not request.is_active.value:
            if not account.entitymonitormapping_set.filter(monitor_id=db_monitor.id).exists():
                update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_STATUS,
                                                  update_monitor_status=UpdateMonitorOp.UpdateMonitorStatus(
                                                      is_active=BoolValue(value=request.is_active.value))))
            else:
                update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_STATUS,
                                                  update_monitor_is_generated=UpdateMonitorOp.UpdateMonitorStatus(
                                                      is_active=BoolValue(value=True))))
                update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_IS_GENERATED,
                                                  update_monitor_is_generated=UpdateMonitorOp.UpdateMonitorIsGenerated(
                                                      is_generated=BoolValue(value=True))))
        else:
            update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_STATUS,
                                              update_monitor_status=UpdateMonitorOp.UpdateMonitorStatus(
                                                  is_active=BoolValue(value=request.is_active.value))))
            if db_monitor.is_generated:
                update_ops.append(UpdateMonitorOp(op=UpdateMonitorOp.Op.UPDATE_MONITOR_IS_GENERATED,
                                                  update_monitor_is_generated=UpdateMonitorOp.UpdateMonitorIsGenerated(
                                                      is_generated=BoolValue(value=False))))

    monitor_update_processor.update(db_monitor, update_ops)
    return CreateOrUpdateMonitorResponse(success=BoolValue(value=True), message=Message(title='Monitor updated'))

event/monitors_crud.py
DrDroidLab/kenobi
from typing import Dict

from django.conf import settings

from event.models import Monitor, Trigger, EventKey, TriggerNotificationConfigMapping, is_filterable_key_type
from event.notification.notifications_crud import create_db_notifications
from protos.event.literal_pb2 import LiteralType
from protos.event.notification_pb2 import NotificationConfig
from protos.event.trigger_pb2 import Trigger as TriggerProto, TriggerPriority, TriggerDefinition, BasicTriggerFilter, \
    DelayedEventTrigger
from utils.proto_utils import proto_to_dict


class TriggerCrudException(ValueError):
    pass


def get_monitor(scope, trigger: TriggerProto) -> Monitor:
    try:
        if not trigger.monitor or not trigger.monitor.id:
            raise TriggerCrudException(f'Incorrect payload - Missing monitor id')
        else:
            monitor: Monitor = scope.monitor_set.get(id=trigger.monitor.id)
            return monitor
    except Monitor.DoesNotExist:
        raise TriggerCrudException(f'No monitor found with id {trigger.monitor.id}')


def create_monitor_triggers(scope, triggers: [TriggerProto], notifications: [NotificationConfig]) -> ([Trigger], str):
    db_triggers: [Trigger] = []
    for trigger in triggers:
        if not trigger.name or not trigger.definition or not trigger.definition.type:
            return [], 'Received invalid Trigger Config'

        trigger_name: str = trigger.name
        trigger_definition: TriggerDefinition = trigger.definition
        priority: TriggerPriority = trigger.priority
        try:
            monitor: Monitor = get_monitor(scope, trigger)
        except TriggerCrudException as tce:
            return [], str(tce)

        which_one_of = trigger_definition.WhichOneof('trigger')
        if trigger_definition.type == TriggerDefinition.Type.MISSING_EVENT and which_one_of == 'missing_event_trigger':

            if trigger_definition.missing_event_trigger.transaction_time_threshold.value <= 0:
                return [], 'Received invalid Missing Event Trigger Config'

            trigger_config = proto_to_dict(trigger_definition.missing_event_trigger)
        elif trigger_definition.type == TriggerDefinition.Type.DELAYED_EVENT and which_one_of == 'delayed_event_trigger':

            if trigger_definition.delayed_event_trigger.trigger_threshold.value < 0 or \
                    trigger_definition.delayed_event_trigger.transaction_time_threshold.value <= 0 or \
                    trigger_definition.delayed_event_trigger.resolution.value < \
                    settings.DELAYED_EVENT_TRIGGER_RESOLUTION_WINDOW_LOWER_BOUND_IN_SEC:
                return [], 'Received invalid Delayed Event Aggregated Trigger Config'

            if trigger_definition.delayed_event_trigger.transaction_time_threshold.value >= \
                    trigger_definition.delayed_event_trigger.resolution.value:
                return [], 'Transaction time cannot be greater than or equal to resolution window'

            if trigger_definition.delayed_event_trigger.type == DelayedEventTrigger.Type.UNKNOWN:
                if trigger_definition.delayed_event_trigger.skip_unfinished_transactions.value:
                    trigger_definition.delayed_event_trigger.type = DelayedEventTrigger.Type.DELAYED_EVENTS
                else:
                    trigger_definition.delayed_event_trigger.type = DelayedEventTrigger.Type.MISSING_AND_DELAYED_EVENTS
            trigger_config = proto_to_dict(trigger_definition.delayed_event_trigger)

        else:
            return [], 'Received invalid trigger definition'

        try:
            primary_event_filters = process_trigger_filters(scope, trigger_definition.primary_event_filters,
                                                            monitor.primary_key.event_type_id)
            secondary_event_filters = process_trigger_filters(scope, trigger_definition.secondary_event_filters,
                                                              monitor.secondary_key.event_type_id)
        except ValueError:
            return [], 'Received invalid event filter keys'

        generated_config = {'primary_event_filters': primary_event_filters,
                            'secondary_event_filters': secondary_event_filters}
        db_triggers.append(
            Trigger(name=trigger_name, type=trigger_definition.type, config=trigger_config, priority=priority,
                    generated_config=generated_config, monitor=monitor))

    duplicate_triggers = []
    created_trigger = []
    for db_t in db_triggers:
        trigger, created = Trigger.objects.get_or_create(account=scope,
                                                         name=db_t.name,
                                                         defaults={
                                                             'monitor': db_t.monitor,
                                                             'type': db_t.type,
                                                             'config': db_t.config,
                                                             'generated_config': db_t.generated_config,
                                                             'priority': db_t.priority,
                                                             "is_active": True
                                                         })
        if created:
            created_trigger.append(trigger)
        else:
            duplicate_triggers.append(trigger)

    map_notifications_for_triggers(scope, created_trigger, notifications)

    return duplicate_triggers, None


def process_trigger_filters(scope, filters: [BasicTriggerFilter], event_type_id) -> Dict:
    filter_map = {'event__event_type_id': event_type_id}
    for f in filters:
        try:
            event_key = scope.eventkey_set.get(id=f.event_key_id)
        except EventKey.DoesNotExist:
            raise ValueError('Event filter key does not exist')

        key = 'event__processed_kvs__' + event_key.name

        f_literal = f.literal
        if not is_filterable_key_type(event_key.type):
            raise ValueError('Invalid event key type encountered')

        if f_literal.literal_type == LiteralType.STRING:
            value = f.literal.string.value
        elif f_literal.literal_type == LiteralType.BOOLEAN:
            value = f.literal.boolean.value
        elif f_literal.literal_type == LiteralType.LONG:
            value = f.literal.long.value
        elif f_literal.literal_type == LiteralType.DOUBLE:
            value = f.literal.double.value
        elif f_literal.literal_type == LiteralType.STRING_ARRAY:
            key += "__in"
            value = [f'{string}' for string in f.literal.string_array]
        else:
            raise ValueError('Invalid key type encountered')

        filter_map[key] = value
    return filter_map


def map_notifications_for_triggers(scope, db_triggers: [Trigger], notifications: [NotificationConfig]):
    db_notifications = create_db_notifications(scope, notifications)
    for db_t in db_triggers:
        for db_n in db_notifications:
            TriggerNotificationConfigMapping.objects.update_or_create(account=scope, trigger=db_t, notification=db_n)

event/monitors_triggers_crud.py
DrDroidLab/kenobi
import requests

from django.core.mail import send_mail
from django.template.loader import render_to_string

from accounts.models import Account, User
from event.models import Notification, Alert
from event.notification.notification_helper import generate_notification_text
from protos.event.notification_pb2 import NotificationConfig, EmailConfiguration
from protos.event.options_pb2 import NotificationOption


class NotificationFacadeException(ValueError):
    pass


class NotificationFacade:

    @classmethod
    def send_notification(cls, scope, slack_notification_client: Notification, trigger, alert: Alert):
        pass

    @classmethod
    def get_notification_options(cls, scope) -> [NotificationOption]:
        pass


class SlackNotificationFacade(NotificationFacade):

    @classmethod
    def send_notification(cls, scope, slack_notification_client: Notification, trigger, alert: Alert):
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        headers = {"Content-type": "application/json"}

        data = {
            "text": generate_notification_text(scope, NotificationConfig.Channel.SLACK, alert, trigger)
        }

        requests.post(slack_notification_client.config["webhook_url"], headers=headers, json=data)

    @classmethod
    def get_notification_options(cls, scope) -> [NotificationOption]:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        notifications = scope.notification_set.filter(channel=NotificationConfig.Channel.SLACK)

        notification_options: [NotificationOption] = []
        for n in notifications:
            notification_options.append(NotificationOption(channel=n.channel, slack_configuration=n.config))

        return notification_options


class EmailNotificationFacade(NotificationFacade):

    @classmethod
    def send_notification(cls, scope, email_notification_client: Notification, trigger, alert: Alert):
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        notification_text_config = generate_notification_text(scope, NotificationConfig.Channel.EMAIL, alert, trigger)

        msg_html = render_to_string('alert_email_template.html',
                                    {'trigger_name': notification_text_config["trigger_name"],
                                     'source_name': notification_text_config["source_name"],
                                     'why': notification_text_config["why"],
                                     'when': notification_text_config["when"],
                                     'alert_link': notification_text_config["alert_link"],
                                     'error_type': notification_text_config["error_type"]})
        recipient_email_id = email_notification_client.config["recipient_email_id"]
        send_mail(
            subject=notification_text_config["subject"],
            message=notification_text_config["body"],
            from_email=None,
            recipient_list=[recipient_email_id],
            html_message=msg_html
        )

    @classmethod
    def get_notification_options(cls, scope) -> [NotificationOption]:
        if type(scope) is not User:
            raise ValueError(f'{scope} needs to be User')

        email = scope.email

        notification_options: [NotificationOption] = [NotificationOption(channel=NotificationConfig.Channel.EMAIL,
                                                                         email_configuration=EmailConfiguration(
                                                                             recipient_email_id=email))]

        return notification_options


class NotificationFacadeFactory:
    def __init__(self):
        self._map = {}

    def register(self, channel: NotificationConfig.Channel, notification_facade):
        self._map[channel] = notification_facade

    def notify(self, scope, notifications: [Notification], trigger, alert: Alert):
        for notification in notifications:
            if self._map[notification.channel]:
                self._map[notification.channel].send_notification(scope, notification, trigger, alert)

    def get_notification_options(self, user: User, account: Account) -> [NotificationOption]:
        notification_options: [NotificationOption] = []

        slack_notification_options = self._map[NotificationConfig.Channel.SLACK].get_notification_options(account)
        notification_options.extend(slack_notification_options)

        email_notification_options = self._map[NotificationConfig.Channel.EMAIL].get_notification_options(user)
        notification_options.extend(email_notification_options)
        return notification_options


notification_client = NotificationFacadeFactory()
notification_client.register(NotificationConfig.Channel.SLACK, SlackNotificationFacade())
notification_client.register(NotificationConfig.Channel.EMAIL, EmailNotificationFacade())

event/notification/notification_facade.py
DrDroidLab/kenobi
import time
from typing import Dict

from django.core.exceptions import ImproperlyConfigured
from django.conf import settings

from accounts.models import Account
from event.models import MonitorTransaction, Alert
from protos.event.entity_pb2 import EntityTriggerDefinition
from protos.event.notification_pb2 import NotificationConfig
from protos.event.trigger_pb2 import TriggerDefinition

from prototype.utils.utils import build_absolute_uri


def generate_alert_hyperlink(alert_id):
    location = settings.ALERT_PAGE_URL.format(str(alert_id))
    protocol = settings.ALERT_SITE_HTTP_PROTOCOL
    enabled = settings.ALERT_USE_SITE
    try:
        uri = build_absolute_uri(None, location, protocol, enabled)
    except ImproperlyConfigured:
        return ""
    return uri


class NotificationHelperException(ValueError):
    pass


def process_alert_time(event_time):
    if int(event_time / 10) <= 0:
        return "few seconds "
    if int(event_time / 60) <= 0:
        return str(int(event_time)) + " seconds "
    event_time = event_time / 60
    if int(event_time / 60) <= 0:
        return str(int(event_time)) + " minutes "
    event_time = event_time / 60
    if int(event_time / 10) <= 0:
        return str(int(event_time)) + " hours "
    event_time = event_time / 24
    return str(int(event_time)) + " days "


def format_duration(seconds):
    if seconds < 0:
        raise ValueError("Input must be a non-negative integer.")

    if seconds == 0:
        return "0 seconds"

    # Calculate the number of hours, minutes, and seconds
    hours, remainder = divmod(seconds, 3600)
    minutes, seconds = divmod(remainder, 60)

    # Create a list to store the time components
    components = []

    # Add components with proper pluralization
    if hours > 1:
        components.append(f"{hours} {'hour' if hours == 1 else 'hours'}")
    if minutes > 2:
        components.append(f"{minutes} {'minute' if minutes == 1 else 'minutes'}")
    if seconds > 0:
        components.append(f"{seconds} {'second' if seconds == 1 else 'seconds'}")

    # Join the components into a human-readable string
    if len(components) == 1:
        return components[0]
    elif len(components) == 2:
        return " and ".join(components)
    else:
        return ""


def format_date_time(dt):
    return dt.strftime("%Y-%m-%d %H:%M:%S")


class NotificationHelper:

    @classmethod
    def generate_missing_event_trigger_notification_text(cls, scope, alert: Alert, trigger):
        pass

    @classmethod
    def generate_delayed_event_trigger_notification_text(cls, scope, alert: Alert, trigger):
        pass


class SlackNotificationHelper(NotificationHelper):

    @classmethod
    def generate_missing_event_trigger_notification_text(cls, scope, alert: Alert, trigger) -> str:
        alert_monitor_transactions = scope.alertmonitortransactionmapping_set.filter(
            alert=alert.id).prefetch_related('monitor_transaction')

        base_text = "*" + "Alert" + "*" + ": " + "`" + trigger["name"] + "`" + " triggered"

        description = "*" + "Why" + "*" + ": "
        for amt in alert_monitor_transactions:
            mt: MonitorTransaction = amt.monitor_transaction
            description += "`" + trigger[
                "monitor__secondary_key__event_type__name"] + "`" + " not received for monitor " + "`" + trigger[
                               "monitor__name"] + "`" + " with in " + "`" + str(
                trigger["config"].get("transaction_time_threshold")) + "`" + " seconds, of " + "`" + trigger[
                               "monitor__primary_key__event_type__name"] + "`" + " for " + "`" + trigger[
                               "monitor__primary_key__name"] + "`" + " = > " + "`" + str(mt.transaction) + "`"

        diff = int(time.time()) - int(alert.triggered_at.timestamp())
        time_diff = process_alert_time(diff)
        time_stamp = "*" + "When" + "*" + ": " + "`" + time_diff + "ago" + "`" + " (" + str(
            format_date_time(alert.triggered_at.replace(microsecond=0))) + ")"

        alert_link = "*" + "See More Details: " + "*" + ": " + generate_alert_hyperlink(alert.id)

        return base_text + "\n" + description + "\n" + time_stamp + "\n" + alert_link

    @classmethod
    def generate_per_event_entity_trigger_notification_text(cls, scope, alert: Alert, trigger) -> str:
        base_text = "*" + "Alert" + "*" + ": " + "`" + trigger["name"] + "`" + " triggered"
        description = "*" + "Why" + "*" + ": "

        entity_instance_value = alert.stats['entity_instance_value']
        event_key_name = alert.stats['event_key_name']
        event_timestamp = alert.stats['event_timestamp']
        event_name = alert.stats['event_name']
        entity_name = trigger['entity__name']

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.LAST_EVENT:
            time_gap = format_duration(int(trigger['config'].get('time_interval', 0)))
            description += (
                    "No event was received for `" + time_gap + "`" + " after event `" + event_name + "` occurred at `" + str(
                format_date_time(alert.triggered_at.replace(
                    microsecond=0))) + " UTC" + "` for entity `" + entity_name + "`" + " with " + "`" + event_key_name + "`" + " -> " + "`" + entity_instance_value + "`")

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.EVENT_COUNT:
            time_gap = format_duration(int(trigger['config'].get('time_interval', 0)))
            threshold_count = int(trigger['config'].get('threshold_count', 0))
            if threshold_count == 1:
                threshold_count_str = "once"
            else:
                threshold_count_str = str(threshold_count) + " times"
            description += (
                    "Event `" + event_name + "` was received more than `" + threshold_count_str + "` within `" + time_gap + "` for entity `" + entity_name + "`" + " with " + "`" + event_key_name + "`" + " -> " + "`" + entity_instance_value + "`")

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.EVENT_OCCURS:
            description += (
                    "Event `" + event_name + "` was received for entity `" + entity_name + "`" + " with " + "`" + event_key_name + "`" + " -> " + "`" + entity_instance_value + "`")

        diff = int(time.time()) - int(alert.triggered_at.timestamp())
        time_diff = process_alert_time(diff)
        time_stamp = "*" + "When" + "*" + ": " + "`" + time_diff + "ago" + "`" + " (" + str(
            format_date_time(alert.triggered_at.replace(microsecond=0))) + " UTC)"

        alert_link = "*" + "See More Details: " + "*" + ": " + generate_alert_hyperlink(alert.id)

        return base_text + "\n" + description + "\n" + time_stamp + "\n" + alert_link

    @classmethod
    def generate_delayed_event_trigger_notification_text(cls, scope, alert: Alert, trigger) -> str:
        base_text = "*" + "Alert" + "*" + ": " + "`" + trigger["name"] + "`" + " triggered"

        resolution_time = process_alert_time(trigger["config"].get("resolution"))
        description = "*" + "Why" + "*" + ": " + str(
            trigger["config"].get("trigger_threshold")) + " % transactions did not receive " + "`" + trigger[
                          "monitor__secondary_key__event_type__name"] + "`" + " from " + "`" + trigger[
                          "monitor__primary_key__event_type__name"] + "`" + " within " + str(trigger["config"].get(
            "transaction_time_threshold")) + " seconds" + " for monitor: " + "`" + trigger[
                          "monitor__name"] + "`" + " in last " + "`" + resolution_time + "`"

        diff = int(time.time()) - int(alert.triggered_at.timestamp())
        time_diff = process_alert_time(diff)
        time_stamp = "*" + "When" + "*" + ": " + "`" + time_diff + "ago" + "`" + " (" + str(
            format_date_time(alert.triggered_at.replace(microsecond=0))) + ")"

        alert_link = "*" + "See More Details: " + "*" + ": " + generate_alert_hyperlink(alert.id)

        return base_text + "\n" + description + "\n" + time_stamp + "\n" + alert_link


class EmailNotificationHelper(NotificationHelper):

    @classmethod
    def generate_missing_event_trigger_notification_text(cls, scope, alert: Alert, trigger) -> Dict:
        alert_monitor_transactions = scope.alertmonitortransactionmapping_set.filter(
            alert=alert.id).prefetch_related('monitor_transaction')

        trigger_name = trigger["name"]
        monitor_name = trigger["monitor__name"]

        description = ""
        for amt in alert_monitor_transactions:
            mt: MonitorTransaction = amt.monitor_transaction
            description += trigger[
                               "monitor__secondary_key__event_type__name"] + " not received for " + trigger[
                               "monitor__primary_key__event_type__name"] + " within " + str(
                trigger["config"].get("transaction_time_threshold")) + " seconds from " + trigger[
                               "monitor__primary_key__name"] + " => " + str(mt.transaction)

        time_stamp = str(format_date_time(alert.triggered_at.replace(microsecond=0)))

        alert_link = generate_alert_hyperlink(alert.id)

        body = description + "\n" + time_stamp + "\n" + alert_link

        subject = "Alert: " + monitor_name + " -> " + "Missing Event Alert"

        return {"trigger_name": trigger_name, "source_name": monitor_name, "subject": subject, "body": body,
                "why": description, "when": time_stamp, "alert_link": alert_link, "error_type": "Missing Event Alert"}

    @classmethod
    def generate_per_event_entity_trigger_notification_text(cls, scope, alert: Alert, trigger) -> Dict:
        description = ""

        entity_instance_value = alert.stats['entity_instance_value']
        event_key_name = alert.stats['event_key_name']
        event_name = alert.stats['event_name']
        entity_name = trigger['entity__name']
        trigger_name = alert.entity_trigger.name

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.LAST_EVENT:
            time_gap = format_duration(int(trigger['config'].get('time_interval', 0)))
            description += (
                    "No event was received for " + time_gap + " after event " + event_name + " occurred at " + str(
                format_date_time(alert.triggered_at.replace(
                    microsecond=0))) + " UTC" + " for entity " + entity_name + " with " + event_key_name + " -> " + entity_instance_value)

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.EVENT_COUNT:
            time_gap = format_duration(int(trigger['config'].get('time_interval', 0)))
            threshold_count = int(trigger['config'].get('threshold_count', 0))
            if threshold_count == 1:
                threshold_count_str = "once"
            else:
                threshold_count_str = str(threshold_count) + " times"
            description += (
                    "Event " + event_name + " was received more than " + threshold_count_str + " within " + time_gap + " for entity " + entity_nam + " with " + event_key_name + " -> " + entity_instance_value)

        if trigger['rule_type'] == EntityTriggerDefinition.EntityTriggerRuleType.EVENT_OCCURS:
            description += (
                    "Event " + event_name + " was received for entity " + entity_name + " with " + event_key_name + " -> " + entity_instance_value)

        time_stamp = str(format_date_time(alert.triggered_at.replace(microsecond=0)))
        alert_link = generate_alert_hyperlink(alert.id)

        subject = "Alert: " + entity_name + " -> " + "Events Alert"
        body = description + "\n" + time_stamp + "\n" + alert_link

        return {"trigger_name": trigger_name, "source_name": entity_name, "subject": subject, "body": body,
                "why": description, "when": time_stamp, "alert_link": alert_link, "error_type": "Event Alert"}

    @classmethod
    def generate_delayed_event_trigger_notification_text(cls, scope, alert: Alert, trigger) -> Dict:
        trigger_name = trigger["name"]
        monitor_name = trigger["monitor__name"]

        resolution_time = process_alert_time(trigger["config"].get("resolution"))
        description = str(
            trigger["config"].get("trigger_threshold")) + " % transactions did not receive " + trigger[
                          "monitor__secondary_key__event_type__name"] + " after " + trigger[
                          "monitor__primary_key__event_type__name"] + " within " + str(trigger["config"].get(
            "transaction_time_threshold")) + " seconds" + " in last " + resolution_time

        time_stamp = str(format_date_time(alert.triggered_at.replace(microsecond=0)))

        alert_link = generate_alert_hyperlink(alert.id)

        body = description + "\n" + time_stamp + "\n" + alert_link

        subject = "Alert: " + monitor_name + " -> " + "Aggregated Events Alert"

        return {"trigger_name": trigger_name, "source_name": monitor_name, "subject": subject, "body": body,
                "why": description, "when": time_stamp, "alert_link": alert_link,
                "error_type": "Aggregated Events Alert"}


class NotificationHelperFactory:

    def __init__(self):
        self._map = {}

    def register(self, channel_type: NotificationConfig.Channel, notification_helper: NotificationHelper):
        self._map[channel_type] = notification_helper

    def generate_missing_event_trigger_text(self, scope, channel_type: NotificationConfig.Channel, alert: Alert,
                                            trigger) -> str:
        return self._map[channel_type].generate_missing_event_trigger_notification_text(scope, alert, trigger)

    def generate_per_event_entity_trigger_text(self, scope, channel_type: NotificationConfig.Channel, alert: Alert,
                                               trigger) -> str:
        return self._map[channel_type].generate_per_event_entity_trigger_notification_text(scope, alert, trigger)

    def generate_delayed_event_trigger_text(self, scope, channel_type: NotificationConfig.Channel, alert: Alert,
                                            trigger) -> str:
        return self._map[channel_type].generate_delayed_event_trigger_notification_text(scope, alert, trigger)


notification_helper_factory = NotificationHelperFactory()
notification_helper_factory.register(TriggerDefinition.Type.MISSING_EVENT, SlackNotificationHelper())
notification_helper_factory.register(TriggerDefinition.Type.DELAYED_EVENT, EmailNotificationHelper())


def generate_notification_text(scope, channel_type: NotificationConfig.Channel, alert: Alert, trigger) -> str:
    if type(scope) is not Account:
        raise ValueError(f'{scope} needs to be Account')

    if alert.trigger:
        if trigger["type"] == TriggerDefinition.MISSING_EVENT:
            return notification_helper_factory.generate_missing_event_trigger_text(scope, channel_type, alert, trigger)
        elif trigger["type"] == TriggerDefinition.DELAYED_EVENT:
            return notification_helper_factory.generate_delayed_event_trigger_text(scope, channel_type, alert, trigger)

    if alert.entity_trigger:
        if trigger["type"] == EntityTriggerDefinition.TriggerType.PER_EVENT:
            return notification_helper_factory.generate_per_event_entity_trigger_text(scope, channel_type, alert,
                                                                                      trigger)

event/notification/notification_helper.py
DrDroidLab/kenobi
from event.models import Notification
from protos.event.notification_pb2 import NotificationConfig
from utils.proto_utils import proto_to_dict


def create_db_notifications(scope, notifications: [NotificationConfig]):
    db_notifications: [Notification] = []
    for nc in notifications:
        channel_config = None
        which_one_of = nc.WhichOneof('config')
        if nc.channel == NotificationConfig.Channel.SLACK and which_one_of == 'slack_configuration':
            channel_config = proto_to_dict(nc.slack_configuration)
        if nc.channel == NotificationConfig.Channel.EMAIL and which_one_of == 'email_configuration':
            channel_config = proto_to_dict(nc.email_configuration)
        if nc.channel and channel_config:
            db_notifications.append(Notification(config=channel_config, channel=nc.channel))

    saved_notifications = []
    for db_n in db_notifications:
        saved_notification, _ = Notification.objects.get_or_create(account=scope,
                                                                   config=db_n.config,
                                                                   defaults={'channel': db_n.channel})

        saved_notifications.append(saved_notification)
    return saved_notifications

event/notification/notifications_crud.py
DrDroidLab/kenobi
from django.conf import settings
from django.core.cache import caches

from event.models import EventType, EventKey


class EventTypeCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def _get_or_create(self, **kwargs):
        return EventType.objects.get_or_create(**kwargs)

    def get_or_create(self, account_id, name):
        if not self._enabled:
            return self._get_or_create(account_id=account_id, name=name)

        key = f'event_type:{account_id}:{name}'
        value = caches[self._cache_key].get(key)
        if value is not None:
            return value, False
        obj, created = self._get_or_create(account_id=account_id, name=name)
        caches[self._cache_key].set(key, obj)
        return obj, created

    def update(self, obj, update_fields=None):
        if not self._enabled:
            obj.save(update_fields=update_fields)

        obj.save(update_fields=update_fields)
        key = f'event_type:{obj.account_id}:{obj.name}'
        caches[self._cache_key].set(key, obj)


class EventKeyCache:
    def __init__(self, cache_key, enabled):
        self._cache_key = cache_key
        self._enabled = enabled

    def _get_or_create(self, **kwargs):
        return EventKey.objects.get_or_create(**kwargs)

    def get_or_create(self, account_id, event_type_id, name, defaults=None):
        if not self._enabled:
            return self._get_or_create(
                account_id=account_id, event_type_id=event_type_id, name=name,
                defaults=defaults
            )

        key = f'event_key:{account_id}:{event_type_id}:{name}'
        value = caches[self._cache_key].get(key)
        if value is not None:
            return value, False
        obj, created = self._get_or_create(
            account_id=account_id, event_type_id=event_type_id, name=name,
            defaults=defaults
        )
        caches[self._cache_key].set(key, obj)
        return obj, created


GLOBAL_EVENT_TYPE_CACHE = EventTypeCache(**settings.GLOBAL_EVENT_TYPE_CACHE)
GLOBAL_EVENT_KEY_CACHE = EventKeyCache(**settings.GLOBAL_EVENT_KEY_CACHE)

event/processors/cache.py
DrDroidLab/kenobi
from google.protobuf.wrappers_pb2 import UInt64Value, StringValue

from accounts.models import Account
from event.models import EntityInstanceEventMapping
from protos.kafka.base_pb2 import DiscoveredEntityInstance


def process_event_entities(account: Account, event_key_id_set, saved_events, event_key_dict_list):
    eiems = []
    event_entity_instance_lists = [[]] * len(saved_events)
    entity_event_keys = list(account.entityeventkeymapping_set.filter(
        is_active=True, entity__is_active=True, event_key_id__in=event_key_id_set
    ).values('entity_id', 'event_key_id'))
    if entity_event_keys:
        event_entity_instance_lists = []
        for idx, (saved_event, event_key_dict) in enumerate(zip(saved_events, event_key_dict_list)):
            event_keys = event_key_dict.keys()

            event_entity_instance_list = []
            for entity_event_key in entity_event_keys:
                entity_id = entity_event_key['entity_id']
                event_key_id = entity_event_key['event_key_id']

                if event_key_id in event_keys:
                    instance_value = str(event_key_dict[event_key_id])
                    instance, _ = account.entityinstance_set.get_or_create(
                        entity_id=entity_id, instance=instance_value
                    )
                    eiem = EntityInstanceEventMapping(
                        account=account,
                        entity_instance=instance,
                        event=saved_event,
                        event_key_id=event_key_id,
                        entity_id=entity_id,
                        instance=instance_value,
                        event_processed_kvs=saved_event.processed_kvs,
                        event_timestamp=saved_event.timestamp,
                    )
                    eiems.append(eiem)
                    event_entity_instance_list.append({
                        'entity_instance_id': instance.id, 'entity_id': entity_id, 'instance': instance_value
                    })
            event_entity_instance_lists.append(event_entity_instance_list)

    if eiems:
        EntityInstanceEventMapping.objects.bulk_create(eiems, batch_size=25)

    return event_entity_instance_lists


def get_discovered_entity_instance_lists(account_id, event_entity_instance_lists):
    return [
        [
            DiscoveredEntityInstance(
                account_id=UInt64Value(value=account_id),
                entity_instance_id=UInt64Value(value=entity_instance['entity_instance_id']),
                entity_id=UInt64Value(value=entity_instance['entity_id']),
                instance=StringValue(value=entity_instance['instance']),
            ) for entity_instance in entity_instance_list
        ]
        for entity_instance_list in event_entity_instance_lists
    ]

event/processors/process_entities.py
DrDroidLab/kenobi
from datetime import datetime, timezone
from typing import List

from django.conf import settings
from google.protobuf.wrappers_pb2 import UInt64Value, StringValue

from accounts.models import Account
from event.models import is_transactional_key_type, Event
from event.processors.cache import GLOBAL_EVENT_KEY_CACHE, GLOBAL_EVENT_TYPE_CACHE
from event.processors.process_entities import process_event_entities, get_discovered_entity_instance_lists
from event.processors.process_monitors import generate_raw_monitor_transactions, \
    process_raw_monitor_transactions
from event.processors.processed_events_processor import is_produce_processed_events, produce_account_processed_events
from event.processors.raw_monitor_transactions_processor import is_monitor_transaction_processing_async, \
    process_raw_monitor_transactions_async
from event.processors.utils import infer_key_type
from protos.event.base_pb2 import Event as EventProto
from protos.event.schema_pb2 import IngestionEvent
from protos.kafka.base_pb2 import ProcessedIngestionEvent
from protos.kafka.raw_monitor_transactions_pb2 import RawMonitorTransactionPayloadKey, RawMonitorTransactionPayloadValue
from utils.proto_utils import get_value, proto_to_dict


def is_entity_processing_enabled():
    if settings.ENTITY_PROCESSING_ENABLED:
        return settings.ENTITY_PROCESSING_ENABLED
    return False


def process_account_events(
        account: Account, events: List[IngestionEvent],
        source: EventProto.EventSource = EventProto.EventSource.SDK,
        is_generated: bool = False, evaluate_event_monitors: bool = True
):
    db_events = []
    event_key_dict_list = []

    event_type_cache = {}
    event_key_cache = {}
    event_key_id_set = set()

    processed_ingestion_event_list: List[ProcessedIngestionEvent] = []

    for event in events:
        if event.timestamp == 0:
            event.timestamp = int(datetime.now(tz=timezone.utc).timestamp() * 1000)
        event_name: str = event.name
        if not event_name:
            continue

        if event_name not in event_type_cache:
            db_event_type, _ = GLOBAL_EVENT_TYPE_CACHE.get_or_create(name=event_name, account_id=account.id)
            event_type_sources = set(db_event_type.event_sources)
            if source not in event_type_sources:
                event_type_sources.add(source)
                db_event_type.event_sources = list(event_type_sources)
                GLOBAL_EVENT_TYPE_CACHE.update(obj=db_event_type, update_fields=["event_sources"])
            event_type_cache[event_name] = db_event_type
        else:
            db_event_type = event_type_cache[event_name]

        kv_dict = {}
        db_key_dict = {}

        event_type_key_cache = event_key_cache.get(event_name, {})
        pkvs = []
        for kv in event.kvs:
            k = kv.key
            k_type = infer_key_type(kv.value)

            if k not in event_type_key_cache:
                db_key, _ = GLOBAL_EVENT_KEY_CACHE.get_or_create(
                    event_type_id=db_event_type.id, name=k, account_id=account.id,
                    defaults={'type': k_type}
                )
                event_type_key_cache[k] = db_key
            else:
                db_key = event_type_key_cache[k]

            value = get_value(kv.value)
            if is_transactional_key_type(k_type):
                db_key_dict[db_key.id] = value
                event_key_id_set.add(db_key.id)
            kv_dict[k] = value
            pkvs.append(ProcessedIngestionEvent.ProcessedKeyValue(
                id=UInt64Value(value=db_key.id),
                key=StringValue(value=k),
                value=kv.value
            ))
        event_key_dict_list.append(db_key_dict)

        event_key_cache[event_name] = event_type_key_cache

        processed_ingestion_event_list.append(
            ProcessedIngestionEvent(
                account_id=UInt64Value(value=account.id),
                event_type=db_event_type.proto_partial,
                kvs=pkvs,
                timestamp=event.timestamp,
                event_source=source,
            )
        )

        db_event = Event(
            account=account,
            event_type=db_event_type,
            event=proto_to_dict(event),
            processed_kvs=kv_dict,
            timestamp=datetime.utcfromtimestamp(event.timestamp / 1000).astimezone(timezone.utc),
            event_source=source,
            is_generated=is_generated
        )
        db_events.append(db_event)

    saved_events: List[Event] = Event.objects.bulk_create(db_events, batch_size=25)
    for idx, saved_event in enumerate(saved_events):
        processed_ingestion_event_list[idx].id.value = saved_event.id

    if not is_generated:
        if is_produce_processed_events():
            processed_event_publish_count = produce_account_processed_events(account, processed_ingestion_event_list)
            print(
                f"Kafka processed-events Publish Count : {processed_event_publish_count} for account: {account.id}")

        event_entity_instance_lists = process_event_entities(
            account, event_key_id_set, saved_events,
            event_key_dict_list
        )
        discovered_entity_instance_lists = get_discovered_entity_instance_lists(account.id, event_entity_instance_lists)

        if evaluate_event_monitors:
            monitor_keys, monitor_values = generate_raw_monitor_transactions(
                account, event_key_id_set,
                processed_ingestion_event_list,
                event_key_dict_list,
                discovered_entity_instance_lists
            )

            process_raw_monitor_transaction_payload(monitor_keys, monitor_values)

    return len(saved_events)


def process_raw_monitor_transaction_payload(
        keys: List[RawMonitorTransactionPayloadKey], values: List[RawMonitorTransactionPayloadValue]
):
    if is_monitor_transaction_processing_async():
        publish_count = process_raw_monitor_transactions_async(keys, values)
        print(
            f"Kafka raw-monitor-transactions Publish Count : {publish_count}")
    else:
        process_raw_monitor_transactions(values)

event/processors/process_events.py
DrDroidLab/kenobi
from typing import List

from accounts.models import Account
from event.processors.process_events import process_account_events
from event.processors.raw_events_processor import is_event_processing_async, process_account_events_async
from event.processors.raw_monitor_transactions_processor import is_monitor_transaction_processing_async, \
    process_raw_monitor_transactions_async
from protos.event.base_pb2 import Event as EventProto
from protos.event.schema_pb2 import IngestionEventPayload
from protos.kafka.raw_monitor_transactions_pb2 import RawMonitorTransactionPayloadValue, RawMonitorTransactionPayloadKey


def process_account_events_payload(
        account: Account, payload: IngestionEventPayload,
        source: EventProto.EventSource = EventProto.EventSource.SDK,
        is_generated: bool = False, evaluate_event_monitors: bool = True
):
    events = payload.events
    if is_generated:
        process_account_events(account, events, source, is_generated, evaluate_event_monitors)
    if is_event_processing_async():
        publish_count = process_account_events_async(account, events, source)
        print(
            f"Kafka raw-events Publish Count : {publish_count} for account: {account.id} and events: {len(events)}")
    else:
        process_account_events(account, events, source, is_generated, evaluate_event_monitors)

    return len(events)

event/processors/process_events_payload.py
DrDroidLab/kenobi
from datetime import datetime, timezone
from typing import List, Dict

from django.db.models import Q, F, Min
from google.protobuf.wrappers_pb2 import UInt64Value, StringValue, DoubleValue

from accounts.models import Account
from event.models import MonitorTransaction, MonitorTransactionStats, MonitorTransactionEventMapping, \
    EntityInstanceMonitorTransactionMapping
from event.processors.processed_monitor_transaction_processor import is_produce_processed_monitor_transactions, \
    produce_account_processed_monitor_transactions
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto, \
    MonitorTransactionStats as MonitorTransactionStatsProto
from protos.kafka.base_pb2 import ProcessedMonitorTransaction
from protos.kafka.raw_monitor_transactions_pb2 import RawMonitorTransactionPayloadKey, RawMonitorTransactionPayloadValue
from utils.proto_utils import proto_to_dict, dict_to_proto
from utils.tracer import tracer


def generate_raw_monitor_transactions(
        account: Account,
        event_key_id_set,
        processed_ingestion_event_list,
        event_key_dict_list,
        discovered_entity_instance_lists
):
    keys = []
    values = []

    account_monitors = list(account.monitor_set.filter(
        Q(primary_key_id__in=event_key_id_set) | Q(secondary_key_id__in=event_key_id_set)
    ).exclude(is_active=False).annotate(
        monitor_id=F('id')
    ).values(
        'monitor_id', 'primary_key_id', 'secondary_key_id'
    ))

    for monitor in account_monitors:
        monitor_id, primary_key_id, secondary_key_id = monitor['monitor_id'], monitor['primary_key_id'], monitor[
            'secondary_key_id']
        for idx, (processed_ingestion_event, event_key_dict, discovered_entity_instance_list) in enumerate(
                zip(processed_ingestion_event_list, event_key_dict_list, discovered_entity_instance_lists)
        ):
            event_keys = event_key_dict.keys()
            transaction_event_type = MonitorTransactionProto.MonitorTransactionEventType.UNKNOWN_MT_ET
            transaction: str = ''
            found = False
            if primary_key_id in event_keys:
                transaction_event_type = MonitorTransactionProto.MonitorTransactionEventType.PRIMARY
                transaction = str(event_key_dict[primary_key_id])
                found = True
            elif secondary_key_id in event_keys:
                transaction_event_type = MonitorTransactionProto.MonitorTransactionEventType.SECONDARY
                transaction = str(event_key_dict[secondary_key_id])
                found = True
            if found:
                key = RawMonitorTransactionPayloadKey(
                    account_id=UInt64Value(value=account.id),
                    monitor_id=UInt64Value(value=monitor_id),
                    transaction=StringValue(value=transaction),
                )
                value = RawMonitorTransactionPayloadValue(
                    account_id=UInt64Value(value=account.id),
                    monitor_id=UInt64Value(value=monitor_id),
                    transaction=StringValue(value=transaction),
                    event=processed_ingestion_event,
                    transaction_event_type=transaction_event_type,
                    discovered_entity_instances=discovered_entity_instance_list
                )
                keys.append(key)
                values.append(value)

    return keys, values


def process_raw_monitor_transactions(
        raw_monitor_transaction_payloads: List[RawMonitorTransactionPayloadValue],
):
    account_monitor_dict = {}
    count = 0
    for raw_monitor_transaction in raw_monitor_transaction_payloads:
        account_id = raw_monitor_transaction.account_id.value
        monitor_id = raw_monitor_transaction.monitor_id.value
        transaction = raw_monitor_transaction.transaction.value

        monitor_transaction_dict = account_monitor_dict.get(account_id, {})
        transaction_payloads_dict = monitor_transaction_dict.get(monitor_id, {})
        transaction_payload_list = transaction_payloads_dict.get(transaction, [])

        transaction_payload_list.append(raw_monitor_transaction)
        transaction_payloads_dict[transaction] = transaction_payload_list
        monitor_transaction_dict[monitor_id] = transaction_payloads_dict
        account_monitor_dict[account_id] = monitor_transaction_dict
        count += 1

    _process_account_monitor_dict(account_monitor_dict)
    print(
        f"Kafka raw-monitor-transactions Consume Count : {count}")

    return


def _process_account_monitor_dict(account_monitor_dict):
    for account_id, monitor_transaction_dict in account_monitor_dict.items():
        account_obj = Account(id=account_id)
        processed_account_monitor_transactions_protos = []
        for monitor_id, transaction_payloads_dict in monitor_transaction_dict.items():
            with tracer.trace('process.monitor_transaction') as span:
                span.set_tag('account_id', account_id)
                span.set_tag('monitor_id', monitor_id)
                span.set_tag('transaction_count', len(transaction_payloads_dict))
                try:
                    processed_monitor_transactions_protos = _process_raw_monitor_transactions_for_monitor_id(
                        account_obj, monitor_id,
                        transaction_payloads_dict
                    )
                    if processed_monitor_transactions_protos:
                        processed_account_monitor_transactions_protos.extend(processed_monitor_transactions_protos)
                except Exception as e:
                    span.set_traceback()
        if is_produce_processed_monitor_transactions():
            processed_monitor_transaction_publish_count = produce_account_processed_monitor_transactions(account_obj,
                                                                                                         processed_account_monitor_transactions_protos)
            print(
                f"Kafka processed-monitor-transactions Publish Count : {processed_monitor_transaction_publish_count} for account: {account_obj.id}")


def _process_raw_monitor_transactions_for_monitor_id(
        account: Account, monitor_id: int, transaction_payloads_dict: Dict[str, List[RawMonitorTransactionPayloadValue]]
):
    discovered_transactions_set = set(transaction_payloads_dict.keys())
    existing_db_transactions = list(account.monitortransaction_set.filter(
        monitor_id=monitor_id,
        transaction__in=discovered_transactions_set,
    ).values('id', 'transaction', 'type'))

    existing_transaction_set = set(
        [existing_transaction['transaction'] for existing_transaction in existing_db_transactions]
    )
    missing_transactions = set(discovered_transactions_set).difference(existing_transaction_set)
    missing_transaction_objs = [
        MonitorTransaction(
            account=account, monitor_id=monitor_id, transaction=missing_transaction,
            type=MonitorTransactionProto.MonitorTransactionStatus.PRIMARY_RECEIVED
        ) for missing_transaction in missing_transactions
    ]
    missing_db_transactions = list(MonitorTransaction.objects.bulk_create(
        missing_transaction_objs, batch_size=50,
    ))

    db_transactions = {
        **{
            existing_db_transaction['transaction']: MonitorTransaction(
                id=existing_db_transaction['id'],
                account=account, monitor_id=monitor_id, transaction=existing_db_transaction['transaction'],
                type=existing_db_transaction['type']
            )
            for existing_db_transaction in existing_db_transactions
        },
        **{
            missing_db_transaction.transaction: missing_db_transaction
            for missing_db_transaction in missing_db_transactions
        }
    }

    mtems = []
    update_db_transaction_type_set = set()
    eimtms = []
    db_transaction_id_set = set()
    processed_monitor_transactions_dict = {}
    for db_transaction_str, db_transaction in db_transactions.items():
        db_transaction_id = db_transaction.id
        db_transaction_id_set.add(db_transaction_id)
        payloads: List[RawMonitorTransactionPayloadValue] = transaction_payloads_dict[db_transaction_str]
        db_transaction_type = db_transaction.type
        is_db_transaction_type_updated = False
        entity_instance_dict = {}
        processed_monitor_transactions_list = processed_monitor_transactions_dict.get(db_transaction_id, [])
        for payload in payloads:
            mtems.append(MonitorTransactionEventMapping(
                account=account,
                monitor_transaction=db_transaction,
                event_id=payload.event.id.value,
                type=payload.transaction_event_type,
                event_timestamp=datetime.utcfromtimestamp(payload.event.timestamp / 1000).astimezone(timezone.utc),
            ))
            transaction_event_type = payload.transaction_event_type
            if transaction_event_type == MonitorTransactionProto.MonitorTransactionEventType.PRIMARY:
                if db_transaction_type == MonitorTransactionProto.MonitorTransactionStatus.UNKNOWN:
                    db_transaction_type = MonitorTransactionProto.MonitorTransactionStatus.PRIMARY_RECEIVED
                    is_db_transaction_type_updated = True
            elif transaction_event_type == MonitorTransactionProto.MonitorTransactionEventType.SECONDARY:
                if db_transaction.type != MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED:
                    db_transaction_type = MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED
                    is_db_transaction_type_updated = True
            if is_db_transaction_type_updated:
                db_transaction.type = db_transaction_type
                update_db_transaction_type_set.add(db_transaction)
            processed_monitor_transactions_list.append(
                {
                    'account_id': account.id,
                    'monitor_transaction': {
                        'id': db_transaction_id,
                        'transaction': db_transaction_str,
                        'monitor': {
                            'id': monitor_id
                        }
                    },
                    'monitor_transaction_status': db_transaction_type,
                    'ingestion_event': proto_to_dict(payload.event),
                    'monitor_transaction_event_type': transaction_event_type,
                    'transaction_time': None
                }
            )
            for discovered_entity in payload.discovered_entity_instances:
                if discovered_entity.entity_instance_id.value not in entity_instance_dict:
                    entity_instance_dict[discovered_entity.entity_instance_id.value] = {
                        'entity_instance_id': discovered_entity.entity_instance_id.value,
                        'entity_id': discovered_entity.entity_id.value,
                        'instance': discovered_entity.instance.value,
                    }
        for entity_instance_id, entity_instance in entity_instance_dict.items():
            eimtms.append(EntityInstanceMonitorTransactionMapping(
                account=account,
                **entity_instance,
                monitor_transaction=db_transaction,
                monitor_id=monitor_id,
                transaction=db_transaction_str
            ))
        processed_monitor_transactions_dict[db_transaction_id] = processed_monitor_transactions_list

    if update_db_transaction_type_set:
        account.monitortransaction_set.bulk_update(
            update_db_transaction_type_set, batch_size=100, fields=['type'],
        )
    if mtems:
        account.monitortransactioneventmapping_set.bulk_create(
            mtems, batch_size=100, ignore_conflicts=True
        )
    if eimtms:
        account.entityinstancemonitortransactionmapping_set.bulk_create(
            eimtms, batch_size=100, ignore_conflicts=True
        )
    monitor_transaction_event_timestamps = list(account.monitortransactioneventmapping_set.filter(
        monitor_transaction_id__in=db_transaction_id_set,
    ).values('monitor_transaction_id', 'type').annotate(
        min_event_timestamp=Min('event_timestamp')
    ))

    primary_timestamps = {
        monitor_transaction_event_timestamp['monitor_transaction_id']:
            monitor_transaction_event_timestamp['min_event_timestamp']
        for monitor_transaction_event_timestamp in monitor_transaction_event_timestamps
        if monitor_transaction_event_timestamp['type'] == MonitorTransactionProto.MonitorTransactionEventType.PRIMARY
    }

    secondary_timestamps = {
        monitor_transaction_event_timestamp['monitor_transaction_id']:
            monitor_transaction_event_timestamp['min_event_timestamp']
        for monitor_transaction_event_timestamp in monitor_transaction_event_timestamps
        if monitor_transaction_event_timestamp['type'] == MonitorTransactionProto.MonitorTransactionEventType.SECONDARY
    }

    monitor_transaction_id_stat_dict = {}
    for monitor_transaction_id, primary_timestamp in primary_timestamps.items():
        if monitor_transaction_id in secondary_timestamps:
            delta = secondary_timestamps[monitor_transaction_id] - primary_timestamps[monitor_transaction_id]
            monitor_transaction_id_stat_dict[monitor_transaction_id] = MonitorTransactionStats(
                account=account,
                monitor_transaction_id=monitor_transaction_id,
                stats=proto_to_dict(MonitorTransactionStatsProto(
                    transaction_time=DoubleValue(value=delta.total_seconds())
                ))
            )
            processed_monitor_transactions = processed_monitor_transactions_dict[monitor_transaction_id]
            for processed_monitor_transaction in processed_monitor_transactions:
                if processed_monitor_transaction['monitor_transaction_event_type'] == \
                        MonitorTransactionProto.MonitorTransactionEventType.SECONDARY:
                    processed_monitor_transaction['transaction_time'] = delta.total_seconds()

    existing_monitor_transaction_stats_ids = list(account.monitortransactionstats_set.filter(
        monitor_transaction_id__in=set(monitor_transaction_id_stat_dict.keys()),
    ).values('id', 'monitor_transaction_id'))

    existing_stats = []
    for existing_monitor_transaction_stats_id in existing_monitor_transaction_stats_ids:
        stat = monitor_transaction_id_stat_dict.pop(existing_monitor_transaction_stats_id['monitor_transaction_id'])
        stat.id = existing_monitor_transaction_stats_id['id']
        existing_stats.append(stat)

    new_stats = [id_stat for monitor_tranaction_id, id_stat in monitor_transaction_id_stat_dict.items()]

    if existing_stats:
        account.monitortransactionstats_set.bulk_update(existing_stats, batch_size=50, fields=['stats'])
    if new_stats:
        account.monitortransactionstats_set.bulk_create(new_stats, batch_size=50, ignore_conflicts=True)

    processed_monitor_transactions_protos: [ProcessedMonitorTransaction] = []
    for db_transaction_id, processed_monitor_transactions_list in processed_monitor_transactions_dict.items():
        for processed_monitor_transaction in processed_monitor_transactions_list:
            processed_monitor_transactions_protos.append(
                dict_to_proto(processed_monitor_transaction, ProcessedMonitorTransaction))
    return processed_monitor_transactions_protos

event/processors/process_monitors.py
DrDroidLab/kenobi
from typing import List

from django.conf import settings
from google.protobuf.wrappers_pb2 import UInt64Value

from accounts.models import Account
from event.clickhouse.model_helper import ingest_processed_events
from protos.kafka.base_pb2 import ProcessedIngestionEvent
from protos.kafka.processed_events_pb2 import ProcessedEventPayloadKey, ProcessedEventPayloadValue
from prototype.kafka.processor import Processor
from prototype.kafka.producer import get_kafka_producer


def is_produce_processed_events():
    if settings.PROCESSED_EVENT_PROCESSING:
        return settings.PROCESSED_EVENT_PROCESSING.get('async', False)
    return False


def processed_event_async_producer():
    if settings.PROCESSED_EVENT_PROCESSING:
        return settings.PROCESSED_EVENT_PROCESSING.get('producer', 'processed_events')
    return 'processed_events'


def produce_account_processed_events(
        account: Account, processed_events: List[ProcessedIngestionEvent]
):
    producer = get_kafka_producer(processed_event_async_producer())
    count = 0
    for event in processed_events:
        key = ProcessedEventPayloadKey()
        value = ProcessedEventPayloadValue(
            account_id=UInt64Value(value=account.id),
            event=event,
        )
        producer.process(key, value)
        count += 1
    producer.flush()

    return count


class ProcessedEventClickhouseIngestProcessor(Processor):
    key_cls = ProcessedEventPayloadKey
    value_cls = ProcessedEventPayloadValue

    def process(self, keys: List[ProcessedEventPayloadKey], values: List[ProcessedEventPayloadValue]):
        account_processed_events_dict = {}
        for key, value in zip(keys, values):
            account_id = value.account_id.value
            processed_event = value.event

            processed_events_list: List[ProcessedIngestionEvent] = account_processed_events_dict.get(account_id, [])
            processed_events_list.append(processed_event)
            account_processed_events_dict[account_id] = processed_events_list

        for account_id, processed_events_list in account_processed_events_dict.items():
            ingest_processed_events(processed_events_list)
            print(
                f"Kafka processed-events Consume Count : {len(processed_events_list)} for account_id: {account_id}")

        return

event/processors/processed_events_processor.py
DrDroidLab/kenobi
from typing import List

from django.conf import settings
from google.protobuf.wrappers_pb2 import UInt64Value

from accounts.models import Account
from event.clickhouse.model_helper import ingest_processed_monitor_transactions
from protos.kafka.base_pb2 import ProcessedMonitorTransaction
from protos.kafka.processed_monitor_transactions_pb2 import ProcessedMonitorTransactionPayloadKey, \
    ProcessedMonitorTransactionPayloadValue
from prototype.kafka.processor import Processor
from prototype.kafka.producer import get_kafka_producer


def is_produce_processed_monitor_transactions():
    if settings.PROCESSED_MONITOR_TRANSACTION_PROCESSING:
        return settings.PROCESSED_MONITOR_TRANSACTION_PROCESSING.get('async', False)
    return False


def processed_monitor_transaction_async_producer():
    if settings.PROCESSED_MONITOR_TRANSACTION_PROCESSING:
        return settings.PROCESSED_MONITOR_TRANSACTION_PROCESSING.get('producer', 'processed_monitor_transactions')
    return 'processed_monitor_transactions'


def produce_account_processed_monitor_transactions(
        account: Account, processed_monitor_transactions: List[ProcessedMonitorTransaction]
):
    producer = get_kafka_producer(processed_monitor_transaction_async_producer())
    count = 0
    for processed_monitor_transaction in processed_monitor_transactions:
        key = ProcessedMonitorTransactionPayloadKey()
        value = ProcessedMonitorTransactionPayloadValue(
            account_id=UInt64Value(value=account.id),
            processed_monitor_transaction=processed_monitor_transaction,
        )
        producer.process(key, value)
        count += 1
    producer.flush()

    return count


class ProcessedMonitorTransactionClickhouseIngestProcessor(Processor):
    key_cls = ProcessedMonitorTransactionPayloadKey
    value_cls = ProcessedMonitorTransactionPayloadValue

    def process(self, keys: List[ProcessedMonitorTransactionPayloadKey],
                values: List[ProcessedMonitorTransactionPayloadValue]):
        account_processed_monitor_transactions_dict = {}
        for key, value in zip(keys, values):
            account_id = value.account_id.value
            processed_monitor_transactions = value.processed_monitor_transaction

            processed_monitor_transactions_list: List[
                ProcessedMonitorTransaction] = account_processed_monitor_transactions_dict.get(account_id, [])
            processed_monitor_transactions_list.append(processed_monitor_transactions)
            account_processed_monitor_transactions_dict[account_id] = processed_monitor_transactions_list

        for account_id, processed_monitor_transactions_list in account_processed_monitor_transactions_dict.items():
            ingest_processed_monitor_transactions(processed_monitor_transactions_list)
            print(f"Kafka processed-monitor-transactions Consume Count : {len(processed_monitor_transactions_list)} "
                  f"for account_id: {account_id}")
        return

event/processors/processed_monitor_transaction_processor.py
DrDroidLab/kenobi
from typing import List

from django.conf import settings
from google.protobuf.wrappers_pb2 import UInt64Value

from accounts.models import Account
from event.processors.process_events import process_account_events
from protos.event.base_pb2 import Event as EventProto
from protos.event.schema_pb2 import IngestionEvent
from protos.kafka.raw_events_pb2 import RawEventPayloadKey, RawEventPayloadValue
from prototype.kafka.processor import Processor
from prototype.kafka.producer import get_kafka_producer
from prototype.utils.utils import current_milli_time


def is_event_processing_async():
    if settings.RAW_EVENT_PROCESSING:
        return settings.RAW_EVENT_PROCESSING.get('async', False)
    return False


def event_processing_async_producer():
    if settings.RAW_EVENT_PROCESSING:
        return settings.RAW_EVENT_PROCESSING.get('producer', 'raw_events')
    return 'raw_events'


def process_account_events_async(
        account: Account, events: List[IngestionEvent],
        source: EventProto.EventSource = EventProto.EventSource.SDK
):
    producer = get_kafka_producer('raw_events')
    count = 0
    processing_time = current_milli_time()
    for event in events:
        event_name: str = event.name
        if not event_name:
            continue

        if not event.timestamp:
            event.timestamp = processing_time

        key = RawEventPayloadKey()
        value = RawEventPayloadValue(
            account_id=UInt64Value(value=account.id),
            event=event,
            event_source=source
        )
        producer.process(key, value)
        count += 1
    producer.flush()

    return count


class RawEventProcessor(Processor):
    key_cls = RawEventPayloadKey
    value_cls = RawEventPayloadValue

    def process(self, keys: List[RawEventPayloadKey], values: List[RawEventPayloadValue]):
        account_source_dict = {}
        for key, value in zip(keys, values):
            account = value.account_id.value
            source = value.event_source

            source_event_dict = account_source_dict.get(account, {})
            events = source_event_dict.get(source, [])
            if value.event:
                events.append(value.event)

            source_event_dict[source] = events
            account_source_dict[account] = source_event_dict

        for account, source_event_dict in account_source_dict.items():
            account_obj = Account(id=account)
            for source, events in source_event_dict.items():
                saved_count = process_account_events(account_obj, events, source)
                print(
                    f"Kafka raw-events Consumer Saved events Count : {saved_count} for account: {account_obj.id} and consumed events count: {len(events)}")

        return

event/processors/raw_events_processor.py
DrDroidLab/kenobi
from typing import List

from django.conf import settings

from event.processors.process_monitors import process_raw_monitor_transactions
from protos.kafka.raw_monitor_transactions_pb2 import RawMonitorTransactionPayloadKey, RawMonitorTransactionPayloadValue
from prototype.kafka.processor import Processor
from prototype.kafka.producer import get_kafka_producer


def is_monitor_transaction_processing_async():
    if settings.RAW_MONITOR_TRANSACTION_PROCESSING:
        return settings.RAW_MONITOR_TRANSACTION_PROCESSING.get('async', False)
    return False


def monitor_transaction_processing_async_producer():
    if settings.RAW_MONITOR_TRANSACTION_PROCESSING:
        return settings.RAW_MONITOR_TRANSACTION_PROCESSING.get('producer', 'raw_monitor_transactions')
    return 'raw_monitor_transactions'


def process_raw_monitor_transactions_async(
        keys: List[RawMonitorTransactionPayloadKey], values: List[RawMonitorTransactionPayloadValue]
):
    producer = get_kafka_producer(monitor_transaction_processing_async_producer())
    count = 0
    for key, value in zip(keys, values):
        producer.process(key, value)
        count += 1
    producer.flush()

    return count


class RawMonitorTransactionProcessor(Processor):
    key_cls = RawMonitorTransactionPayloadKey
    value_cls = RawMonitorTransactionPayloadValue

    def process(self, keys: List[RawMonitorTransactionPayloadKey], values: List[RawMonitorTransactionPayloadValue]):
        return process_raw_monitor_transactions(values)

event/processors/raw_monitor_transactions_processor.py
DrDroidLab/kenobi
from protos.event.base_pb2 import EventKey as EventKeyProto
from protos.event.schema_pb2 import Value


def infer_key_type(v: Value):
    which_one_of = v.WhichOneof('value')
    if which_one_of == 'string_value':
        return EventKeyProto.KeyType.STRING
    elif which_one_of == 'int_value':
        return EventKeyProto.KeyType.LONG
    elif which_one_of == 'double_value':
        return EventKeyProto.KeyType.DOUBLE
    elif which_one_of == 'bool_value':
        return EventKeyProto.KeyType.BOOLEAN
    elif which_one_of == 'byte_value':
        return EventKeyProto.KeyType.BYTE
    elif which_one_of == 'array_value':
        return EventKeyProto.KeyType.ARRAY
    elif which_one_of == 'kvlist_value':
        return EventKeyProto.KeyType.OBJECT
    return EventKeyProto.KeyType.UNKNOWN

event/processors/utils.py
DrDroidLab/kenobi
import csv
import datetime
import json
import time

import pytz
from celery import shared_task, group
from django.conf import settings
from django.core.mail import EmailMessage
from django.db.models import F, FloatField
from django.db.models.functions import Cast
from django.http import HttpResponse

from accounts.models import Account
from event.cache import GLOBAL_EXPORT_CONTEXT_CACHE
from event.clickhouse.models import Events
from event.engine.query_engine import global_event_search_clickhouse_engine, monitor_transaction_search_engine
from event.models import Alert, MonitorTransactionEventMapping
from event.notification.notification_facade import notification_client
from event.triggers.trigger_processor import process_trigger
from event.triggers.entity_trigger_processor import process_trigger as process_entity_trigger
from management.models import PeriodicTaskStatus
from management.utils.celery_task_signal_utils import publish_pre_run_task, publish_post_run_task, publish_failure_task, \
    get_or_create_task, check_scheduled_or_running_task_run_for_task, \
    create_task_run
from protos.event.api_pb2 import GetEventExportQueryRequest
from protos.event.base_pb2 import Context
from protos.event.entity_pb2 import EntityTriggerDefinition
from protos.event.query_base_pb2 import QueryRequest
from protos.event.trigger_pb2 import TriggerDefinition, DelayedEventTrigger
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto
from prototype.db.decorator import use_read_replica
from prototype.utils.timerange import filter_dtr, to_dtr, DateTimeRange
from utils.proto_utils import dict_to_proto, get_value


def get_monitor_transaction_status(status):
    if status == 2:
        return 'Finished'
    return 'Active'


@shared_task
def missing_event_trigger_cron(lookback_interval=None):
    current_time = time.time()
    # publish_pre_run_task(sender='evaluate_missing_event_triggers_for')
    if not lookback_interval or lookback_interval <= 0:
        # adding backlog step to populate alerts for last 3 hours in seconds
        lookback_interval = 10

    tasks = []
    for ac in Account.objects.all():
        # All monitor triggers
        triggers = ac.trigger_set.filter(type=TriggerDefinition.Type.MISSING_EVENT).exclude(is_active=False).exclude(
            monitor__isnull=True).exclude(monitor__is_active=False).values('id', 'type', 'monitor_id', 'config', 'name',
                                                                           'generated_config',
                                                                           'monitor__name',
                                                                           'monitor__primary_key__name',
                                                                           'monitor__secondary_key__event_type__name',
                                                                           'monitor__primary_key__event_type__name')
        for trigger in triggers:
            expected_transaction_time_sec = trigger['config']['transaction_time_threshold']
            time_lower_bound = current_time - expected_transaction_time_sec
            time_upper_bound = time_lower_bound - lookback_interval

            saved_task = get_or_create_task(evaluate_missing_event_triggers_for.__name__, ac.id, trigger,
                                            time_lower_bound, time_upper_bound)

            if check_scheduled_or_running_task_run_for_task(saved_task):
                continue

            task_run = evaluate_missing_event_triggers_for.s(ac.id, trigger, time_lower_bound,
                                                             time_upper_bound)
            task_run_meta = task_run.freeze()

            # Avoid alert noise if expected transaction time is less than a second
            if expected_transaction_time_sec <= 0:
                create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                                status=PeriodicTaskStatus.SKIPPED)
                continue

            create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                            status=PeriodicTaskStatus.SCHEDULED)

            tasks.append(task_run)

        # All Entity triggers
        triggers = ac.entitytrigger_set.filter(type=EntityTriggerDefinition.TriggerType.PER_EVENT).exclude(
            is_active=False).exclude(
            entity__isnull=True).exclude(entity__is_active=False).values('id', 'type', 'rule_type', 'entity_id', 'config', 'name',
                                                                         'generated_config',
                                                                         'entity__name')

        for trigger in triggers:
            config_time_interval_sec = int(trigger['config'].get('time_interval', 0.0))
            time_lower_bound = current_time - config_time_interval_sec
            time_upper_bound = time_lower_bound - lookback_interval

            saved_task = get_or_create_task(evaluate_entity_event_level_triggers.__name__, ac.id, trigger,
                                            time_lower_bound, time_upper_bound)

            if check_scheduled_or_running_task_run_for_task(saved_task):
                continue

            task_run = evaluate_entity_event_level_triggers.s(ac.id, trigger, time_lower_bound,
                                                              time_upper_bound)
            task_run_meta = task_run.freeze()

            create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                            status=PeriodicTaskStatus.SCHEDULED)

            tasks.append(task_run)

    group(tasks).apply_async()


@shared_task(max_retries=3)
def evaluate_entity_event_level_triggers(account_id, trigger, time_lower_bound, time_upper_bound):
    ac: Account = Account.objects.get(id=account_id)
    time_lower_bound_datetime = datetime.datetime.fromtimestamp(time_lower_bound, tz=pytz.utc)
    time_upper_bound_datetime = datetime.datetime.fromtimestamp(time_upper_bound, tz=pytz.utc)
    alerts: [Alert] = process_entity_trigger(ac, trigger, time_lower_bound_datetime, time_upper_bound_datetime)

    notify(ac, trigger, alerts)


@shared_task(max_retries=3)
def evaluate_missing_event_triggers_for(account_id, trigger, time_lower_bound, time_upper_bound):
    ac: Account = Account.objects.get(id=account_id)
    time_lower_bound_datetime = datetime.datetime.fromtimestamp(time_lower_bound, tz=pytz.utc)
    time_upper_bound_datetime = datetime.datetime.fromtimestamp(time_upper_bound, tz=pytz.utc)
    alerts: [Alert] = process_trigger(ac, trigger, time_lower_bound_datetime, time_upper_bound_datetime)

    notify(ac, trigger, alerts)


missing_event_trigger_cron_task_prerun_notifier = publish_pre_run_task(evaluate_missing_event_triggers_for)
missing_event_trigger_cron_task_failure_notifier = publish_failure_task(evaluate_missing_event_triggers_for)
missing_event_trigger_cron_task_postrun_notifier = publish_post_run_task(evaluate_missing_event_triggers_for)


@shared_task
def delayed_event_trigger_cron(lookback_interval=None):
    current_time = time.time()
    if not lookback_interval or lookback_interval <= 0:
        # adding backlog step to populate alerts for last 60 seconds
        lookback_interval = 60

    tasks = []
    for ac in Account.objects.all():
        triggers = ac.trigger_set.filter(type=TriggerDefinition.Type.DELAYED_EVENT).exclude(is_active=False).exclude(
            monitor__isnull=True).exclude(monitor__is_active=False).values('id', 'type', 'monitor_id', 'config',
                                                                           'generated_config',
                                                                           'name',
                                                                           'monitor__name',
                                                                           'monitor__secondary_key__event_type__name',
                                                                           'monitor__primary_key__event_type__name')
        for trigger in triggers:
            trigger_config = dict_to_proto(trigger["config"], DelayedEventTrigger)
            expected_transaction_time_sec = trigger_config.transaction_time_threshold.value
            trigger_resolution = trigger_config.resolution.value
            trigger_threshold = trigger_config.trigger_threshold.value

            time_lower_bound = current_time - expected_transaction_time_sec - lookback_interval
            time_upper_bound = time_lower_bound - trigger_resolution

            saved_task = get_or_create_task(evaluate_delayed_event_triggers_for.__name__, ac.id, trigger,
                                            time_lower_bound, time_upper_bound)

            if check_scheduled_or_running_task_run_for_task(saved_task):
                continue

            task_run = evaluate_delayed_event_triggers_for.s(ac.id, trigger, time_lower_bound, time_upper_bound)
            task_run_meta = task_run.freeze()

            # Avoid alert noise if resolution window is zero or transaction time is less than a second
            if trigger_resolution < settings.DELAYED_EVENT_TRIGGER_RESOLUTION_WINDOW_LOWER_BOUND_IN_SEC or \
                    expected_transaction_time_sec <= 0 or trigger_threshold < 0:
                create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                                status=PeriodicTaskStatus.SKIPPED)
                continue

            # Avoid alert noise if alert already generated in the current resolution window
            check_alert_time_lower_bound = datetime.datetime.fromtimestamp(current_time, tz=pytz.utc)
            check_alert_time_upper_bound = datetime.datetime.fromtimestamp(current_time - trigger_resolution,
                                                                           tz=pytz.utc)
            if ac.alert_set.filter(trigger_id=trigger['id']).filter(
                    triggered_at__range=[check_alert_time_upper_bound, check_alert_time_lower_bound]).count() > 0:
                create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                                status=PeriodicTaskStatus.SKIPPED)
                continue

            create_task_run(task=saved_task, task_uuid=task_run_meta.task_id,
                            status=PeriodicTaskStatus.SCHEDULED)
            tasks.append(task_run)

    group(tasks).apply_async()


@shared_task(max_retries=3)
def evaluate_delayed_event_triggers_for(account_id, trigger, time_lower_bound, time_upper_bound):
    ac: Account = Account.objects.get(id=account_id)
    time_lower_bound_datetime = datetime.datetime.fromtimestamp(time_lower_bound, tz=pytz.utc)
    time_upper_bound_datetime = datetime.datetime.fromtimestamp(time_upper_bound, tz=pytz.utc)
    alerts: [Alert] = process_trigger(ac, trigger, time_lower_bound_datetime, time_upper_bound_datetime)

    notify(ac, trigger, alerts)


delayed_event_trigger_cron_task_prerun_notifier = publish_pre_run_task(evaluate_delayed_event_triggers_for)
delayed_event_trigger_cron_task_failure_notifier = publish_failure_task(evaluate_delayed_event_triggers_for)
delayed_event_trigger_cron_task_postrun_notifier = publish_post_run_task(evaluate_delayed_event_triggers_for)


def notify(account: Account, trigger, alerts: [Alert]):
    for alert in alerts:
        notifications = []
        if alert.trigger:
            notifications = account.notification_set.filter(trigger=trigger["id"])
        if alert.entity_trigger:
            notifications = account.notification_set.filter(entity_trigger=trigger["id"])
        notification_client.notify(account, notifications, trigger, alert)


@shared_task(max_retries=3)
def export_task(account_id, export_context: Context, user_email, request_message_json):
    if not account_id or not export_context or not user_email or not request_message_json:
        return False, "Invalid request"

    if not GLOBAL_EXPORT_CONTEXT_CACHE.get(account_id, export_context, user_email):
        return False, "Export context not found"

    export_id = GLOBAL_EXPORT_CONTEXT_CACHE.get(account_id, export_context, user_email)
    if not export_id:
        return False, "Export id not found"

    if export_context == Context.EVENT:
        event_export_task.s(account_id, export_id, user_email, request_message_json).apply_async()
    elif export_context == Context.MONITOR_TRANSACTION:
        monitor_transaction_export_task.s(account_id, export_id, user_email, request_message_json).apply_async()

    GLOBAL_EXPORT_CONTEXT_CACHE.delete(account_id, export_context, user_email)
    return True, ""


@shared_task(max_retries=3)
def event_export_task(account_id, export_id, user_email, request_message_json):
    request_message = dict_to_proto(request_message_json, GetEventExportQueryRequest)
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request: QueryRequest = request_message.query_request

    qs = Events.objects.all()
    qs = filter_dtr(qs, dtr, 'timestamp')
    qs = qs.filter(account_id=account_id)
    qs = global_event_search_clickhouse_engine.process_query(qs, query_request)
    qs = global_event_search_clickhouse_engine.process_ordering(qs, query_request)

    event_protos = [e.proto for e in qs]

    csv_data = [['ID', 'Timestamp', 'Event Type', 'Attributes']]
    for event in event_protos:
        attributes = {}
        for kv in event.kvs:
            attributes[kv.key] = get_value(kv.value)
        csv_data.append([event.id, event.timestamp, event.event_type.name, json.dumps(attributes)])

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="file.csv"'
    writer = csv.writer(response)

    for row in csv_data:
        writer.writerow(row)

    email = EmailMessage(
        subject="[Doctor Droid] Event Export: {}".format(export_id),
        body="PFA the exported file for export request: {}".format(export_id),
        from_email=None,
        to=[user_email],
    )

    email.attach('drdroid_events_export.csv', response.getvalue(), 'text/csv')
    email.send()


@shared_task(max_retries=3)
def monitor_transaction_export_task(account_id, export_id, user_email, request_message_json):
    request_message = dict_to_proto(request_message_json, GetEventExportQueryRequest)
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request: QueryRequest = request_message.query_request

    account = Account.objects.get(id=account_id)
    qs = account.monitortransaction_set.all()
    qs = filter_dtr(qs, dtr, 'created_at')
    qs = monitor_transaction_search_engine.process_query(qs, query_request)
    qs = qs.annotate(transaction_time=(Cast(F('monitortransactionstats__stats__transaction_time'), FloatField())))

    qs = qs.select_related('monitor')
    mt_protos = [mt.proto_for_export for mt in qs]

    mt_ids = list(x.id for x in mt_protos)
    mt_events_mapping = MonitorTransactionEventMapping.objects.filter(account_id=account.id,
                                                                      monitor_transaction_id__in=mt_ids).values_list(
        'monitor_transaction_id', 'type', 'event_id')

    mt_events_map = {}
    for mapping in mt_events_mapping:
        if mapping[0] not in mt_events_map:
            mt_events_map[mapping[0]] = []
        mt_events_map[mapping[0]].append(mapping)

    events_map = {}
    events = Events.objects.filter(account_id=account.id, id__in=list(x[2] for x in mt_events_mapping)).values_list(
        'id', 'event_type_name', 'timestamp', 'ingested_event', 'processed_kvs')
    for event in events:
        events_map[event[0]] = event

    csv_data = []

    primary_event_keys_seq = []
    secondary_event_keys_seq = []

    finished_txn = qs.filter(type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED).first()
    if finished_txn:
        sample_mt_proto = finished_txn.proto_for_export
    else:
        sample_mt_proto = mt_protos[0]

    csv_data_row = ['Transaction Value', 'Monitor Name', 'Status', 'Has Alerts', 'Transaction Time']
    primary_event = list(filter(lambda x: x[1] == 1, mt_events_map[sample_mt_proto.id]))
    if primary_event:
        primary_event = primary_event[0]
        event_object = events_map[primary_event[2]]
        csv_data_row.append('primary_event_type')
        csv_data_row.append('primary_event_timestamp')
        for kv in json.loads(event_object[3])['kvs']:
            csv_data_row.append("primary_" + kv['key'])
            primary_event_keys_seq.append(kv['key'])

    secondary_event = list(filter(lambda x: x[1] == 2, mt_events_map[sample_mt_proto.id]))
    if secondary_event:
        secondary_event = secondary_event[0]
        event_object = events_map[secondary_event[2]]
        csv_data_row.append('secondary_event_type')
        csv_data_row.append('secondary_event_timestamp')
        for kv in json.loads(event_object[3])['kvs']:
            csv_data_row.append("secondary_" + kv['key'])
            secondary_event_keys_seq.append(kv['key'])

    csv_data.append(csv_data_row)

    for mt in mt_protos:
        csv_data_row = [mt.transaction, mt.monitor.name, get_monitor_transaction_status(mt.status), mt.has_alerts.value,
                        mt.transaction_time.value]
        primary_event = list(filter(lambda x: x[1] == 1, mt_events_map[mt.id]))
        if primary_event:
            primary_event = primary_event[0]
            event_object = events_map[primary_event[2]]
            csv_data_row.append(event_object[1])
            csv_data_row.append(event_object[2].strftime('%Y-%m-%d %H:%M:%S'))
            for k in primary_event_keys_seq:
                csv_data_row.append(event_object[4].get(k, None))

        secondary_event = list(filter(lambda x: x[1] == 2, mt_events_map[mt.id]))
        if secondary_event:
            secondary_event = secondary_event[0]
            event_object = events_map[secondary_event[2]]
            csv_data_row.append(event_object[1])
            csv_data_row.append(event_object[2].strftime('%Y-%m-%d %H:%M:%S'))
            for k in secondary_event_keys_seq:
                csv_data_row.append(event_object[4].get(k, None))

        csv_data.append(csv_data_row)

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="file.csv"'
    writer = csv.writer(response)

    for row in csv_data:
        writer.writerow(row)

    email = EmailMessage(
        subject="[Doctor Droid] Event Export: {}".format(export_id),
        body="PFA the exported file for export request: {}".format(export_id),
        from_email=None,
        to=[user_email],
    )

    email.attach('drdroid_monitor_transaction_export.csv', response.getvalue(), 'text/csv')
    email.send()

event/tasks.py
DrDroidLab/kenobi
import datetime
import statistics
from typing import Dict, List

from django.db.models import ExpressionWrapper, F, fields, Subquery, OuterRef
from google.protobuf.wrappers_pb2 import UInt32Value, DoubleValue, UInt64Value

from accounts.models import Account
from event.models import MonitorTransaction, Alert, AlertMonitorTransactionMapping, MonitorTransactionEventMapping, \
    EntityInstanceEventMapping, Event, AlertEntityInstanceMapping
from event.triggers.trigger_processor import TriggerProcessor
from protos.event.alert_pb2 import AlertStats, AlertMonitorTransaction, DelayedMonitorTransactionStats, \
    AlertEntityInstance
from protos.event.entity_pb2 import EntityTriggerDefinition
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto
from protos.event.options_pb2 import TriggerOption
from protos.event.query_base_pb2 import OpDescription, Op, Filter
from protos.event.trigger_pb2 import TriggerDefinition, DelayedEventTrigger, MissingEventTrigger, \
    BasicTriggerFilterOptions, DelayedEventTriggerOptions
from utils.proto_utils import proto_to_dict, dict_to_proto

from event.engine.query_engine import global_event_search_engine

_BASIC_TRIGGER_FILTER_OP_DESCRIPTIONS = [
    OpDescription(op=Op.EQ, label='=', is_unary=False)
]


def process_last_event_entity_trigger(scope, entity_instance_mapping_list, config_event_id):
    alert_data = []
    for instance_mapping in entity_instance_mapping_list:
        event_name = instance_mapping.event_key.event_type.name
        event_key_name = instance_mapping.event_key.name
        entity_instance_id = instance_mapping.entity_instance_id
        later_event_mappings = (EntityInstanceEventMapping.objects.using('replica1').filter(account_id=scope.id,
                                                                            entity_instance_id=entity_instance_id,
                                                                            event_timestamp__gt=instance_mapping.event_timestamp)
                                .exclude(event_key__event_type_id=config_event_id))
        if not later_event_mappings or len(later_event_mappings) == 0:
            alert_data.append({"entity_instance_value": instance_mapping.entity_instance.instance, "entity_instance_id": instance_mapping.entity_instance.id, "event_key_name": event_key_name,
                               "event_timestamp": int(instance_mapping.event_timestamp.timestamp()), "event_name": event_name})
    return alert_data


def process_event_count_entity_trigger(scope, entity_instance_mapping_list, config_event_id, config_time_interval_sec, config_threshold_count):
    alert_data = []
    for instance_mapping in entity_instance_mapping_list:
        event_name = instance_mapping.event_key.event_type.name
        event_key_name = instance_mapping.event_key.name
        entity_instance_id = instance_mapping.entity_instance_id
        later_event_mappings = EntityInstanceEventMapping.objects.using('replica1').filter(account_id=scope.id,
                                                                            entity_instance_id=entity_instance_id,
                                                                            event_timestamp__lt=instance_mapping.event_timestamp,
                                                                          event_timestamp__gt=(instance_mapping.event_timestamp - datetime.timedelta(seconds=config_time_interval_sec)),
                                                                          event_key__event_type_id=config_event_id)
        if len(later_event_mappings) > config_threshold_count - 1:
            alert_data.append({"entity_instance_value": instance_mapping.entity_instance.instance, "entity_instance_id": instance_mapping.entity_instance.id, "event_key_name": event_key_name,
                               "event_timestamp": int(instance_mapping.event_timestamp.timestamp()), "event_name": event_name})
    return alert_data


def process_event_occurrence_entity_trigger(entity_instance_mapping_list):
    alert_data = []
    for instance_mapping in entity_instance_mapping_list:
        event_name = instance_mapping.event_key.event_type.name
        event_key_name = instance_mapping.event_key.name
        alert_data.append({"entity_instance_value": instance_mapping.entity_instance.instance, "entity_instance_id": instance_mapping.entity_instance.id, "event_key_name": event_key_name,
                               "event_timestamp": int(instance_mapping.event_timestamp.timestamp()), "event_name": event_name})
    return alert_data


class PerEventTriggerProcessor(TriggerProcessor):

    @classmethod
    def get_default_config(cls):
        return MissingEventTrigger(transaction_time_threshold=DoubleValue(value=10.0))

    @classmethod
    def evaluate_triggers(cls, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        generated_alerts: [Alert] = []
        trigger_id = trigger['id']
        trigger_name = trigger['name']
        entity_id = trigger['entity_id']
        config_time_interval_sec = int(trigger['config'].get('time_interval', 0))
        config_event_id = trigger['config'].get('event_id')
        config_threshold_count = int(trigger['config'].get('threshold_count', 0))
        trigger_rule_type = trigger['rule_type']
        filter_config = trigger['generated_config']

        if trigger_rule_type in [EntityTriggerDefinition.EntityTriggerRuleType.LAST_EVENT]:
            last_event_check_start_time = time_upper_bound
            last_event_check_end_time = time_lower_bound
            check_interval_entity_instance_mapping_list = EntityInstanceEventMapping.objects.using('replica1').filter(account_id=scope.id, entity_id=entity_id,
                                                                                                    event_key__event_type_id=config_event_id,
                                                                                                    event_timestamp__range=[
                                                                                                        last_event_check_start_time,
                                                                                                        last_event_check_end_time])
            filter_events = global_event_search_engine.process_query_with_filter(Event.objects.using('replica1').filter(account=scope, timestamp__range=[
                                                                                                        last_event_check_start_time,
                                                                                                        last_event_check_end_time]), dict_to_proto(filter_config, Filter))
            check_interval_entity_instance_mapping_list = check_interval_entity_instance_mapping_list.filter(event__in=filter_events)

            last_event_alerts = process_last_event_entity_trigger(scope, check_interval_entity_instance_mapping_list, config_event_id)
            for alert_data in last_event_alerts:
                saved_alert = Alert.objects.create(account=scope, entity_trigger_id=trigger_id, stats=alert_data)

                AlertEntityInstanceMapping.objects.get_or_create(account=scope,
                                                                     alert=saved_alert,
                                                                     entity_instance_id=alert_data.get('entity_instance_id'),
                                                                     type=AlertEntityInstance.Type.PER_EVENT)
                generated_alerts.append(saved_alert)

        if trigger_rule_type in [EntityTriggerDefinition.EntityTriggerRuleType.EVENT_COUNT]:
            last_event_check_start_time = time_upper_bound
            last_event_check_end_time = time_lower_bound
            check_interval_entity_instance_mapping_list = EntityInstanceEventMapping.objects.using('replica1').filter(account_id=scope.id,
                                                                                                    entity_id=entity_id,
                                                                                                    event_key__event_type_id=config_event_id,
                                                                                                    event_timestamp__range=[
                                                                                                        last_event_check_start_time,
                                                                                                        last_event_check_end_time])
            filter_events = global_event_search_engine.process_query_with_filter(
                Event.objects.using('replica1').filter(account=scope, timestamp__range=[
                            last_event_check_start_time,
                            last_event_check_end_time]), dict_to_proto(filter_config, Filter))
            check_interval_entity_instance_mapping_list = check_interval_entity_instance_mapping_list.filter(event__in=filter_events)
            event_count_alerts = process_event_count_entity_trigger(scope, check_interval_entity_instance_mapping_list, config_event_id, config_time_interval_sec, config_threshold_count)
            for alert_data in event_count_alerts:
                saved_alert = Alert.objects.create(account=scope, entity_trigger_id=trigger_id, stats=alert_data)

                AlertEntityInstanceMapping.objects.get_or_create(account=scope,
                                                                 alert=saved_alert,
                                                                 entity_instance_id=alert_data.get(
                                                                     'entity_instance_id'),
                                                                 type=AlertEntityInstance.Type.PER_EVENT)
                generated_alerts.append(saved_alert)

        if trigger_rule_type in [EntityTriggerDefinition.EntityTriggerRuleType.EVENT_OCCURS]:
            check_interval_entity_instance_mapping_list = EntityInstanceEventMapping.objects.using('replica1').filter(account_id=scope.id, entity_id=entity_id,
                                                                                                    event_key__event_type_id=config_event_id,
                                                                                                    event_timestamp__range=[
                                                                                                        time_upper_bound,
                                                                                                        time_lower_bound])
            filter_events = global_event_search_engine.process_query_with_filter(
                Event.objects.using('replica1').filter(account=scope, timestamp__range=[
                        time_upper_bound,
                        time_lower_bound]), dict_to_proto(filter_config, Filter))
            check_interval_entity_instance_mapping_list = check_interval_entity_instance_mapping_list.filter(event__in=filter_events)
            event_occurrence_alerts = process_event_occurrence_entity_trigger(check_interval_entity_instance_mapping_list)
            for alert_data in event_occurrence_alerts:
                saved_alert = Alert.objects.create(account=scope, entity_trigger_id=trigger_id, stats=alert_data)

                AlertEntityInstanceMapping.objects.get_or_create(account=scope,
                                                                 alert=saved_alert,
                                                                 entity_instance_id=alert_data.get(
                                                                     'entity_instance_id'),
                                                                 type=AlertEntityInstance.Type.PER_EVENT)
                generated_alerts.append(saved_alert)

        return generated_alerts


per_event_trigger_processor = PerEventTriggerProcessor()


class TriggerProcessorFacade:

    def __init__(self):
        self._map = {}

    def register(self, trigger_type: TriggerDefinition.Type, typed_trigger_processor: TriggerProcessor):
        self._map[trigger_type] = typed_trigger_processor

    def process_triggers(self, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        return self._map[trigger["type"]].evaluate_triggers(scope, trigger, time_lower_bound, time_upper_bound)


trigger_processor = TriggerProcessorFacade()
trigger_processor.register(EntityTriggerDefinition.TriggerType.PER_EVENT, PerEventTriggerProcessor())


def process_trigger(scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
    return trigger_processor.process_triggers(scope, trigger, time_lower_bound, time_upper_bound)

event/triggers/entity_trigger_processor.py
DrDroidLab/kenobi
import datetime
import statistics
from typing import Dict, List

from django.db.models import ExpressionWrapper, F, fields, Subquery, OuterRef
from google.protobuf.wrappers_pb2 import UInt32Value, DoubleValue, UInt64Value

from accounts.models import Account
from event.models import MonitorTransaction, Alert, AlertMonitorTransactionMapping, MonitorTransactionEventMapping
from protos.event.alert_pb2 import AlertStats, AlertMonitorTransaction, DelayedMonitorTransactionStats
from protos.event.monitor_pb2 import MonitorTransaction as MonitorTransactionProto
from protos.event.options_pb2 import TriggerOption
from protos.event.query_base_pb2 import OpDescription, Op
from protos.event.trigger_pb2 import TriggerDefinition, DelayedEventTrigger, MissingEventTrigger, \
    BasicTriggerFilterOptions, DelayedEventTriggerOptions
from utils.proto_utils import proto_to_dict, dict_to_proto

_BASIC_TRIGGER_FILTER_OP_DESCRIPTIONS = [
    OpDescription(op=Op.EQ, label='=', is_unary=False)
]


def process_missing_event_monitor_transactions(scope, trigger, time_upper_bound, time_lower_bound,
                                               expected_transaction_time):
    delay_duration = ExpressionWrapper(F('secondary_event_timestamp') - F('primary_event_timestamp'),
                                       output_field=fields.DurationField())

    primary_filter = trigger["generated_config"].get('primary_event_filters', {})
    secondary_filter = trigger["generated_config"].get('secondary_event_filters', {})

    if not primary_filter or not secondary_filter:
        return {"monitor_transactions_qs": [],
                "missing_event_monitor_transactions_qs": [],
                "delayed_event_monitor_transactions_qs": []}

    monitor_transactions: [MonitorTransaction] = scope.monitortransaction_set.using('replica1').filter(
        monitor_id=trigger['monitor_id']).filter(created_at__range=[time_upper_bound, time_lower_bound]).annotate(
        primary_event_timestamp=Subquery(
            MonitorTransactionEventMapping.objects.using('replica1').filter(monitor_transaction=OuterRef("pk")).filter(
                **primary_filter).values('event__timestamp').order_by('event__timestamp')[:1]
        )
    ).filter(primary_event_timestamp__isnull=False)

    missing_event_monitor_transactions: [MonitorTransaction] = monitor_transactions.filter(
        type=MonitorTransactionProto.MonitorTransactionStatus.PRIMARY_RECEIVED)

    # handle cases where secondary event is received but not satisfying the secondary filters
    missing_filtered_event_monitor_transactions = []
    if secondary_filter:
        missing_filtered_event_monitor_transactions: [MonitorTransaction] = monitor_transactions.annotate(
            secondary_event_timestamp=Subquery(
                MonitorTransactionEventMapping.objects.using('replica1').filter(monitor_transaction=OuterRef("pk")).filter(
                    **secondary_filter).values('event__timestamp')[:1])
        ).filter(secondary_event_timestamp__isnull=True).filter(
            type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED)

    delayed_event_monitor_transactions: [MonitorTransaction] = monitor_transactions.annotate(
        secondary_event_timestamp=Subquery(
            MonitorTransactionEventMapping.objects.using('replica1').filter(monitor_transaction=OuterRef("pk")).filter(
                **secondary_filter).values('event__timestamp')[:1])
    ).filter(secondary_event_timestamp__isnull=False).filter(
        type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED).annotate(
        duration=delay_duration).filter(
        duration__gt=datetime.timedelta(seconds=expected_transaction_time))

    return {"monitor_transactions_qs": monitor_transactions,
            "missing_event_monitor_transactions_qs": missing_event_monitor_transactions,
            "missing_filtered_event_monitor_transactions_qs": missing_filtered_event_monitor_transactions,
            "delayed_event_monitor_transactions_qs": delayed_event_monitor_transactions}


def process_delayed_event_monitor_transactions(scope, trigger, time_upper_bound, time_lower_bound,
                                               expected_transaction_time,
                                               trigger_config_type: DelayedEventTrigger.Type):
    delay_duration = ExpressionWrapper(F('secondary_event_timestamp') - F('primary_event_timestamp'),
                                       output_field=fields.DurationField())

    primary_filter = trigger["generated_config"].get('primary_event_filters', {})
    secondary_filter = trigger["generated_config"].get('secondary_event_filters', {})

    if not primary_filter or not secondary_filter:
        return {"monitor_transactions_qs": [],
                "missing_event_monitor_transactions_qs": [],
                "delayed_event_monitor_transactions_qs": []}

    monitor_transactions: [MonitorTransaction] = scope.monitortransaction_set.using('replica1').filter(
        monitor_id=trigger['monitor_id']).filter(created_at__range=[time_upper_bound, time_lower_bound]).annotate(
        primary_event_timestamp=Subquery(
            MonitorTransactionEventMapping.objects.using('replica1').filter(monitor_transaction=OuterRef("pk")).filter(
                **primary_filter).values('event__timestamp').order_by('event__timestamp')[:1]
        )
    ).filter(primary_event_timestamp__isnull=False)

    missing_event_monitor_transactions: [MonitorTransaction] = monitor_transactions.filter(
        type=MonitorTransactionProto.MonitorTransactionStatus.PRIMARY_RECEIVED)

    delayed_event_monitor_transactions: [MonitorTransaction] = monitor_transactions.annotate(
        secondary_event_timestamp=Subquery(
            MonitorTransactionEventMapping.objects.filter(monitor_transaction=OuterRef("pk")).filter(
                **secondary_filter).values('event__timestamp')[:1])
    ).filter(secondary_event_timestamp__isnull=False).filter(
        type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED).annotate(
        duration=delay_duration).filter(
        duration__gt=datetime.timedelta(seconds=expected_transaction_time))

    if trigger_config_type == DelayedEventTrigger.Type.DELAYED_EVENTS:
        missing_event_monitor_transactions = []

        monitor_transactions: [MonitorTransaction] = scope.monitortransaction_set.using('replica1').filter(
            monitor_id=trigger['monitor_id']).filter(created_at__range=[time_upper_bound, time_lower_bound]).filter(
            type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED).annotate(
            primary_event_timestamp=Subquery(
                MonitorTransactionEventMapping.objects.using('replica1').filter(monitor_transaction=OuterRef("pk")).filter(
                    **primary_filter).values('event__timestamp').order_by('event__timestamp')[:1]
            )
        ).filter(primary_event_timestamp__isnull=False)

    elif trigger_config_type == DelayedEventTrigger.Type.MISSING_EVENTS:
        delayed_event_monitor_transactions: [MonitorTransaction] = []

    return {"monitor_transactions_qs": monitor_transactions,
            "missing_event_monitor_transactions_qs": missing_event_monitor_transactions,
            "delayed_event_monitor_transactions_qs": delayed_event_monitor_transactions}


class TriggerProcessorException(ValueError):
    pass


class TriggerProcessor:

    @classmethod
    def get_default_config(cls) -> Dict:
        pass

    @classmethod
    def evaluate_triggers(cls, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        pass

    @classmethod
    def get_default_trigger_options(cls, scope) -> TriggerOption:
        pass

    # TODO: add contextual trigger options


class MissingEventTriggerProcessor(TriggerProcessor):

    @classmethod
    def get_default_config(cls):
        return MissingEventTrigger(transaction_time_threshold=DoubleValue(value=10.0))

    @classmethod
    def evaluate_triggers(cls, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        generated_alerts: [Alert] = []
        expected_transaction_time_sec = trigger['config'].get("transaction_time_threshold")

        qs_set = process_missing_event_monitor_transactions(scope, trigger, time_upper_bound, time_lower_bound,
                                                            expected_transaction_time_sec)

        for mt in qs_set["missing_event_monitor_transactions_qs"]:
            alert_stats = AlertStats(total_transaction_count=UInt64Value(value=1),
                                     missed_transaction_count=UInt64Value(value=1))

            saved_alert = Alert.objects.create(account=scope, trigger_id=trigger['id'],
                                               stats=proto_to_dict(alert_stats))

            AlertMonitorTransactionMapping.objects.get_or_create(account=scope,
                                                                 alert=saved_alert,
                                                                 monitor_transaction=mt,
                                                                 type=AlertMonitorTransaction.Type.MISSED_TRANSACTION)
            generated_alerts.append(saved_alert)

        for mt in qs_set["missing_filtered_event_monitor_transactions_qs"]:
            alert_stats = AlertStats(total_transaction_count=UInt64Value(value=1),
                                     missed_transaction_count=UInt64Value(value=1))

            saved_alert = Alert.objects.create(account=scope, trigger_id=trigger['id'],
                                               stats=proto_to_dict(alert_stats))

            AlertMonitorTransactionMapping.objects.get_or_create(account=scope,
                                                                 alert=saved_alert,
                                                                 monitor_transaction=mt,
                                                                 type=AlertMonitorTransaction.Type.MISSED_TRANSACTION)
            generated_alerts.append(saved_alert)

        for mt in qs_set["delayed_event_monitor_transactions_qs"]:
            alert_stats = AlertStats(total_transaction_count=UInt64Value(value=1),
                                     delayed_transaction_count=UInt64Value(value=1),
                                     median_delay=DoubleValue(
                                         value=mt.duration.total_seconds()))

            saved_alert = Alert.objects.create(account=scope, trigger_id=trigger['id'],
                                               stats=proto_to_dict(alert_stats))
            amt_stats = DelayedMonitorTransactionStats(
                delay_duration=DoubleValue(value=mt.duration.total_seconds()))

            AlertMonitorTransactionMapping.objects.get_or_create(account=scope,
                                                                 alert=saved_alert,
                                                                 monitor_transaction=mt,
                                                                 type=AlertMonitorTransaction.Type.DELAYED_TRANSACTION,
                                                                 stats=proto_to_dict(amt_stats))
            generated_alerts.append(saved_alert)
        return generated_alerts

    @classmethod
    def get_default_trigger_options(cls, scope) -> TriggerOption:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        default_config = cls.get_default_config()
        return TriggerOption(
            trigger_type=TriggerDefinition.Type.MISSING_EVENT,
            default_missing_event_trigger_config=default_config,
            basic_trigger_filter_options=BasicTriggerFilterOptions(
                op_description=_BASIC_TRIGGER_FILTER_OP_DESCRIPTIONS
            )
        )


class DelayedEventTriggerProcessor(TriggerProcessor):
    @classmethod
    def get_default_config(cls):

        delayed_event_trigger_types = []
        for k, _ in DelayedEventTrigger.Type.items():
            delayed_event_trigger_types.append(k)

        return DelayedEventTriggerOptions(transaction_time_threshold=DoubleValue(value=10.0),
                                          trigger_threshold=DoubleValue(value=10.0),
                                          resolution=UInt32Value(value=60),
                                          type=delayed_event_trigger_types)

    @classmethod
    def evaluate_triggers(cls, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        generated_alerts: [Alert] = []

        trigger_config = dict_to_proto(trigger["config"], DelayedEventTrigger)
        expected_transaction_time_sec = trigger_config.transaction_time_threshold.value
        trigger_threshold = trigger_config.trigger_threshold.value
        delayed_event_trigger_type = trigger_config.type

        qs_set = process_delayed_event_monitor_transactions(scope, trigger, time_upper_bound, time_lower_bound,
                                                            expected_transaction_time_sec,
                                                            delayed_event_trigger_type)

        total_transaction_count = len(qs_set["monitor_transactions_qs"])
        missing_event_transactions = qs_set["missing_event_monitor_transactions_qs"]
        delayed_event_transactions = qs_set["delayed_event_monitor_transactions_qs"]

        if total_transaction_count > 0 and (
                (len(delayed_event_transactions) + len(missing_event_transactions)) / total_transaction_count) \
                * 100 >= trigger_threshold:

            saved_alert = Alert.objects.create(account=scope, trigger_id=trigger['id'])

            for mt in qs_set["missing_event_monitor_transactions_qs"]:
                AlertMonitorTransactionMapping.objects.create(account=scope, alert=saved_alert,
                                                              monitor_transaction=mt,
                                                              type=AlertMonitorTransaction.Type.MISSED_TRANSACTION)

            delay_duration_list = []
            for mt in qs_set["delayed_event_monitor_transactions_qs"]:
                delay_duration = mt.duration.total_seconds()
                delay_duration_list.append(delay_duration)
                amt_stats = DelayedMonitorTransactionStats(delay_duration=DoubleValue(value=delay_duration))
                AlertMonitorTransactionMapping.objects.create(account=scope, alert=saved_alert,
                                                              monitor_transaction=mt,
                                                              type=AlertMonitorTransaction.Type.DELAYED_TRANSACTION,
                                                              stats=proto_to_dict(amt_stats))

            if len(delay_duration_list) > 0:
                stats = AlertStats(total_transaction_count=UInt64Value(value=total_transaction_count),
                                   missed_transaction_count=UInt64Value(value=len(missing_event_transactions)),
                                   delayed_transaction_count=UInt64Value(value=len(delayed_event_transactions)),
                                   median_delay=DoubleValue(value=statistics.median(delay_duration_list)))
            else:
                stats = AlertStats(total_transaction_count=UInt64Value(value=total_transaction_count),
                                   missed_transaction_count=UInt64Value(value=len(missing_event_transactions)),
                                   delayed_transaction_count=UInt64Value(value=len(delayed_event_transactions)))

            Alert.objects.update_or_create(id=saved_alert.id, defaults={'stats': proto_to_dict(stats)})

            generated_alerts.append(saved_alert)

        return generated_alerts

    @classmethod
    def get_default_trigger_options(cls, scope) -> TriggerOption:
        if type(scope) is not Account:
            raise ValueError(f'{scope} needs to be Account')

        default_config = cls.get_default_config()
        return TriggerOption(
            trigger_type=TriggerDefinition.Type.DELAYED_EVENT,
            default_delayed_event_trigger_config=default_config,
            basic_trigger_filter_options=BasicTriggerFilterOptions(
                op_description=_BASIC_TRIGGER_FILTER_OP_DESCRIPTIONS
            )
        )


missing_event_trigger_processor = MissingEventTriggerProcessor()
delayed_event_trigger_processor = DelayedEventTriggerProcessor()


class TriggerProcessorFacade:

    def __init__(self):
        self._map = {}

    def register(self, trigger_type: TriggerDefinition.Type, typed_trigger_processor: TriggerProcessor):
        self._map[trigger_type] = typed_trigger_processor

    def process_triggers(self, scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
        return self._map[trigger["type"]].evaluate_triggers(scope, trigger, time_lower_bound, time_upper_bound)

    def get_default_triggers_options(self, scope, trigger_type: TriggerDefinition.Type = None) -> List:
        if trigger_type:
            return self._map[trigger_type].get_default_trigger_options(scope)
        else:
            trigger_options: [TriggerOption] = []
            for key, value in self._map.items():
                trigger_options.append(value.get_default_trigger_options(scope))
            return trigger_options


trigger_processor = TriggerProcessorFacade()
trigger_processor.register(TriggerDefinition.Type.MISSING_EVENT, MissingEventTriggerProcessor())
trigger_processor.register(TriggerDefinition.Type.DELAYED_EVENT, DelayedEventTriggerProcessor())


def process_trigger(scope, trigger, time_lower_bound, time_upper_bound) -> [Alert]:
    return trigger_processor.process_triggers(scope, trigger, time_lower_bound, time_upper_bound)


def get_default_trigger_options(scope, trigger_type: TriggerDefinition.Type = None):
    return trigger_processor.get_default_triggers_options(scope, trigger_type)

event/triggers/trigger_processor.py
DrDroidLab/kenobi
import copy
from typing import List, Callable, Dict

from django.conf import settings
from django.db import transaction as dj_transaction, IntegrityError
from google.protobuf.message import Message

from event.models import Entity, EntityEventKeyMapping, Trigger, TriggerNotificationConfigMapping, Monitor
from event.monitors_triggers_crud import process_trigger_filters
from event.notification.notifications_crud import create_db_notifications
from protos.event.entity_pb2 import UpdateEntityOp
from protos.event.monitor_pb2 import UpdateMonitorOp
from protos.event.trigger_pb2 import UpdateTriggerOp, TriggerDefinition
from utils.proto_utils import proto_to_dict


def msg_getter(field_name):
    def getter(msg):
        return getattr(msg, field_name, None)

    getter.__name__ = f'{field_name}_getter'
    return getter


class UpdateProcessorError(ValueError):
    pass


class UpdateProcessorMixin:
    update_op_cls = None

    def __init__(self, *args, **kwargs):
        if self.update_op_cls is None:
            raise Exception("update_op_cls must be set")
        op_enum = self.update_op_cls.DESCRIPTOR.enum_types_by_name['Op']
        self._op_name_dict: Dict[int, Callable] = {
            op.number: op.name.lower() for op in op_enum.values if op.number > 0
        }
        self._op_display_str_dict: Dict[int, Callable] = {
            op.number: op.name.replace('_', ' ').capitalize() for op in op_enum.values if op.number > 0
        }
        self._op_fn_dict: Dict[int, Callable] = {
            op.number: getattr(self, op.name.lower(), None) for op in op_enum.values if op.number > 0
        }
        self._op_msg_field_getter_dict: Dict[int, Callable] = {
            op.number: msg_getter(op.name.lower()) for op in op_enum.values if op.number > 0
        }

    def update(self, elem, update_ops: List[Message]):
        clone_elem = copy.deepcopy(elem)

        with dj_transaction.atomic():
            for update_op in update_ops:
                op = update_op.op
                op_fn = self._op_fn_dict.get(op, None)
                if op_fn is None:
                    raise Exception(f"Unknown op: {op}")
                op_name: str = self._op_name_dict.get(op, '')
                msg_field_getter = self._op_msg_field_getter_dict.get(op, None)
                update_op_msg = msg_field_getter(update_op)
                try:
                    elem = op_fn(elem, update_op_msg)
                except Exception as ex:
                    raise UpdateProcessorError(
                        f"{self._op_display_str_dict.get(op)} error for: {clone_elem.name} - {ex}"
                    )


class EntityUpdateProcessor(UpdateProcessorMixin):
    update_op_cls = UpdateEntityOp

    @staticmethod
    def update_entity_name(elem: Entity, update_op: UpdateEntityOp.UpdateEntityName) -> Entity:
        if not update_op.name.value:
            raise Exception(f"New entity name missing for update entity name op")

        if update_op.name.value == elem.name:
            return elem
        elem.name = update_op.name.value
        try:
            elem.save(update_fields=['name'])
        except IntegrityError as ex:
            raise Exception(f"Entity name {update_op.name.value} already exists")
        return elem

    @staticmethod
    def update_entity_status(elem: Entity, update_op: UpdateEntityOp.UpdateEntityStatus) -> Entity:
        elem.is_active = update_op.is_active.value
        elem.save(update_fields=['is_active'])
        return elem

    @staticmethod
    def update_entity_event_key_mapping_status(elem: Entity,
                                               update_op: UpdateEntityOp.UpdateEntityEventKeyMappingStatus) -> Entity:
        if not update_op.entity_event_key_mapping_id.value:
            raise Exception(f"Entity event key mapping id missing for update key status op")

        entity_event_key_mapping_id: int = update_op.entity_event_key_mapping_id.value

        qs = elem.entityeventkeymapping_set.filter(id=entity_event_key_mapping_id)
        if not qs.exists():
            raise Exception(f"Entity event key mapping id {entity_event_key_mapping_id} not found")

        entity_event_key_mapping: EntityEventKeyMapping = qs.first()
        entity_event_key_mapping.is_active = update_op.is_active.value
        entity_event_key_mapping.save(update_fields=['is_active'])

        return elem

    @staticmethod
    def add_entity_event_key_mappings(elem: Entity, update_op: UpdateEntityOp.AddEntityEventKeyMappings) -> Entity:
        event_key_ids: List[int] = update_op.event_key_ids
        if not event_key_ids:
            raise Exception(f"Event key ids missing for add entity event key mapping op")

        if len(event_key_ids) != elem.account.eventkey_set.filter(id__in=event_key_ids).count():
            raise Exception(f"Invalid event key ids for add entity event key mapping op")

        eekms = [
            EntityEventKeyMapping(
                account=elem.account, entity=elem, event_key_id=event_key_id
            ) for event_key_id in event_key_ids
        ]

        if eekms:
            EntityEventKeyMapping.objects.bulk_create(
                eekms,
                ignore_conflicts=True,
                batch_size=25
            )

        return elem

    @staticmethod
    def remove_entity_event_key_mappings(elem: Entity,
                                         update_op: UpdateEntityOp.RemoveEntityEventKeyMappings) -> Entity:
        entity_event_key_mapping_ids: List[int] = update_op.entity_event_key_mapping_ids
        if not entity_event_key_mapping_ids:
            raise Exception(f"Entity event key mapping missing for remove entity event key mapping op")

        qs = elem.entityeventkeymapping_set.filter(id__in=entity_event_key_mapping_ids)
        if len(entity_event_key_mapping_ids) != qs.count():
            raise Exception(f"Invalid entity event key mapping for remove entity event key mapping op")
        qs.delete()
        return elem


class MonitorUpdateProcessor(UpdateProcessorMixin):
    update_op_cls = UpdateMonitorOp

    @staticmethod
    def update_monitor_name(elem: Monitor, update_op: UpdateMonitorOp.UpdateMonitorName) -> Monitor:
        if not update_op.name.value:
            raise Exception(f"New monitor name missing for update monitor name op")

        if update_op.name.value == elem.name:
            return elem
        if elem.account.monitor_set.filter(name=update_op.name.value).exists():
            raise Exception(f"Monitor with name {update_op.name.value} already exists")
        elem.name = update_op.name.value
        elem.save(update_fields=['name'])
        return elem

    @staticmethod
    def update_monitor_primary_key(elem: Monitor, update_op: UpdateMonitorOp.UpdateMonitorPrimaryKey) -> Monitor:
        if update_op.primary_event_key.id == elem.primary_key.id:
            return elem
        raise Exception(f"Monitor primary key cannot be updated")

    @staticmethod
    def update_monitor_secondary_key(elem: Monitor, update_op: UpdateMonitorOp.UpdateMonitorSecondaryKey) -> Monitor:
        if update_op.secondary_event_key.id == elem.secondary_key.id:
            return elem
        raise Exception(f"Monitor secondary key cannot be updated")

    @staticmethod
    def update_monitor_status(elem: Monitor, update_op: UpdateTriggerOp.UpdateTriggerDefinition) -> Monitor:
        elem.is_active = update_op.is_active.value
        elem.save(update_fields=['is_active'])
        return elem

    @staticmethod
    def update_monitor_is_generated(elem: Monitor, update_op: UpdateMonitorOp.UpdateMonitorIsGenerated) -> Monitor:
        elem.is_generated = update_op.is_generated.value
        elem.save(update_fields=['is_generated'])
        return elem


class MonitorTriggerUpdateProcessor(UpdateProcessorMixin):
    update_op_cls = UpdateTriggerOp

    @staticmethod
    def update_trigger_name(elem: Trigger, update_op: UpdateTriggerOp.UpdateTriggerName) -> Trigger:
        if not update_op.name.value:
            raise Exception(f"New trigger name missing for update trigger name op")

        if update_op.name.value == elem.name:
            return elem
        elem.name = update_op.name.value
        elem.save(update_fields=['name'])
        return elem

    @staticmethod
    def update_trigger_status(elem: Trigger, update_op: UpdateTriggerOp.UpdateTriggerStatus) -> Trigger:
        elem.is_active = update_op.is_active.value
        elem.save(update_fields=['is_active'])
        return elem

    @staticmethod
    def update_trigger_priority(elem: Trigger, update_op: UpdateTriggerOp.UpdateTriggerPriority) -> Trigger:

        elem.priority = update_op.priority
        elem.save(update_fields=['priority'])
        return elem

    @staticmethod
    def update_trigger_definition(elem: Trigger, update_op: UpdateTriggerOp.UpdateTriggerDefinition) -> Trigger:
        trigger_definition = update_op.definition
        td_which_one_of = trigger_definition.WhichOneof('trigger')
        if trigger_definition.type == TriggerDefinition.Type.MISSING_EVENT and td_which_one_of == 'missing_event_trigger':
            if trigger_definition.missing_event_trigger.transaction_time_threshold.value <= 0:
                raise Exception(f"Invalid missing event trigger config")
            trigger_config = proto_to_dict(trigger_definition.missing_event_trigger)
        elif trigger_definition.type == TriggerDefinition.Type.DELAYED_EVENT and td_which_one_of == 'delayed_event_trigger':
            if trigger_definition.delayed_event_trigger.trigger_threshold.value < 0 or \
                    trigger_definition.delayed_event_trigger.transaction_time_threshold.value <= 0 or \
                    trigger_definition.delayed_event_trigger.resolution.value < \
                    settings.DELAYED_EVENT_TRIGGER_RESOLUTION_WINDOW_LOWER_BOUND_IN_SEC:
                raise Exception(f"Invalid delayed aggregated event trigger config")
            elif trigger_definition.delayed_event_trigger.transaction_time_threshold.value >= \
                    trigger_definition.delayed_event_trigger.resolution.value:
                raise Exception(f"Delayed aggregated event trigger transaction time cannot be >= to resolution window")
            trigger_config = proto_to_dict(trigger_definition.delayed_event_trigger)
        else:
            raise Exception(f"Invalid trigger definition")

        try:
            primary_event_filters = process_trigger_filters(elem.account, trigger_definition.primary_event_filters,
                                                            elem.monitor.primary_key.event_type_id)
            secondary_event_filters = process_trigger_filters(elem.account, trigger_definition.secondary_event_filters,
                                                              elem.monitor.secondary_key.event_type_id)
        except ValueError:
            raise Exception(f"Received invalid event filter keys")

        generated_config = {'primary_event_filters': primary_event_filters,
                            'secondary_event_filters': secondary_event_filters}

        elem.type = trigger_definition.type
        elem.config = trigger_config
        elem.generated_config = generated_config
        elem.save(update_fields=['type', 'config', 'generated_config'])
        return elem

    @staticmethod
    def update_trigger_notifications(elem: Trigger, update_op: UpdateTriggerOp.UpdateTriggerNotifications) -> Trigger:
        notifications = update_op.notifications

        db_notifications = create_db_notifications(elem.account, notifications)
        elem.account.triggernotificationconfigmapping_set.filter(trigger=elem).delete()
        for db_n in db_notifications:
            TriggerNotificationConfigMapping.objects.update_or_create(account=elem.account, trigger=elem,
                                                                      notification=db_n)
        return elem


entity_update_processor = EntityUpdateProcessor()
monitor_update_processor = MonitorUpdateProcessor()
monitor_trigger_update_processor = MonitorTriggerUpdateProcessor()

event/update_processor.py
DrDroidLab/kenobi
from django.urls import path

from . import views

urlpatterns = [
    path('ingest/events/v1', views.ingest_events_v1),
    path('ingest/events/v2', views.ingest_events_v2),
    path('ingest/events/aws_kinesis', views.aws_kinesis_ingest_events),

    path('api/search/options', views.global_search_options_get),
    path('api/events', views.events_get),
    path('api/search/options/events', views.event_search_options_get),
    path('api/events/search', views.events_search),
    path('api/events/search/save', views.save_events_search),
    path('api/events/search/v2', views.events_search_v2),
    path('api/events/export', views.events_export),
    path('api/event_keys/search', views.event_keys_search),
    path('api/event_types/summary', views.event_types_summary),
    path('api/event_types/definition', views.event_types_definition),
    path('api/monitors/create', views.monitors_create),
    path('api/monitors/update', views.monitors_update),
    path('api/monitors', views.monitors_get),
    path('api/monitors/options', views.monitors_option),
    path('api/monitors/definition', views.monitors_definition),
    path('api/monitors/transactions', views.monitors_transactions_get),
    path('api/monitors/transactions/events', views.monitors_transactions_events_get),
    path('api/monitors/transactions/details', views.monitors_transactions_details),
    path('api/search/options/monitors/transactions', views.monitor_transaction_search_options_get),
    path('api/search/options/monitors/transactions/v2', views.monitor_transaction_search_options_get_v2),
    path('api/monitors/transactions/search', views.monitors_transactions_search),
    path('api/monitors/transactions/export', views.monitors_transactions_export),
    path('api/monitors/transactions/export/v2', views.monitors_transactions_export_v2),
    path('api/monitors/transactions/search/v2', views.monitors_transactions_search_v2),
    path('api/monitors/transactions/search/save', views.save_monitor_transactions_search),
    path('api/monitors/triggers', views.monitors_triggers_get),
    path('api/monitors/triggers/options', views.monitors_triggers_options),
    path('api/monitors/triggers/create', views.monitors_triggers_create),
    path('api/monitors/triggers/update', views.monitors_triggers_update),

    path('api/metrics/options', views.metrics_options),
    path('api/metrics', views.metrics),

    path('api/panels_v1', views.panels_v1_get),
    path('api/panels_v1/create_or_update', views.panels_v1_create_or_update),
    path('api/panels_v1/delete', views.panels_v1_delete),

    path('api/dashboard_v1', views.dashboards_v1_get),
    path('api/dashboard_v1/create_or_update', views.dashboard_v1_create_or_update),
    path('api/dashboard_v1/delete', views.dashboards_v1_delete),

    path('api/alerts/summary', views.alerts_summary_get),
    path('api/alerts/details', views.alerts_details_get),
    path('api/alerts/monitor_transactions', views.alerts_monitor_transactions_get),

    path('api/notifications_get', views.notifications_get),
    path('api/notifications_create', views.notifications_create),

    path('api/entity/create/options', views.entity_options_get),
    path('api/entity/create', views.entity_create),
    path('api/entity/update', views.entity_update),
    path('api/entity', views.entity_summary_get),
    path('api/entity/details', views.entity_detail_get),
    path('api/entity/instances', views.entity_instances_summary_get),
    path('api/entity/instances/details', views.entity_instances_details_get),
    path('api/entity/instances/timeline', views.entity_instances_timeline_get),
    path('api/search/options/entity/instances', views.entity_instance_search_options_get),
    path('api/search/options/entity/instances/v2', views.entity_instance_search_options_get_v2),
    path('api/entity/instances/search', views.entity_instances_search),

    path('api/entities/triggers/get', views.entity_triggers_get),
    path('api/entities/triggers/options', views.entity_triggers_options),
    path('api/entities/triggers/create', views.entity_triggers_create),
    path('api/entities/triggers/update', views.entity_triggers_update),
    path('api/entities/triggers/inactivate', views.entity_triggers_inactivate),

    path('api/entity/workflow', views.entity_workflow_get),
    path('api/entity/build_workflow', views.workflow_builder),
    path('api/entity/node_metrics_timeseries', views.workflow_node_metrics_timeseries),

    path('api/funnel/v2', views.funnel_get_v2),
    path('api/funnel/drop_off_distribution', views.funnel_drop_off_distribution_get),
    path('api/funnel/drop_off_distribution/v2', views.funnel_drop_off_distribution_get_v2),
    path('api/funnel/drop_off', views.funnel_drop_off_get),

    path('api/funnel/v3', views.funnel_get_v3),

    # Funnel Revamp
    path('api/entity_funnels/view', views.funnel_get_v2),
    path('api/entity_funnels/create', views.entity_funnel_create),
    path('api/entity_funnels/get', views.entity_funnel_get),
]

event/urls.py
DrDroidLab/kenobi
from datetime import datetime, timedelta

from accounts.models import Account
from event.processors.process_events_payload import process_account_events_payload
from protos.event.base_pb2 import Event as EventProto
from protos.event.schema_pb2 import IngestionEventPayload, IngestionEvent, KeyValue, Value


def get_sample_event_payload():
    current_time = datetime.utcnow()
    lookback_120s = int((current_time - timedelta(seconds=120)).timestamp() * 1000)
    lookback_115s = int((current_time - timedelta(seconds=115)).timestamp() * 1000)
    lookback_112s = int((current_time - timedelta(seconds=112)).timestamp() * 1000)
    lookback_60s = int((current_time - timedelta(seconds=60)).timestamp() * 1000)
    lookback_55s = int((current_time - timedelta(seconds=55)).timestamp() * 1000)
    lookback_now = int(current_time.timestamp() * 1000)
    return IngestionEventPayload(
        events=[
            IngestionEvent(
                name="payment_started",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_LULhDM2vvvt7yZ")
                    ),
                    KeyValue(
                        key="amount",
                        value=Value(double_value=54.8)
                    ),
                    KeyValue(
                        key="currency",
                        value=Value(string_value="usd")
                    ),
                    KeyValue(
                        key="invoice_id",
                        value=Value(string_value="in_1KnN0G589O8KAxCGfVSpD0Pj")
                    )
                ],
                timestamp=lookback_120s
            ),
            IngestionEvent(
                name="payment_started",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_ekpsHJ9GhQd7OU")
                    ),
                    KeyValue(
                        key="amount",
                        value=Value(double_value=12.4)
                    ),
                    KeyValue(
                        key="currency",
                        value=Value(string_value="usd")
                    ),
                    KeyValue(
                        key="invoice_id",
                        value=Value(string_value="in_98363ixIwXnh07L5DuLvQE6c")
                    )
                ],
                timestamp=lookback_115s
            ),
            IngestionEvent(
                name="payment_started",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_4i7w711u93uH46")
                    ),
                    KeyValue(
                        key="amount",
                        value=Value(double_value=8.7)
                    ),
                    KeyValue(
                        key="currency",
                        value=Value(string_value="usd")
                    ),
                    KeyValue(
                        key="invoice_id",
                        value=Value(string_value="in_QezX99uPQSrgbgpP82U6YUu6")
                    )
                ],
                timestamp=lookback_112s
            ),
            IngestionEvent(
                name="payment_started",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_2v0fm8274y9nem")
                    ),
                    KeyValue(
                        key="amount",
                        value=Value(double_value=23.9)
                    ),
                    KeyValue(
                        key="currency",
                        value=Value(string_value="usd")
                    ),
                    KeyValue(
                        key="invoice_id",
                        value=Value(string_value="in_B31kppK5qEF7Dv5S13D0VZ6B")
                    )
                ],
                timestamp=lookback_60s
            ),
            IngestionEvent(
                name="payment_started",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_tWaIb3aK3vlHn9")
                    ),
                    KeyValue(
                        key="amount",
                        value=Value(double_value=6.8)
                    ),
                    KeyValue(
                        key="currency",
                        value=Value(string_value="usd")
                    ),
                    KeyValue(
                        key="invoice_id",
                        value=Value(string_value="in_U0T8TO2dDAKxxtzO4cG3Dr3l")
                    )
                ],
                timestamp=lookback_55s
            ),
            IngestionEvent(
                name="payment_callback",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_LULhDM2vvvt7yZ")
                    ),
                    KeyValue(
                        key="status",
                        value=Value(string_value="paid")
                    ),
                    KeyValue(
                        key="payment_method",
                        value=Value(string_value="cc")
                    )
                ],
                timestamp=lookback_60s
            ),
            IngestionEvent(
                name="payment_callback",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_ekpsHJ9GhQd7OU")
                    ),
                    KeyValue(
                        key="status",
                        value=Value(string_value="paid")
                    ),
                    KeyValue(
                        key="payment_method",
                        value=Value(string_value="venmo")
                    )
                ],
                timestamp=lookback_55s
            ),
            IngestionEvent(
                name="payment_callback",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_4i7w711u93uH46")
                    ),
                    KeyValue(
                        key="status",
                        value=Value(string_value="paid")
                    ),
                    KeyValue(
                        key="payment_method",
                        value=Value(string_value="cc")
                    )
                ],
                timestamp=lookback_now
            ),
            IngestionEvent(
                name="payment_callback",
                kvs=[
                    KeyValue(
                        key="transaction_id",
                        value=Value(string_value="si_2v0fm8274y9nem")
                    ),
                    KeyValue(
                        key="status",
                        value=Value(string_value="failed")
                    ),
                ],
                timestamp=lookback_now
            )
        ]
    )


def setup_account_event_data(account: Account):
    process_account_events_payload(account=account, payload=get_sample_event_payload(),
                                   source=EventProto.EventSource.SAMPLE, is_generated=True,
                                   evaluate_event_monitors=False)

    return

event/utils/account_data.py
DrDroidLab/kenobi
import json
import traceback
import uuid
from datetime import timedelta, datetime
from typing import Union
from collections import defaultdict
import csv

from django.db import transaction as dj_transaction
from django.db.models import F, Avg, FloatField, OuterRef, Subquery, Count
from django.db.models import QuerySet
from django.db.models.functions import Cast
from django.http import HttpResponse, HttpRequest, JsonResponse
from django.views.decorators.csrf import csrf_exempt
from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value, StringValue, DoubleValue
from rest_framework.decorators import api_view, authentication_classes

from accounts.authentication import AccountApiTokenAuthentication
from accounts.models import get_request_account, Account, get_request_user, User
from accounts.utils import do_process_events, AccountDailyEventQuotaReached, get_account_quota_error_message
from connectors.integrations.datadog import DatadogConnector
from connectors.integrations.newrelic import NewRelicConnector
from connectors.models import TransformerMapping
from connectors.transformers.transformer import transformer_facade
from event.base.op import OP_DESCRIPTIONS, OP_MAPPINGS, LITERAL_TYPE_DESCRIPTIONS
from event.cache import GLOBAL_PANEL_CACHE, GLOBAL_DASHBOARD_CACHE, GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE, \
    GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE, GLOBAL_EXPORT_CONTEXT_CACHE
from event.engine.fuzzy_match_engine import event_type_fuzzy_match_engine, monitor_fuzzy_match_engine, \
    entity_fuzzy_match_engine
from event.engine.metric_engine import process_metric_expressions, get_metric_options, get_default_metric_expression, \
    process_workflow_metric_expression
from event.engine.query_engine import global_event_search_engine, global_event_search_clickhouse_engine, \
    monitor_transaction_search_engine, \
    entity_instance_search_engine
from event.entity_funnels_crud import entity_funnels_create, entity_funnels_get, EntityFunnelCrudNotFoundException, \
    entity_funnels_get_clickhouse_events
from event.models import is_transactional_key_type, Notification, Alert, is_filterable_key_type, \
    TRANSACTIONAL_KEY_TYPES, Entity, EntityEventKeyMapping, Trigger, MonitorTransactionStats, \
    MonitorTransactionEventMapping, EventKey, EntityTrigger, EntityTriggerNotificationConfigMapping
from event.monitors_crud import create_monitors, update_monitors
from event.monitors_triggers_crud import create_monitor_triggers
from event.notification.notification_facade import notification_client
from event.notification.notifications_crud import create_db_notifications
from event.processors.process_events_payload import process_account_events_payload
from event.tasks import export_task
from event.triggers.trigger_processor import get_default_trigger_options
from event.update_processor import entity_update_processor, monitor_trigger_update_processor
from event.workflows.builder import WorkflowBuilder
from event.workflows.funnel import Funnel
from event.workflows.graph import Graph, NewGraph
from protos.event.alert_pb2 import AlertDetail
from protos.event.api_pb2 import *
from protos.event.base_pb2 import TimeRange, Page, EventKey as EventKeyProto, Event as EventProto, EventTypePartial, \
    EventTypeStats, EventTypeSummary, Context
from protos.event.engine_options_pb2 import GlobalQueryOptions
from protos.event.timeline_pb2 import EntityInstanceTimelineRecord
from protos.event.monitor_pb2 import MonitorPartial, MonitorTransactionPartial, MonitorStats, \
    MonitorTransaction as MonitorTransactionProto
from protos.event.notification_pb2 import NotificationConfig
from protos.event.options_pb2 import MonitorTriggerOptions
from protos.event.options_pb2 import TriggerOption, MonitorOptions, NotificationOption, EntityOptions
from protos.event.panel_pb2 import PanelV1, DashboardV1, ComplexTableData
from protos.event.query_base_pb2 import QueryRequest
from protos.event.schema_pb2 import IngestionEventPayloadRequest, IngestionEventPayloadResponse, IngestionEventPayload, \
    KeyValue, IngestionEvent, Value as ValueProto, AWSKinesisIngestionStreamPayloadRequest, \
    AWSKinesisIngestionStreamPayloadResponse, \
    AWSKinesisEventPayload
from protos.event.trigger_pb2 import Trigger as TriggerProto, MonitorTriggerNotificationDetail, \
    EntityTriggerNotificationDetail
from prototype.aggregates import Percentile
from prototype.db.decorator import use_read_replica
from prototype.entity_utils import annotate_entity_instance_stats, annotate_entity_timeline
from prototype.threadlocal import get_current_request
from prototype.utils.decorators import account_data_api, web_api, api_auth_check, aws_kinesis_data_api, \
    ProtoJsonResponse
from prototype.utils.meta import get_meta
from prototype.utils.queryset import filter_time_range, filter_page
from prototype.utils.timerange import to_dtr, DateTimeRange, to_tr, filter_dtr, to_str
from prototype.utils.utils import current_milli_time
from utils.proto_utils import proto_to_dict, dict_to_proto

from event.clickhouse.models import Events

EVENT_TIME_FIELD = 'timestamp'


def infer_value_type(v):
    if isinstance(v, str):
        return EventKeyProto.KeyType.STRING, ValueProto(string_value=v)
    elif isinstance(v, bool):
        return EventKeyProto.KeyType.BOOLEAN, ValueProto(bool_value=v)
    elif isinstance(v, int):
        return EventKeyProto.KeyType.LONG, ValueProto(int_value=v)
    elif isinstance(v, float):
        return EventKeyProto.KeyType.DOUBLE, ValueProto(double_value=v)
    return EventKeyProto.KeyType.UNKNOWN, None


def transform_event_json(event_json: dict):
    event_name: str = event_json.get('name', '')
    if not event_json:
        return None
    payload = event_json.get('payload', None)
    if not isinstance(payload, dict):
        return None

    timestamp_in_ms = event_json.get('timestamp', current_milli_time())

    kvlist = []
    for key, value in payload.items():
        _, proto_value = infer_value_type(value)
        if not proto_value:
            continue
        kvlist.append(KeyValue(key=key, value=proto_value))

    return IngestionEvent(name=event_name, kvs=kvlist, timestamp=timestamp_in_ms)


@csrf_exempt
@api_view(['POST'])
@authentication_classes([AccountApiTokenAuthentication])
@api_auth_check
def ingest_events_v1(request: HttpRequest):
    # Process the events and metadata and put it into db
    body = request.body
    parsed_body = json.loads(body)
    event = transform_event_json(parsed_body)
    if not event:
        return JsonResponse({'count': 0}, status=200)
    account: Account = get_request_account()

    event_count = 1
    try:
        do_process_events(account.id, event_count)
    except AccountDailyEventQuotaReached as e:
        return JsonResponse({'count': 0, 'errorMessage': get_account_quota_error_message()}, status=200)

    count = process_account_events_payload(account, IngestionEventPayload(events=[event]), EventProto.EventSource.API)
    return JsonResponse({'count': count}, status=200)


@account_data_api(IngestionEventPayloadRequest)
def ingest_events_v2(request_message: IngestionEventPayloadRequest) -> Union[
    IngestionEventPayloadResponse, HttpResponse]:
    # Process the events and metadata and put it into db
    account: Account = get_request_account()

    data: IngestionEventPayload = request_message.data
    if not data:
        return IngestionEventPayloadResponse(count=0)

    event_count = len(data.events)
    try:
        do_process_events(account.id, event_count)
    except AccountDailyEventQuotaReached as e:
        return IngestionEventPayloadResponse(count=0, errorMessage=get_account_quota_error_message())

    count = process_account_events_payload(account, data)
    return JsonResponse({'count': count}, status=200)


@aws_kinesis_data_api(AWSKinesisIngestionStreamPayloadRequest)
def aws_kinesis_ingest_events(request_message: AWSKinesisIngestionStreamPayloadRequest) -> Union[
    AWSKinesisIngestionStreamPayloadResponse, HttpResponse]:
    # Process the kinesis delivery stream of segment events and put it into db

    account: Account = get_request_account()

    request_headers = get_current_request().headers
    common_attributes_headers = request_headers.get('X-Amz-Firehose-Common-Attributes', None)
    if not common_attributes_headers:
        return JsonResponse({'error': 'Missing Common Attributes Header'}, status=400,
                            content_type='application/json')

    common_attributes = json.loads(common_attributes_headers).get('commonAttributes')
    if not common_attributes:
        return JsonResponse({'error': 'Missing Common Attributes'}, status=400,
                            content_type='application/json')

    source = EventProto.EventSource.Value(common_attributes.get('source', 'UNKNOWN'))
    transformer_id = int(common_attributes.get('transformer_id', None))
    if not transformer_id:
        return JsonResponse({'error': 'Invalid transformer id'}, status=400,
                            content_type='application/json')

    transformer_mapping: TransformerMapping = account.transformermapping_set.get(id=transformer_id)
    if not transformer_mapping:
        print(
            f"No transformer mapping found for AWS Kinesis Ingestion for account id: {account.id} and transformer id: {transformer_id}")

    request_id: str = request_message.requestId
    records: [AWSKinesisEventPayload] = request_message.records
    timestamp = request_message.timestamp
    transformed_event_list = []
    for record in records:
        try:
            events = transformer_facade.transform(transformer_mapping, record)
            if not events:
                continue
            for event in events:
                transformed_event = transform_event_json(event)
                transformed_event_list.append(transformed_event)
        except Exception as ex:
            print(
                f"Transformer Error in AWS Kinesis Ingestion: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")

    count = 0
    if transformed_event_list:
        count = process_account_events_payload(account, IngestionEventPayload(events=transformed_event_list), source)

    print("Processed: {} AWS Kinesis Ingestion Events for account id: {}".format(count, account.id))
    return JsonResponse({'requestId': request_id, 'timestamp': timestamp}, status=200,
                        content_type='application/json')


@web_api(GetEventsRequest)
@use_read_replica
def events_get(request_message: GetEventsRequest) -> Union[GetEventsResponse, HttpResponse]:
    # Process the events and metadata and put it into db
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    qs = Events.objects.filter(account_id=account.id)

    if request_message.event_type_ids:
        qs = qs.filter(event_type_id__in=request_message.event_type_ids)

    qs = filter_time_range(qs, tr, 'timestamp')
    total_count = qs.count()
    qs = qs.order_by('-timestamp')
    qs = filter_page(qs, page)

    event_protos = [e.proto for e in qs]

    return GetEventsResponse(meta=get_meta(tr=tr, page=page, total_count=total_count), events=event_protos)


@web_api(GetEventSearchQueryRequest)
@use_read_replica
def events_search(request_message: GetEventSearchQueryRequest) -> Union[GetEventSearchQueryResponse, HttpResponse]:
    # Search events given a search request
    account: Account = get_request_account()
    page = request_message.meta.page
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request = request_message.query_request

    generated_search_context_id = None
    if request_message.save_query.value:
        generated_search_context_id = uuid.uuid4().hex
        query_request_dict = proto_to_dict(query_request)
        GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE.create_or_update(account.id, generated_search_context_id,
                                                                 query_request_dict, None)

    qs = Events.objects.all()
    qs = filter_dtr(qs, dtr, 'timestamp')
    qs = qs.filter(account_id=account.id)
    qs = global_event_search_clickhouse_engine.process_query(qs, query_request)
    total_count = qs.count()

    qs = filter_page(qs, page)
    event_protos = [e.proto for e in qs]

    return GetEventSearchQueryResponse(meta=get_meta(total_count=total_count, tr=dtr.to_tr(), page=page),
                                       events=event_protos, query_context_id=generated_search_context_id)


@web_api(SaveEventSearchQueryRequest)
@use_read_replica
def save_events_search(request_message: SaveEventSearchQueryRequest) -> Union[
    SaveEventSearchQueryResponse, HttpResponse]:
    # Save events search context
    account: Account = get_request_account()

    generated_search_context_id = uuid.uuid4().hex
    query_request_dict = proto_to_dict(request_message.query_request)
    GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE.create_or_update(account.id, generated_search_context_id,
                                                             query_request_dict, None)

    return SaveEventSearchQueryResponse(meta=get_meta(), query_context_id=generated_search_context_id)


@web_api(GetEventSearchQueryRequest)
@use_read_replica
def events_search_v2(request_message: GetEventSearchQueryRequest) -> Union[GetEventSearchQueryResponse, HttpResponse]:
    # Search events given a search request
    account: Account = get_request_account()
    page = request_message.meta.page
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request = request_message.query_request

    generated_search_context_id = None
    if request_message.save_query.value:
        generated_search_context_id = uuid.uuid4().hex
        query_request_dict = proto_to_dict(request_message.query_request)
        GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE.create_or_update(account.id, generated_search_context_id,
                                                                 query_request_dict, None)

    qs = Events.objects.all()
    qs = filter_dtr(qs, dtr, 'timestamp')
    qs = qs.filter(account_id=account.id)
    qs = global_event_search_clickhouse_engine.process_query(qs, query_request)
    total_count = qs.count()

    qs = global_event_search_clickhouse_engine.process_ordering(qs, query_request)

    qs = filter_page(qs, page)
    event_protos = [e.proto for e in qs]

    return GetEventSearchQueryResponse(meta=get_meta(total_count=total_count, tr=dtr.to_tr(), page=page),
                                       events=event_protos, query_context_id=generated_search_context_id)


@web_api(GetEventExportQueryRequest)
@use_read_replica
def events_export(request_message: GetEventExportQueryRequest) -> Union[
    GetEventExportQueryResponse, HttpResponse]:
    # Export events given a search request

    account: Account = get_request_account()
    user: User = get_request_user()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request = request_message.query_request

    export_id = uuid.uuid4().hex
    if GLOBAL_EXPORT_CONTEXT_CACHE.get(account.id, Context.EVENT, user.email):
        export_id = GLOBAL_EXPORT_CONTEXT_CACHE.get(account.id, Context.EVENT, user.email)
    else:
        qs = Events.objects.all()
        qs = filter_dtr(qs, dtr, 'timestamp')
        qs = qs.filter(account_id=account.id)
        qs = global_event_search_clickhouse_engine.process_query(qs, query_request)
        if qs.count() > 10000:
            return GetEventExportQueryResponse(meta=get_meta(), success=BoolValue(value=False),
                                               message=Message(title="Event Export Failure",
                                                               description="Export size too large"))
        else:
            GLOBAL_EXPORT_CONTEXT_CACHE.create_or_update(account.id, Context.EVENT, user.email, export_id)
            export_task_request, message = export_task(account.id, Context.EVENT, user.email,
                                                       proto_to_dict(request_message))

            if not export_task_request:
                return GetEventExportQueryResponse(meta=get_meta(), success=BoolValue(value=False),
                                                   message=Message(title="Event Export Failure",
                                                                   description=message))

    return GetEventExportQueryResponse(meta=get_meta(), success=BoolValue(value=True), export_context_id=export_id)


@web_api(GetEventKeysSearchRequest)
@use_read_replica
def event_keys_search(request_message: GetEventKeysSearchRequest) -> Union[
    GetEventKeysSearchResponse, HttpResponse]:
    account: Account = get_request_account()
    event_key_ids = request_message.event_key_ids

    event_keys = EventKey.objects.filter(account_id=account.id, id__in=event_key_ids).values_list('name', flat=True)
    similar_event_keys = EventKey.objects.filter(account_id=account.id, name__in=event_keys).exclude(
        id__in=event_key_ids)

    similar_event_keys_protos = [e.proto for e in similar_event_keys]
    return GetEventKeysSearchResponse(event_keys=similar_event_keys_protos)


@web_api(GetEventTypeSummaryRequest)
@use_read_replica
def event_types_summary(request_message: GetEventTypeSummaryRequest) -> Union[
    GetEventTypeSummaryResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    is_event_type_page = False

    qs: QuerySet = account.eventtype_set.all()
    if request_message.fuzzy_search_request.pattern:
        qs: QuerySet = event_type_fuzzy_match_engine.match_pattern(account,
                                                                   request_message.fuzzy_search_request.pattern)
    if request_message.event_type_ids:
        qs = qs.filter(id__in=request_message.event_type_ids)
        if len(request_message.event_type_ids) == 1:
            is_event_type_page = True

    total_count = qs.count()
    qs = filter_page(qs, page)

    summaries = []

    if is_event_type_page:
        event_type_info = [EventTypePartial(id=event_type.id, name=str(event_type.name)) for event_type in qs][0]
        summaries.append(EventTypeSummary(event_type=event_type_info, stats=None))
    else:
        monitor_and_keys_summaries = [event_type.monitor_and_keys_summary() for event_type in qs]

        event_type_ids = [event_type.id for event_type in qs]
        e_qs = Events.objects.filter(account_id=account.id)
        e_qs = filter_dtr(e_qs, tr, 'timestamp')
        e_qs = e_qs.filter(event_type_id__in=event_type_ids)
        e_qs = e_qs.values('event_type_id').annotate(count=Count('id'))

        event_type_counts = list(e_qs)
        event_type_counts_map = {e['event_type_id']: e['count'] for e in event_type_counts}

        for summary in monitor_and_keys_summaries:
            event_count = event_type_counts_map.get(summary[0].id, 0)
            stats = EventTypeStats(keys_count=UInt64Value(value=summary[1].get('keys_count')),
                                   event_count=UInt64Value(value=event_count),
                                   monitor_count=UInt64Value(value=summary[1].get('monitor_count')))
            summaries.append(EventTypeSummary(event_type=summary[0], stats=stats))

    return GetEventTypeSummaryResponse(meta=get_meta(tr=tr, page=page, total_count=total_count),
                                       event_type_summary=summaries)


@web_api(GetEventTypeDefinitionRequest)
@use_read_replica
def event_types_definition(request_message: GetEventTypeDefinitionRequest) -> Union[
    GetEventTypeDefinitionResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    qs: QuerySet = account.eventtype_set.all()
    if request_message.event_type_ids:
        qs = qs.filter(id__in=request_message.event_type_ids)

    total_count = qs.count()
    qs = filter_page(qs, page)

    definitions = [event_type.definition(tr) for event_type in qs]

    return GetEventTypeDefinitionResponse(meta=get_meta(tr=tr, page=page, total_count=total_count),
                                          event_type_definitions=definitions)


@web_api(CreateOrUpdateMonitorRequest)
def monitors_create(request_message: CreateOrUpdateMonitorRequest) -> Union[
    CreateOrUpdateMonitorResponse, HttpResponse]:
    account: Account = get_request_account()

    monitor_name: str = request_message.name
    primary_event_key_id = request_message.primary_event_key_id
    secondary_event_key_id = request_message.secondary_event_key_id

    db_monitor, created, error = create_monitors(account, monitor_name, primary_event_key_id, secondary_event_key_id)
    if error:
        if db_monitor:
            return CreateOrUpdateMonitorResponse(success=BoolValue(value=False),
                                                 message=Message(title="Monitor Creation Failure",
                                                                 description=error),
                                                 monitor=db_monitor.proto_partial)
        else:
            return CreateOrUpdateMonitorResponse(success=BoolValue(value=False),
                                                 message=Message(title="Monitor Creation Failure",
                                                                 description=error))
    return CreateOrUpdateMonitorResponse(success=BoolValue(value=True), monitor=db_monitor.proto_partial)


@web_api(CreateOrUpdateMonitorRequest)
def monitors_update(request_message: CreateOrUpdateMonitorRequest) -> Union[
    CreateOrUpdateMonitorResponse, HttpResponse]:
    account: Account = get_request_account()

    return update_monitors(account, request_message)


@web_api(GetMonitorDefinitionRequest)
@use_read_replica
def monitors_definition(request_message: GetMonitorDefinitionRequest) -> Union[
    GetMonitorDefinitionResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    qs: QuerySet = account.monitor_set.all()
    if request_message.monitor_ids:
        qs = qs.filter(id__in=request_message.monitor_ids)

    total_count = qs.count()
    qs = qs.order_by('-created_at')
    qs = filter_page(qs, page)

    qs = qs.select_related('primary_key', 'primary_key__event_type', 'secondary_key', 'secondary_key__event_type')

    definitions = [monitor.definition(tr) for monitor in qs]

    return GetMonitorDefinitionResponse(meta=get_meta(tr=tr, page=page, total_count=total_count),
                                        monitor_definitions=definitions)


@web_api(GetMonitorRequest)
@use_read_replica
def monitors_get(request_message: GetMonitorRequest) -> Union[GetMonitorResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    show_inactive = meta.show_inactive
    page: Page = meta.page

    qs: QuerySet = account.monitor_set.filter(is_generated=False)
    if request_message.fuzzy_search_request.pattern:
        qs: QuerySet = monitor_fuzzy_match_engine.match_pattern(account, request_message.fuzzy_search_request.pattern)

    if request_message.monitor_ids:
        qs = qs.filter(id__in=request_message.monitor_ids)
    elif not show_inactive or not show_inactive.value:
        qs = qs.filter(is_active=True)

    total_count = qs.count()
    qs = qs.order_by('-created_at')
    qs = filter_page(qs, page)

    qs = qs.select_related('primary_key', 'primary_key__event_type', 'secondary_key', 'secondary_key__event_type')

    monitors = [monitor.proto for monitor in qs]

    return GetMonitorResponse(meta=get_meta(page=page, total_count=total_count, show_inactive=show_inactive),
                              monitors=monitors)


@web_api(GetMonitorOptionsRequest)
@use_read_replica
def monitors_option(request_message: GetMonitorOptionsRequest) -> Union[GetMonitorOptionsResponse, HttpResponse]:
    account: Account = get_request_account()

    qs: QuerySet = account.eventkey_set.filter()
    qs = qs.prefetch_related('event_type')

    total_count = qs.count()

    event_key_options = [ek.proto for ek in qs if is_transactional_key_type(ek.type)]

    return GetMonitorOptionsResponse(
        meta=get_meta(total_count=total_count),
        monitor_options=MonitorOptions(event_key_options=event_key_options)
    )


@web_api(GetMonitorTransactionsRequest)
@use_read_replica
def monitors_transactions_get(request_message: GetMonitorTransactionsRequest) -> Union[
    GetMonitorTransactionsResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    monitor_id = request_message.monitor_id

    qs: QuerySet = account.monitortransaction_set.all()

    if monitor_id:
        qs = qs.filter(monitor_id=monitor_id)

    if request_message.params:
        params = request_message.params
        transaction_ids = params.transaction_ids
        transactions = params.transactions
        transaction_status = params.transaction_status
        if transaction_ids:
            qs = qs.filter(id__in=transaction_ids)
        elif transactions:
            qs = qs.filter(trasaction__in=transactions)
        elif transaction_status:
            qs = qs.filter(type=transaction_status)

    qs = filter_time_range(qs, tr, 'updated_at')
    total_count = qs.count()
    qs = qs.order_by('-updated_at')
    qs = filter_page(qs, page)

    qs = qs.prefetch_related('monitor')

    monitor_transactions = [monitor_transaction.proto for monitor_transaction in qs]

    qs: QuerySet = account.alertmonitortransactionmapping_set.all()
    qs = qs.prefetch_related('alert', 'alert__trigger')
    qs = qs.filter(monitor_transaction_id__in=transaction_ids)
    alert_monitor_transaction_mappings = [amtm.miniproto for amtm in qs]

    return GetMonitorTransactionsResponse(meta=get_meta(tr=tr, page=page, total_count=total_count),
                                          monitor_transactions=monitor_transactions,
                                          alerts=alert_monitor_transaction_mappings)


@web_api(GetMonitorTransactionEventsRequest)
@use_read_replica
def monitors_transactions_events_get(request_message: GetMonitorTransactionEventsRequest) -> Union[
    GetMonitorTransactionEventsResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    tr: TimeRange = meta.time_range
    page: Page = meta.page

    transaction_id = request_message.transaction_id

    if not transaction_id:
        return GetMonitorTransactionEventsResponse(meta=get_meta(tr=tr, page=page))

    qs: QuerySet = account.monitortransactioneventmapping_set.all()

    qs = qs.filter(monitor_transaction_id=transaction_id)
    events = [mte.event.proto for mte in qs]

    return GetMonitorTransactionEventsResponse(meta=get_meta(tr=tr, page=page),
                                               events=events)


@web_api(GetMonitorTransactionDetailsRequest)
@use_read_replica
def monitors_transactions_details(request_message: GetMonitorTransactionDetailsRequest) -> Union[
    GetMonitorTransactionDetailsResponse, HttpResponse]:
    account: Account = get_request_account()

    transaction_id = request_message.transaction_id

    if not transaction_id:
        return GetMonitorTransactionDetailsResponse(meta=get_meta())

    qs: QuerySet = account.monitortransactionstats_set.all()

    qs = qs.filter(monitor_transaction_id=transaction_id)
    qs = qs.select_related('monitor_transaction__monitor')
    details = [mte.proto for mte in qs]

    return GetMonitorTransactionDetailsResponse(meta=get_meta(),
                                                monitor_transaction_details=details)


@web_api(GetMonitorsTransactionsSearchOptionsRequest)
@use_read_replica
def monitor_transaction_search_options_get(request_message: GetMonitorsTransactionsSearchOptionsRequest) -> Union[
    GetMonitorsTransactionsSearchOptionsResponse, HttpResponse]:
    account: Account = get_request_account()

    default_query = None
    saved_query_request_context_id = request_message.saved_query_request_context_id.string.value
    if saved_query_request_context_id:
        saved_context = GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE.get(account.id,
                                                                                  saved_query_request_context_id)
        if saved_context:
            default_query = dict_to_proto(saved_context, QueryRequest)

    obj = monitor_transaction_search_engine.get_obj(account, request_message.id_literal)
    if not default_query:
        default_query = monitor_transaction_search_engine.get_default_query(account, obj)

    options = monitor_transaction_search_engine.get_search_options(account, obj)

    return GetMonitorsTransactionsSearchOptionsResponse(meta=get_meta(),
                                                        monitor_transaction_query_options=options,
                                                        default_query_request=default_query)


@web_api(GetMonitorsTransactionsSearchOptionsV2Request)
@use_read_replica
def monitor_transaction_search_options_get_v2(request_message: GetMonitorsTransactionsSearchOptionsV2Request) -> Union[
    GetMonitorsTransactionsSearchOptionsV2Response, HttpResponse]:
    account: Account = get_request_account()

    default_query = None
    saved_query_request_context_id = request_message.saved_query_request_context_id.string.value
    if saved_query_request_context_id:
        saved_context = GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE.get(account.id,
                                                                                  saved_query_request_context_id)
        if saved_context:
            default_query = dict_to_proto(saved_context, QueryRequest)

    obj = monitor_transaction_search_engine.get_obj(account, request_message.id_literal)
    if not default_query:
        default_query = monitor_transaction_search_engine.get_default_query(account, obj)

    options = monitor_transaction_search_engine.get_search_options_v2(account, obj)

    return GetMonitorsTransactionsSearchOptionsV2Response(meta=get_meta(),
                                                          monitor_transaction_query_options=options,
                                                          default_query_request=default_query)


@web_api(GetMonitorsTransactionsSearchQueryRequest)
@use_read_replica
def monitors_transactions_search(request_message: GetMonitorsTransactionsSearchQueryRequest) -> Union[
    GetMonitorsTransactionsSearchQueryResponse, HttpResponse]:
    # Search monitor transactions given a search request
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    page = request_message.meta.page

    qs = account.monitortransaction_set.all()
    qs = filter_dtr(qs, dtr, 'created_at')
    qs = monitor_transaction_search_engine.process_query(qs, request_message.query_request)
    total_count = qs.count()
    monitor_transactions_finished = qs.filter(type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED)

    monitor_transactions_aggregates = monitor_transactions_finished.annotate(
        transaction_time=(Cast(F('monitortransactionstats__stats__transaction_time'), FloatField()))
    ).aggregate(
        p95=Percentile('transaction_time', percentile=0.95),
        p99=Percentile('transaction_time', percentile=0.99),
        avg_delay=Avg(F('transaction_time'))
    )

    avg_delay = monitor_transactions_aggregates.get('avg_delay')
    if not avg_delay:
        avg_delay = 0

    p95 = monitor_transactions_aggregates.get('p95', timedelta(seconds=0))
    if not p95:
        p95 = 0
    p99 = monitor_transactions_aggregates.get('p99', timedelta(seconds=0))
    if not p99:
        p99 = 0

    search_result_monitor_stats = MonitorStats(
        transaction_count=UInt64Value(value=total_count),
        finished_transaction_count=UInt64Value(value=monitor_transactions_finished.count()),
        transaction_avg_delay=DoubleValue(value=round(avg_delay, 3)),
        percentiles={'p95': round(p95, 3), 'p99': round(p99, 3)}
    )

    qs = qs.order_by('-created_at')

    qs = filter_page(qs, page)
    qs = qs.select_related('monitor')
    mt_protos = [mt.proto for mt in qs]

    return GetMonitorsTransactionsSearchQueryResponse(meta=get_meta(total_count=total_count, tr=dtr.to_tr(), page=page),
                                                      monitor_transactions=mt_protos,
                                                      search_result_monitor_stats=search_result_monitor_stats)


def get_monitor_transaction_status(status):
    if status == 2:
        return 'Finished'
    return 'Active'


@web_api(GetMonitorsTransactionsSearchQueryRequest)
@use_read_replica
def monitors_transactions_export(request_message: GetMonitorsTransactionsSearchQueryRequest) -> Union[
    GetMonitorsTransactionsSearchQueryResponse, HttpResponse]:
    # Search monitor transactions given a search request
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    qs = account.monitortransaction_set.all()
    qs = filter_dtr(qs, dtr, 'created_at')
    qs = monitor_transaction_search_engine.process_query(qs, request_message.query_request)
    qs = qs.annotate(transaction_time=(Cast(F('monitortransactionstats__stats__transaction_time'), FloatField())))

    qs = qs.select_related('monitor')
    mt_protos = [mt.proto_for_export for mt in qs]

    mt_ids = list(x.id for x in mt_protos)
    mt_events_mapping = MonitorTransactionEventMapping.objects.filter(account_id=account.id,
                                                                      monitor_transaction_id__in=mt_ids).values_list(
        'monitor_transaction_id', 'type', 'event_id')

    mt_events_map = {}
    for mapping in mt_events_mapping:
        if mapping[0] not in mt_events_map:
            mt_events_map[mapping[0]] = []
        mt_events_map[mapping[0]].append(mapping)

    events_map = {}
    events = Events.objects.filter(account_id=account.id, id__in=list(x[2] for x in mt_events_mapping)).values_list(
        'id', 'event_type_name', 'timestamp', 'ingested_event', 'processed_kvs')
    for event in events:
        events_map[event[0]] = event

    csv_data = []

    primary_event_keys_seq = []
    secondary_event_keys_seq = []

    finished_txn = qs.filter(type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED).first()
    if finished_txn:
        sample_mt_proto = finished_txn.proto_for_export
    else:
        sample_mt_proto = mt_protos[0]

    csv_data_row = ['Transaction Value', 'Monitor Name', 'Status', 'Has Alerts', 'Transaction Time']
    primary_event = list(filter(lambda x: x[1] == 1, mt_events_map[sample_mt_proto.id]))
    if primary_event:
        primary_event = primary_event[0]
        event_object = events_map[primary_event[2]]
        csv_data_row.append('primary_event_type')
        csv_data_row.append('primary_event_timestamp')
        for kv in json.loads(event_object[3])['kvs']:
            csv_data_row.append("primary_" + kv['key'])
            primary_event_keys_seq.append(kv['key'])

    secondary_event = list(filter(lambda x: x[1] == 2, mt_events_map[sample_mt_proto.id]))
    if secondary_event:
        secondary_event = secondary_event[0]
        event_object = events_map[secondary_event[2]]
        csv_data_row.append('secondary_event_type')
        csv_data_row.append('secondary_event_timestamp')
        for kv in json.loads(event_object[3])['kvs']:
            csv_data_row.append("secondary_" + kv['key'])
            secondary_event_keys_seq.append(kv['key'])

    csv_data.append(csv_data_row)

    for mt in mt_protos:
        csv_data_row = [mt.transaction, mt.monitor.name, get_monitor_transaction_status(mt.status), mt.has_alerts.value,
                        mt.transaction_time.value]
        primary_event = list(filter(lambda x: x[1] == 1, mt_events_map[mt.id]))
        if primary_event:
            primary_event = primary_event[0]
            event_object = events_map[primary_event[2]]
            csv_data_row.append(event_object[1])
            csv_data_row.append(event_object[2].strftime('%Y-%m-%d %H:%M:%S'))
            for k in primary_event_keys_seq:
                csv_data_row.append(event_object[4].get(k, None))

        secondary_event = list(filter(lambda x: x[1] == 2, mt_events_map[mt.id]))
        if secondary_event:
            secondary_event = secondary_event[0]
            event_object = events_map[secondary_event[2]]
            csv_data_row.append(event_object[1])
            csv_data_row.append(event_object[2].strftime('%Y-%m-%d %H:%M:%S'))
            for k in secondary_event_keys_seq:
                csv_data_row.append(event_object[4].get(k, None))

        csv_data.append(csv_data_row)

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="file.csv"'
    writer = csv.writer(response)

    for row in csv_data:
        writer.writerow(row)
    return response


@web_api(GetMonitorsTransactionsExportQueryRequest)
@use_read_replica
def monitors_transactions_export_v2(request_message: GetMonitorsTransactionsExportQueryRequest) -> Union[
    GetMonitorsTransactionsExportQueryResponse, HttpResponse]:
    # Search monitor transactions given a search request
    account: Account = get_request_account()
    user: User = get_request_user()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    query_request = request_message.query_request

    export_id = uuid.uuid4().hex
    if GLOBAL_EXPORT_CONTEXT_CACHE.get(account.id, Context.MONITOR_TRANSACTION, user.email):
        export_id = GLOBAL_EXPORT_CONTEXT_CACHE.get(account.id, Context.EVENT, user.email)
    else:
        qs = account.monitortransaction_set.all()
        qs = filter_dtr(qs, dtr, 'created_at')
        qs = monitor_transaction_search_engine.process_query(qs, query_request)
        if qs.count() > 10000:
            return GetMonitorsTransactionsExportQueryResponse(meta=get_meta(), success=BoolValue(value=False),
                                                              message=Message(
                                                                  title="Monitor Transaction Export Failure",
                                                                  description="Export size too large"))
        else:
            GLOBAL_EXPORT_CONTEXT_CACHE.create_or_update(account.id, Context.MONITOR_TRANSACTION, user.email, export_id)
            export_task_request, message = export_task(account.id, Context.MONITOR_TRANSACTION, user.email,
                                                       proto_to_dict(request_message))

            if not export_task_request:
                return GetMonitorsTransactionsExportQueryResponse(meta=get_meta(), success=BoolValue(value=False),
                                                                  message=Message(
                                                                      title="Monitor Transaction Export Failure",
                                                                      description=message))

    return GetMonitorsTransactionsExportQueryResponse(meta=get_meta(), success=BoolValue(value=True),
                                                      export_context_id=export_id)


@web_api(SaveMonitorTransactionsSearchQueryRequest)
@use_read_replica
def save_monitor_transactions_search(request_message: SaveMonitorTransactionsSearchQueryRequest) -> Union[
    SaveMonitorTransactionsSearchQueryResponse, HttpResponse]:
    # Save events search context
    account: Account = get_request_account()

    generated_search_context_id = uuid.uuid4().hex
    query_request_dict = proto_to_dict(request_message.query_request)
    GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE.create_or_update(account.id, generated_search_context_id,
                                                                           query_request_dict, None)

    return SaveMonitorTransactionsSearchQueryResponse(meta=get_meta(), query_context_id=generated_search_context_id)


@web_api(GetMonitorsTransactionsSearchV2QueryRequest)
@use_read_replica
def monitors_transactions_search_v2(request_message: GetMonitorsTransactionsSearchV2QueryRequest) -> Union[
    GetMonitorsTransactionsSearchV2QueryResponse, HttpResponse]:
    # V2 search monitor transactions given a search request
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    page = request_message.meta.page

    generated_search_context_id = None
    if request_message.save_query.value:
        generated_search_context_id = uuid.uuid4().hex
        query_request_dict = proto_to_dict(request_message.query_request)
        GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE.create_or_update(account.id, generated_search_context_id,
                                                                               query_request_dict, None)

    qs = account.monitortransaction_set.all()
    qs = filter_dtr(qs, dtr, 'created_at')
    qs = monitor_transaction_search_engine.process_query(qs, request_message.query_request)
    total_count = qs.count()
    monitor_transactions_finished = qs.filter(type=MonitorTransactionProto.MonitorTransactionStatus.SECONDARY_RECEIVED)

    monitor_transactions_aggregates = monitor_transactions_finished.annotate(
        transaction_time=(Cast(F('monitortransactionstats__stats__transaction_time'), FloatField()))
    ).aggregate(
        p95=Percentile('transaction_time', percentile=0.95),
        p99=Percentile('transaction_time', percentile=0.99),
        avg_delay=Avg(F('transaction_time'))
    )

    avg_delay = monitor_transactions_aggregates.get('avg_delay')
    if not avg_delay:
        avg_delay = 0

    p95 = monitor_transactions_aggregates.get('p95', timedelta(seconds=0))
    if not p95:
        p95 = 0
    p99 = monitor_transactions_aggregates.get('p99', timedelta(seconds=0))
    if not p99:
        p99 = 0

    search_result_monitor_stats = MonitorStats(
        transaction_count=UInt64Value(value=total_count),
        finished_transaction_count=UInt64Value(value=monitor_transactions_finished.count()),
        transaction_avg_delay=DoubleValue(value=round(avg_delay, 3)),
        percentiles={'p95': round(p95, 3), 'p99': round(p99, 3)}
    )

    qs = monitor_transaction_search_engine.process_ordering(qs, request_message.query_request)
    qs = qs.select_related('monitor')
    qs = qs.annotate(stats=Subquery(
        MonitorTransactionStats.objects.filter(monitor_transaction=OuterRef("pk")).values('stats').order_by(
            'created_at')[:1]
    ))

    qs = filter_page(qs, page)
    mt_details_proto = [mt.details_proto for mt in qs]

    return GetMonitorsTransactionsSearchV2QueryResponse(
        meta=get_meta(total_count=total_count, tr=dtr.to_tr(), page=page),
        monitor_transaction_details=mt_details_proto,
        search_result_monitor_stats=search_result_monitor_stats,
        query_context_id=generated_search_context_id
    )


@web_api(GetMetricExpressionOptionsRequest)
@use_read_replica
def metrics_options(request_message: GetMetricExpressionOptionsRequest) -> Union[
    GetMetricExpressionOptionsResponse, HttpResponse]:
    account: Account = get_request_account()

    metrics_options = get_metric_options(account, request_message.context, request_message.id_literal)
    metric_expression = get_default_metric_expression(account, request_message.context, request_message.id_literal)

    return GetMetricExpressionOptionsResponse(
        meta=get_meta(),
        metric_options=metrics_options,
        default_metric_expression=metric_expression
    )


@web_api(GetMetricRequest)
@use_read_replica
def metrics(request_message: GetMetricRequest) -> Union[GetMetricResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    metric_expressions = request_message.metric_expressions

    try:
        metric_data = process_metric_expressions(account, request_message.context, metric_expressions, dtr)
        return GetMetricResponse(success=BoolValue(value=True), metric_data=metric_data)
    except Exception as e:
        return GetMetricResponse(success=BoolValue(value=False), message=Message(description='The metric expression is invalid. Try changing the aggregation data or grouping keys'))


@web_api(GetPanelsV1Request)
def panels_v1_get(request_message: GetPanelsV1Request) -> Union[GetPanelsV1Response, HttpResponse]:
    account: Account = get_request_account()
    name: str = request_message.name
    panels = GLOBAL_PANEL_CACHE.get(account_id=account.id, name=name)

    if not panels:
        return GetPanelsV1Response(meta=get_meta(), panels=[])

    panel_protos: PanelV1 = [dict_to_proto(panel, PanelV1) for panel in panels]

    return GetPanelsV1Response(meta=get_meta(), panels=panel_protos)


@web_api(CreateOrUpdatePanelV1Request)
def panels_v1_create_or_update(request_message: CreateOrUpdatePanelV1Request) -> Union[
    CreateOrUpdatePanelV1Response, HttpResponse]:
    account: Account = get_request_account()

    panel: PanelV1 = request_message.panel

    if not panel or not panel.meta_info.name:
        return CreateOrUpdatePanelV1Response(success=BoolValue(value=False))

    GLOBAL_PANEL_CACHE.create_or_update(account_id=account.id, name=panel.meta_info.name, panel=proto_to_dict(panel))
    return CreateOrUpdatePanelV1Response(success=BoolValue(value=True))


@web_api(DeletePanelV1Request)
def panels_v1_delete(request_message: DeletePanelV1Request) -> Union[
    DeletePanelV1Response, HttpResponse]:
    account: Account = get_request_account()

    name: str = request_message.name

    if not name:
        return DeletePanelV1Response(success=BoolValue(value=False))

    dashboards = GLOBAL_DASHBOARD_CACHE.get(account_id=account.id)
    for dashboard in dashboards:
        panel_names = dashboard.get('panels', [])
        for d_p_name in panel_names:
            if d_p_name == name:
                panel_names.remove(name)
        GLOBAL_DASHBOARD_CACHE.create_or_update(account_id=account.id, name=dashboard.get('name'), dashboard=dashboard)

    GLOBAL_PANEL_CACHE.delete(account_id=account.id, name=name)
    return DeletePanelV1Response(success=BoolValue(value=True))


@web_api(GetDashboardsV1Request)
def dashboards_v1_get(request_message: GetDashboardsV1Request) -> Union[GetDashboardsV1Response, HttpResponse]:
    account: Account = get_request_account()
    name: str = request_message.name
    dashboards = GLOBAL_DASHBOARD_CACHE.get(account_id=account.id, name=name)

    if not dashboards:
        return GetDashboardsV1Response(meta=get_meta(), dashboards=[])

    for dashboard in dashboards:
        panels = []
        panel_names = dashboard.get('panels', [])
        for name in panel_names:
            panel = GLOBAL_PANEL_CACHE.get(account_id=account.id, name=name)
            panels.extend(panel)
        dashboard['panels'] = panels

    dashboard_protos: DashboardV1 = [dict_to_proto(dashboard, DashboardV1) for dashboard in dashboards]

    return GetDashboardsV1Response(meta=get_meta(), dashboards=dashboard_protos)


@web_api(CreateOrUpdateDashboardV1Request)
def dashboard_v1_create_or_update(request_message: CreateOrUpdateDashboardV1Request) -> Union[
    CreateOrUpdateDashboardV1Response, HttpResponse]:
    account: Account = get_request_account()

    dashboard: DashboardV1 = request_message.dashboard

    if not dashboard or not dashboard.name or not dashboard.panels:
        return CreateOrUpdateDashboardV1Response(success=BoolValue(value=False))

    panels = dashboard.panels
    panel_names = []
    for panel in panels:
        if not panel.meta_info or not panel.meta_info.name:
            return CreateOrUpdateDashboardV1Response(success=BoolValue(value=False))
        panel_names.append(panel.meta_info.name)
        GLOBAL_PANEL_CACHE.create_or_update(account_id=account.id, name=panel.meta_info.name,
                                            panel=proto_to_dict(panel))

    dashboard_dict = proto_to_dict(dashboard)
    dashboard_dict['panels'] = panel_names
    GLOBAL_DASHBOARD_CACHE.create_or_update(account_id=account.id, name=dashboard.name, dashboard=dashboard_dict)

    return CreateOrUpdateDashboardV1Response(success=BoolValue(value=True))


@web_api(DeleteDashboardV1Request)
def dashboards_v1_delete(request_message: DeleteDashboardV1Request) -> Union[
    DeleteDashboardV1Response, HttpResponse]:
    account: Account = get_request_account()

    name: str = request_message.name

    if not name:
        return DeletePanelV1Response(success=BoolValue(value=False))

    GLOBAL_DASHBOARD_CACHE.delete(account_id=account.id, name=name)
    return DeleteDashboardV1Response(success=BoolValue(value=True))


@web_api(GetMonitorTriggerRequest)
def monitors_triggers_get(request_message: GetMonitorTriggerRequest) -> Union[GetMonitorTriggerResponse, HttpResponse]:
    account: Account = get_request_account()

    tr: TimeRange = request_message.meta.time_range
    page: Page = request_message.meta.page
    show_inactive = request_message.meta.show_inactive

    trigger_ids = request_message.trigger_ids
    monitor_id = request_message.monitor_id
    if monitor_id:
        event_triggers = account.trigger_set.filter(monitor_id=monitor_id)
        if not show_inactive or not show_inactive.value:
            event_triggers = event_triggers.filter(is_active=True)
    elif trigger_ids:
        event_triggers = account.trigger_set.filter(id__in=trigger_ids)
    else:
        return GetMonitorTriggerResponse(meta=get_meta())

    total_count = event_triggers.count()
    event_triggers = filter_page(event_triggers, page)
    trigger_protos = [t.proto for t in event_triggers]

    monitor_trigger_notifications: [MonitorTriggerNotificationDetail] = []
    for t in trigger_protos:
        notifications = account.notification_set.filter(trigger=t.id)
        notification_protos = [n.proto for n in notifications]
        monitor_trigger_notifications.append(
            MonitorTriggerNotificationDetail(trigger=t, notifications=notification_protos))

    return GetMonitorTriggerResponse(meta=get_meta(tr, page, total_count, show_inactive),
                                     monitor_trigger_notification_details=monitor_trigger_notifications)


@web_api(CreateMonitorTriggerRequest)
def monitors_triggers_create(request_message: CreateMonitorTriggerRequest) -> Union[
    CreateMonitorTriggerResponse, HttpResponse]:
    account: Account = get_request_account()

    triggers: [TriggerProto] = request_message.triggers
    notifications: [NotificationConfig] = request_message.notifications
    duplicate_triggers, err = create_monitor_triggers(account, triggers, notifications)
    if err:
        return CreateMonitorTriggerResponse(success=BoolValue(value=False), message=Message(description=err))
    elif duplicate_triggers:
        message = ''
        for dp_triggers in duplicate_triggers:
            message += dp_triggers.name
            message += ', '
        return CreateMonitorTriggerResponse(success=BoolValue(value=False),
                                            message=Message(title='Found duplicate trigger names', description=message))

    return CreateMonitorTriggerResponse(success=BoolValue(value=True))


@web_api(UpdateMonitorTriggerRequest)
def monitors_triggers_update(request_message: UpdateMonitorTriggerRequest) -> Union[
    UpdateMonitorTriggerResponse, HttpResponse]:
    account: Account = get_request_account()

    trigger_id = request_message.trigger_id.value
    if not trigger_id:
        return ProtoJsonResponse(
            UpdateMonitorTriggerResponse(
                success=BoolValue(value=False),
                message=Message(title='No trigger id defined in the request')
            ),
            status=400
        )

    qs: QuerySet = account.trigger_set.filter(id=trigger_id).select_related('monitor', 'monitor__primary_key',
                                                                            'monitor__secondary_key')
    if not qs.exists():
        return ProtoJsonResponse(
            UpdateMonitorTriggerResponse(
                success=BoolValue(value=False),
                message=Message(title=f'{trigger_id} defined in the request not found')
            ),
            status=400
        )

    trigger: Trigger = qs.first()
    try:
        monitor_trigger_update_processor.update(trigger, request_message.update_trigger_ops)
    except Exception as ex:
        return ProtoJsonResponse(
            UpdateMonitorTriggerResponse(
                success=BoolValue(value=False),
                message=Message(
                    title=f'Error updating entity {trigger.name}',
                    # description=f'{traceback.format_exception(type(ex), ex, ex.__traceback__)}'
                )
            ),
            status=400
        )
    return UpdateMonitorTriggerResponse(
        success=BoolValue(value=True),
        message=Message(title=f'Entity "{trigger.name}" updated successfully')
    )


@web_api(GetMonitorTriggerOptionsRequest)
def monitors_triggers_options(request_message: GetMonitorTriggerOptionsRequest) -> Union[
    GetMonitorTriggerOptionsResponse, HttpResponse]:
    user: User = get_request_user()
    account: Account = get_request_account()

    if not request_message.monitor_id:
        return GetMonitorTriggerOptionsResponse(meta=get_meta(), success=BoolValue(value=False),
                                                message=Message(description="Invalid request body",
                                                                title="Missing monitor id"))
    else:
        monitor = \
            account.monitor_set.filter(id=request_message.monitor_id).values('id', 'name', 'primary_key__event_type_id',
                                                                             'secondary_key__event_type_id')[0]

        event_keys = account.eventkey_set.filter(event_type_id__in=[monitor['primary_key__event_type_id'],
                                                                    monitor[
                                                                        'secondary_key__event_type_id']])
        event_key_options = [ek.proto for ek in event_keys if is_filterable_key_type(ek.type)]

    trigger_options: [TriggerOption] = get_default_trigger_options(scope=account)
    notification_options: [NotificationOption] = notification_client.get_notification_options(user, account)

    return GetMonitorTriggerOptionsResponse(meta=get_meta(), success=BoolValue(value=True),
                                            monitor=MonitorPartial(id=monitor['id'], name=monitor["name"]),
                                            primary_event_type=EventTypePartial(
                                                id=monitor['primary_key__event_type_id']),
                                            secondary_event_type=EventTypePartial(
                                                id=monitor['secondary_key__event_type_id']),
                                            monitor_trigger_options=MonitorTriggerOptions(
                                                default_trigger_options=trigger_options,
                                                notification_options=notification_options,
                                                event_key_filter_options=event_key_options))


@web_api(GetAlertsSummaryRequest)
def alerts_summary_get(request_message: GetAlertsSummaryRequest) -> Union[GetAlertsSummaryResponse, HttpResponse]:
    account: Account = get_request_account()

    tr: TimeRange = request_message.meta.time_range
    page: Page = request_message.meta.page

    if request_message.monitor_id:
        triggers = account.trigger_set.filter(monitor_id=request_message.monitor_id).values('id')
        trigger_ids = [trigger["id"] for trigger in triggers]
        event_alerts = account.alert_set.filter(trigger_id__in=trigger_ids)
    elif request_message.alert_ids:
        event_alerts = account.alert_set.filter(id__in=request_message.alert_ids)
    else:
        event_alerts = account.alert_set.all()

    qs = filter_time_range(event_alerts, tr, 'triggered_at')
    qs = qs.order_by('-triggered_at')
    qs = qs.select_related('trigger', 'trigger__monitor', 'trigger__monitor__primary_key',
                           'trigger__monitor__secondary_key', 'trigger__monitor__primary_key__event_type',
                           'trigger__monitor__secondary_key__event_type')
    total_count = qs.count()

    event_alerts = filter_page(qs, page)
    alert_summary_protos = [a.proto_summary for a in event_alerts]

    return GetAlertsSummaryResponse(meta=get_meta(tr=tr, page=page, total_count=total_count),
                                    alerts_summary=alert_summary_protos)


@web_api(GetAlertDetailsRequest)
def alerts_details_get(request_message: GetAlertDetailsRequest) -> Union[GetAlertDetailsResponse, HttpResponse]:
    account: Account = get_request_account()

    alert_id = request_message.alert_id

    if not alert_id:
        return GetAlertDetailsResponse(meta=get_meta(), alert_detail=AlertDetail())

    alert: Alert = account.alert_set.select_related('trigger', 'trigger__monitor', 'trigger__monitor__primary_key',
                                                    'trigger__monitor__secondary_key',
                                                    'trigger__monitor__primary_key__event_type',
                                                    'trigger__monitor__secondary_key__event_type').get(id=alert_id)

    if alert:
        return GetAlertDetailsResponse(meta=get_meta(), alert_detail=alert.proto_details)

    return GetAlertDetailsResponse(meta=get_meta(), alert_detail=AlertDetail())


@web_api(GetAlertMonitorTransactionsRequest)
def alerts_monitor_transactions_get(request_message: GetAlertMonitorTransactionsRequest) -> Union[
    GetAlertMonitorTransactionsResponse, HttpResponse]:
    account: Account = get_request_account()

    page: Page = request_message.meta.page
    alert_id = request_message.alert_id

    if not alert_id:
        return GetAlertMonitorTransactionsResponse(meta=get_meta(page=page))

    alert_mts = account.alertmonitortransactionmapping_set.filter(alert=alert_id).prefetch_related(
        'monitor_transaction').select_related('monitor_transaction__monitor')
    total_count = alert_mts.count()
    alert_mts = alert_mts.order_by('-created_at')
    alert_mts = filter_page(alert_mts, page)
    alert_mts_proto = [amt.proto for amt in alert_mts]
    return GetAlertMonitorTransactionsResponse(meta=get_meta(page=page, total_count=total_count),
                                               monitor_transactions=alert_mts_proto)


@web_api(GetNotificationConfigsRequest)
def notifications_get(request_message: GetNotificationConfigsRequest) -> Union[
    GetNotificationConfigsResponse, HttpResponse]:
    account: Account = get_request_account()

    if request_message.trigger_id:
        notifications = account.notification_set.filter(
            trigger=request_message.trigger_id)
    else:
        notifications = account.notification_set.all()

    notification_protos = [n.proto for n in notifications]

    return GetNotificationConfigsResponse(notification_configs=notification_protos)


@web_api(CreateNotificationConfigRequest)
def notifications_create(request_message: CreateNotificationConfigRequest) -> Union[
    CreateNotificationConfigResponse, HttpResponse]:
    account: Account = get_request_account()

    notification_config: NotificationConfig = request_message.notification_config

    channel_config = None
    which_one_of = notification_config.WhichOneof('config')
    if notification_config.channel == NotificationConfig.Channel.SLACK and which_one_of == 'slack_configuration':
        channel_config = proto_to_dict(notification_config.slack_configuration)

    if not notification_config.channel or not channel_config:
        return CreateNotificationConfigResponse(success=BoolValue(value=False),
                                                message=Message(title='Error creating notification',
                                                                description='Invalid payload'))

    Notification.objects.update_or_create(account=account,
                                          config=channel_config,
                                          defaults={
                                              'channel': notification_config.channel,
                                          })

    return CreateNotificationConfigResponse(success=BoolValue(value=True))


@web_api(GetGlobalSearchOptionsRequest)
def global_search_options_get(request_message: GetGlobalSearchOptionsRequest) -> Union[
    GetGlobalSearchOptionsResponse, HttpResponse]:
    return GetGlobalSearchOptionsResponse(meta=get_meta(),
                                          global_query_options=GlobalQueryOptions(op_descriptions=OP_DESCRIPTIONS,
                                                                                  op_mapping=OP_MAPPINGS,
                                                                                  literal_type_description=LITERAL_TYPE_DESCRIPTIONS))


@web_api(GetEventSearchOptionsRequest)
@use_read_replica
def event_search_options_get(request_message: GetEventSearchOptionsRequest) -> Union[
    GetEventSearchOptionsResponse, HttpResponse]:
    account: Account = get_request_account()

    default_query = None
    saved_query_request_context_id = request_message.saved_query_request_context_id.string.value
    if saved_query_request_context_id:
        saved_context = GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE.get(account.id, saved_query_request_context_id)
        if saved_context:
            default_query = dict_to_proto(saved_context, QueryRequest)

    obj = global_event_search_engine.get_obj(account, request_message.id_literal)
    if not default_query:
        default_query = global_event_search_engine.get_default_query(account, obj)

    options = global_event_search_engine.get_search_options(account, obj)
    return GetEventSearchOptionsResponse(meta=get_meta(),
                                         event_query_options=options,
                                         default_query_request=default_query)


@web_api(GetEntityOptionsRequest)
def entity_options_get(request_message: GetEntityOptionsRequest) -> Union[GetEntityOptionsResponse, HttpResponse]:
    account: Account = get_request_account()
    qs: QuerySet = account.eventkey_set.filter(type__in=TRANSACTIONAL_KEY_TYPES)
    qs = qs.prefetch_related('event_type')
    event_key_options = [ek.proto for ek in qs]
    return GetEntityOptionsResponse(entity_options=EntityOptions(event_key_options=event_key_options))


@web_api(CreateEntityRequest)
def entity_create(request_message: CreateEntityRequest) -> Union[CreateEntityResponse, HttpResponse]:
    account: Account = get_request_account()

    name = request_message.name.value
    if not name:
        return ProtoJsonResponse(
            CreateEntityResponse(
                success=BoolValue(value=False),
                message=Message(title='Entity creation failed', description='Missing name defined for the entity')
            ),
            status=400
        )
    event_key_ids = set(request_message.event_key_ids)
    if account.eventkey_set.filter(id__in=event_key_ids).count() != len(event_key_ids):
        return ProtoJsonResponse(
            CreateEntityResponse(
                success=BoolValue(value=False),
                message=Message(title='Entity creation failed', description='Invalid event keys selected')
            ),
            status=400
        )
    with dj_transaction.atomic():
        if account.entity_set.filter(name=name).exists():
            return CreateEntityResponse(
                success=BoolValue(value=False),
                message=Message(title='Entity creation failed', description=f'Entity with name {name} already exists')
            )
        e = Entity(
            account=account,
            name=name,
            is_active=request_message.is_active.value
        )
        e.save()

        eekms = [
            EntityEventKeyMapping(
                account=account, entity=e, event_key_id=event_key_id
            ) for event_key_id in request_message.event_key_ids
        ]

        if eekms:
            EntityEventKeyMapping.objects.bulk_create(
                eekms,
                ignore_conflicts=True,
                batch_size=25
            )

    return CreateEntityResponse(
        success=BoolValue(value=True),
        message=Message(
            title=f'Entity {name} created successfully'
        ),
        entity=e.proto_partial
    )


@web_api(UpdateEntityRequest)
def entity_update(request_message: UpdateEntityRequest) -> Union[UpdateEntityResponse, HttpResponse]:
    account: Account = get_request_account()

    entity_id = request_message.entity_id.value
    if not entity_id:
        return ProtoJsonResponse(
            UpdateEntityResponse(
                success=BoolValue(value=False),
                message=Message(title='No entity id defined in the request')
            ),
            status=400
        )

    qs: QuerySet = account.entity_set.filter(id=entity_id)
    if not qs.exists():
        return ProtoJsonResponse(
            UpdateEntityResponse(
                success=BoolValue(value=False),
                message=Message(title=f'{entity_id} defined in the request not found')
            ),
            status=400
        )

    entity: Entity = qs.first()
    entity_name = entity.name
    try:
        entity_update_processor.update(entity, request_message.update_entity_ops)
    except Exception as ex:
        return ProtoJsonResponse(
            UpdateEntityResponse(
                success=BoolValue(value=False),
                message=Message(
                    title=f'Error updating entity {entity_name}',
                    # description=f'{traceback.format_exception(type(ex), ex, ex.__traceback__)}'
                    description=f'{ex}'
                )
            ),
            status=400
        )
    return UpdateEntityResponse(
        success=BoolValue(value=True),
        message=Message(title=f'Entity "{entity.name}" updated successfully')
    )


@web_api(EntityTriggerOptionsRequest)
def entity_triggers_options(request_message: EntityTriggerOptionsRequest) -> Union[
    EntityTriggerOptionsResponse, HttpResponse]:
    account: Account = get_request_account()
    meta: Meta = request_message.meta
    dtr: DateTimeRange = to_dtr(meta.time_range)

    entity_id = request_message.entity_id
    qs: QuerySet = account.entity_set.filter(id=entity_id)
    entity = qs.first()

    event_type_ids = EntityEventKeyMapping.objects.prefetch_related('event_key__event_type').filter(
        entity=entity).values_list('event_key__event_type_id', flat=True)
    event_keys = account.eventkey_set.filter(event_type_id__in=event_type_ids)
    event_key_options = [ek.proto for ek in event_keys if is_filterable_key_type(ek.type)]

    return EntityTriggerOptionsResponse(meta=get_meta(tr=to_tr(dtr)), event_key_options=event_key_options,
                                        entity_name=entity.name)


@web_api(EntityTriggerCreateRequest)
def entity_triggers_create(request_message: EntityTriggerCreateRequest) -> Union[
    EntityTriggerCreateResponse, HttpResponse]:
    account: Account = get_request_account()
    trigger_name = request_message.trigger.name
    entity_id = request_message.trigger.entity_id
    notifications = request_message.notifications

    trigger_definition = request_message.trigger.definition
    trigger_filter = request_message.trigger.filter

    if not trigger_name:
        return EntityTriggerCreateResponse(
            message=Message(title='Trigger creation failed', description='Missing name defined for the trigger'))

    if not entity_id:
        return EntityTriggerCreateResponse(
            message=Message(title='Trigger creation failed', description='Missing entity id defined for the trigger'))

    entity = account.entity_set.filter(id=entity_id).first()
    db_trigger = EntityTrigger.objects.create(
        account=account, name=trigger_name, entity=entity, type=trigger_definition.type,
        rule_type=trigger_definition.rule_type,
        config=proto_to_dict(trigger_definition.trigger_rule_config), generated_config=proto_to_dict(trigger_filter)
    )

    if notifications:
        db_notifications = create_db_notifications(account, notifications)
        for db_n in db_notifications:
            EntityTriggerNotificationConfigMapping.objects.update_or_create(account=account, entity_trigger=db_trigger,
                                                                            notification=db_n)

    return EntityTriggerCreateResponse(message=Message(title='Trigger created'))


@web_api(EntityTriggerCreateRequest)
def entity_triggers_update(request_message: EntityTriggerCreateRequest) -> Union[
    EntityTriggerCreateResponse, HttpResponse]:
    account: Account = get_request_account()
    trigger_name = request_message.trigger.name
    trigger_id = request_message.trigger.id
    is_trigger_active = request_message.trigger.is_active.value
    notifications = request_message.notifications

    trigger_definition = request_message.trigger.definition
    trigger_filter = request_message.trigger.filter

    if not trigger_name:
        return EntityTriggerCreateResponse(
            message=Message(title='Trigger creation failed', description='Missing name defined for the trigger'))

    entity_trigger = EntityTrigger.objects.get(account=account, id=trigger_id)
    entity_trigger.name = trigger_name
    entity_trigger.type = trigger_definition.type
    entity_trigger.rule_type = trigger_definition.rule_type
    entity_trigger.config = proto_to_dict(trigger_definition.trigger_rule_config)
    entity_trigger.generated_config = proto_to_dict(trigger_filter)
    entity_trigger.is_active = is_trigger_active
    entity_trigger.save()

    # remove notification mappings
    EntityTriggerNotificationConfigMapping.objects.filter(account=account, entity_trigger=entity_trigger).delete()

    if notifications:
        db_notifications = create_db_notifications(account, notifications)
        for db_n in db_notifications:
            EntityTriggerNotificationConfigMapping.objects.update_or_create(account=account,
                                                                            entity_trigger=entity_trigger,
                                                                            notification=db_n)

    return EntityTriggerCreateResponse(message=Message(title='Trigger Updated'))


@web_api(EntityTriggerInactiveRequest)
def entity_triggers_inactivate(request_message: EntityTriggerInactiveRequest) -> Union[
    EntityTriggerInactiveResponse, HttpResponse]:
    account: Account = get_request_account()
    entity_trigger_id = request_message.entity_trigger_id

    if not entity_trigger_id:
        return EntityTriggerInactiveResponse(
            message=Message(title='Trigger inactivation failed', description='Missing entity trigger id'))

    EntityTrigger.objects.filter(
        account=account, id=entity_trigger_id
    ).update(is_active=False)

    return EntityTriggerInactiveResponse(message=Message(title='Trigger deactivated'))


@web_api(GetEntityTriggersRequest)
def entity_triggers_get(request_message: GetEntityTriggersRequest) -> Union[GetEntityTriggersResponse, HttpResponse]:
    account: Account = get_request_account()

    meta: Meta = request_message.meta
    dtr: DateTimeRange = to_dtr(meta.time_range)
    page = meta.page

    entity_id = request_message.entity_id
    entity_trigger_id = request_message.entity_trigger_id

    event_triggers = account.entitytrigger_set.filter(entity_id=entity_id, is_active=True)
    if entity_trigger_id:
        event_triggers = event_triggers.filter(id=entity_trigger_id)

    total_count = event_triggers.count()
    event_triggers = filter_page(event_triggers, page)
    trigger_protos = [t.proto for t in event_triggers]

    entity_trigger_notifications = []
    for t in trigger_protos:
        notifications = account.notification_set.filter(entity_trigger=t.id)
        notification_protos = [n.proto for n in notifications]
        entity_trigger_notifications.append(
            EntityTriggerNotificationDetail(trigger=t, notifications=notification_protos))

    return GetEntityTriggersResponse(meta=get_meta(tr=to_tr(dtr), page=page, total_count=total_count),
                                     entity_trigger_notification_details=entity_trigger_notifications)


@web_api(GetEntitySummaryRequest)
def entity_summary_get(request_message: GetEntitySummaryRequest) -> Union[GetEntitySummaryResponse, HttpResponse]:
    account: Account = get_request_account()

    meta: Meta = request_message.meta
    dtr: DateTimeRange = to_dtr(meta.time_range)
    page = meta.page
    show_inactive = meta.show_inactive

    qs: QuerySet = account.entity_set.filter(is_generated=False)
    if request_message.fuzzy_search_request.pattern:
        qs: QuerySet = entity_fuzzy_match_engine.match_pattern(account, request_message.fuzzy_search_request.pattern)

    if request_message.entity_ids:
        qs = qs.filter(id__in=request_message.entity_ids)

    if not show_inactive or not show_inactive.value:
        qs = qs.filter(is_active=True)

    total_count = qs.count()
    qs = filter_page(qs, page)

    entities = [entity.summary for entity in qs]
    return GetEntitySummaryResponse(
        meta=get_meta(tr=to_tr(dtr), page=page, total_count=total_count, show_inactive=show_inactive),
        entities=entities,
    )


@web_api(GetEntityDetailRequest)
def entity_detail_get(request_message: GetEntityDetailRequest) -> Union[GetEntityDetailResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    entity_id = request_message.entity_id.value
    if not entity_id:
        return JsonResponse({'message': 'No entity id defined in the request'}, status=400)

    qs: QuerySet = account.entity_set.filter(id=entity_id)
    if not qs.exists():
        return JsonResponse({'message': f'{entity_id} defined in the request not found'}, status=400)

    qs = qs.prefetch_related(
        'entityeventkeymapping_set',
        'entityeventkeymapping_set__event_key',
        'entityeventkeymapping_set__event_key__event_type'
    )
    entity: Entity = qs.first()

    return GetEntityDetailResponse(meta=get_meta(tr=to_tr(dtr)), entity=entity.detail_without_stats)


@web_api(GetEntityInstancesSummaryRequest)
def entity_instances_summary_get(request_message: GetEntityInstancesSummaryRequest) -> Union[
    GetEntityInstancesSummaryResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    page: Page = request_message.meta.page

    entity_id = request_message.entity_id.value
    if not entity_id:
        return JsonResponse({'message': 'No entity id defined in the request'}, status=400)
    if not account.entity_set.filter(id=entity_id).exists():
        return JsonResponse({'message': f'{entity_id} defined in the request not found'}, status=400)

    dtr_entity_instances = filter_dtr(
        account.entityinstanceeventmapping_set.filter(entity_id=entity_id),
        dtr,
        'created_at'
    )
    dtr_entity_instances = dtr_entity_instances.values('entity_instance_id').distinct().values_list(
        'entity_instance_id', flat=True
    )

    qs: QuerySet = account.entityinstance_set.filter(id__in=dtr_entity_instances)
    total_count = qs.count()
    qs = filter_page(qs.order_by('-created_at'), page)
    qs = annotate_entity_instance_stats(account, qs, dtr)

    entity_instances = [entity_instance.summary for entity_instance in qs]

    return GetEntityInstancesSummaryResponse(
        meta=get_meta(tr=to_tr(dtr), page=page, total_count=total_count),
        entity_instances=entity_instances
    )


@web_api(GetEntityInstancesDetailRequest)
def entity_instances_details_get(request_message: GetEntityInstancesDetailRequest) -> Union[
    GetEntityInstancesDetailResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    page: Page = request_message.meta.page

    entity_instance_id = request_message.entity_instance_id.value
    if not entity_instance_id:
        return JsonResponse({'message': 'No entity instance id defined in the request'}, status=400)
    qs: QuerySet = account.entityinstance_set.filter(id=entity_instance_id)

    if not qs.exists():
        return JsonResponse({'message': f'{entity_instance_id} defined in the request not found'}, status=400)

    qs = annotate_entity_instance_stats(account, qs, dtr)
    qs = qs.prefetch_related('entity')
    entity_instance = qs.first()

    return GetEntityInstancesDetailResponse(
        meta=get_meta(tr=to_tr(dtr), page=page),
        entity_instance=entity_instance.detail
    )


@web_api(GetEntityInstancesTimelineRequest)
@use_read_replica
def entity_instances_timeline_get(request_message: GetEntityInstancesTimelineRequest) -> Union[
    GetEntityInstancesTimelineResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    page: Page = request_message.meta.page

    entity_instance_id = request_message.entity_instance_id.value
    if not entity_instance_id:
        return JsonResponse({'message': 'No entity instance id defined in the request'}, status=400)
    qs: QuerySet = account.entityinstance_set.filter(id=entity_instance_id)

    if not qs.exists():
        return JsonResponse({'message': f'{entity_instance_id} defined in the request not found'}, status=400)

    entity_instance = qs.first()

    qs = account.entityinstanceeventmapping_set.filter(
        entity_instance=entity_instance
    ).order_by('-event_timestamp')

    total_count = qs.count()
    qs = annotate_entity_timeline(qs)

    event_id_record_dict = {}
    for rec in qs:
        event_id = rec['event_id']
        if event_id not in event_id_record_dict:
            event_id_record_dict[event_id] = EntityInstanceTimelineRecord(
                timestamp=int(rec['event_timestamp'].timestamp() * 1000),
                event=EventProto(
                    id=event_id,
                    event_type=EventTypePartial(
                        id=rec['event_type_id'],
                        name=rec['event_type_name']
                    ),
                    kvs=dict_to_proto(rec['event_event'], IngestionEvent).kvs
                ),
                transaction_event_type_mapping=[]
            )

        timeline_record: EntityInstanceTimelineRecord = event_id_record_dict[event_id]
        monitor = None
        transaction = None
        if 'monitor_id' in rec and rec['monitor_id']:
            monitor = MonitorPartial(
                id=rec['monitor_id'],
                name=rec['monitor_name']
            )
        if 'monitor_transaction_id' in rec and rec['monitor_transaction_id']:
            transaction = MonitorTransactionPartial(
                id=UInt64Value(value=rec['monitor_transaction_id']),
                transaction=StringValue(
                    value=rec['transaction']
                ),
                monitor=monitor
            )
            transaction_event_type = rec['transaction_event_type']

        qc: QuerySet = account.alertmonitortransactionmapping_set.all()
        qc = qc.prefetch_related('alert', 'alert__trigger')
        qc = qc.filter(monitor_transaction_id=rec['monitor_transaction_id'])

        for alert_monitor_transaction_mapping in qc:
            timeline_record.alerts.append(alert_monitor_transaction_mapping.miniproto)

        # if transaction:
        #     timeline_record.transaction_event_type_mapping.append(
        #         EntityInstanceTimelineRecord.MonitorTransactionEventTypeMapping(
        #             monitor_transaction=transaction,
        #             transaction_event_type=transaction_event_type,
        #             has_alerts=BoolValue(value=rec['monitor_transaction_has_alert'])
        #         )
        #     )

    sorted_events = sorted(event_id_record_dict.items(), key=lambda record: record[1].timestamp, reverse=True)
    recs = [rec for event_id, rec in sorted_events]

    return GetEntityInstancesTimelineResponse(
        meta=get_meta(tr=to_tr(dtr), page=page, total_count=total_count),
        entity_instance_timeline_records=recs,
    )


@web_api(GetEntityInstancesSearchOptionsRequest)
def entity_instance_search_options_get(request_message: GetEntityInstancesSearchOptionsRequest) -> Union[
    GetEntityInstancesSearchOptionsResponse, HttpResponse]:
    account: Account = get_request_account()

    obj = entity_instance_search_engine.get_obj(account, request_message.id_literal)
    options = entity_instance_search_engine.get_search_options(account, obj)
    default_query = entity_instance_search_engine.get_default_query(account, obj)

    return GetEntityInstancesSearchOptionsResponse(meta=get_meta(),
                                                   event_query_options=options,
                                                   default_query_request=default_query)


@web_api(GetEntityInstancesSearchOptionsV2Request)
def entity_instance_search_options_get_v2(request_message: GetEntityInstancesSearchOptionsV2Request) -> Union[
    GetEntityInstancesSearchOptionsV2Response, HttpResponse]:
    account: Account = get_request_account()

    obj = entity_instance_search_engine.get_obj(account, request_message.id_literal)
    options = entity_instance_search_engine.get_search_options_v2(account, obj)
    default_query = entity_instance_search_engine.get_default_query(account, obj)

    return GetEntityInstancesSearchOptionsV2Response(meta=get_meta(),
                                                     event_query_options=options,
                                                     default_query_request=default_query)


@web_api(GetEntityInstanceSearchQueryRequest)
@use_read_replica
def entity_instances_search(request_message: GetEntityInstanceSearchQueryRequest) -> Union[
    GetEntityInstanceSearchQueryResponse, HttpResponse]:
    # Search entity instances given a search request
    account: Account = get_request_account()
    page = request_message.meta.page
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    qs = account.entityinstance_set.all()
    qs = filter_dtr(qs, dtr, 'created_at')
    qs = entity_instance_search_engine.process_query(qs, request_message.query_request)

    total_count = qs.count()
    qs = qs.order_by('-created_at')
    qs = filter_page(qs, page)

    qs = annotate_entity_instance_stats(account, qs, dtr)
    qs = qs.prefetch_related('entity')

    entity_instances_protos = [ei.summary for ei in qs]

    return GetEntityInstanceSearchQueryResponse(meta=get_meta(total_count=total_count, tr=dtr.to_tr(), page=page),
                                                entity_instances=entity_instances_protos)


@web_api(GetEntityWorkflowSearchQueryRequest)
@use_read_replica
def entity_workflow_get(request_message: GetEntityWorkflowSearchQueryRequest) -> Union[
    GetEntityWorkflowSearchQueryResponse, HttpResponse]:
    # Search entity instances given a search request
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    entity_id = request_message.entity_id.value
    entity_instance_value = request_message.entity_instance_value

    is_single_key_name = False

    entity_event_key_type_metadata = account.entityeventkeymapping_set.filter(entity_id=entity_id).values(
        'event_key__name', 'event_key__event_type_id')
    event_type_ids = set([e['event_key__event_type_id'] for e in entity_event_key_type_metadata])
    event_type_key_name_mapping = {e['event_key__event_type_id']: e['event_key__name'] for e in
                                   entity_event_key_type_metadata}

    if len(set(event_type_key_name_mapping.values())) == 1:
        is_single_key_name = True

    if is_single_key_name:
        key_name = list(set(event_type_key_name_mapping.values()))[0]

        if entity_instance_value:
            events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and processed_kvs.{} = '{}' and timestamp between '{}' and '{}' order by timestamp) group by e_id;".format(
                key_name, ','.join([str(e) for e in event_type_ids]), account.id, key_name, entity_instance_value,
                dtr.to_tr_str()[0], dtr.to_tr_str()[1]
            )
        else:
            events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' order by timestamp) group by e_id;".format(
                key_name, ','.join([str(e) for e in event_type_ids]), account.id, dtr.to_tr_str()[0], dtr.to_tr_str()[1]
            )

        qs = Events.objects.raw(events_group_query)
        events_group_set = list(qs)

        g = NewGraph()

        for grouped_events in events_group_set:
            g.add_flow_grouped_events(grouped_events)

        wf_view = g.get_view()
    else:
        if entity_instance_value:
            entity_instance_value_query_string_arr = []
            for type_id in event_type_key_name_mapping:
                key_name = event_type_key_name_mapping[type_id]
                entity_instance_value_query_string_arr.append(
                    "(event_type_id = {} and processed_kvs.{} = '{}')".format(type_id, key_name, entity_instance_value))
            entity_instance_value_query_string = "( {} )".format(' OR '.join(entity_instance_value_query_string_arr))

            events_group_query = "SELECT id, account_id, created_at, timestamp, event_type_id, event_type_name, event_source, processed_kvs, ingested_event FROM events WHERE (account_id = {} AND {} AND timestamp >= '{}' AND timestamp < '{}');".format(
                account.id, entity_instance_value_query_string, dtr.to_tr_str()[0], dtr.to_tr_str()[1]
            )
        else:
            events_group_query = "SELECT id, account_id, created_at, timestamp, event_type_id, event_type_name, event_source, processed_kvs, ingested_event FROM events WHERE (account_id = {} AND event_type_id IN ({}) AND timestamp >= '{}' AND timestamp < '{}');".format(
                account.id, ','.join([str(e) for e in event_type_ids]), dtr.to_tr_str()[0], dtr.to_tr_str()[1]
            )

        qs = Events.objects.raw(events_group_query)
        events_set = list(qs)
        event_groups = defaultdict(list)

        g = Graph()

        for event in events_set:
            event_group_key_val = event.processed_kvs.get(event_type_key_name_mapping[event.event_type_id])
            event_groups[event_group_key_val].append(event)

        for grouped_events in event_groups.values():
            grouped_events.sort(key=lambda e: e.timestamp)
            g.add_flow_events(grouped_events)

        wf_view = g.get_view()

    return GetEntityWorkflowSearchQueryResponse(meta=get_meta(tr=dtr.to_tr()),
                                                workflow_view=wf_view)


@web_api(GetFunnelRequest)
@use_read_replica
def funnel_get_v2(request_message: GetFunnelRequest) -> Union[
    GetFunnelResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    funnel_event_type_ids = request_message.event_type_ids
    event_key_name = request_message.event_key_name
    filter_key_name = request_message.filter_key_name
    filter_value = request_message.filter_value

    events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}' ) order by timestamp asc, created_at asc) group by e_id;".format(
        event_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id, dtr.to_tr_str()[0],
        to_str(datetime.now()), event_key_name, event_key_name, funnel_event_type_ids[0], account.id,
        dtr.to_tr_str()[0],
        dtr.to_tr_str()[1]
    )

    if filter_key_name and filter_value:
        events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, groupArray(e_filter_value) as filter_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id, processed_kvs.{} as e_filter_value from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}') order by timestamp asc, created_at asc) group by e_id;".format(
            event_key_name, filter_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id,
            dtr.to_tr_str()[0],
            to_str(datetime.now()), event_key_name, event_key_name, funnel_event_type_ids[0], account.id,
            dtr.to_tr_str()[0],
            dtr.to_tr_str()[1]
        )

    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    f = Funnel()

    for grouped_event in events_group_set:
        f.add_to_node_map(grouped_event, filter_key_name=filter_key_name, filter_value=filter_value)

    ordered_funnel_data = f.get_ordered_funnel_data(funnel_event_type_ids)
    return GetFunnelResponse(meta=get_meta(tr=dtr.to_tr()), workflow_view=ordered_funnel_data)


@web_api(GetFunnelDropOffRequest)
@use_read_replica
def funnel_drop_off_distribution_get(request_message: GetFunnelDropOffRequest) -> Union[
    GetFunnelDropOffDistributionResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    funnel_event_type_ids = request_message.funnel_event_type_ids
    funnel_key_name = request_message.funnel_key_name
    start_event_type_id = request_message.start_event_type_id
    end_event_type_id = request_message.end_event_type_id

    filter_key_name = request_message.filter_key_name
    filter_value = request_message.filter_value

    events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}' ) order by timestamp asc, created_at asc) group by e_id;".format(
        funnel_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id, dtr.to_tr_str()[0],
        to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
        dtr.to_tr_str()[0],
        dtr.to_tr_str()[1]
    )

    if filter_key_name and filter_value:
        events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, groupArray(e_filter_value) as filter_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id, processed_kvs.{} as e_filter_value from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}') order by timestamp asc, created_at asc) group by e_id;".format(
            funnel_key_name, filter_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id,
            dtr.to_tr_str()[0],
            to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
            dtr.to_tr_str()[0],
            dtr.to_tr_str()[1]
        )

    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    f = Funnel()

    for grouped_event in events_group_set:
        f.add_to_node_map(grouped_event, filter_key_name=filter_key_name, filter_value=filter_value)

    dropped_funnel_records, start_event_type_event_count = f.get_funnel_drop_records(funnel_event_type_ids,
                                                                                     start_event_type_id,
                                                                                     end_event_type_id)
    if not dropped_funnel_records:
        return GetFunnelDropOffDistributionResponse(meta=get_meta(tr=dtr.to_tr()))
    else:
        dropped_funnel_records_str = ', '.join(f"'{x}'" for x in list(dropped_funnel_records))
        grouped_by_eventType_dropped_records = f"SELECT count(processed_kvs.{funnel_key_name}) as count, event_type_name as id FROM (SELECT processed_kvs.{funnel_key_name},timestamp, event_type_name, ROW_NUMBER() OVER (PARTITION BY processed_kvs.{funnel_key_name} ORDER BY timestamp DESC) AS row_num FROM events where processed_kvs.{funnel_key_name} in ({dropped_funnel_records_str}) and timestamp >= '{dtr.to_tr_str()[0]}' and account_id = {account.id}) WHERE row_num = 1 group by event_type_name"
        funnel_event_type_distribution = []
        qs = Events.objects.raw(grouped_by_eventType_dropped_records)
        for row in qs:
            funnel_event_type_distribution_object = GetFunnelDropOffDistributionResponse.FunnelEventTypeDistribution(
                event_type_name=row.id, count=row.count)
            funnel_event_type_distribution.append(funnel_event_type_distribution_object)

        return GetFunnelDropOffDistributionResponse(meta=get_meta(tr=dtr.to_tr()),
                                                    funnel_event_type_distribution=funnel_event_type_distribution,
                                                    previous_node_count=len(start_event_type_event_count))


@web_api(GetFunnelDropOffRequest)
@use_read_replica
def funnel_drop_off_distribution_get_v2(request_message: GetFunnelDropOffRequest) -> Union[
    GetFunnelDropOffDistributionResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    funnel_event_type_ids = request_message.funnel_event_type_ids
    funnel_key_name = request_message.funnel_key_name
    start_event_type_id = request_message.start_event_type_id
    end_event_type_id = request_message.end_event_type_id

    filter_key_name = request_message.filter_key_name
    filter_value = request_message.filter_value

    events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}' ) order by timestamp asc, created_at asc) group by e_id;".format(
        funnel_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id, dtr.to_tr_str()[0],
        to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
        dtr.to_tr_str()[0],
        dtr.to_tr_str()[1]
    )

    if filter_key_name and filter_value:
        events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, groupArray(e_filter_value) as filter_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id, processed_kvs.{} as e_filter_value from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}') order by timestamp asc, created_at asc) group by e_id;".format(
            funnel_key_name, filter_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id,
            dtr.to_tr_str()[0],
            to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
            dtr.to_tr_str()[0],
            dtr.to_tr_str()[1]
        )

    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    f = Funnel()

    for grouped_event in events_group_set:
        f.add_to_node_map(grouped_event, filter_key_name=filter_key_name, filter_value=filter_value)

    dropped_funnel_records, start_event_type_event_count = f.get_funnel_drop_records(funnel_event_type_ids,
                                                                                     start_event_type_id,
                                                                                     end_event_type_id)
    if not dropped_funnel_records:
        return GetFunnelDropOffDistributionResponse(meta=get_meta(tr=dtr.to_tr()))
    else:
        dropped_funnel_records_str = ', '.join(f"'{x}'" for x in list(dropped_funnel_records))
        grouped_by_eventType_dropped_records = f"SELECT id, processed_kvs.{funnel_key_name} as p_e_id, event_type_id, event_type_name, ROW_NUMBER() OVER (PARTITION BY processed_kvs.{funnel_key_name} ORDER BY timestamp DESC) AS outer_row_num FROM (SELECT id, processed_kvs.{funnel_key_name}, event_type_id, event_type_name, timestamp FROM (SELECT id, processed_kvs.{funnel_key_name}, event_type_id, event_type_name, timestamp, ROW_NUMBER() OVER (PARTITION BY processed_kvs.{funnel_key_name}, event_type_name ORDER BY timestamp DESC) AS row_num FROM events where processed_kvs.{funnel_key_name} in ({dropped_funnel_records_str}) and timestamp >= '{dtr.to_tr_str()[0]}' and account_id = {account.id}) WHERE row_num = 1)"
        funnel_event_type_distribution = {}
        qs = Events.objects.raw(grouped_by_eventType_dropped_records)
        row = 0
        end_of_rows = False
        while row < len(qs):
            if end_of_rows:
                break
            curr_pkv_event_types = []
            start_event_type_id_found = False
            current_record_id = qs[row].p_e_id
            for i in range(row, len(qs)):
                if qs[i].p_e_id != current_record_id:
                    row = i
                    break
                if not start_event_type_id_found:
                    curr_pkv_event_types.append(qs[i].event_type_name)
                if qs[i].event_type_id == start_event_type_id:
                    start_event_type_id_found = True
                    if len(curr_pkv_event_types) > 0:
                        curr_pkv_event_types.reverse()
                        ev_type_key = curr_pkv_event_types[0]
                        if len(curr_pkv_event_types) > 1:
                            ev_type_key = ' -> '.join([str(ev_type) for ev_type in curr_pkv_event_types])
                        if ev_type_key in funnel_event_type_distribution:
                            funnel_event_type_distribution[ev_type_key] += 1
                        else:
                            funnel_event_type_distribution[ev_type_key] = 1
                if i >= len(qs) - 1:
                    end_of_rows = True
                    break
        result_set = []
        for k, v in funnel_event_type_distribution.items():
            funnel_event_type_distribution_object = GetFunnelDropOffDistributionResponse.FunnelEventTypeDistribution(
                event_type_name=k, count=v)
            result_set.append(funnel_event_type_distribution_object)

        return GetFunnelDropOffDistributionResponse(meta=get_meta(tr=dtr.to_tr()),
                                                    funnel_event_type_distribution=result_set,
                                                    previous_node_count=len(start_event_type_event_count))


@web_api(GetFunnelDropOffRequest)
@use_read_replica
def funnel_drop_off_get(request_message: GetFunnelDropOffRequest) -> Union[
    GetFunnelDropOffResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)

    funnel_event_type_ids = request_message.funnel_event_type_ids
    funnel_key_name = request_message.funnel_key_name
    start_event_type_id = request_message.start_event_type_id
    end_event_type_id = request_message.end_event_type_id

    filter_key_name = request_message.filter_key_name
    filter_value = request_message.filter_value

    events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}' ) order by timestamp asc, created_at asc) group by e_id;".format(
        funnel_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id, dtr.to_tr_str()[0],
        to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
        dtr.to_tr_str()[0],
        dtr.to_tr_str()[1]
    )

    if filter_key_name and filter_value:
        events_group_query = "select groupArray(id) as id, groupArray(event_type_id) as event_type_id_group, groupArray(event_type_name) as event_type_name_group, groupArray(timestamp) as timestamp_group, groupArray(e_filter_value) as filter_group, e_id from (select id, event_type_id, event_type_name, timestamp, processed_kvs.{} as e_id, processed_kvs.{} as e_filter_value from events where event_type_id in ({}) and account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} in (select distinct processed_kvs.{} from events where event_type_id = {} and account_id = {} and timestamp between '{}' and '{}') order by timestamp asc, created_at asc) group by e_id;".format(
            funnel_key_name, filter_key_name, ','.join([str(e) for e in funnel_event_type_ids]), account.id,
            dtr.to_tr_str()[0],
            to_str(datetime.now()), funnel_key_name, funnel_key_name, funnel_event_type_ids[0], account.id,
            dtr.to_tr_str()[0],
            dtr.to_tr_str()[1]
        )

    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    f = Funnel()

    for grouped_event in events_group_set:
        f.add_to_node_map(grouped_event, filter_key_name=filter_key_name, filter_value=filter_value)

    dropped_funnel_records, _ = f.get_funnel_drop_records(funnel_event_type_ids, start_event_type_id, end_event_type_id)

    csv_data = [[funnel_key_name, 'path']]

    dropped_funnel_records_str = ', '.join(f"'{x}'" for x in list(dropped_funnel_records))
    grouped_by_eventType_dropped_records = f"SELECT id, processed_kvs.{funnel_key_name} as p_e_id, event_type_id, event_type_name, ROW_NUMBER() OVER (PARTITION BY processed_kvs.{funnel_key_name} ORDER BY timestamp DESC) AS outer_row_num FROM (SELECT id, processed_kvs.{funnel_key_name}, event_type_id, event_type_name, timestamp FROM (SELECT id, processed_kvs.{funnel_key_name}, event_type_id, event_type_name, timestamp, ROW_NUMBER() OVER (PARTITION BY processed_kvs.{funnel_key_name}, event_type_name ORDER BY timestamp DESC) AS row_num FROM events where processed_kvs.{funnel_key_name} in ({dropped_funnel_records_str}) and timestamp >= '{dtr.to_tr_str()[0]}' and account_id = {account.id}) WHERE row_num = 1)"
    qs = Events.objects.raw(grouped_by_eventType_dropped_records)
    row = 0
    end_of_rows = False
    while row < len(qs):
        if end_of_rows:
            break
        curr_pkv_event_types = []
        start_event_type_id_found = False
        current_record_id = qs[row].p_e_id
        for i in range(row, len(qs)):
            if qs[i].p_e_id != current_record_id:
                row = i
                break
            if not start_event_type_id_found:
                curr_pkv_event_types.append(qs[i].event_type_name)
            if qs[i].event_type_id == start_event_type_id:
                start_event_type_id_found = True
                if len(curr_pkv_event_types) > 0:
                    curr_pkv_event_types.reverse()
                    ev_type_key = curr_pkv_event_types[0]
                    if len(curr_pkv_event_types) > 1:
                        ev_type_key = ' -> '.join([str(ev_type) for ev_type in curr_pkv_event_types])
                    csv_data.append([current_record_id, ev_type_key])
            if i >= len(qs) - 1:
                end_of_rows = True
                break

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="file.csv"'
    writer = csv.writer(response)

    for row in csv_data:
        writer.writerow(row)

    return response


@web_api(WorkflowBuilderRequest)
@use_read_replica
def workflow_builder(request_message: WorkflowBuilderRequest) -> Union[
    GetFunnelResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    workflow_config = json.loads(request_message.workflow_config)

    try:
        wb = WorkflowBuilder(dtr, account)
        wb.process_workflow_config(workflow_config)
        w_view = wb.get_workflow_view()

        return GetFunnelResponse(meta=get_meta(tr=dtr.to_tr()), workflow_view=w_view)
    except Exception as e:
        print(str(e))
        return GetFunnelResponse(
            message=str("Something went wrong. Please check your config. If you think it is correct, report it."))


@web_api(WorkflowBuilderRequest)
@use_read_replica
def workflow_node_metrics_timeseries(request_message: WorkflowBuilderRequest) -> Union[
    WorkflowTimeSeriesResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    workflow_config = json.loads(request_message.workflow_config)

    parsed_workflow_config = workflow_config
    config = parsed_workflow_config['config']
    compare_to = parsed_workflow_config['compare_to']

    name = config.get('name')
    metrics = config.get('metrics')

    metric_series_responses = []

    for metric in metrics:
        metric_name = metric.get('name')
        metric_source = metric.get('source')
        metric_expr = metric.get('expr')

        if metric_source == 'Datadog':
            dd_connector = DatadogConnector(dtr.to_tr(), account.id)
            metric_series = dd_connector.fetch_metric_timeseries(metric_expr)
            output_metric_series = [
                WorkflowTimeSeriesResponse.MetricValue(timestamp=int(item['timestamp']),
                                                       value=item['value'])
                for item in metric_series['series']
            ]

            if compare_to:
                prev_dtr = dtr.get_prev_dtr(compare_to)
                past_metric_series = dd_connector.fetch_metric_timeseries(metric_expr, prev_dtr.to_tr())
                past_output_metric_series = [
                    WorkflowTimeSeriesResponse.MetricValue(timestamp=int(item['timestamp']),
                                                           value=item['value'])
                    for item in past_metric_series['series']
                ]
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series,
                                                                past_series=past_output_metric_series))
            else:
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series))

        if metric_source == 'Newrelic':
            nr_connector = NewRelicConnector(dtr.to_tr(), account.id)
            metric_series = nr_connector.fetch_metric_timeseries(metric_expr)
            output_metric_series = [
                WorkflowTimeSeriesResponse.MetricValue(
                    timestamp=int(datetime.fromisoformat(item['from']).timestamp() * 1000),
                    value=item['values'][metric_series["metric_type"]])
                for item in metric_series['series']
            ]

            if compare_to:
                prev_dtr = dtr.get_prev_dtr(compare_to)
                past_metric_series = nr_connector.fetch_metric_timeseries(metric_expr, prev_dtr.to_tr())
                past_output_metric_series = [
                    WorkflowTimeSeriesResponse.MetricValue(
                        timestamp=int(datetime.fromisoformat(item['from']).timestamp() * 1000),
                        value=item['values'][past_metric_series["metric_type"]])
                    for item in past_metric_series['series']
                ]
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series,
                                                                past_series=past_output_metric_series))
            else:
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series))

        if metric_source == 'DrDroid':
            metric_series = process_workflow_metric_expression(account, metric_expr, dtr)
            output_metric_series = [
                WorkflowTimeSeriesResponse.MetricValue(
                    timestamp=item['timestamp'],
                    value=item['value'])
                for item in metric_series
            ]

            if compare_to:
                prev_dtr = dtr.get_prev_dtr(compare_to)
                past_metric_series = process_workflow_metric_expression(account, metric_expr, prev_dtr)
                past_output_metric_series = [
                    WorkflowTimeSeriesResponse.MetricValue(
                        timestamp=item['timestamp'],
                        value=item['value'])
                    for item in past_metric_series
                ]
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series,
                                                                past_series=past_output_metric_series))
            else:
                metric_series_responses.append(
                    WorkflowTimeSeriesResponse.MetricTimeSeries(metric_name=metric_name, metric_source=metric_source,
                                                                current_series=output_metric_series))

    return WorkflowTimeSeriesResponse(meta=get_meta(tr=dtr.to_tr()), compare_to=compare_to, node_name=name,
                                      metric_time_series=metric_series_responses)


def get_index_in_array(arr, elem):
    if elem in arr:
        return arr.index(elem)
    return 100


@web_api(GetFunnelRequestV3)
@use_read_replica
def funnel_get_v3(request_message: GetFunnelRequestV3) -> Union[
    GetFunnelResponseV3, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    event_key_name = 'recordId'
    global_key_name = 'connectorId'
    filter_value = request_message.filter_value

    EVENTS_ORDER = {
        'Ingestion': ['p1-startIngestion', 'p1-unarchiveFile', 'p1-unarchive', ['p1-metadataRead', 'p2-audioReceived'],
                      'p1-transform', 'p1-normalize'],
        'p1': ['p1-startEvent', ['p1-success', 'p1-failed', 'p1-error'], 'p1-endEvent'],
        'p2': ['p2-startEvent', ['p2-success', 'p2-failed', 'p2-error'], 'p2-endEvent'],
        'p3': ['p3-startEvent', 'p3-filteredRecord', 'p3-success', 'p3-endEvent'],
        'p4': ['p4-startEvent', 'p4-asrInit', 'p4-success', 'p4-endEvent'],
        'ad': ['ad-startEvent', 'ad-dgInit', 'ad-dgCallbackStart', ['ad-dgCallbackSuccess', 'ad-dgCallbackError'],
               'ad-dgCallbackEnd', 'ad-success', 'ad-endEvent', ],
        'p5': ['p5-startEvent', 'p5-success', 'p5-endEvent'],
        'p5_1': ['p5_1-startEvent', ['p5_1-success', 'p5_1-failed'], 'p5_1-endEvent'],
        'p6': ['p6-startEvent', ['p6-success', 'p6-failed'], 'p6-endEvent'],
        'INGESTION_COMPLETE': ['INGESTION_COMPLETE']
    }

    events_group_query = "select count(distinct id) as id, count(distinct processed_kvs.{}) as unique_records, event_type_id, event_type_name from events where account_id = {} and timestamp between '{}' and '{}' group by event_type_id, event_type_name;".format(
        event_key_name, account.id, dtr.to_tr_str()[0],
        dtr.to_tr_str()[1]
    )

    if filter_value:
        events_group_query = "select count(distinct id) as id, count(distinct processed_kvs.{}) as unique_records, event_type_id, event_type_name from events where account_id = {} and timestamp between '{}' and '{}' and processed_kvs.{} = '{}' group by event_type_id, event_type_name;".format(
            event_key_name, account.id, dtr.to_tr_str()[0],
            dtr.to_tr_str()[1],
            global_key_name,
            filter_value
        )

    print(events_group_query)
    qs = Events.objects.raw(events_group_query)
    events_group_set = list(qs)

    events_group_dict = {}
    for node in events_group_set:
        events_group_dict[node.event_type_name] = node

    groups_dict = {}
    for stage in EVENTS_ORDER.keys():
        groups_dict[stage] = []
        for event in EVENTS_ORDER[stage]:
            if isinstance(event, list):
                temp_e = []
                for e in event:
                    if e in events_group_dict.keys():
                        temp_e.append(events_group_dict[e])
                groups_dict[stage].append(temp_e)
            else:
                if event in events_group_dict.keys():
                    groups_dict[stage].append(events_group_dict[event])

    stages = []
    for prefix, nodes in groups_dict.items():
        events = []
        for node in nodes:
            if isinstance(node, list):
                child_nodes = []
                for n in node:
                    child_nodes.append(GetFunnelResponseV3.stage_data.event_data(event_name=n.event_type_name,
                                                                                 event_id=n.event_type_id,
                                                                                 unique_records_count=n.unique_records,
                                                                                 count=n.id))
                events.append(GetFunnelResponseV3.stage_data.event_data(child_events=child_nodes))
            else:
                events.append(GetFunnelResponseV3.stage_data.event_data(event_name=node.event_type_name,
                                                                        event_id=node.event_type_id,
                                                                        unique_records_count=node.id, count=node.id))
        stages.append(GetFunnelResponseV3.stage_data(stage=prefix, events=events))

    return GetFunnelResponseV3(meta=get_meta(tr=dtr.to_tr()), stages=stages)


@web_api(CreateEntityFunnelRequest)
@use_read_replica
def entity_funnel_create(request_message: CreateEntityFunnelRequest) -> Union[CreateEntityFunnelResponse, HttpResponse]:
    account: Account = get_request_account()
    try:
        entity_funnel, error = entity_funnels_create(account, request_message.entity_funnel_panel)
        if entity_funnel:
            return CreateEntityFunnelResponse(success=BoolValue(value=True), funnel_entity=entity_funnel.proto_partial)
        if error:
            return CreateEntityFunnelResponse(success=BoolValue(value=False),
                                              message=Message(title="Entity Funnel Panel Creation Failure",
                                                              description=error))
    except Exception as e:
        return CreateEntityFunnelResponse(success=BoolValue(value=False),
                                          message=Message(title="Entity Funnel Panel Creation Failure",
                                                          description=str(e)))


@web_api(GetEntityFunnelRequest)
@use_read_replica
def entity_funnel_get(request_message: GetEntityFunnelRequest) -> Union[GetEntityFunnelResponse, HttpResponse]:
    account: Account = get_request_account()
    dtr: DateTimeRange = to_dtr(request_message.meta.time_range)
    entity_funnel_name = request_message.entity_funnel_name
    filter_key_name = request_message.filter_key_name
    filter_value = request_message.filter_value
    if request_message.meta.time_range.time_geq < 1695907800:
        try:
            workflow_view = entity_funnels_get_clickhouse_events(account, dtr, entity_funnel_name, filter_key_name,
                                                                 filter_value)
            if workflow_view:
                return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=True),
                                               workflow_view=workflow_view)
            else:
                return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=True),
                                               message=Message(title="No Funnel Data Found"))
        except Exception as e:
            return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=False),
                                           message=Message(title="Failed to load Entity Funnel Panel",
                                                           traceback=str(e)))
    try:
        workflow_view = entity_funnels_get(account, dtr, entity_funnel_name, filter_key_name, filter_value)
        if workflow_view:
            return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=True),
                                           workflow_view=workflow_view)
        else:
            return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=True),
                                           message=Message(title="No Funnel Data Found"))
    except EntityFunnelCrudNotFoundException:
        try:
            workflow_view = entity_funnels_get_clickhouse_events(account, dtr, entity_funnel_name, filter_key_name,
                                                                 filter_value)
            if workflow_view:
                return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=True),
                                               workflow_view=workflow_view)
            else:
                return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=False),
                                               message=Message(title="No Funnel Data Found"))
        except Exception as e:
            return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=False),
                                           message=Message(title="Failed to load Entity Funnel Panel",
                                                           traceback=str(e)))
    except Exception as e:
        return GetEntityFunnelResponse(meta=get_meta(tr=dtr.to_tr()), success=BoolValue(value=False),
                                       message=Message(title="Failed to load Entity Funnel Panel", traceback=str(e)))

event/views.py
DrDroidLab/kenobi
import json
import math
from collections import defaultdict

from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value, StringValue, DoubleValue

from connectors.integrations.datadog import DatadogConnector
from connectors.integrations.newrelic import NewRelicConnector

from event.clickhouse.models import Events
from event.engine.metric_engine import process_metric_expressions
from event.workflows.workflow_utils import get_metric_key_context, get_event_metric_expr_query, \
    get_transaction_metric_expr_query
from protos.event.entity_pb2 import WorkflowView

class Node:

    def __init__(self, event_type_name, event_type_id):
        self.name = event_type_name
        self.id = event_type_id
        self.has_in_connections = False
        self.has_out_connections = False
        self.metrics = []
        self.status = WorkflowView.NodeStatus.GOOD
        self.workflow_node_config = None

    def mark_in_connection(self):
        self.has_in_connections = True

    def mark_out_connection(self):
        self.has_out_connections = True

    def add_metric(self, metric):
        self.metrics.append(metric)

    def set_status(self, status):
        self.status = status

    def get_proto(self):
        conn_types = []

        if self.has_in_connections:
            conn_types.append(WorkflowView.NodeConnectionType.IN)
        if self.has_out_connections:
            conn_types.append(WorkflowView.NodeConnectionType.OUT)

        return WorkflowView.Node(
            node_id=UInt64Value(value=self.id), label=StringValue(value=self.name),
            status=self.status,
            connection_types=conn_types,
            metrics=self.metrics,
            workflow_node_config=StringValue(value=json.dumps(self.workflow_node_config)),
        )


class Edge:

    def __init__(self, start: Node, end: Node):
        self.start = start
        self.end = end
        start.mark_out_connection()
        end.mark_in_connection()

    def __str__(self):
        return "{} - {} (Count: {})".format(self.start.name, self.end.name, self.counter)

    def print(self):
        print("{} -> {} ({}, {})".format(self.start.name, self.end.name, len(self.owners), self.owners[:1]))

    def get_proto(self):
        return WorkflowView.Edge(
            start_node_id=UInt64Value(value=self.start.id),
            end_node_id=UInt64Value(value=self.end.id),
            metrics=[],
        )

class WorkflowBuilder:

    def __init__(self, dtr, account):
        self.nodes = []
        self.edges = []
        self.dtr = dtr
        self.account = account
        self.dd_connector = None
        self.nr_connector = None

    def get_metric_from_datadog(self, metric_expr, compare_to=None):
        metric_value, delta, delta_type = None, None, None

        self.dd_connector = DatadogConnector(self.dtr.to_tr(), self.account.id)
        metric_value_json = self.dd_connector.fetch_metric(metric_expr)
        metric_value_compare = metric_value_json.get('value')
        if metric_value_compare is not None:
            metric_value = "{} {}".format(metric_value_json.get('value'), metric_value_json.get('unit'))
        else:
            metric_value = None

        if compare_to:
            prev_dtr = self.dtr.get_prev_dtr(compare_to)
            prev_metric_value_json = self.dd_connector.fetch_metric(metric_expr, prev_dtr.to_tr())
            prev_metric_value_compare = prev_metric_value_json.get('value')
            if metric_value_compare and not prev_metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.UP
            elif prev_metric_value_compare and not metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.DOWN
            elif not prev_metric_value_compare and not metric_value_compare:
                delta = None
                delta_type = None
            else:
                delta = abs(round((metric_value_compare / prev_metric_value_compare) - 1.0, 2)) * 100
                if metric_value_compare >= prev_metric_value_compare:
                    delta_type = WorkflowView.DeltaType.UP
                else:
                    delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type


    def get_metric_from_drdroid(self, metric_expr, compare_to=None):
        metric_value, delta, delta_type = None, None, None

        aggregation_type = metric_expr.split(':')[0]
        aggregation_field = metric_expr.split(':')[1].split('{')[0]
        filters = list({'lhs': x.split(':')[0], 'rhs': x.split(':')[1]} for x in metric_expr.split('{')[1].split('}')[0].split(','))

        aggregation_object = aggregation_field.split('.')[0]
        metric_expr_query = None
        if aggregation_object == 'event':
            metric_expr_query = get_event_metric_expr_query(aggregation_type, aggregation_field, filters, self.account.id, self.dtr)
        if aggregation_object == 'transaction':
            metric_expr_query = get_transaction_metric_expr_query(aggregation_type, aggregation_field, filters, self.account.id, self.dtr)

        if metric_expr_query:
            qs = Events.objects.raw(metric_expr_query)
            metric_value = list(qs)[0].id
            metric_value = round(metric_value, 2)

            if compare_to:
                prev_dtr = self.dtr.get_prev_dtr(compare_to)
                prev_metric_expr_query = None
                if aggregation_object == 'event':
                    prev_metric_expr_query = get_event_metric_expr_query(aggregation_type, aggregation_field, filters,
                                                                    self.account.id, prev_dtr)
                if aggregation_object == 'transaction':
                    prev_metric_expr_query = get_transaction_metric_expr_query(aggregation_type, aggregation_field, filters,
                                                                          self.account.id, prev_dtr)

                prev_qs = Events.objects.raw(prev_metric_expr_query)
                prev_metric_value = list(prev_qs)[0].id
                if metric_value and not prev_metric_value:
                    delta = 100
                    delta_type = WorkflowView.DeltaType.UP
                elif prev_metric_value and not metric_value:
                    delta = 100
                    delta_type = WorkflowView.DeltaType.DOWN
                elif not prev_metric_value and not metric_value:
                    delta = None
                    delta_type = None
                else:
                    delta = abs(round((metric_value / prev_metric_value) - 1.0, 2)) * 100
                    if metric_value >= prev_metric_value:
                        delta_type = WorkflowView.DeltaType.UP
                    else:
                        delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type

    def process_workflow_config(self, workflow_config):

        compare_to = workflow_config.get('compare_to')
        nodes = workflow_config.get('nodes')

        for idx, node in enumerate(nodes):
            name = node.get('name')
            metrics = node.get('metrics')
            wb_node = Node(name, idx)

            for metric in metrics:
                metric_name = metric.get('name')
                metric_source = metric.get('source')
                metric_expr = metric.get('expr')

                metric_value, delta, delta_type = None, None, None

                if metric_source == 'Datadog':
                    metric_value, delta, delta_type = self.get_metric_from_datadog(metric_expr, compare_to)
                if metric_source == 'DrDroid':
                    metric_value, delta, delta_type = self.get_metric_from_drdroid(metric_expr, compare_to)
                if metric_source == 'Newrelic':
                    metric_value, delta, delta_type = self.get_metric_from_newrelic(metric_expr, compare_to)

                metric_status = WorkflowView.MetricStatus.GREEN
                if delta_type:
                    metric_context = get_metric_key_context(metric_name)

                    if metric_context == 'up' and delta_type == WorkflowView.DeltaType.DOWN:
                        metric_status = WorkflowView.MetricStatus.RED
                    if metric_context == 'down' and delta_type == WorkflowView.DeltaType.UP:
                        metric_status = WorkflowView.MetricStatus.RED

                if metric_status == WorkflowView.MetricStatus.RED and delta and delta > 15.0:
                    wb_node.set_status(WorkflowView.NodeStatus.BAD)

                wb_node.add_metric(WorkflowView.Node.NodeMetric(name=StringValue(value=metric_name),
                                                                      value=StringValue(value=str(metric_value)),
                                                                      metric_status=metric_status,
                                                                      delta=DoubleValue(value=delta),
                                                                      delta_type=delta_type
                                                                    ))
                wb_node.workflow_node_config = {"config": node, "compare_to": compare_to}

            self.nodes.append(wb_node)
            if idx > 0:
                edge = Edge(self.nodes[idx - 1], wb_node)
                self.edges.append(edge)
                self.nodes[idx - 1].mark_out_connection()
                self.nodes[idx].mark_in_connection()


    def get_workflow_view(self):
        data = []

        for node in self.nodes:
            data.append(
                node.get_proto()
            )

        links = []
        for edge in self.edges:
            links.append(
                edge.get_proto()
            )

        return WorkflowView(
            nodes=data,
            edges=links
        )

    def get_metric_from_newrelic(self, metric_expr, compare_to):
        metric_value, delta, delta_type = None, None, None

        self.nr_connector = NewRelicConnector(self.dtr.to_tr(), self.account.id)
        metric_value_json = self.nr_connector.fetch_metric(metric_expr)
        metric_value_compare = metric_value_json.get('value')
        if metric_value_compare is not None:
            metric_value = "{} {}".format(metric_value_json.get('value'), metric_value_json.get('unit'))
        else:
            metric_value = None

        if compare_to:
            prev_dtr = self.dtr.get_prev_dtr(compare_to)
            prev_metric_value_json = self.nr_connector.fetch_metric(metric_expr, prev_dtr.to_tr())
            prev_metric_value_compare = prev_metric_value_json.get('value')
            if metric_value_compare and not prev_metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.UP
            elif prev_metric_value_compare and not metric_value_compare:
                delta = 100
                delta_type = WorkflowView.DeltaType.DOWN
            elif not prev_metric_value_compare and not metric_value_compare:
                delta = None
                delta_type = None
            else:
                delta = abs(round(((metric_value_compare / prev_metric_value_compare) - 1.0), 2)) * 100
                if metric_value_compare >= prev_metric_value_compare:
                    delta_type = WorkflowView.DeltaType.UP
                else:
                    delta_type = WorkflowView.DeltaType.DOWN

        return metric_value, delta, delta_type




event/workflows/builder.py
DrDroidLab/kenobi
from collections import defaultdict

from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value, StringValue, DoubleValue

from protos.event.entity_pb2 import FunnelView, WorkflowView


def get_duration(ts1, ts2):
    return (ts2 - ts1).seconds


def get_dag_values(event_type_ids, event_type_names, event_timestamps):
    unique_dict = {}
    result = []

    for timestamp, type_name, type_id in zip(event_timestamps, event_type_names, event_type_ids):
        type_tuple = (type_id, type_name)
        if type_tuple not in unique_dict:
            unique_dict[type_tuple] = []
        unique_dict[type_tuple].append(timestamp)

    for type_tuple, timestamps in unique_dict.items():
        timestamp_elements_count = len(timestamps)
        result.append((type_tuple[0], type_tuple[1], timestamps[timestamp_elements_count - 1]))

    return result


class Node:

    def __init__(self, event_type_name, event_type_id):
        self.name = event_type_name
        self.id = event_type_id
        self.counter = 0
        self.owners = set()
        self.has_in_connections = False
        self.has_out_connections = False

    def __str__(self):
        return "{} (Count: {})".format(self.name, self.counter)

    def incr(self, owner):
        self.counter += 1
        self.owners.add(owner)

    def mark_in_connection(self):
        self.has_in_connections = True

    def mark_out_connection(self):
        self.has_out_connections = True

    def get_proto(self):
        conn_types = []

        if self.has_in_connections:
            conn_types.append(WorkflowView.NodeConnectionType.IN)
        if self.has_out_connections:
            conn_types.append(WorkflowView.NodeConnectionType.OUT)

        return WorkflowView.Node(
            node_id=UInt64Value(value=self.id), label=StringValue(value=self.name),
            status=WorkflowView.NodeStatus.GOOD,
            connection_types=conn_types,
            metrics=[
                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                             value=StringValue(value=str(self.counter)),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ]
        )


class Edge:

    def __init__(self, start: Node, end: Node):
        self.start = start
        self.end = end
        self.counter = 0
        self.owners = []
        self.durations = []
        start.mark_out_connection()
        end.mark_in_connection()

    def __str__(self):
        return "{} - {} (Count: {})".format(self.start.name, self.end.name, self.counter)

    def add_edge(self, edge_owner, edge_duration):
        self.counter += 1
        self.owners.append(edge_owner)
        self.durations.append(edge_duration)

    def print(self):
        print("{} -> {} ({}, {})".format(self.start.name, self.end.name, len(self.owners), self.owners[:1]))

    def get_proto(self):
        avg_duration = round((sum(self.durations) / self.counter), 2)
        return WorkflowView.Edge(
            start_node_id=UInt64Value(value=self.start.id),
            end_node_id=UInt64Value(value=self.end.id),
            metrics=[
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Txn Count"),
                                             value=StringValue(value=str(self.counter)),
                                             metric_status=WorkflowView.MetricStatus.GREEN),
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Avg Time"),
                                             value=StringValue(value="{} sec".format(str(avg_duration))),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ],
        )


class Funnel:

    def __init__(self):
        self.node_map = {}
        self.edge_map = {}
        pass

    def add_to_node_map(self, grouped_event, filter_key_name=None, filter_value=None):
        event_type_id_group = grouped_event.event_type_id_group
        event_type_name_group = grouped_event.event_type_name_group
        timestamp_group = grouped_event.timestamp_group
        dag_values = get_dag_values(event_type_id_group, event_type_name_group, timestamp_group)
        owner = grouped_event.e_id

        if filter_key_name and filter_value:
            filter_group = grouped_event.filter_group
            if filter_value.lower() not in list(x.lower() for x in filter_group):
                return

        for idx in range(0, len(dag_values)):
            if dag_values[idx][0] not in self.node_map:
                self.node_map[dag_values[idx][0]] = Node(dag_values[idx][1], dag_values[idx][0])
            self.node_map[dag_values[idx][0]].incr(owner)
            if idx + 1 < len(dag_values):
                if dag_values[idx + 1][0] not in self.node_map:
                    self.node_map[dag_values[idx + 1][0]] = Node(dag_values[idx + 1][1], dag_values[idx + 1][0])
                if dag_values[idx][0] != dag_values[idx + 1][0]:
                    edge_key = "{}-{}".format(dag_values[idx][0], dag_values[idx + 1][0])
                    if edge_key not in self.edge_map:
                        self.edge_map[edge_key] = Edge(self.node_map[dag_values[idx][0]],
                                                       self.node_map[dag_values[idx + 1][0]])
                    self.edge_map[edge_key].add_edge(owner,
                                                     get_duration(dag_values[idx][2], dag_values[idx + 1][2]))

    def get_funnel_data(self):
        data = []
        for node in self.node_map.values():
            data.append(
                node.get_proto()
            )

        links = []
        acyclic_edges = self.edge_map.values()
        for edge in acyclic_edges:
            links.append(
                edge.get_proto()
            )

        return WorkflowView(
            nodes=data,
            edges=links
        )

    def get_funnel_drop_records(self, funnel_event_type_ids, start_event_type_id, end_event_type_id):
        prev_node = self.node_map.get(funnel_event_type_ids[0])
        funnel_records = prev_node.owners

        for idx, event_type_id in enumerate(funnel_event_type_ids[1:]):
            node = self.node_map.get(event_type_id)
            if node:
                if prev_node.id == start_event_type_id and node.id == end_event_type_id:
                    dropped_funnel_records = funnel_records - node.owners
                    return dropped_funnel_records, funnel_records

                prev_node = node
                funnel_records = funnel_records & node.owners

        return set(), set()

    def get_ordered_funnel_data(self, funnel_event_type_ids):
        data = []
        links = []

        prev_node = self.node_map.get(funnel_event_type_ids[0])
        if prev_node:
            funnel_records = prev_node.owners
            prev_record_count = len(funnel_records)
            data.append(
                WorkflowView.Node(
                    node_id=UInt64Value(value=prev_node.id), label=StringValue(value=prev_node.name),
                    connection_types=[WorkflowView.NodeConnectionType.OUT],
                    metrics=[
                        WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                                     value=StringValue(value=str(prev_record_count)),
                                                     metric_status=WorkflowView.MetricStatus.GREEN)
                    ]
                )
            )

            for idx, event_type_id in enumerate(funnel_event_type_ids[1:]):
                node = self.node_map.get(event_type_id)
                if node:
                    funnel_records = funnel_records & node.owners
                    funnel_records_count = len(funnel_records)

                    conn_types = [WorkflowView.NodeConnectionType.IN, WorkflowView.NodeConnectionType.OUT]
                    if idx == len(funnel_event_type_ids) - 2:
                        conn_types = [WorkflowView.NodeConnectionType.IN]

                    data.append(
                        WorkflowView.Node(
                            node_id=UInt64Value(value=node.id), label=StringValue(value=node.name),
                            connection_types=conn_types,
                            metrics=[
                                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                                             value=StringValue(value=str(funnel_records_count)),
                                                             metric_status=WorkflowView.MetricStatus.GREEN)
                            ]
                        )
                    )
                    edge = self.edge_map.get("{}-{}".format(prev_node.id, node.id))
                    drop_percent = 0
                    if prev_record_count > 0 and funnel_records_count > 0:
                        drop_percent = round(((prev_record_count - funnel_records_count) * 100 / prev_record_count), 2)
                    if edge:
                        avg_duration = round((sum(edge.durations) / edge.counter), 2)
                        links.append(
                            WorkflowView.Edge(
                                start_node_id=UInt64Value(value=prev_node.id),
                                end_node_id=UInt64Value(value=node.id),
                                metrics=[
                                    WorkflowView.Edge.EdgeMetric(name=StringValue(value="Drop"),
                                                                 value=StringValue(
                                                                     value="{}%".format(str(drop_percent))),
                                                                 metric_status=WorkflowView.MetricStatus.GREEN),
                                    WorkflowView.Edge.EdgeMetric(name=StringValue(value="Avg Time"),
                                                                 value=StringValue(
                                                                     value="{} sec".format(str(avg_duration))),
                                                                 metric_status=WorkflowView.MetricStatus.GREEN)
                                ],
                            )
                        )
                    prev_node = node
                    prev_record_count = funnel_records_count

        return WorkflowView(
            nodes=data,
            edges=links
        )

event/workflows/funnel.py
DrDroidLab/kenobi
import itertools
from typing import Dict

from sortedcontainers import SortedList
from protos.event.entity_pb2 import WorkflowView
from event.models import Event, EventType

from google.protobuf.wrappers_pb2 import BoolValue, UInt64Value, StringValue, DoubleValue

class Node:
    def __init__(self, event_type_name, event_type_id):
        self.event_type_name = event_type_name
        self.events = SortedList(key=lambda e: e.timestamp)
        self.id = event_type_id
        self.edges = {}
        self.has_in_connections = False
        self.has_out_connections = False

    def __str__(self):
        return f'{self.id}:{self.event_type_name}'

    def __eq__(self, other):
        return self.id == other.id

    def __hash__(self):
        return self.id

    def mark_in_connection(self):
        self.has_in_connections = True

    def mark_out_connection(self):
        self.has_out_connections = True

    def add_event(self, event):
        self.events.add(event)

    def view_proto(self) -> WorkflowView.Node:
        total_count = len(self.events)

        conn_types = []
        if self.has_in_connections:
            conn_types.append(WorkflowView.NodeConnectionType.IN)
        if self.has_out_connections:
            conn_types.append(WorkflowView.NodeConnectionType.OUT)

        return WorkflowView.Node(
            node_id=UInt64Value(value=self.id), label=StringValue(value=self.event_type_name),
            status=WorkflowView.NodeStatus.GOOD,
            connection_types=conn_types,
            metrics=[
                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                             value=StringValue(value=str(total_count)),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ]
        )


class Edge:
    def __init__(self, start: Node, end: Node):
        self.start = start
        self.end = end
        self.event_pairs = SortedList(key=lambda t: (t[0].timestamp, t[1].timestamp))

    def __str__(self):
        return f'{self.start} --> {self.end}'

    def __eq__(self, other):
        return self.start == other.start and self.end == other.end

    def __hash__(self):
        return self.start, self.end

    def add_event(self, start_event, end_event):
        self.event_pairs.add((start_event, end_event))

    def view_proto(self) -> WorkflowView.Edge:
        total_transitions = len(self.event_pairs)
        time_delta = map(lambda t: t.total_seconds(), map(lambda t: t[1].timestamp - t[0].timestamp, self.event_pairs))
        return WorkflowView.Edge(
            start_node_id=UInt64Value(value=self.start.id),
            end_node_id=UInt64Value(value=self.end.id),
            metrics=[
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Txn Count"),
                                             value=StringValue(value=str(total_transitions)),
                                             metric_status=WorkflowView.MetricStatus.GREEN),
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Avg Time"),
                                             value=StringValue(value="{}s".format(str(int(sum(list(time_delta))/total_transitions)))),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ],
        )

class Graph:
    def __init__(self):
        self._state_node_dict: Dict[str, Node] = {}
        return

    def _add_events(self, events):
        for event in events:
            self._add_event(event)

        return

    def _add_event(self, event: Event):
        if event.event_type_name not in self._state_node_dict:
            self._state_node_dict[event.event_type_name] = Node(event.event_type_name, event.event_type_id)
        node = self._state_node_dict[event.event_type_name]
        node.add_event(event)
        return

    def add_flow_events(self, events):
        # This populates the nodes
        self._add_events(events)

        for start, end in itertools.zip_longest(events, events[1:]):
            if not end:
                break
            start_node = self._state_node_dict[start.event_type_name]
            end_node = self._state_node_dict[end.event_type_name]

            if start_node.id == end_node.id:
                continue

            start_node.mark_out_connection()
            end_node.mark_in_connection()

            if end_node not in start_node.edges:
                e = Edge(start_node, end_node)
                start_node.edges[end_node] = e
            edge: Edge = start_node.edges[end_node]
            edge.add_event(start, end)

    def get_view(self) -> WorkflowView:
        nodes = []
        edges = []
        for node in self._state_node_dict.values():
            nodes.append(node.view_proto())
            for edge in node.edges.values():
                edges.append(edge.view_proto())

        return WorkflowView(
            nodes=nodes,
            edges=edges
        )


class NewNode:
    def __init__(self, event_type_name, event_type_id):
        self.event_type_name = event_type_name
        self.events = []
        self.id = event_type_id
        self.edges = {}
        self.has_in_connections = False
        self.has_out_connections = False

    def __str__(self):
        return f'{self.id}:{self.event_type_name}'

    def __eq__(self, other):
        return self.id == other.id

    def __hash__(self):
        return self.id

    def mark_in_connection(self):
        self.has_in_connections = True

    def mark_out_connection(self):
        self.has_out_connections = True

    def add_event(self, event):
        self.events.append(event)

    def view_proto(self) -> WorkflowView.Node:
        total_count = len(self.events)

        conn_types = []
        if self.has_in_connections:
            conn_types.append(WorkflowView.NodeConnectionType.IN)
        if self.has_out_connections:
            conn_types.append(WorkflowView.NodeConnectionType.OUT)

        return WorkflowView.Node(
            node_id=UInt64Value(value=self.id), label=StringValue(value=self.event_type_name),
            status=WorkflowView.NodeStatus.GOOD,
            connection_types=conn_types,
            metrics=[
                WorkflowView.Node.NodeMetric(name=StringValue(value="Count"),
                                             value=StringValue(value=str(total_count)),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ]
        )

class NewEdge:
    def __init__(self, start: NewNode, end: NewNode):
        self.start = start
        self.end = end
        self.event_pairs = SortedList(key=lambda t: (t[0][2], t[1][2]))

    def __str__(self):
        return f'{self.start} --> {self.end}'

    def __eq__(self, other):
        return self.start == other.start and self.end == other.end

    def __hash__(self):
        return self.start, self.end

    def add_event(self, start_event, end_event):
        self.event_pairs.add((start_event, end_event))

    def view_proto(self) -> WorkflowView.Edge:
        total_transitions = len(self.event_pairs)
        time_delta = map(lambda t: t.total_seconds(), map(lambda t: t[1][2] - t[0][2], self.event_pairs))
        return WorkflowView.Edge(
            start_node_id=UInt64Value(value=self.start.id),
            end_node_id=UInt64Value(value=self.end.id),
            metrics=[
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Txn Count"),
                                             value=StringValue(value=str(total_transitions)),
                                             metric_status=WorkflowView.MetricStatus.GREEN),
                WorkflowView.Edge.EdgeMetric(name=StringValue(value="Avg Time"),
                                             value=StringValue(value="{}s".format(str(int(sum(list(time_delta))/total_transitions)))),
                                             metric_status=WorkflowView.MetricStatus.GREEN)
            ],
        )

class NewGraph:
    def __init__(self):
        self._state_node_dict: Dict[str, NewNode] = {}
        return

    def _add_grouped_events(self, events):
        for event in events:
            self._add_event(event)
        return

    def _add_event(self, event: tuple):
        if event[1] not in self._state_node_dict:
            self._state_node_dict[event[1]] = NewNode(event[1], event[0])
        node = self._state_node_dict[event[1]]
        node.add_event(event)
        return

    def add_flow_grouped_events(self, grouped_event):
        # This populates the nodes
        events = []
        event_type_id_group = grouped_event.event_type_id_group
        event_type_name_group = grouped_event.event_type_name_group
        timestamp_group = grouped_event.timestamp_group
        for event_type_id, event_type_name, timestamp in zip(event_type_id_group, event_type_name_group, timestamp_group):
            events.append((event_type_id, event_type_name, timestamp))

        self._add_grouped_events(events)

        for start, end in itertools.zip_longest(events, events[1:]):
            if not end:
                break

            start_node = self._state_node_dict[start[1]]
            end_node = self._state_node_dict[end[1]]

            if start_node.id == end_node.id:
                continue

            start_node.mark_out_connection()
            end_node.mark_in_connection()

            if end_node not in start_node.edges:
                e = NewEdge(start_node, end_node)
                start_node.edges[end_node] = e
            edge: NewEdge = start_node.edges[end_node]
            edge.add_event(start, end)

    def get_view(self) -> WorkflowView:
        nodes = []
        edges = []

        for node in self._state_node_dict.values():
            nodes.append(node.view_proto())
            for edge in node.edges.values():
                edges.append(edge.view_proto())

        return WorkflowView(
            nodes=nodes,
            edges=edges
        )

event/workflows/graph.py
DrDroidLab/kenobi


def get_workflow_for_events(account_id, events_data, time_range):
    """
    Returns the workflow for the given event.
    """
    print(1)
event/workflows/utils.py
DrDroidLab/kenobi
BAD_METRICS = ['error', 'latency', 'duration', 'memory', 'cpu', 'failure']
GOOD_METRICS = ['count', 'throughput', 'hit', 'success']
AGGR_MAP = {'count': 'count({})', 'avg': 'avg({})', 'sum': 'sum({})', 'min': 'min({})', 'max': 'max({})',
            'count_distinct': 'count(distinct {})', 'distinct_count': 'count(distinct {})'}
AGGR_FUNCTION_MAP = {'count': 'COUNT', 'avg': 'AVG', 'sum': 'SUM', 'min': 'MIN', 'max': 'MAX',
            'count_distinct': 'COUNT_DISTINCT', 'distinct_count': 'COUNT_DISTINCT'}


def get_metric_key_context(key):
    for m in BAD_METRICS:
        if m in key.lower():
            return 'down'
    for m in GOOD_METRICS:
        if m in key.lower():
            return 'up'
    return 'up'


def get_metric_aggr_function_mapping(aggr):
    return AGGR_FUNCTION_MAP.get(aggr, 'COUNT')


def get_event_metric_expr_query(aggr_type, aggr_field, filters, account_id, dtr):
    start_time, end_time = dtr.to_tr_str()

    selector = 'count(id)'
    if '.' in aggr_field and aggr_field.split('.')[0] == 'event' and aggr_field.split('.')[1]:
        selector = AGGR_MAP.get(aggr_type, 'count({})').format('processed_kvs.{}'.format(aggr_field.split('.')[1]))

    q_filter = 'account_id = {}'.format(account_id)
    for f in filters:
        if f['lhs'] == 'event_name':
            q_filter += ' and event_type_name = \'{}\''.format(f['rhs'])
        else:
            q_filter += ' and processed_kvs.{} = \'{}\''.format(f['lhs'], f['rhs'])
    q_filter = q_filter + ' and '

    query = "select {} as id from events where {} timestamp between '{}' and '{}'".format(selector, q_filter, start_time,
                                                                                    end_time)
    return query


def get_transaction_metric_expr_query(aggr_type, aggr_field, filters, account_id, dtr):
    start_time, end_time = dtr.to_tr_str()

    metric_type = aggr_field.split('.')[1]
    metric_object = aggr_field.split('.')[0]
    primary_event_name = list(filter(lambda x: x['lhs'] == 'primary_event_name', filters))[0]['rhs']
    secondary_event_name = list(filter(lambda x: x['lhs'] == 'secondary_event_name', filters))[0]['rhs']
    joining_key = list(filter(lambda x: x['lhs'] == 'joining_key', filters))[0]['rhs']

    filter_clause = ''
    event_filters = list(filter(lambda x: x['lhs'] != 'primary_event_name' and x['lhs'] != 'secondary_event_name' and x['lhs'] != 'joining_key', filters))
    for f in event_filters:
        filter_clause += ' and processed_kvs.{} = \'{}\''.format(f['lhs'], f['rhs'])

    query = None
    if aggr_type == 'percent':
        if metric_type == 'active':
            query = "select (1 - count(distinct B.s_j_key) / count(distinct A.p_j_key))*100 as id from (select processed_kvs.{} as p_j_key from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key".format(
                joining_key, account_id, primary_event_name, start_time, end_time, filter_clause, joining_key, account_id, secondary_event_name, start_time
            )
        if metric_type == 'finished':
            query = "select count(distinct B.s_j_key) * 100 / count(distinct A.p_j_key) as id from (select processed_kvs.{} as p_j_key from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key".format(
                joining_key, account_id, primary_event_name, start_time, end_time, filter_clause, joining_key, account_id, secondary_event_name, start_time
            )

    if aggr_type == 'avg':
        if metric_type == 'time':
            query = "select avg(date_diff('ms', p_timestamp, s_timestamp)) as id from (select processed_kvs.{} as p_j_key, timestamp as p_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp between '{}' and '{}' {}) A left join (select processed_kvs.{} as s_j_key, timestamp as s_timestamp from events where account_id = {} and event_type_name = '{}' and timestamp >= '{}') B on A.p_j_key = B.s_j_key where A.p_j_key = B.s_j_key".format(
                joining_key, account_id, primary_event_name, start_time, end_time, filter_clause, joining_key, account_id, secondary_event_name, start_time
            )

    return query
event/workflows/workflow_utils.py
DrDroidLab/kenobi
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys


def main():
    """Run administrative tasks."""
    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'prototype.base_settings')
    try:
        from django.core.management import execute_from_command_line
    except ImportError as exc:
        raise ImportError(
            "Couldn't import Django. Are you sure it's installed and "
            "available on your PYTHONPATH environment variable? Did you "
            "forget to activate a virtual environment?"
        ) from exc
    execute_from_command_line(sys.argv)


if __name__ == '__main__':
    main()

manage.py
DrDroidLab/kenobi
from django.contrib import admin

from management.models import Task, TaskRun


@admin.register(Task)
class TaskAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "task",
        "max_retries",
        "retry_deadline",
        "created_at",
        "updated_at",
    ]
    list_filter = ("task",)


@admin.register(TaskRun)
class TaskRunAdmin(admin.ModelAdmin):
    list_display = [
        "id",
        "task",
        "task_uuid",
        "result",
        "status",
        "scheduled_at",
        "started_at",
        "completed_at",
    ]
    list_filter = ("task", "task_uuid", "result", "status",)

management/admin.py
DrDroidLab/kenobi
# Generated by Django 4.1.4 on 2023-05-02 08:22

from django.db import migrations, models
import django.db.models.deletion
import management.models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Task',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('task', models.TextField(db_index=True)),
                ('fargs', models.JSONField(blank=True, null=True)),
                ('md5_fargs', models.CharField(blank=True, db_index=True, max_length=256, null=True)),
                ('max_retries', models.IntegerField(default=0)),
                ('retry_deadline', models.DateTimeField(blank=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
            ],
            options={
                'unique_together': {('task', 'md5_fargs')},
            },
        ),
        migrations.CreateModel(
            name='TaskRun',
            fields=[
                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('task_uuid', models.CharField(db_index=True, max_length=256)),
                ('status', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SKIPPED'), (2, 'SCHEDULED'), (3, 'RUNNING'), (4, 'COMPLETED')], db_index=True, default=management.models.PeriodicTaskStatus['UNKNOWN'], null=True)),
                ('result', models.IntegerField(blank=True, choices=[(0, 'UNKNOWN'), (1, 'SUCCESS'), (2, 'FAILED')], db_index=True, default=management.models.PeriodicTaskResult['SUCCESS'], null=True)),
                ('scheduled_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('started_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
                ('completed_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
                ('updated_at', models.DateTimeField(auto_now=True, db_index=True, null=True)),
                ('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='management.task')),
            ],
            options={
                'unique_together': {('task', 'task_uuid')},
            },
        ),
    ]

management/migrations/0001_initial.py
DrDroidLab/kenobi
from enum import IntEnum
from hashlib import md5

from django.db import models


class PeriodicTaskStatus(IntEnum):
    UNKNOWN = 0
    SKIPPED = 1
    SCHEDULED = 2
    RUNNING = 3
    COMPLETED = 4

    @classmethod
    def choices(cls):
        return [(key.value, key.name) for key in cls]


class PeriodicTaskResult(IntEnum):
    UNKNOWN = 0
    SUCCESS = 1
    FAILED = 2

    @classmethod
    def choices(cls):
        return [(key.value, key.name) for key in cls]


class Task(models.Model):
    task = models.TextField(db_index=True)
    fargs = models.JSONField(null=True, blank=True)
    md5_fargs = models.CharField(max_length=256, null=True, blank=True, db_index=True)

    max_retries = models.IntegerField(default=0)
    retry_deadline = models.DateTimeField(null=True, blank=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    class Meta:
        unique_together = [['task', 'md5_fargs']]

    def __str__(self):
        return f'{self.task}'

    def save(self, **kwargs):
        if self.fargs:
            self.md5_fargs = md5(str(self.fargs).encode('utf-8')).hexdigest()
        super().save(**kwargs)


class TaskRun(models.Model):
    task = models.ForeignKey(Task, on_delete=models.CASCADE)
    task_uuid = models.CharField(max_length=256, db_index=True)
    status = models.IntegerField(null=True, blank=True,
                                 choices=PeriodicTaskStatus.choices(),
                                 default=PeriodicTaskStatus.UNKNOWN,
                                 db_index=True)
    result = models.IntegerField(null=True, blank=True, choices=PeriodicTaskResult.choices(),
                                 default=PeriodicTaskResult.SUCCESS, db_index=True)
    scheduled_at = models.DateTimeField(auto_now_add=True, db_index=True)
    started_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)
    completed_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    created_at = models.DateTimeField(auto_now_add=True, db_index=True)
    updated_at = models.DateTimeField(null=True, blank=True, auto_now=True, db_index=True)

    class Meta:
        unique_together = [['task', 'task_uuid']]

    def __str__(self):
        return f'{self.task}:{self.task_uuid}'

management/models.py
DrDroidLab/kenobi
import hashlib
from datetime import datetime

from celery.signals import task_prerun, task_postrun, task_failure

from management.models import TaskRun, PeriodicTaskStatus, PeriodicTaskResult, Task


def get_fargs_dict(*args, **kwargs):
    return {'args': args, 'kwargs': kwargs}


def get_fargs_json_mds(fargs):
    return hashlib.md5(str(fargs).encode('utf-8')).hexdigest()


def publish_pre_run_task(sender):
    @task_prerun.connect(sender=sender)
    def trigger_cron_task_pre_run_notifier(signal=None, sender=None, task_id=None, task=None, args=None,
                                           **kwargs):
        task_run = TaskRun.objects.filter(task_uuid=task_id).first()
        if task_run:
            task_run.status = PeriodicTaskStatus.RUNNING
            task_run.started_at = datetime.utcnow()
            task_run.save()

    return trigger_cron_task_pre_run_notifier


def publish_post_run_task(sender):
    @task_postrun.connect(sender=sender)
    def trigger_cron_task_post_run_notifier(signal=None, sender=None, task_id=None, task=None, args=None,
                                            **kwargs):
        task_run = TaskRun.objects.filter(task_uuid=task_id).first()
        if task_run:
            task_run.status = PeriodicTaskStatus.COMPLETED
            task_run.completed_at = datetime.utcnow()
            task_run.save()

    return trigger_cron_task_post_run_notifier


def publish_failure_task(sender):
    @task_failure.connect(sender=sender)
    def trigger_cron_task_failure_notifier(signal=None, sender=None, task_id=None, task=None, args=None,
                                           **kwargs):
        task_run = TaskRun.objects.filter(task_uuid=task_id).first()
        if task_run:
            task_run.result = PeriodicTaskResult.FAILED
            task_run.save()

    return trigger_cron_task_failure_notifier


def get_or_create_task(task, *args, **kwargs):
    fargs = get_fargs_dict(*args, **kwargs)
    md5_farags = get_fargs_json_mds(fargs)
    saved_task, _ = Task.objects.get_or_create(task=task,
                                               md5_fargs=md5_farags,
                                               defaults={
                                                   'fargs': fargs,
                                               })

    return saved_task


def check_scheduled_or_running_task_run_for_task(saved_task):
    return TaskRun.objects.filter(task=saved_task).filter(
        status__in=[PeriodicTaskStatus.SCHEDULED, PeriodicTaskStatus.RUNNING]).count() > 0


def create_task_run(**kwargs):
    task_run = TaskRun.objects.create(**kwargs)
    return task_run

management/utils/celery_task_signal_utils.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/config/collector.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dprotos/config/collector.proto\x12\rprotos.config\x1a\x1egoogle/protobuf/wrappers.proto\"\xe0\x01\n\x0bSqsConsumer\x12+\n\x07\x65nabled\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12/\n\tqueue_url\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12;\n\x16max_number_of_messages\x18\x03 \x01(\x0b\x32\x1b.google.protobuf.Int64Value\x12\x36\n\x11wait_time_seconds\x18\x04 \x01(\x0b\x32\x1b.google.protobuf.Int64Value\"\xcd\x01\n\x0ePipelineConfig\x12.\n\x0bhttp_server\x18\x01 \x01(\x0b\x32\x19.protos.config.HttpServer\x12\'\n\x07\x62\x61tcher\x18\x02 \x01(\x0b\x32\x16.protos.config.Batcher\x12\x30\n\x08\x65xporter\x18\x03 \x01(\x0b\x32\x1e.protos.config.WrappedExporter\x12\x30\n\x0csqs_consumer\x18\x04 \x01(\x0b\x32\x1a.protos.config.SqsConsumer\"j\n\x07NetAddr\x12.\n\x08\x65ndpoint\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\ttransport\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"6\n\nHttpServer\x12(\n\x08net_addr\x18\x01 \x01(\x0b\x32\x16.protos.config.NetAddr\"\x9b\x01\n\x07\x42\x61tcher\x12/\n\tchan_size\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x30\n\nbatch_size\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12-\n\x07timeout\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xb3\x02\n\x0fWrappedExporter\x12>\n\x08wrappers\x18\x01 \x03(\x0b\x32,.protos.config.WrappedExporter.WrappersEntry\x1a\x86\x01\n\x07Wrapper\x12:\n\x10\x63onsole_exporter\x18\x01 \x01(\x0b\x32\x1e.protos.config.ConsoleExporterH\x00\x12\x34\n\rhttp_exporter\x18\x02 \x01(\x0b\x32\x1b.protos.config.HttpExporterH\x00\x42\t\n\x07wrapper\x1aW\n\rWrappersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x35\n\x05value\x18\x02 \x01(\x0b\x32&.protos.config.WrappedExporter.Wrapper:\x02\x38\x01\">\n\x0f\x43onsoleExporter\x12+\n\x07\x65nabled\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"g\n\nBearerAuth\x12+\n\x05token\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06scheme\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"J\n\x0e\x41uthentication\x12\x30\n\x0b\x62\x65\x61rer_auth\x18\x01 \x01(\x0b\x32\x19.protos.config.BearerAuthH\x00\x42\x06\n\x04\x61uth\"\xbd\x02\n\x0cHttpExporter\x12*\n\x04host\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12.\n\x08\x65ndpoint\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12/\n\tchan_size\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x35\n\x0e\x61uthentication\x18\x04 \x01(\x0b\x32\x1d.protos.config.Authentication\x12\x39\n\x07headers\x18\x05 \x03(\x0b\x32(.protos.config.HttpExporter.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x11Z\x0f./protos/configb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.config.collector_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  DESCRIPTOR._serialized_options = b'Z\017./protos/config'
  _WRAPPEDEXPORTER_WRAPPERSENTRY._options = None
  _WRAPPEDEXPORTER_WRAPPERSENTRY._serialized_options = b'8\001'
  _HTTPEXPORTER_HEADERSENTRY._options = None
  _HTTPEXPORTER_HEADERSENTRY._serialized_options = b'8\001'
  _SQSCONSUMER._serialized_start=81
  _SQSCONSUMER._serialized_end=305
  _PIPELINECONFIG._serialized_start=308
  _PIPELINECONFIG._serialized_end=513
  _NETADDR._serialized_start=515
  _NETADDR._serialized_end=621
  _HTTPSERVER._serialized_start=623
  _HTTPSERVER._serialized_end=677
  _BATCHER._serialized_start=680
  _BATCHER._serialized_end=835
  _WRAPPEDEXPORTER._serialized_start=838
  _WRAPPEDEXPORTER._serialized_end=1145
  _WRAPPEDEXPORTER_WRAPPER._serialized_start=922
  _WRAPPEDEXPORTER_WRAPPER._serialized_end=1056
  _WRAPPEDEXPORTER_WRAPPERSENTRY._serialized_start=1058
  _WRAPPEDEXPORTER_WRAPPERSENTRY._serialized_end=1145
  _CONSOLEEXPORTER._serialized_start=1147
  _CONSOLEEXPORTER._serialized_end=1209
  _BEARERAUTH._serialized_start=1211
  _BEARERAUTH._serialized_end=1314
  _AUTHENTICATION._serialized_start=1316
  _AUTHENTICATION._serialized_end=1390
  _HTTPEXPORTER._serialized_start=1393
  _HTTPEXPORTER._serialized_end=1710
  _HTTPEXPORTER_HEADERSENTRY._serialized_start=1664
  _HTTPEXPORTER_HEADERSENTRY._serialized_end=1710
# @@protoc_insertion_point(module_scope)

protos/config/collector_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/account.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aprotos/event/account.proto\x12\x0cprotos.event\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\"v\n\x0f\x41\x63\x63ountApiToken\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x12\n\ncreated_by\x18\x02 \x01(\t\x12.\n\ncreated_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\naccount_id\x18\x04 \x01(\x04\"x\n\tUserFlags\x12\x35\n\x11is_email_verified\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x34\n\x10is_account_owner\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"u\n\x04User\x12\n\n\x02id\x18\x01 \x01(\x04\x12\r\n\x05\x65mail\x18\x02 \x01(\t\x12\x12\n\nfirst_name\x18\x03 \x01(\t\x12\x11\n\tlast_name\x18\x04 \x01(\t\x12+\n\nuser_flags\x18\x05 \x01(\x0b\x32\x17.protos.event.UserFlagsb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.account_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _ACCOUNTAPITOKEN._serialized_start=109
  _ACCOUNTAPITOKEN._serialized_end=227
  _USERFLAGS._serialized_start=229
  _USERFLAGS._serialized_end=349
  _USER._serialized_start=351
  _USER._serialized_end=468
# @@protoc_insertion_point(module_scope)

protos/event/account_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/alert.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import trigger_pb2 as protos_dot_event_dot_trigger__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2
from protos.event import entity_pb2 as protos_dot_event_dot_entity__pb2
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18protos/event/alert.proto\x12\x0cprotos.event\x1a\x1aprotos/event/trigger.proto\x1a\x1aprotos/event/monitor.proto\x1a\x19protos/event/entity.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x94\x01\n\x0c\x41lertSummary\x12\n\n\x02id\x18\x01 \x01(\x04\x12-\n\x07trigger\x18\x02 \x01(\x0b\x32\x1c.protos.event.TriggerSummary\x12\x33\n\x0e\x65ntity_trigger\x18\x03 \x01(\x0b\x32\x1b.protos.event.EntityTrigger\x12\x14\n\x0ctriggered_at\x18\x04 \x01(\x10\"\xdc\x03\n\nAlertStats\x12=\n\x17total_transaction_count\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12>\n\x18missed_transaction_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12?\n\x19\x64\x65layed_transaction_count\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x32\n\x0cmedian_delay\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x30\n\nevent_name\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\x0e\x65vent_key_name\x18\x06 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x0f\x65vent_timestamp\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12;\n\x15\x65ntity_instance_value\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xd1\x01\n\x0b\x41lertDetail\x12\n\n\x02id\x18\x01 \x01(\x04\x12&\n\x07trigger\x18\x02 \x01(\x0b\x32\x15.protos.event.Trigger\x12\x33\n\x0e\x65ntity_trigger\x18\x03 \x01(\x0b\x32\x1b.protos.event.EntityTrigger\x12\x30\n\x0ctriggered_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x05stats\x18\x05 \x01(\x0b\x32\x18.protos.event.AlertStats\"V\n\x1e\x44\x65layedMonitorTransactionStats\x12\x34\n\x0e\x64\x65lay_duration\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\"\xe4\x02\n\x17\x41lertMonitorTransaction\x12\x38\n\x04type\x18\x01 \x01(\x0e\x32*.protos.event.AlertMonitorTransaction.Type\x12=\n\x13monitor_transaction\x18\x02 \x01(\x0b\x32 .protos.event.MonitorTransaction\x12\x10\n\x08\x61lert_id\x18\x03 \x01(\x04\x12Y\n!delayed_monitor_transaction_stats\x18\x04 \x01(\x0b\x32,.protos.event.DelayedMonitorTransactionStatsH\x00\x12\x14\n\x0ctrigger_name\x18\x05 \x01(\t\"D\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x16\n\x12MISSED_TRANSACTION\x10\x01\x12\x17\n\x13\x44\x45LAYED_TRANSACTION\x10\x02\x42\x07\n\x05stats\"\xe5\x01\n\x13\x41lertEntityInstance\x12\x34\n\x04type\x18\x01 \x01(\x0e\x32&.protos.event.AlertEntityInstance.Type\x12\x35\n\x0f\x65ntity_instance\x18\x02 \x01(\x0b\x32\x1c.protos.event.EntityInstance\x12\x10\n\x08\x61lert_id\x18\x03 \x01(\x04\x12\x14\n\x0ctrigger_name\x18\x04 \x01(\t\"9\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\r\n\tPER_EVENT\x10\x01\x12\x15\n\x11\x41GGREGATED_EVENTS\x10\x02\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.alert_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _ALERTSUMMARY._serialized_start=191
  _ALERTSUMMARY._serialized_end=339
  _ALERTSTATS._serialized_start=342
  _ALERTSTATS._serialized_end=818
  _ALERTDETAIL._serialized_start=821
  _ALERTDETAIL._serialized_end=1030
  _DELAYEDMONITORTRANSACTIONSTATS._serialized_start=1032
  _DELAYEDMONITORTRANSACTIONSTATS._serialized_end=1118
  _ALERTMONITORTRANSACTION._serialized_start=1121
  _ALERTMONITORTRANSACTION._serialized_end=1477
  _ALERTMONITORTRANSACTION_TYPE._serialized_start=1400
  _ALERTMONITORTRANSACTION_TYPE._serialized_end=1468
  _ALERTENTITYINSTANCE._serialized_start=1480
  _ALERTENTITYINSTANCE._serialized_end=1709
  _ALERTENTITYINSTANCE_TYPE._serialized_start=1652
  _ALERTENTITYINSTANCE_TYPE._serialized_end=1709
# @@protoc_insertion_point(module_scope)

protos/event/alert_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/api.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import account_pb2 as protos_dot_event_dot_account__pb2
from protos.event import panel_pb2 as protos_dot_event_dot_panel__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2
from protos.event import metric_pb2 as protos_dot_event_dot_metric__pb2
from protos.event import trigger_pb2 as protos_dot_event_dot_trigger__pb2
from protos.event import alert_pb2 as protos_dot_event_dot_alert__pb2
from protos.event import options_pb2 as protos_dot_event_dot_options__pb2
from protos.event import notification_pb2 as protos_dot_event_dot_notification__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import engine_options_pb2 as protos_dot_event_dot_engine__options__pb2
from protos.event import schema_pb2 as protos_dot_event_dot_schema__pb2
from protos.event import literal_pb2 as protos_dot_event_dot_literal__pb2
from protos.event import entity_pb2 as protos_dot_event_dot_entity__pb2
from protos.event import connectors_pb2 as protos_dot_event_dot_connectors__pb2
from protos.event import search_pb2 as protos_dot_event_dot_search__pb2
from protos.event import timeline_pb2 as protos_dot_event_dot_timeline__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16protos/event/api.proto\x12\x0cprotos.event\x1a\x17protos/event/base.proto\x1a\x1aprotos/event/account.proto\x1a\x18protos/event/panel.proto\x1a\x1aprotos/event/monitor.proto\x1a\x19protos/event/metric.proto\x1a\x1aprotos/event/trigger.proto\x1a\x18protos/event/alert.proto\x1a\x1aprotos/event/options.proto\x1a\x1fprotos/event/notification.proto\x1a\x1dprotos/event/query_base.proto\x1a!protos/event/engine_options.proto\x1a\x19protos/event/schema.proto\x1a\x1aprotos/event/literal.proto\x1a\x19protos/event/entity.proto\x1a\x1dprotos/event/connectors.proto\x1a\x19protos/event/search.proto\x1a\x1bprotos/event/timeline.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\xbb\x01\n\x04Meta\x12+\n\ntime_range\x18\x01 \x01(\x0b\x32\x17.protos.event.TimeRange\x12 \n\x04page\x18\x02 \x01(\x0b\x32\x12.protos.event.Page\x12\x31\n\x0btotal_count\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x31\n\rshow_inactive\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"@\n\x07Message\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x11\n\ttraceback\x18\x03 \x01(\t\"a\n\x0c\x45rrorMessage\x12)\n\x05\x65rror\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"L\n\x10GetEventsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0e\x65vent_type_ids\x18\x02 \x03(\x04\"Z\n\x11GetEventsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12#\n\x06\x65vents\x18\x02 \x03(\x0b\x32\x13.protos.event.Event\"\xa1\x01\n\x1aGetEventSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\x12.\n\nsave_query\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb0\x01\n\x1bGetEventSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12#\n\x06\x65vents\x18\x02 \x03(\x0b\x32\x13.protos.event.Event\x12\x30\n\x10selected_columns\x18\x03 \x03(\x0b\x32\x16.protos.event.KeyValue\x12\x18\n\x10query_context_id\x18\x04 \x01(\t\"\xa3\x01\n\x1cGetEventSearchV2QueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\x12.\n\nsave_query\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xb2\x01\n\x1dGetEventSearchV2QueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12#\n\x06\x65vents\x18\x02 \x03(\x0b\x32\x13.protos.event.Event\x12\x30\n\x10selected_columns\x18\x03 \x03(\x0b\x32\x16.protos.event.KeyValue\x12\x18\n\x10query_context_id\x18\x04 \x01(\t\"r\n\x1bSaveEventSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"Z\n\x1cSaveEventSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x18\n\x10query_context_id\x18\x04 \x01(\t\"q\n\x1aGetEventExportQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xaf\x01\n\x1bGetEventExportQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\x07success\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x03 \x01(\x0b\x32\x15.protos.event.Message\x12\x19\n\x11\x65xport_context_id\x18\x04 \x01(\t\"\x96\x01\n\x1aGetEventTypeSummaryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0e\x65vent_type_ids\x18\x02 \x03(\x04\x12>\n\x14\x66uzzy_search_request\x18\x03 \x01(\x0b\x32 .protos.event.FuzzySearchRequest\"{\n\x1bGetEventTypeSummaryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12:\n\x12\x65vent_type_summary\x18\x02 \x03(\x0b\x32\x1e.protos.event.EventTypeSummary\"K\n\x19GetEventKeysSearchRequest\x12\x15\n\revent_key_ids\x18\x01 \x03(\x04\x12\x17\n\x0f\x65vent_key_names\x18\x02 \x03(\t\"H\n\x1aGetEventKeysSearchResponse\x12*\n\nevent_keys\x18\x01 \x03(\x0b\x32\x16.protos.event.EventKey\"\x93\x01\n\x10GetFunnelRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0e\x65vent_key_name\x18\x02 \x01(\t\x12\x16\n\x0e\x65vent_type_ids\x18\x03 \x03(\x04\x12\x17\n\x0f\x66ilter_key_name\x18\x04 \x01(\t\x12\x14\n\x0c\x66ilter_value\x18\x05 \x01(\t\"S\n\x16WorkflowBuilderRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x17\n\x0fworkflow_config\x18\x02 \x01(\t\"\xc7\x03\n\x1aWorkflowTimeSeriesResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x11\n\tnode_name\x18\x02 \x01(\t\x12U\n\x12metric_time_series\x18\x03 \x03(\x0b\x32\x39.protos.event.WorkflowTimeSeriesResponse.MetricTimeSeries\x12\x12\n\ncompare_to\x18\x04 \x01(\t\x1a/\n\x0bMetricValue\x12\x11\n\ttimestamp\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x01\x1a\xd7\x01\n\x10MetricTimeSeries\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x15\n\rmetric_source\x18\x02 \x01(\t\x12L\n\x0e\x63urrent_series\x18\x03 \x03(\x0b\x32\x34.protos.event.WorkflowTimeSeriesResponse.MetricValue\x12I\n\x0bpast_series\x18\x04 \x03(\x0b\x32\x34.protos.event.WorkflowTimeSeriesResponse.MetricValue\"\xa8\x01\n\x11GetFunnelResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12-\n\x0b\x66unnel_view\x18\x02 \x01(\x0b\x32\x18.protos.event.FunnelView\x12\x31\n\rworkflow_view\x18\x03 \x01(\x0b\x32\x1a.protos.event.WorkflowView\x12\x0f\n\x07message\x18\x04 \x01(\t\"\xda\x01\n\x17GetFunnelDropOffRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x1d\n\x15\x66unnel_event_type_ids\x18\x02 \x03(\x04\x12\x17\n\x0f\x66unnel_key_name\x18\x03 \x01(\t\x12\x1b\n\x13start_event_type_id\x18\x04 \x01(\x04\x12\x19\n\x11\x65nd_event_type_id\x18\x05 \x01(\x04\x12\x17\n\x0f\x66ilter_key_name\x18\x06 \x01(\t\x12\x14\n\x0c\x66ilter_value\x18\x07 \x01(\t\"<\n\x18GetFunnelDropOffResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\"\xa4\x02\n$GetFunnelDropOffDistributionResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x1b\n\x13previous_node_count\x18\x02 \x01(\x04\x12v\n\x1e\x66unnel_event_type_distribution\x18\x03 \x03(\x0b\x32N.protos.event.GetFunnelDropOffDistributionResponse.FunnelEventTypeDistribution\x1a\x45\n\x1b\x46unnelEventTypeDistribution\x12\x17\n\x0f\x65vent_type_name\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"L\n\x12GetFunnelRequestV3\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x14\n\x0c\x66ilter_value\x18\x02 \x01(\t\"\x8d\x03\n\x13GetFunnelResponseV3\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12<\n\x06stages\x18\x02 \x03(\x0b\x32,.protos.event.GetFunnelResponseV3.stage_data\x1a\x95\x02\n\nstage_data\x12\r\n\x05stage\x18\x01 \x01(\t\x12G\n\x06\x65vents\x18\x02 \x03(\x0b\x32\x37.protos.event.GetFunnelResponseV3.stage_data.event_data\x1a\xae\x01\n\nevent_data\x12\x12\n\nevent_name\x18\x01 \x01(\t\x12\x10\n\x08\x65vent_id\x18\x02 \x01(\x04\x12\x1c\n\x14unique_records_count\x18\x03 \x01(\x04\x12\r\n\x05\x63ount\x18\x04 \x01(\x04\x12M\n\x0c\x63hild_events\x18\x05 \x03(\x0b\x32\x37.protos.event.GetFunnelResponseV3.stage_data.event_data\"N\n\x17GetConnectorKeysRequest\x12\x33\n\x0e\x63onnector_type\x18\x01 \x01(\x0e\x32\x1b.protos.event.ConnectorType\"_\n\x18GetConnectorKeysResponse\x12\x32\n\x0e\x63onnector_keys\x18\x01 \x03(\x0b\x32\x1a.protos.event.ConnectorKey\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x83\x01\n\x18SaveConnectorKeysRequest\x12\x33\n\x0e\x63onnector_type\x18\x01 \x01(\x0e\x32\x1b.protos.event.ConnectorType\x12\x32\n\x0e\x63onnector_keys\x18\x02 \x03(\x0b\x32\x1a.protos.event.ConnectorKey\",\n\x19SaveConnectorKeysResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\"Y\n\x1dGetEventTypeDefinitionRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0e\x65vent_type_ids\x18\x02 \x03(\x04\"\x85\x01\n\x1eGetEventTypeDefinitionResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x41\n\x16\x65vent_type_definitions\x18\x02 \x03(\x0b\x32!.protos.event.EventTypeDefinition\"\xb9\x02\n\x1c\x43reateOrUpdateMonitorRequest\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x1c\n\x14primary_event_key_id\x18\x02 \x01(\x04\x12\x1e\n\x16secondary_event_key_id\x18\x03 \x01(\x04\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\'\n\x08triggers\x18\x05 \x03(\x0b\x32\x15.protos.event.Trigger\x12\x37\n\rnotifications\x18\x06 \x03(\x0b\x32 .protos.event.NotificationConfig\x12-\n\tis_active\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\x0cis_generated\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xa3\x01\n\x1d\x43reateOrUpdateMonitorResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\x12-\n\x07monitor\x18\x03 \x01(\x0b\x32\x1c.protos.event.MonitorPartial\"\x8a\x01\n\x11GetMonitorRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x13\n\x0bmonitor_ids\x18\x02 \x03(\x04\x12>\n\x14\x66uzzy_search_request\x18\x03 \x01(\x0b\x32 .protos.event.FuzzySearchRequest\"_\n\x12GetMonitorResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\'\n\x08monitors\x18\x02 \x03(\x0b\x32\x15.protos.event.Monitor\"T\n\x1bGetMonitorDefinitionRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x13\n\x0bmonitor_ids\x18\x02 \x03(\x04\"~\n\x1cGetMonitorDefinitionResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12<\n\x13monitor_definitions\x18\x02 \x03(\x0b\x32\x1f.protos.event.MonitorDefinition\"<\n\x18GetMonitorOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\"t\n\x19GetMonitorOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x35\n\x0fmonitor_options\x18\x02 \x01(\x0b\x32\x1c.protos.event.MonitorOptions\"\x90\x01\n\x1dGetMonitorTransactionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\nmonitor_id\x18\x02 \x01(\x04\x12\x39\n\x06params\x18\x03 \x01(\x0b\x32).protos.event.MonitorTransactionGetParams\"\xb9\x01\n\x1eGetMonitorTransactionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12>\n\x14monitor_transactions\x18\x02 \x03(\x0b\x32 .protos.event.MonitorTransaction\x12\x35\n\x06\x61lerts\x18\x03 \x03(\x0b\x32%.protos.event.AlertMonitorTransaction\"^\n\"GetMonitorTransactionEventsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0etransaction_id\x18\x02 \x01(\x04\"l\n#GetMonitorTransactionEventsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12#\n\x06\x65vents\x18\x02 \x03(\x0b\x32\x13.protos.event.Event\"_\n#GetMonitorTransactionDetailsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x16\n\x0etransaction_id\x18\x02 \x01(\x04\"\x96\x01\n$GetMonitorTransactionDetailsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12L\n\x1bmonitor_transaction_details\x18\x02 \x03(\x0b\x32\'.protos.event.MonitorTransactionDetails\"\x9a\x01\n!GetMetricExpressionOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12&\n\x07\x63ontext\x18\x02 \x01(\x0e\x32\x15.protos.event.Context\x12+\n\nid_literal\x18\x03 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xbe\x01\n\"GetMetricExpressionOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x33\n\x0emetric_options\x18\x02 \x01(\x0b\x32\x1b.protos.event.MetricOptions\x12\x41\n\x19\x64\x65\x66\x61ult_metric_expression\x18\x03 \x01(\x0b\x32\x1e.protos.event.MetricExpression\"\x98\x01\n\x10GetMetricRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12&\n\x07\x63ontext\x18\x02 \x01(\x0e\x32\x15.protos.event.Context\x12:\n\x12metric_expressions\x18\x03 \x03(\x0b\x32\x1e.protos.event.MetricExpression\"\xb9\x01\n\x11GetMetricResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12-\n\x0bmetric_data\x18\x02 \x03(\x0b\x32\x18.protos.event.MetricData\x12+\n\x07success\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x04 \x01(\x0b\x32\x15.protos.event.Message\"D\n\x12GetPanelsV1Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x0c\n\x04name\x18\x02 \x01(\t\"^\n\x13GetPanelsV1Response\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12%\n\x06panels\x18\x02 \x03(\x0b\x32\x15.protos.event.PanelV1\"D\n\x1c\x43reateOrUpdatePanelV1Request\x12$\n\x05panel\x18\x01 \x01(\x0b\x32\x15.protos.event.PanelV1\"t\n\x1d\x43reateOrUpdatePanelV1Response\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"F\n\x14\x44\x65letePanelV1Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x0c\n\x04name\x18\x02 \x01(\t\"l\n\x15\x44\x65letePanelV1Response\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"H\n\x16GetDashboardsV1Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x0c\n\x04name\x18\x02 \x01(\t\"j\n\x17GetDashboardsV1Response\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12-\n\ndashboards\x18\x02 \x03(\x0b\x32\x19.protos.event.DashboardV1\"P\n CreateOrUpdateDashboardV1Request\x12,\n\tdashboard\x18\x01 \x01(\x0b\x32\x19.protos.event.DashboardV1\"x\n!CreateOrUpdateDashboardV1Response\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"J\n\x18\x44\x65leteDashboardV1Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x0c\n\x04name\x18\x02 \x01(\t\"p\n\x19\x44\x65leteDashboardV1Response\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\">\n\x1aGetAccountApiTokensRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\"z\n\x1bGetAccountApiTokensResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x39\n\x12\x61\x63\x63ount_api_tokens\x18\x02 \x03(\x0b\x32\x1d.protos.event.AccountApiToken\"\x1e\n\x1c\x43reateAccountApiTokenRequest\"\x86\x01\n\x1d\x43reateAccountApiTokenResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x38\n\x11\x61\x63\x63ount_api_token\x18\x02 \x01(\x0b\x32\x1d.protos.event.AccountApiToken\"=\n\x1c\x44\x65leteAccountApiTokenRequest\x12\x1d\n\x15\x61\x63\x63ount_api_token_key\x18\x01 \x01(\t\"L\n\x1d\x44\x65leteAccountApiTokenResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"e\n\x18GetMonitorTriggerRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\nmonitor_id\x18\x02 \x01(\x04\x12\x13\n\x0btrigger_ids\x18\x03 \x03(\x04\"\x9b\x01\n\x19GetMonitorTriggerResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\\\n$monitor_trigger_notification_details\x18\x02 \x03(\x0b\x32..protos.event.MonitorTriggerNotificationDetail\"W\n\x1fGetMonitorTriggerOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\nmonitor_id\x18\x02 \x01(\x04\"\x88\x03\n GetMonitorTriggerOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\x07success\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x03 \x01(\x0b\x32\x15.protos.event.Message\x12-\n\x07monitor\x18\x04 \x01(\x0b\x32\x1c.protos.event.MonitorPartial\x12:\n\x12primary_event_type\x18\x05 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12<\n\x14secondary_event_type\x18\x06 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12\x44\n\x17monitor_trigger_options\x18\x07 \x01(\x0b\x32#.protos.event.MonitorTriggerOptions\"\x7f\n\x1b\x43reateMonitorTriggerRequest\x12\'\n\x08triggers\x18\x01 \x03(\x0b\x32\x15.protos.event.Trigger\x12\x37\n\rnotifications\x18\x02 \x03(\x0b\x32 .protos.event.NotificationConfig\"s\n\x1c\x43reateMonitorTriggerResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"\x8a\x01\n\x1bUpdateMonitorTriggerRequest\x12\x30\n\ntrigger_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x39\n\x12update_trigger_ops\x18\x02 \x03(\x0b\x32\x1d.protos.event.UpdateTriggerOp\"s\n\x1cUpdateMonitorTriggerResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"b\n\x17GetAlertsSummaryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\nmonitor_id\x18\x03 \x01(\x04\x12\x11\n\talert_ids\x18\x04 \x03(\x04\"p\n\x18GetAlertsSummaryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x32\n\x0e\x61lerts_summary\x18\x04 \x03(\x0b\x32\x1a.protos.event.AlertSummary\"L\n\x16GetAlertDetailsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x10\n\x08\x61lert_id\x18\x02 \x01(\x04\"l\n\x17GetAlertDetailsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12/\n\x0c\x61lert_detail\x18\x02 \x01(\x0b\x32\x19.protos.event.AlertDetail\"X\n\"GetAlertMonitorTransactionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x10\n\x08\x61lert_id\x18\x02 \x01(\x04\"\x8c\x01\n#GetAlertMonitorTransactionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x43\n\x14monitor_transactions\x18\x02 \x03(\x0b\x32%.protos.event.AlertMonitorTransaction\"\x10\n\x0eGetUserRequest\"3\n\x0fGetUserResponse\x12 \n\x04user\x18\x01 \x01(\x0b\x32\x12.protos.event.User\"U\n\x1dGetNotificationConfigsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\ntrigger_id\x18\x02 \x01(\x04\"\x82\x01\n\x1eGetNotificationConfigsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12>\n\x14notification_configs\x18\x02 \x03(\x0b\x32 .protos.event.NotificationConfig\"`\n\x1f\x43reateNotificationConfigRequest\x12=\n\x13notification_config\x18\x01 \x01(\x0b\x32 .protos.event.NotificationConfig\"w\n CreateNotificationConfigResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"A\n\x1dGetGlobalSearchOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\"\x82\x01\n\x1eGetGlobalSearchOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12>\n\x14global_query_options\x18\x02 \x01(\x0b\x32 .protos.event.GlobalQueryOptions\"\xae\x01\n\x1cGetEventSearchOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\nid_literal\x18\x02 \x01(\x0b\x32\x17.protos.event.IdLiteral\x12?\n\x1esaved_query_request_context_id\x18\x03 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xb5\x01\n\x1dGetEventSearchOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x37\n\x13\x65vent_query_options\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryOptions\x12\x39\n\x15\x64\x65\x66\x61ult_query_request\x18\x03 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xbd\x01\n+GetMonitorsTransactionsSearchOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\nid_literal\x18\x02 \x01(\x0b\x32\x17.protos.event.IdLiteral\x12?\n\x1esaved_query_request_context_id\x18\x03 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xd2\x01\n,GetMonitorsTransactionsSearchOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x45\n!monitor_transaction_query_options\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryOptions\x12\x39\n\x15\x64\x65\x66\x61ult_query_request\x18\x03 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xbf\x01\n-GetMonitorsTransactionsSearchOptionsV2Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\nid_literal\x18\x02 \x01(\x0b\x32\x17.protos.event.IdLiteral\x12?\n\x1esaved_query_request_context_id\x18\x03 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xd6\x01\n.GetMonitorsTransactionsSearchOptionsV2Response\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12G\n!monitor_transaction_query_options\x18\x02 \x01(\x0b\x32\x1c.protos.event.QueryOptionsV2\x12\x39\n\x15\x64\x65\x66\x61ult_query_request\x18\x03 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\x80\x01\n)GetMonitorsTransactionsSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xcf\x01\n*GetMonitorsTransactionsSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12>\n\x14monitor_transactions\x18\x02 \x03(\x0b\x32 .protos.event.MonitorTransaction\x12?\n\x1bsearch_result_monitor_stats\x18\x03 \x01(\x0b\x32\x1a.protos.event.MonitorStats\"\xb2\x01\n+GetMonitorsTransactionsSearchV2QueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\x12.\n\nsave_query\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xf9\x01\n,GetMonitorsTransactionsSearchV2QueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12L\n\x1bmonitor_transaction_details\x18\x02 \x03(\x0b\x32\'.protos.event.MonitorTransactionDetails\x12?\n\x1bsearch_result_monitor_stats\x18\x03 \x01(\x0b\x32\x1a.protos.event.MonitorStats\x12\x18\n\x10query_context_id\x18\x04 \x01(\t\"\x80\x01\n)SaveMonitorTransactionsSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"h\n*SaveMonitorTransactionsSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x18\n\x10query_context_id\x18\x04 \x01(\t\"\x80\x01\n)GetMonitorsTransactionsExportQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xbe\x01\n*GetMonitorsTransactionsExportQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\x07success\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x03 \x01(\x0b\x32\x15.protos.event.Message\x12\x19\n\x11\x65xport_context_id\x18\x04 \x01(\t\"x\n\x16\x43reateConnectorRequest\x12*\n\tconnector\x18\x01 \x01(\x0b\x32\x17.protos.event.Connector\x12\x32\n\x0e\x63onnector_keys\x18\x02 \x03(\x0b\x32\x1a.protos.event.ConnectorKey\"n\n\x17\x43reateConnectorResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"Q\n\x1aRequestConnectorAPIRequest\x12\x33\n\x0e\x63onnector_type\x18\x01 \x01(\x0e\x32\x1b.protos.event.ConnectorType\"r\n\x1bRequestConnectorAPIResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"\xbd\x01\n\x1f\x43reateTransformerMappingRequest\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12/\n\x0c\x64\x65\x63oder_type\x18\x02 \x01(\x0e\x32\x19.protos.event.DecoderType\x12\x37\n\x10transformer_type\x18\x03 \x01(\x0e\x32\x1d.protos.event.TransformerType\"w\n CreateTransformerMappingResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\";\n\x17GetEntityOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\"q\n\x18GetEntityOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x33\n\x0e\x65ntity_options\x18\x02 \x01(\x0b\x32\x1b.protos.event.EntityOptions\"\x87\x01\n\x13\x43reateEntityRequest\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\tis_active\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x15\n\revent_key_ids\x18\x03 \x03(\x04\"\x98\x01\n\x14\x43reateEntityResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\x12+\n\x06\x65ntity\x18\x03 \x01(\x0b\x32\x1b.protos.event.EntityPartial\"\x7f\n\x13UpdateEntityRequest\x12/\n\tentity_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x37\n\x11update_entity_ops\x18\x02 \x03(\x0b\x32\x1c.protos.event.UpdateEntityOp\"k\n\x14UpdateEntityResponse\x12+\n\x07success\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"R\n\x1b\x45ntityTriggerOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x11\n\tentity_id\x18\x02 \x01(\x04\"\x88\x01\n\x1c\x45ntityTriggerOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x13\n\x0b\x65ntity_name\x18\x02 \x01(\t\x12\x31\n\x11\x65vent_key_options\x18\x03 \x03(\x0b\x32\x16.protos.event.EventKey\"\xa5\x01\n\x1a\x45ntityTriggerCreateRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12,\n\x07trigger\x18\x02 \x01(\x0b\x32\x1b.protos.event.EntityTrigger\x12\x37\n\rnotifications\x18\x03 \x03(\x0b\x32 .protos.event.NotificationConfig\"g\n\x1b\x45ntityTriggerCreateResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"[\n\x1c\x45ntityTriggerInactiveRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x19\n\x11\x65ntity_trigger_id\x18\x02 \x01(\x04\"i\n\x1d\x45ntityTriggerInactiveResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12&\n\x07message\x18\x02 \x01(\x0b\x32\x15.protos.event.Message\"j\n\x18GetEntityTriggersRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x11\n\tentity_id\x18\x02 \x01(\x04\x12\x19\n\x11\x65ntity_trigger_id\x18\x03 \x01(\x04\"\x99\x01\n\x19GetEntityTriggersResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12Z\n#entity_trigger_notification_details\x18\x02 \x03(\x0b\x32-.protos.event.EntityTriggerNotificationDetail\"\x8f\x01\n\x17GetEntitySummaryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x12\n\nentity_ids\x18\x02 \x03(\x04\x12>\n\x14\x66uzzy_search_request\x18\x03 \x01(\x0b\x32 .protos.event.FuzzySearchRequest\"k\n\x18GetEntitySummaryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12-\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x1b.protos.event.EntitySummary\"k\n\x16GetEntityDetailRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12/\n\tentity_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"g\n\x17GetEntityDetailResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12*\n\x06\x65ntity\x18\x02 \x01(\x0b\x32\x1a.protos.event.EntityDetail\"\xac\x01\n GetEntityInstancesSummaryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12/\n\tentity_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x35\n\x06params\x18\x03 \x01(\x0b\x32%.protos.event.EntityInstanceGetParams\"\x84\x01\n!GetEntityInstancesSummaryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12=\n\x10\x65ntity_instances\x18\x02 \x03(\x0b\x32#.protos.event.EntityInstanceSummary\"}\n\x1fGetEntityInstancesDetailRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x38\n\x12\x65ntity_instance_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"\x81\x01\n GetEntityInstancesDetailResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12;\n\x0f\x65ntity_instance\x18\x02 \x01(\x0b\x32\".protos.event.EntityInstanceDetail\"\x7f\n!GetEntityInstancesTimelineRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x38\n\x12\x65ntity_instance_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"\x9c\x01\n\"GetEntityInstancesTimelineResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12T\n entity_instance_timeline_records\x18\x02 \x03(\x0b\x32*.protos.event.EntityInstanceTimelineRecord\"w\n&GetEntityInstancesSearchOptionsRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\nid_literal\x18\x02 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xbf\x01\n\'GetEntityInstancesSearchOptionsResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x37\n\x13\x65vent_query_options\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryOptions\x12\x39\n\x15\x64\x65\x66\x61ult_query_request\x18\x03 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"y\n(GetEntityInstancesSearchOptionsV2Request\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\nid_literal\x18\x02 \x01(\x0b\x32\x17.protos.event.IdLiteral\"\xc3\x01\n)GetEntityInstancesSearchOptionsV2Response\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x39\n\x13\x65vent_query_options\x18\x02 \x01(\x0b\x32\x1c.protos.event.QueryOptionsV2\x12\x39\n\x15\x64\x65\x66\x61ult_query_request\x18\x03 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"z\n#GetEntityInstanceSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rquery_request\x18\x02 \x01(\x0b\x32\x1a.protos.event.QueryRequest\"\xb9\x01\n$GetEntityInstanceSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12=\n\x10\x65ntity_instances\x18\x02 \x03(\x0b\x32#.protos.event.EntityInstanceSummary\x12\x30\n\x10selected_columns\x18\x03 \x03(\x0b\x32\x16.protos.event.KeyValue\"\x97\x01\n#GetEntityWorkflowSearchQueryRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12/\n\tentity_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x1d\n\x15\x65ntity_instance_value\x18\x03 \x01(\t\"{\n$GetEntityWorkflowSearchQueryResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x31\n\rworkflow_view\x18\x02 \x01(\x0b\x32\x1a.protos.event.WorkflowView\"\x99\x01\n\x19\x43reateEntityFunnelRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12&\n\x07\x63ontext\x18\x02 \x01(\x0e\x32\x15.protos.event.Context\x12\x32\n\x13\x65ntity_funnel_panel\x18\x03 \x01(\x0b\x32\x15.protos.event.PanelV1\"\xc7\x01\n\x1a\x43reateEntityFunnelResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\x07success\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x03 \x01(\x0b\x32\x15.protos.event.Message\x12\x32\n\rfunnel_entity\x18\x04 \x01(\x0b\x32\x1b.protos.event.EntityPartial\"\xab\x01\n\x16GetEntityFunnelRequest\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12\x1a\n\x12\x65ntity_funnel_name\x18\x02 \x01(\t\x12\x17\n\x0f\x66ilter_key_name\x18\x03 \x01(\t\x12\x14\n\x0c\x66ilter_value\x18\x04 \x01(\t\x12$\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x14.protos.event.Filter\"\xc3\x01\n\x17GetEntityFunnelResponse\x12 \n\x04meta\x18\x01 \x01(\x0b\x32\x12.protos.event.Meta\x12+\n\x07success\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12&\n\x07message\x18\x03 \x01(\x0b\x32\x15.protos.event.Message\x12\x31\n\rworkflow_view\x18\x04 \x01(\x0b\x32\x1a.protos.event.WorkflowViewb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.api_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _META._serialized_start=557
  _META._serialized_end=744
  _MESSAGE._serialized_start=746
  _MESSAGE._serialized_end=810
  _ERRORMESSAGE._serialized_start=812
  _ERRORMESSAGE._serialized_end=909
  _GETEVENTSREQUEST._serialized_start=911
  _GETEVENTSREQUEST._serialized_end=987
  _GETEVENTSRESPONSE._serialized_start=989
  _GETEVENTSRESPONSE._serialized_end=1079
  _GETEVENTSEARCHQUERYREQUEST._serialized_start=1082
  _GETEVENTSEARCHQUERYREQUEST._serialized_end=1243
  _GETEVENTSEARCHQUERYRESPONSE._serialized_start=1246
  _GETEVENTSEARCHQUERYRESPONSE._serialized_end=1422
  _GETEVENTSEARCHV2QUERYREQUEST._serialized_start=1425
  _GETEVENTSEARCHV2QUERYREQUEST._serialized_end=1588
  _GETEVENTSEARCHV2QUERYRESPONSE._serialized_start=1591
  _GETEVENTSEARCHV2QUERYRESPONSE._serialized_end=1769
  _SAVEEVENTSEARCHQUERYREQUEST._serialized_start=1771
  _SAVEEVENTSEARCHQUERYREQUEST._serialized_end=1885
  _SAVEEVENTSEARCHQUERYRESPONSE._serialized_start=1887
  _SAVEEVENTSEARCHQUERYRESPONSE._serialized_end=1977
  _GETEVENTEXPORTQUERYREQUEST._serialized_start=1979
  _GETEVENTEXPORTQUERYREQUEST._serialized_end=2092
  _GETEVENTEXPORTQUERYRESPONSE._serialized_start=2095
  _GETEVENTEXPORTQUERYRESPONSE._serialized_end=2270
  _GETEVENTTYPESUMMARYREQUEST._serialized_start=2273
  _GETEVENTTYPESUMMARYREQUEST._serialized_end=2423
  _GETEVENTTYPESUMMARYRESPONSE._serialized_start=2425
  _GETEVENTTYPESUMMARYRESPONSE._serialized_end=2548
  _GETEVENTKEYSSEARCHREQUEST._serialized_start=2550
  _GETEVENTKEYSSEARCHREQUEST._serialized_end=2625
  _GETEVENTKEYSSEARCHRESPONSE._serialized_start=2627
  _GETEVENTKEYSSEARCHRESPONSE._serialized_end=2699
  _GETFUNNELREQUEST._serialized_start=2702
  _GETFUNNELREQUEST._serialized_end=2849
  _WORKFLOWBUILDERREQUEST._serialized_start=2851
  _WORKFLOWBUILDERREQUEST._serialized_end=2934
  _WORKFLOWTIMESERIESRESPONSE._serialized_start=2937
  _WORKFLOWTIMESERIESRESPONSE._serialized_end=3392
  _WORKFLOWTIMESERIESRESPONSE_METRICVALUE._serialized_start=3127
  _WORKFLOWTIMESERIESRESPONSE_METRICVALUE._serialized_end=3174
  _WORKFLOWTIMESERIESRESPONSE_METRICTIMESERIES._serialized_start=3177
  _WORKFLOWTIMESERIESRESPONSE_METRICTIMESERIES._serialized_end=3392
  _GETFUNNELRESPONSE._serialized_start=3395
  _GETFUNNELRESPONSE._serialized_end=3563
  _GETFUNNELDROPOFFREQUEST._serialized_start=3566
  _GETFUNNELDROPOFFREQUEST._serialized_end=3784
  _GETFUNNELDROPOFFRESPONSE._serialized_start=3786
  _GETFUNNELDROPOFFRESPONSE._serialized_end=3846
  _GETFUNNELDROPOFFDISTRIBUTIONRESPONSE._serialized_start=3849
  _GETFUNNELDROPOFFDISTRIBUTIONRESPONSE._serialized_end=4141
  _GETFUNNELDROPOFFDISTRIBUTIONRESPONSE_FUNNELEVENTTYPEDISTRIBUTION._serialized_start=4072
  _GETFUNNELDROPOFFDISTRIBUTIONRESPONSE_FUNNELEVENTTYPEDISTRIBUTION._serialized_end=4141
  _GETFUNNELREQUESTV3._serialized_start=4143
  _GETFUNNELREQUESTV3._serialized_end=4219
  _GETFUNNELRESPONSEV3._serialized_start=4222
  _GETFUNNELRESPONSEV3._serialized_end=4619
  _GETFUNNELRESPONSEV3_STAGE_DATA._serialized_start=4342
  _GETFUNNELRESPONSEV3_STAGE_DATA._serialized_end=4619
  _GETFUNNELRESPONSEV3_STAGE_DATA_EVENT_DATA._serialized_start=4445
  _GETFUNNELRESPONSEV3_STAGE_DATA_EVENT_DATA._serialized_end=4619
  _GETCONNECTORKEYSREQUEST._serialized_start=4621
  _GETCONNECTORKEYSREQUEST._serialized_end=4699
  _GETCONNECTORKEYSRESPONSE._serialized_start=4701
  _GETCONNECTORKEYSRESPONSE._serialized_end=4796
  _SAVECONNECTORKEYSREQUEST._serialized_start=4799
  _SAVECONNECTORKEYSREQUEST._serialized_end=4930
  _SAVECONNECTORKEYSRESPONSE._serialized_start=4932
  _SAVECONNECTORKEYSRESPONSE._serialized_end=4976
  _GETEVENTTYPEDEFINITIONREQUEST._serialized_start=4978
  _GETEVENTTYPEDEFINITIONREQUEST._serialized_end=5067
  _GETEVENTTYPEDEFINITIONRESPONSE._serialized_start=5070
  _GETEVENTTYPEDEFINITIONRESPONSE._serialized_end=5203
  _CREATEORUPDATEMONITORREQUEST._serialized_start=5206
  _CREATEORUPDATEMONITORREQUEST._serialized_end=5519
  _CREATEORUPDATEMONITORRESPONSE._serialized_start=5522
  _CREATEORUPDATEMONITORRESPONSE._serialized_end=5685
  _GETMONITORREQUEST._serialized_start=5688
  _GETMONITORREQUEST._serialized_end=5826
  _GETMONITORRESPONSE._serialized_start=5828
  _GETMONITORRESPONSE._serialized_end=5923
  _GETMONITORDEFINITIONREQUEST._serialized_start=5925
  _GETMONITORDEFINITIONREQUEST._serialized_end=6009
  _GETMONITORDEFINITIONRESPONSE._serialized_start=6011
  _GETMONITORDEFINITIONRESPONSE._serialized_end=6137
  _GETMONITOROPTIONSREQUEST._serialized_start=6139
  _GETMONITOROPTIONSREQUEST._serialized_end=6199
  _GETMONITOROPTIONSRESPONSE._serialized_start=6201
  _GETMONITOROPTIONSRESPONSE._serialized_end=6317
  _GETMONITORTRANSACTIONSREQUEST._serialized_start=6320
  _GETMONITORTRANSACTIONSREQUEST._serialized_end=6464
  _GETMONITORTRANSACTIONSRESPONSE._serialized_start=6467
  _GETMONITORTRANSACTIONSRESPONSE._serialized_end=6652
  _GETMONITORTRANSACTIONEVENTSREQUEST._serialized_start=6654
  _GETMONITORTRANSACTIONEVENTSREQUEST._serialized_end=6748
  _GETMONITORTRANSACTIONEVENTSRESPONSE._serialized_start=6750
  _GETMONITORTRANSACTIONEVENTSRESPONSE._serialized_end=6858
  _GETMONITORTRANSACTIONDETAILSREQUEST._serialized_start=6860
  _GETMONITORTRANSACTIONDETAILSREQUEST._serialized_end=6955
  _GETMONITORTRANSACTIONDETAILSRESPONSE._serialized_start=6958
  _GETMONITORTRANSACTIONDETAILSRESPONSE._serialized_end=7108
  _GETMETRICEXPRESSIONOPTIONSREQUEST._serialized_start=7111
  _GETMETRICEXPRESSIONOPTIONSREQUEST._serialized_end=7265
  _GETMETRICEXPRESSIONOPTIONSRESPONSE._serialized_start=7268
  _GETMETRICEXPRESSIONOPTIONSRESPONSE._serialized_end=7458
  _GETMETRICREQUEST._serialized_start=7461
  _GETMETRICREQUEST._serialized_end=7613
  _GETMETRICRESPONSE._serialized_start=7616
  _GETMETRICRESPONSE._serialized_end=7801
  _GETPANELSV1REQUEST._serialized_start=7803
  _GETPANELSV1REQUEST._serialized_end=7871
  _GETPANELSV1RESPONSE._serialized_start=7873
  _GETPANELSV1RESPONSE._serialized_end=7967
  _CREATEORUPDATEPANELV1REQUEST._serialized_start=7969
  _CREATEORUPDATEPANELV1REQUEST._serialized_end=8037
  _CREATEORUPDATEPANELV1RESPONSE._serialized_start=8039
  _CREATEORUPDATEPANELV1RESPONSE._serialized_end=8155
  _DELETEPANELV1REQUEST._serialized_start=8157
  _DELETEPANELV1REQUEST._serialized_end=8227
  _DELETEPANELV1RESPONSE._serialized_start=8229
  _DELETEPANELV1RESPONSE._serialized_end=8337
  _GETDASHBOARDSV1REQUEST._serialized_start=8339
  _GETDASHBOARDSV1REQUEST._serialized_end=8411
  _GETDASHBOARDSV1RESPONSE._serialized_start=8413
  _GETDASHBOARDSV1RESPONSE._serialized_end=8519
  _CREATEORUPDATEDASHBOARDV1REQUEST._serialized_start=8521
  _CREATEORUPDATEDASHBOARDV1REQUEST._serialized_end=8601
  _CREATEORUPDATEDASHBOARDV1RESPONSE._serialized_start=8603
  _CREATEORUPDATEDASHBOARDV1RESPONSE._serialized_end=8723
  _DELETEDASHBOARDV1REQUEST._serialized_start=8725
  _DELETEDASHBOARDV1REQUEST._serialized_end=8799
  _DELETEDASHBOARDV1RESPONSE._serialized_start=8801
  _DELETEDASHBOARDV1RESPONSE._serialized_end=8913
  _GETACCOUNTAPITOKENSREQUEST._serialized_start=8915
  _GETACCOUNTAPITOKENSREQUEST._serialized_end=8977
  _GETACCOUNTAPITOKENSRESPONSE._serialized_start=8979
  _GETACCOUNTAPITOKENSRESPONSE._serialized_end=9101
  _CREATEACCOUNTAPITOKENREQUEST._serialized_start=9103
  _CREATEACCOUNTAPITOKENREQUEST._serialized_end=9133
  _CREATEACCOUNTAPITOKENRESPONSE._serialized_start=9136
  _CREATEACCOUNTAPITOKENRESPONSE._serialized_end=9270
  _DELETEACCOUNTAPITOKENREQUEST._serialized_start=9272
  _DELETEACCOUNTAPITOKENREQUEST._serialized_end=9333
  _DELETEACCOUNTAPITOKENRESPONSE._serialized_start=9335
  _DELETEACCOUNTAPITOKENRESPONSE._serialized_end=9411
  _GETMONITORTRIGGERREQUEST._serialized_start=9413
  _GETMONITORTRIGGERREQUEST._serialized_end=9514
  _GETMONITORTRIGGERRESPONSE._serialized_start=9517
  _GETMONITORTRIGGERRESPONSE._serialized_end=9672
  _GETMONITORTRIGGEROPTIONSREQUEST._serialized_start=9674
  _GETMONITORTRIGGEROPTIONSREQUEST._serialized_end=9761
  _GETMONITORTRIGGEROPTIONSRESPONSE._serialized_start=9764
  _GETMONITORTRIGGEROPTIONSRESPONSE._serialized_end=10156
  _CREATEMONITORTRIGGERREQUEST._serialized_start=10158
  _CREATEMONITORTRIGGERREQUEST._serialized_end=10285
  _CREATEMONITORTRIGGERRESPONSE._serialized_start=10287
  _CREATEMONITORTRIGGERRESPONSE._serialized_end=10402
  _UPDATEMONITORTRIGGERREQUEST._serialized_start=10405
  _UPDATEMONITORTRIGGERREQUEST._serialized_end=10543
  _UPDATEMONITORTRIGGERRESPONSE._serialized_start=10545
  _UPDATEMONITORTRIGGERRESPONSE._serialized_end=10660
  _GETALERTSSUMMARYREQUEST._serialized_start=10662
  _GETALERTSSUMMARYREQUEST._serialized_end=10760
  _GETALERTSSUMMARYRESPONSE._serialized_start=10762
  _GETALERTSSUMMARYRESPONSE._serialized_end=10874
  _GETALERTDETAILSREQUEST._serialized_start=10876
  _GETALERTDETAILSREQUEST._serialized_end=10952
  _GETALERTDETAILSRESPONSE._serialized_start=10954
  _GETALERTDETAILSRESPONSE._serialized_end=11062
  _GETALERTMONITORTRANSACTIONSREQUEST._serialized_start=11064
  _GETALERTMONITORTRANSACTIONSREQUEST._serialized_end=11152
  _GETALERTMONITORTRANSACTIONSRESPONSE._serialized_start=11155
  _GETALERTMONITORTRANSACTIONSRESPONSE._serialized_end=11295
  _GETUSERREQUEST._serialized_start=11297
  _GETUSERREQUEST._serialized_end=11313
  _GETUSERRESPONSE._serialized_start=11315
  _GETUSERRESPONSE._serialized_end=11366
  _GETNOTIFICATIONCONFIGSREQUEST._serialized_start=11368
  _GETNOTIFICATIONCONFIGSREQUEST._serialized_end=11453
  _GETNOTIFICATIONCONFIGSRESPONSE._serialized_start=11456
  _GETNOTIFICATIONCONFIGSRESPONSE._serialized_end=11586
  _CREATENOTIFICATIONCONFIGREQUEST._serialized_start=11588
  _CREATENOTIFICATIONCONFIGREQUEST._serialized_end=11684
  _CREATENOTIFICATIONCONFIGRESPONSE._serialized_start=11686
  _CREATENOTIFICATIONCONFIGRESPONSE._serialized_end=11805
  _GETGLOBALSEARCHOPTIONSREQUEST._serialized_start=11807
  _GETGLOBALSEARCHOPTIONSREQUEST._serialized_end=11872
  _GETGLOBALSEARCHOPTIONSRESPONSE._serialized_start=11875
  _GETGLOBALSEARCHOPTIONSRESPONSE._serialized_end=12005
  _GETEVENTSEARCHOPTIONSREQUEST._serialized_start=12008
  _GETEVENTSEARCHOPTIONSREQUEST._serialized_end=12182
  _GETEVENTSEARCHOPTIONSRESPONSE._serialized_start=12185
  _GETEVENTSEARCHOPTIONSRESPONSE._serialized_end=12366
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSREQUEST._serialized_start=12369
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSREQUEST._serialized_end=12558
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSRESPONSE._serialized_start=12561
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSRESPONSE._serialized_end=12771
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSV2REQUEST._serialized_start=12774
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSV2REQUEST._serialized_end=12965
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSV2RESPONSE._serialized_start=12968
  _GETMONITORSTRANSACTIONSSEARCHOPTIONSV2RESPONSE._serialized_end=13182
  _GETMONITORSTRANSACTIONSSEARCHQUERYREQUEST._serialized_start=13185
  _GETMONITORSTRANSACTIONSSEARCHQUERYREQUEST._serialized_end=13313
  _GETMONITORSTRANSACTIONSSEARCHQUERYRESPONSE._serialized_start=13316
  _GETMONITORSTRANSACTIONSSEARCHQUERYRESPONSE._serialized_end=13523
  _GETMONITORSTRANSACTIONSSEARCHV2QUERYREQUEST._serialized_start=13526
  _GETMONITORSTRANSACTIONSSEARCHV2QUERYREQUEST._serialized_end=13704
  _GETMONITORSTRANSACTIONSSEARCHV2QUERYRESPONSE._serialized_start=13707
  _GETMONITORSTRANSACTIONSSEARCHV2QUERYRESPONSE._serialized_end=13956
  _SAVEMONITORTRANSACTIONSSEARCHQUERYREQUEST._serialized_start=13959
  _SAVEMONITORTRANSACTIONSSEARCHQUERYREQUEST._serialized_end=14087
  _SAVEMONITORTRANSACTIONSSEARCHQUERYRESPONSE._serialized_start=14089
  _SAVEMONITORTRANSACTIONSSEARCHQUERYRESPONSE._serialized_end=14193
  _GETMONITORSTRANSACTIONSEXPORTQUERYREQUEST._serialized_start=14196
  _GETMONITORSTRANSACTIONSEXPORTQUERYREQUEST._serialized_end=14324
  _GETMONITORSTRANSACTIONSEXPORTQUERYRESPONSE._serialized_start=14327
  _GETMONITORSTRANSACTIONSEXPORTQUERYRESPONSE._serialized_end=14517
  _CREATECONNECTORREQUEST._serialized_start=14519
  _CREATECONNECTORREQUEST._serialized_end=14639
  _CREATECONNECTORRESPONSE._serialized_start=14641
  _CREATECONNECTORRESPONSE._serialized_end=14751
  _REQUESTCONNECTORAPIREQUEST._serialized_start=14753
  _REQUESTCONNECTORAPIREQUEST._serialized_end=14834
  _REQUESTCONNECTORAPIRESPONSE._serialized_start=14836
  _REQUESTCONNECTORAPIRESPONSE._serialized_end=14950
  _CREATETRANSFORMERMAPPINGREQUEST._serialized_start=14953
  _CREATETRANSFORMERMAPPINGREQUEST._serialized_end=15142
  _CREATETRANSFORMERMAPPINGRESPONSE._serialized_start=15144
  _CREATETRANSFORMERMAPPINGRESPONSE._serialized_end=15263
  _GETENTITYOPTIONSREQUEST._serialized_start=15265
  _GETENTITYOPTIONSREQUEST._serialized_end=15324
  _GETENTITYOPTIONSRESPONSE._serialized_start=15326
  _GETENTITYOPTIONSRESPONSE._serialized_end=15439
  _CREATEENTITYREQUEST._serialized_start=15442
  _CREATEENTITYREQUEST._serialized_end=15577
  _CREATEENTITYRESPONSE._serialized_start=15580
  _CREATEENTITYRESPONSE._serialized_end=15732
  _UPDATEENTITYREQUEST._serialized_start=15734
  _UPDATEENTITYREQUEST._serialized_end=15861
  _UPDATEENTITYRESPONSE._serialized_start=15863
  _UPDATEENTITYRESPONSE._serialized_end=15970
  _ENTITYTRIGGEROPTIONSREQUEST._serialized_start=15972
  _ENTITYTRIGGEROPTIONSREQUEST._serialized_end=16054
  _ENTITYTRIGGEROPTIONSRESPONSE._serialized_start=16057
  _ENTITYTRIGGEROPTIONSRESPONSE._serialized_end=16193
  _ENTITYTRIGGERCREATEREQUEST._serialized_start=16196
  _ENTITYTRIGGERCREATEREQUEST._serialized_end=16361
  _ENTITYTRIGGERCREATERESPONSE._serialized_start=16363
  _ENTITYTRIGGERCREATERESPONSE._serialized_end=16466
  _ENTITYTRIGGERINACTIVEREQUEST._serialized_start=16468
  _ENTITYTRIGGERINACTIVEREQUEST._serialized_end=16559
  _ENTITYTRIGGERINACTIVERESPONSE._serialized_start=16561
  _ENTITYTRIGGERINACTIVERESPONSE._serialized_end=16666
  _GETENTITYTRIGGERSREQUEST._serialized_start=16668
  _GETENTITYTRIGGERSREQUEST._serialized_end=16774
  _GETENTITYTRIGGERSRESPONSE._serialized_start=16777
  _GETENTITYTRIGGERSRESPONSE._serialized_end=16930
  _GETENTITYSUMMARYREQUEST._serialized_start=16933
  _GETENTITYSUMMARYREQUEST._serialized_end=17076
  _GETENTITYSUMMARYRESPONSE._serialized_start=17078
  _GETENTITYSUMMARYRESPONSE._serialized_end=17185
  _GETENTITYDETAILREQUEST._serialized_start=17187
  _GETENTITYDETAILREQUEST._serialized_end=17294
  _GETENTITYDETAILRESPONSE._serialized_start=17296
  _GETENTITYDETAILRESPONSE._serialized_end=17399
  _GETENTITYINSTANCESSUMMARYREQUEST._serialized_start=17402
  _GETENTITYINSTANCESSUMMARYREQUEST._serialized_end=17574
  _GETENTITYINSTANCESSUMMARYRESPONSE._serialized_start=17577
  _GETENTITYINSTANCESSUMMARYRESPONSE._serialized_end=17709
  _GETENTITYINSTANCESDETAILREQUEST._serialized_start=17711
  _GETENTITYINSTANCESDETAILREQUEST._serialized_end=17836
  _GETENTITYINSTANCESDETAILRESPONSE._serialized_start=17839
  _GETENTITYINSTANCESDETAILRESPONSE._serialized_end=17968
  _GETENTITYINSTANCESTIMELINEREQUEST._serialized_start=17970
  _GETENTITYINSTANCESTIMELINEREQUEST._serialized_end=18097
  _GETENTITYINSTANCESTIMELINERESPONSE._serialized_start=18100
  _GETENTITYINSTANCESTIMELINERESPONSE._serialized_end=18256
  _GETENTITYINSTANCESSEARCHOPTIONSREQUEST._serialized_start=18258
  _GETENTITYINSTANCESSEARCHOPTIONSREQUEST._serialized_end=18377
  _GETENTITYINSTANCESSEARCHOPTIONSRESPONSE._serialized_start=18380
  _GETENTITYINSTANCESSEARCHOPTIONSRESPONSE._serialized_end=18571
  _GETENTITYINSTANCESSEARCHOPTIONSV2REQUEST._serialized_start=18573
  _GETENTITYINSTANCESSEARCHOPTIONSV2REQUEST._serialized_end=18694
  _GETENTITYINSTANCESSEARCHOPTIONSV2RESPONSE._serialized_start=18697
  _GETENTITYINSTANCESSEARCHOPTIONSV2RESPONSE._serialized_end=18892
  _GETENTITYINSTANCESEARCHQUERYREQUEST._serialized_start=18894
  _GETENTITYINSTANCESEARCHQUERYREQUEST._serialized_end=19016
  _GETENTITYINSTANCESEARCHQUERYRESPONSE._serialized_start=19019
  _GETENTITYINSTANCESEARCHQUERYRESPONSE._serialized_end=19204
  _GETENTITYWORKFLOWSEARCHQUERYREQUEST._serialized_start=19207
  _GETENTITYWORKFLOWSEARCHQUERYREQUEST._serialized_end=19358
  _GETENTITYWORKFLOWSEARCHQUERYRESPONSE._serialized_start=19360
  _GETENTITYWORKFLOWSEARCHQUERYRESPONSE._serialized_end=19483
  _CREATEENTITYFUNNELREQUEST._serialized_start=19486
  _CREATEENTITYFUNNELREQUEST._serialized_end=19639
  _CREATEENTITYFUNNELRESPONSE._serialized_start=19642
  _CREATEENTITYFUNNELRESPONSE._serialized_end=19841
  _GETENTITYFUNNELREQUEST._serialized_start=19844
  _GETENTITYFUNNELREQUEST._serialized_end=20015
  _GETENTITYFUNNELRESPONSE._serialized_start=20018
  _GETENTITYFUNNELRESPONSE._serialized_end=20213
# @@protoc_insertion_point(module_scope)

protos/event/api_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/base.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import schema_pb2 as protos_dot_event_dot_schema__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17protos/event/base.proto\x12\x0cprotos.event\x1a\x19protos/event/schema.proto\x1a\x1egoogle/protobuf/wrappers.proto\".\n\tTimeRange\x12\x10\n\x08time_geq\x18\x01 \x01(\x04\x12\x0f\n\x07time_lt\x18\x02 \x01(\x04\"a\n\x04Page\x12+\n\x05limit\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12,\n\x06offset\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\xf1\x01\n\x08\x45ventKey\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x30\n\x08key_type\x18\x03 \x01(\x0e\x32\x1e.protos.event.EventKey.KeyType\x12\x32\n\nevent_type\x18\x04 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\"f\n\x07KeyType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04LONG\x10\x02\x12\n\n\x06\x44OUBLE\x10\x03\x12\x0b\n\x07\x42OOLEAN\x10\x04\x12\x08\n\x04\x42YTE\x10\x05\x12\t\n\x05\x41RRAY\x10\x06\x12\n\n\x06OBJECT\x10\x07\"\x83\x01\n\tEventType\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12$\n\x04keys\x18\x03 \x03(\x0b\x32\x16.protos.event.EventKey\x12\x36\n\revent_sources\x18\x04 \x03(\x0e\x32\x1f.protos.event.Event.EventSource\"\xaa\x01\n\x0e\x45ventTypeStats\x12\x30\n\nkeys_count\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0b\x65vent_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x33\n\rmonitor_count\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\",\n\x10\x45ventTypePartial\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\"s\n\x10\x45ventTypeSummary\x12\x32\n\nevent_type\x18\x01 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12+\n\x05stats\x18\x02 \x01(\x0b\x32\x1c.protos.event.EventTypeStats\"o\n\x13\x45ventTypeDefinition\x12+\n\nevent_type\x18\x01 \x01(\x0b\x32\x17.protos.event.EventType\x12+\n\x05stats\x18\x02 \x01(\x0b\x32\x1c.protos.event.EventTypeStats\"\xa5\x02\n\x05\x45vent\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x32\n\nevent_type\x18\x02 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12#\n\x03kvs\x18\x03 \x03(\x0b\x32\x16.protos.event.KeyValue\x12\x11\n\ttimestamp\x18\x04 \x01(\x10\x12\x35\n\x0c\x65vent_source\x18\x05 \x01(\x0e\x32\x1f.protos.event.Event.EventSource\"m\n\x0b\x45ventSource\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06SAMPLE\x10\x01\x12\x07\n\x03\x41PI\x10\x02\x12\x07\n\x03SDK\x10\x03\x12\x0b\n\x07SEGMENT\x10\x04\x12\r\n\tAMPLITUDE\x10\x05\x12\x07\n\x03SNS\x10\x06\x12\x0e\n\nCLOUDWATCH\x10\x07*\x91\x01\n\x07\x43ontext\x12\r\n\tUNKNOWN_C\x10\x00\x12\t\n\x05\x45VENT\x10\x01\x12\x17\n\x13MONITOR_TRANSACTION\x10\x02\x12\x13\n\x0f\x45NTITY_INSTANCE\x10\x03\x12\x0e\n\nEVENT_TYPE\x10\x04\x12\x0b\n\x07MONITOR\x10\x05\x12\n\n\x06\x45NTITY\x10\x06\x12\x15\n\x11\x45VENTS_CLICKHOUSE\x10\x07\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.base_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _CONTEXT._serialized_start=1371
  _CONTEXT._serialized_end=1516
  _TIMERANGE._serialized_start=100
  _TIMERANGE._serialized_end=146
  _PAGE._serialized_start=148
  _PAGE._serialized_end=245
  _EVENTKEY._serialized_start=248
  _EVENTKEY._serialized_end=489
  _EVENTKEY_KEYTYPE._serialized_start=387
  _EVENTKEY_KEYTYPE._serialized_end=489
  _EVENTTYPE._serialized_start=492
  _EVENTTYPE._serialized_end=623
  _EVENTTYPESTATS._serialized_start=626
  _EVENTTYPESTATS._serialized_end=796
  _EVENTTYPEPARTIAL._serialized_start=798
  _EVENTTYPEPARTIAL._serialized_end=842
  _EVENTTYPESUMMARY._serialized_start=844
  _EVENTTYPESUMMARY._serialized_end=959
  _EVENTTYPEDEFINITION._serialized_start=961
  _EVENTTYPEDEFINITION._serialized_end=1072
  _EVENT._serialized_start=1075
  _EVENT._serialized_end=1368
  _EVENT_EVENTSOURCE._serialized_start=1259
  _EVENT_EVENTSOURCE._serialized_end=1368
# @@protoc_insertion_point(module_scope)

protos/event/base_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/connectors.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dprotos/event/connectors.proto\x12\x0cprotos.event\x1a\x1egoogle/protobuf/wrappers.proto\"\x83\x02\n\tConnector\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12)\n\x04type\x18\x02 \x01(\x0e\x32\x1b.protos.event.ConnectorType\x12-\n\tis_active\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12*\n\x04name\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12<\n\rsentry_config\x18\n \x01(\x0b\x32#.protos.event.SentryConnectorConfigH\x00\x42\x08\n\x06\x63onfig\"P\n\x15SentryConnectorConfig\x12\x37\n\x11polling_frequency\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"\xc9\x02\n\x0c\x43onnectorKey\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x34\n\x08key_type\x18\x02 \x01(\x0e\x32\".protos.event.ConnectorKey.KeyType\x12)\n\x03key\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\tis_active\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\x7f\n\x07KeyType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x12\n\x0eSENTRY_API_KEY\x10\x01\x12\x13\n\x0f\x44\x41TADOG_APP_KEY\x10\x02\x12\x13\n\x0f\x44\x41TADOG_API_KEY\x10\x03\x12\x14\n\x10NEWRELIC_API_KEY\x10\x04\x12\x13\n\x0fNEWRELIC_APP_ID\x10\x05\"T\n\x11PeriodicRunStatus\"?\n\nStatusType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0b\n\x07STARTED\x10\x01\x12\x0c\n\x08\x46INISHED\x10\x02\x12\t\n\x05\x45RROR\x10\x03*\xa1\x02\n\rConnectorType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06SENTRY\x10\x01\x12\x0b\n\x07SEGMENT\x10\x02\x12\x12\n\x0e\x45LASTIC_SEARCH\x10\x03\x12\r\n\tAMPLITUDE\x10\x04\x12\x0f\n\x0b\x41WS_KINESIS\x10\x05\x12\x0e\n\nCLOUDWATCH\x10\x06\x12\r\n\tCLEVERTAP\x10\x07\x12\x0f\n\x0bRUDDERSTACK\x10\x08\x12\x0c\n\x08MOENGAGE\x10\t\x12\t\n\x05\x43RIBL\x10\n\x12\t\n\x05KAFKA\x10\x0b\x12\x0b\n\x07\x44\x41TADOG\x10\x0c\x12\x0c\n\x08\x46ILEBEAT\x10\r\x12\x0c\n\x08LOGSTASH\x10\x0e\x12\x0b\n\x07\x46LUENTD\x10\x0f\x12\r\n\tFLUENTBIT\x10\x10\x12\x0e\n\nPAGER_DUTY\x10\x11\x12\r\n\tNEW_RELIC\x10\x12*\x89\x01\n\x0fTransformerType\x12\x0e\n\nUNKNOWN_TT\x10\x00\x12\x1e\n\x1aSEGMENT_DFAULT_TRANSFORMER\x10\x02\x12!\n\x1d\x41MPLITUDE_DEFAULT_TRANSFORMER\x10\x03\x12#\n\x1f\x43LOUDWATCH_JSON_LOG_TRANSFORMER\x10\x04*Z\n\x0b\x44\x65\x63oderType\x12\x0e\n\nUNKNOWN_DT\x10\x00\x12\x17\n\x13\x41WS_KINESIS_DECODER\x10\x01\x12\"\n\x1e\x41WS_CLOUDWATCH_KINESIS_DECODER\x10\x02\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.connectors_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _CONNECTORTYPE._serialized_start=842
  _CONNECTORTYPE._serialized_end=1131
  _TRANSFORMERTYPE._serialized_start=1134
  _TRANSFORMERTYPE._serialized_end=1271
  _DECODERTYPE._serialized_start=1273
  _DECODERTYPE._serialized_end=1363
  _CONNECTOR._serialized_start=80
  _CONNECTOR._serialized_end=339
  _SENTRYCONNECTORCONFIG._serialized_start=341
  _SENTRYCONNECTORCONFIG._serialized_end=421
  _CONNECTORKEY._serialized_start=424
  _CONNECTORKEY._serialized_end=753
  _CONNECTORKEY_KEYTYPE._serialized_start=626
  _CONNECTORKEY_KEYTYPE._serialized_end=753
  _PERIODICRUNSTATUS._serialized_start=755
  _PERIODICRUNSTATUS._serialized_end=839
  _PERIODICRUNSTATUS_STATUSTYPE._serialized_start=776
  _PERIODICRUNSTATUS_STATUSTYPE._serialized_end=839
# @@protoc_insertion_point(module_scope)

protos/event/connectors_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/engine_options.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import literal_pb2 as protos_dot_event_dot_literal__pb2
from protos.event import metric_pb2 as protos_dot_event_dot_metric__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n!protos/event/engine_options.proto\x12\x0cprotos.event\x1a\x1dprotos/event/query_base.proto\x1a\x1aprotos/event/literal.proto\x1a\x19protos/event/metric.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\xbf\x01\n\x12GlobalQueryOptions\x12\x34\n\x0fop_descriptions\x18\x01 \x03(\x0b\x32\x1b.protos.event.OpDescription\x12+\n\nop_mapping\x18\x02 \x03(\x0b\x32\x17.protos.event.OpMapping\x12\x46\n\x18literal_type_description\x18\x03 \x03(\x0b\x32$.protos.event.LiteralTypeDescription\"\xa3\x04\n\x0c\x43olumnOption\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x61lias\x18\x02 \x01(\t\x12\'\n\x04type\x18\x03 \x01(\x0e\x32\x19.protos.event.LiteralType\x12\x14\n\x0cis_groupable\x18\x05 \x01(\x08\x12@\n\x15\x61ggregation_functions\x18\x06 \x03(\x0e\x32!.protos.event.AggregationFunction\x12\x36\n\tid_option\x18\n \x01(\x0b\x32#.protos.event.ColumnOption.IdOption\x1a\xbc\x02\n\x08IdOption\x12*\n\x04type\x18\x01 \x01(\x0e\x32\x1c.protos.event.IdLiteral.Type\x12J\n\x0clong_options\x18\x02 \x03(\x0b\x32\x34.protos.event.ColumnOption.IdOption.LongOptionsEntry\x12N\n\x0estring_options\x18\x03 \x03(\x0b\x32\x36.protos.event.ColumnOption.IdOption.StringOptionsEntry\x1a\x32\n\x10LongOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x34\n\x12StringOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xdb\x02\n\x0f\x41ttributeOption\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\'\n\x04type\x18\x03 \x01(\x0e\x32\x19.protos.event.LiteralType\x12\r\n\x05\x61lias\x18\x04 \x01(\t\x12\x43\n\x0e\x63olumn_context\x18\x05 \x03(\x0b\x32+.protos.event.AttributeOption.ColumnContext\x12\x12\n\npath_alias\x18\x06 \x01(\t\x12\x14\n\x0cis_groupable\x18\x07 \x01(\x08\x12@\n\x15\x61ggregation_functions\x18\x08 \x03(\x0e\x32!.protos.event.AggregationFunction\x1a\x43\n\rColumnContext\x12#\n\x02id\x18\x01 \x01(\x0b\x32\x17.protos.event.IdLiteral\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"X\n\x11\x41ttributeOptionV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\'\n\x04type\x18\x03 \x01(\x0e\x32\x19.protos.event.LiteralType\"|\n\x0cQueryOptions\x12\x32\n\x0e\x63olumn_options\x18\x01 \x03(\x0b\x32\x1a.protos.event.ColumnOption\x12\x38\n\x11\x61ttribute_options\x18\x02 \x03(\x0b\x32\x1d.protos.event.AttributeOption\"\x83\x01\n\x0eQueryOptionsV2\x12\x32\n\x0e\x63olumn_options\x18\x01 \x03(\x0b\x32\x1a.protos.event.ColumnOption\x12=\n\x14\x61ttribute_options_v2\x18\x02 \x03(\x0b\x32\x1f.protos.event.AttributeOptionV2\"\xae\x03\n\rMetricOptions\x12[\n\x1c\x61ggregation_function_options\x18\x01 \x03(\x0b\x32\x35.protos.event.MetricOptions.AggregationFunctionOption\x12\x32\n\x0e\x63olumn_options\x18\x02 \x03(\x0b\x32\x1a.protos.event.ColumnOption\x12\x38\n\x11\x61ttribute_options\x18\x03 \x03(\x0b\x32\x1d.protos.event.AttributeOption\x1a\xd1\x01\n\x19\x41ggregationFunctionOption\x12?\n\x14\x61ggregation_function\x18\x01 \x01(\x0e\x32!.protos.event.AggregationFunction\x12\x46\n\"supports_empty_expression_selector\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12+\n\x05label\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValueb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.engine_options_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _COLUMNOPTION_IDOPTION_LONGOPTIONSENTRY._options = None
  _COLUMNOPTION_IDOPTION_LONGOPTIONSENTRY._serialized_options = b'8\001'
  _COLUMNOPTION_IDOPTION_STRINGOPTIONSENTRY._options = None
  _COLUMNOPTION_IDOPTION_STRINGOPTIONSENTRY._serialized_options = b'8\001'
  _GLOBALQUERYOPTIONS._serialized_start=170
  _GLOBALQUERYOPTIONS._serialized_end=361
  _COLUMNOPTION._serialized_start=364
  _COLUMNOPTION._serialized_end=911
  _COLUMNOPTION_IDOPTION._serialized_start=595
  _COLUMNOPTION_IDOPTION._serialized_end=911
  _COLUMNOPTION_IDOPTION_LONGOPTIONSENTRY._serialized_start=807
  _COLUMNOPTION_IDOPTION_LONGOPTIONSENTRY._serialized_end=857
  _COLUMNOPTION_IDOPTION_STRINGOPTIONSENTRY._serialized_start=859
  _COLUMNOPTION_IDOPTION_STRINGOPTIONSENTRY._serialized_end=911
  _ATTRIBUTEOPTION._serialized_start=914
  _ATTRIBUTEOPTION._serialized_end=1261
  _ATTRIBUTEOPTION_COLUMNCONTEXT._serialized_start=1194
  _ATTRIBUTEOPTION_COLUMNCONTEXT._serialized_end=1261
  _ATTRIBUTEOPTIONV2._serialized_start=1263
  _ATTRIBUTEOPTIONV2._serialized_end=1351
  _QUERYOPTIONS._serialized_start=1353
  _QUERYOPTIONS._serialized_end=1477
  _QUERYOPTIONSV2._serialized_start=1480
  _QUERYOPTIONSV2._serialized_end=1611
  _METRICOPTIONS._serialized_start=1614
  _METRICOPTIONS._serialized_end=2044
  _METRICOPTIONS_AGGREGATIONFUNCTIONOPTION._serialized_start=1835
  _METRICOPTIONS_AGGREGATIONFUNCTIONOPTION._serialized_end=2044
# @@protoc_insertion_point(module_scope)

protos/event/engine_options_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/entity.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import notification_pb2 as protos_dot_event_dot_notification__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19protos/event/entity.proto\x12\x0cprotos.event\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/event/base.proto\x1a\x1aprotos/event/monitor.proto\x1a\x1dprotos/event/query_base.proto\x1a\x1fprotos/event/notification.proto\"\x9b\x01\n\x15\x45ntityEventKeyMapping\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12)\n\tevent_key\x18\x03 \x01(\x0b\x32\x16.protos.event.EventKey\x12-\n\tis_active\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\x8b\x04\n\x17\x45ntityTriggerDefinition\x12N\n\trule_type\x18\x01 \x01(\x0e\x32;.protos.event.EntityTriggerDefinition.EntityTriggerRuleType\x12?\n\x04type\x18\x02 \x01(\x0e\x32\x31.protos.event.EntityTriggerDefinition.TriggerType\x12T\n\x13trigger_rule_config\x18\x03 \x01(\x0b\x32\x37.protos.event.EntityTriggerDefinition.TriggerRuleConfig\x1ai\n\x11TriggerRuleConfig\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\x04\x12\x15\n\rtime_interval\x18\x02 \x01(\x04\x12\x17\n\x0fthreshold_count\x18\x03 \x01(\x04\x12\x12\n\nevent_name\x18\x04 \x01(\t\"E\n\x0bTriggerType\x12\x10\n\x0cUNKNOWN_TYPE\x10\x00\x12\r\n\tPER_EVENT\x10\x01\x12\x15\n\x11\x41GGREGATED_EVENTS\x10\x02\"W\n\x15\x45ntityTriggerRuleType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0e\n\nLAST_EVENT\x10\x01\x12\x0f\n\x0b\x45VENT_COUNT\x10\x02\x12\x10\n\x0c\x45VENT_OCCURS\x10\x03\"\x9a\x02\n\rEntityTrigger\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x11\n\tentity_id\x18\x02 \x01(\x04\x12\x13\n\x0b\x65ntity_name\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x39\n\ndefinition\x18\x05 \x01(\x0b\x32%.protos.event.EntityTriggerDefinition\x12$\n\x06\x66ilter\x18\x06 \x01(\x0b\x32\x14.protos.event.Filter\x12\x37\n\rnotifications\x18\x07 \x03(\x0b\x32 .protos.event.NotificationConfig\x12-\n\tis_active\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\x8c\x02\n\x06\x45ntity\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12*\n\x04name\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\tis_active\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x12\n\ncreated_at\x18\x05 \x01(\x10\x12\x12\n\nupdated_at\x18\x06 \x01(\x10\x12\'\n\x04type\x18\x07 \x01(\x0e\x32\x19.protos.event.Entity.Type\",\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x01\x12\n\n\x06\x46UNNEL\x10\x02\"e\n\rEntityPartial\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12*\n\x04name\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xf0\x01\n\x0b\x45ntityStats\x12\x38\n\x12new_instance_count\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12;\n\x15\x61\x63tive_instance_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0b\x65vent_count\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x37\n\x11transaction_count\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"_\n\rEntitySummary\x12$\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x14.protos.event.Entity\x12(\n\x05stats\x18\x02 \x01(\x0b\x32\x19.protos.event.EntityStats\"\xa6\x01\n\x0c\x45ntityDetail\x12$\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x14.protos.event.Entity\x12\x46\n\x19\x65ntity_event_key_mappings\x18\x02 \x03(\x0b\x32#.protos.event.EntityEventKeyMapping\x12(\n\x05stats\x18\x03 \x01(\x0b\x32\x19.protos.event.EntityStats\"\xab\x01\n\x0e\x45ntityInstance\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12.\n\x08instance\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x06\x65ntity\x18\x03 \x01(\x0b\x32\x1b.protos.event.EntityPartial\x12\x12\n\ncreated_at\x18\x04 \x01(\x10\"q\n\x15\x45ntityInstancePartial\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12.\n\x08instance\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xae\x02\n\x13\x45ntityInstanceStats\x12\x31\n\x0b\x65vent_count\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x37\n\x11transaction_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12.\n\nhas_alerts\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12=\n\x06\x61lerts\x18\x04 \x03(\x0b\x32-.protos.event.EntityInstanceStats.EntityAlert\x1a<\n\x0b\x45ntityAlert\x12\x10\n\x08\x61lert_id\x18\x01 \x01(\x04\x12\x1b\n\x13\x65ntity_trigger_name\x18\x02 \x01(\t\"\x87\x01\n\x15\x45ntityInstanceSummary\x12<\n\x0f\x65ntity_instance\x18\x01 \x01(\x0b\x32#.protos.event.EntityInstancePartial\x12\x30\n\x05stats\x18\x02 \x01(\x0b\x32!.protos.event.EntityInstanceStats\"\x7f\n\x14\x45ntityInstanceDetail\x12\x35\n\x0f\x65ntity_instance\x18\x01 \x01(\x0b\x32\x1c.protos.event.EntityInstance\x12\x30\n\x05stats\x18\x02 \x01(\x0b\x32!.protos.event.EntityInstanceStats\",\n\x17\x45ntityInstanceGetParams\x12\x11\n\tinstances\x18\x01 \x03(\t\"\xf1\x08\n\x0eUpdateEntityOp\x12+\n\x02op\x18\x01 \x01(\x0e\x32\x1f.protos.event.UpdateEntityOp.Op\x12K\n\x12update_entity_name\x18\x02 \x01(\x0b\x32-.protos.event.UpdateEntityOp.UpdateEntityNameH\x00\x12O\n\x14update_entity_status\x18\x03 \x01(\x0b\x32/.protos.event.UpdateEntityOp.UpdateEntityStatusH\x00\x12p\n&update_entity_event_key_mapping_status\x18\x04 \x01(\x0b\x32>.protos.event.UpdateEntityOp.UpdateEntityEventKeyMappingStatusH\x00\x12_\n\x1d\x61\x64\x64_entity_event_key_mappings\x18\x05 \x01(\x0b\x32\x36.protos.event.UpdateEntityOp.AddEntityEventKeyMappingsH\x00\x12\x65\n remove_entity_event_key_mappings\x18\x06 \x01(\x0b\x32\x39.protos.event.UpdateEntityOp.RemoveEntityEventKeyMappingsH\x00\x1a>\n\x10UpdateEntityName\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x1a\x43\n\x12UpdateEntityStatus\x12-\n\tis_active\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x1a\x95\x01\n!UpdateEntityEventKeyMappingStatus\x12\x41\n\x1b\x65ntity_event_key_mapping_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12-\n\tis_active\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x1a\x32\n\x19\x41\x64\x64\x45ntityEventKeyMappings\x12\x15\n\revent_key_ids\x18\x01 \x03(\x04\x1a\x44\n\x1cRemoveEntityEventKeyMappings\x12$\n\x1c\x65ntity_event_key_mapping_ids\x18\x01 \x03(\x04\"\xb8\x01\n\x02Op\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x16\n\x12UPDATE_ENTITY_NAME\x10\x01\x12\x18\n\x14UPDATE_ENTITY_STATUS\x10\x02\x12*\n&UPDATE_ENTITY_EVENT_KEY_MAPPING_STATUS\x10\x03\x12!\n\x1d\x41\x44\x44_ENTITY_EVENT_KEY_MAPPINGS\x10\x04\x12$\n REMOVE_ENTITY_EVENT_KEY_MAPPINGS\x10\x05\x42\x08\n\x06update\"\xd8\n\n\x0cWorkflowView\x12.\n\x05nodes\x18\x01 \x03(\x0b\x32\x1f.protos.event.WorkflowView.Node\x12.\n\x05\x65\x64ges\x18\x02 \x03(\x0b\x32\x1f.protos.event.WorkflowView.Edge\x1a\x9d\x05\n\x04Node\x12-\n\x07node_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12+\n\x05label\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x35\n\x06status\x18\x03 \x01(\x0e\x32%.protos.event.WorkflowView.NodeStatus\x12G\n\x10\x63onnection_types\x18\x04 \x03(\x0e\x32-.protos.event.WorkflowView.NodeConnectionType\x12;\n\x07metrics\x18\x05 \x03(\x0b\x32*.protos.event.WorkflowView.Node.NodeMetric\x12:\n\x14workflow_node_config\x18\x06 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x1a\xbf\x02\n\nNodeMetric\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12>\n\rmetric_status\x18\x03 \x01(\x0e\x32\'.protos.event.WorkflowView.MetricStatus\x12\x31\n\x0bmetric_link\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05\x64\x65lta\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x38\n\ndelta_type\x18\x06 \x01(\x0e\x32$.protos.event.WorkflowView.DeltaType\x1a\xd3\x02\n\x04\x45\x64ge\x12\x33\n\rstart_node_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0b\x65nd_node_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12;\n\x07metrics\x18\x03 \x03(\x0b\x32*.protos.event.WorkflowView.Edge.EdgeMetric\x1a\xa5\x01\n\nEdgeMetric\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12>\n\rmetric_status\x18\x03 \x01(\x0e\x32\'.protos.event.WorkflowView.MetricStatus\"=\n\x0cMetricStatus\x12\x19\n\x15UNKNOWN_METRIC_STATUS\x10\x00\x12\t\n\x05GREEN\x10\x01\x12\x07\n\x03RED\x10\x02\"8\n\nNodeStatus\x12\x17\n\x13UNKNOWN_NODE_STATUS\x10\x00\x12\x08\n\x04GOOD\x10\x01\x12\x07\n\x03\x42\x41\x44\x10\x02\"B\n\x12NodeConnectionType\x12\x1b\n\x17UNKNOWN_CONNECTION_TYPE\x10\x00\x12\x06\n\x02IN\x10\x01\x12\x07\n\x03OUT\x10\x02\"5\n\tDeltaType\x12\x16\n\x12UNKNOWN_DELTA_TYPE\x10\x00\x12\x06\n\x02UP\x10\x01\x12\x08\n\x04\x44OWN\x10\x02\"\xda\x02\n\nFunnelView\x12+\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1d.protos.event.FunnelView.Node\x12,\n\x05links\x18\x02 \x03(\x0b\x32\x1d.protos.event.FunnelView.Edge\x1a_\n\x04Node\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x1a\x8f\x01\n\x04\x45\x64ge\x12,\n\x06source\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12,\n\x06target\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12+\n\x05value\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt64Valueb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.entity_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _ENTITYEVENTKEYMAPPING._serialized_start=193
  _ENTITYEVENTKEYMAPPING._serialized_end=348
  _ENTITYTRIGGERDEFINITION._serialized_start=351
  _ENTITYTRIGGERDEFINITION._serialized_end=874
  _ENTITYTRIGGERDEFINITION_TRIGGERRULECONFIG._serialized_start=609
  _ENTITYTRIGGERDEFINITION_TRIGGERRULECONFIG._serialized_end=714
  _ENTITYTRIGGERDEFINITION_TRIGGERTYPE._serialized_start=716
  _ENTITYTRIGGERDEFINITION_TRIGGERTYPE._serialized_end=785
  _ENTITYTRIGGERDEFINITION_ENTITYTRIGGERRULETYPE._serialized_start=787
  _ENTITYTRIGGERDEFINITION_ENTITYTRIGGERRULETYPE._serialized_end=874
  _ENTITYTRIGGER._serialized_start=877
  _ENTITYTRIGGER._serialized_end=1159
  _ENTITY._serialized_start=1162
  _ENTITY._serialized_end=1430
  _ENTITY_TYPE._serialized_start=1386
  _ENTITY_TYPE._serialized_end=1430
  _ENTITYPARTIAL._serialized_start=1432
  _ENTITYPARTIAL._serialized_end=1533
  _ENTITYSTATS._serialized_start=1536
  _ENTITYSTATS._serialized_end=1776
  _ENTITYSUMMARY._serialized_start=1778
  _ENTITYSUMMARY._serialized_end=1873
  _ENTITYDETAIL._serialized_start=1876
  _ENTITYDETAIL._serialized_end=2042
  _ENTITYINSTANCE._serialized_start=2045
  _ENTITYINSTANCE._serialized_end=2216
  _ENTITYINSTANCEPARTIAL._serialized_start=2218
  _ENTITYINSTANCEPARTIAL._serialized_end=2331
  _ENTITYINSTANCESTATS._serialized_start=2334
  _ENTITYINSTANCESTATS._serialized_end=2636
  _ENTITYINSTANCESTATS_ENTITYALERT._serialized_start=2576
  _ENTITYINSTANCESTATS_ENTITYALERT._serialized_end=2636
  _ENTITYINSTANCESUMMARY._serialized_start=2639
  _ENTITYINSTANCESUMMARY._serialized_end=2774
  _ENTITYINSTANCEDETAIL._serialized_start=2776
  _ENTITYINSTANCEDETAIL._serialized_end=2903
  _ENTITYINSTANCEGETPARAMS._serialized_start=2905
  _ENTITYINSTANCEGETPARAMS._serialized_end=2949
  _UPDATEENTITYOP._serialized_start=2952
  _UPDATEENTITYOP._serialized_end=4089
  _UPDATEENTITYOP_UPDATEENTITYNAME._serialized_start=3487
  _UPDATEENTITYOP_UPDATEENTITYNAME._serialized_end=3549
  _UPDATEENTITYOP_UPDATEENTITYSTATUS._serialized_start=3551
  _UPDATEENTITYOP_UPDATEENTITYSTATUS._serialized_end=3618
  _UPDATEENTITYOP_UPDATEENTITYEVENTKEYMAPPINGSTATUS._serialized_start=3621
  _UPDATEENTITYOP_UPDATEENTITYEVENTKEYMAPPINGSTATUS._serialized_end=3770
  _UPDATEENTITYOP_ADDENTITYEVENTKEYMAPPINGS._serialized_start=3772
  _UPDATEENTITYOP_ADDENTITYEVENTKEYMAPPINGS._serialized_end=3822
  _UPDATEENTITYOP_REMOVEENTITYEVENTKEYMAPPINGS._serialized_start=3824
  _UPDATEENTITYOP_REMOVEENTITYEVENTKEYMAPPINGS._serialized_end=3892
  _UPDATEENTITYOP_OP._serialized_start=3895
  _UPDATEENTITYOP_OP._serialized_end=4079
  _WORKFLOWVIEW._serialized_start=4092
  _WORKFLOWVIEW._serialized_end=5460
  _WORKFLOWVIEW_NODE._serialized_start=4205
  _WORKFLOWVIEW_NODE._serialized_end=4874
  _WORKFLOWVIEW_NODE_NODEMETRIC._serialized_start=4555
  _WORKFLOWVIEW_NODE_NODEMETRIC._serialized_end=4874
  _WORKFLOWVIEW_EDGE._serialized_start=4877
  _WORKFLOWVIEW_EDGE._serialized_end=5216
  _WORKFLOWVIEW_EDGE_EDGEMETRIC._serialized_start=5051
  _WORKFLOWVIEW_EDGE_EDGEMETRIC._serialized_end=5216
  _WORKFLOWVIEW_METRICSTATUS._serialized_start=5218
  _WORKFLOWVIEW_METRICSTATUS._serialized_end=5279
  _WORKFLOWVIEW_NODESTATUS._serialized_start=5281
  _WORKFLOWVIEW_NODESTATUS._serialized_end=5337
  _WORKFLOWVIEW_NODECONNECTIONTYPE._serialized_start=5339
  _WORKFLOWVIEW_NODECONNECTIONTYPE._serialized_end=5405
  _WORKFLOWVIEW_DELTATYPE._serialized_start=5407
  _WORKFLOWVIEW_DELTATYPE._serialized_end=5460
  _FUNNELVIEW._serialized_start=5463
  _FUNNELVIEW._serialized_end=5809
  _FUNNELVIEW_NODE._serialized_start=5568
  _FUNNELVIEW_NODE._serialized_end=5663
  _FUNNELVIEW_EDGE._serialized_start=5666
  _FUNNELVIEW_EDGE._serialized_end=5809
# @@protoc_insertion_point(module_scope)

protos/event/entity_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/literal.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aprotos/event/literal.proto\x12\x0cprotos.event\x1a\x1egoogle/protobuf/wrappers.proto\"\xc6\x01\n\tIdLiteral\x12*\n\x04type\x18\x01 \x01(\x0e\x32\x1c.protos.event.IdLiteral.Type\x12,\n\x04long\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64ValueH\x00\x12.\n\x06string\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValueH\x00\")\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x08\n\x04LONG\x10\x01\x12\n\n\x06STRING\x10\x02\x42\x04\n\x02id\"\xbd\x03\n\x07Literal\x12/\n\x0cliteral_type\x18\x01 \x01(\x0e\x32\x19.protos.event.LiteralType\x12,\n\x06string\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12)\n\x04long\x18\x03 \x01(\x0b\x32\x1b.google.protobuf.Int64Value\x12,\n\x06\x64ouble\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12+\n\x07\x62oolean\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x11\n\ttimestamp\x18\x06 \x01(\x10\x12#\n\x02id\x18\x07 \x01(\x0b\x32\x17.protos.event.IdLiteral\x12\x14\n\x0cstring_array\x18\x08 \x03(\t\x12\x12\n\nlong_array\x18\t \x03(\x03\x12\x14\n\x0c\x64ouble_array\x18\n \x03(\x01\x12\x13\n\x0b\x62ytes_array\x18\x0b \x03(\x0c\x12\x15\n\rboolean_array\x18\x0c \x03(\x08\x12)\n\x08id_array\x18\r \x03(\x0b\x32\x17.protos.event.IdLiteral\"X\n\x16LiteralTypeDescription\x12/\n\x0cliteral_type\x18\x01 \x01(\x0e\x32\x19.protos.event.LiteralType\x12\r\n\x05label\x18\x02 \x01(\t*\xda\x01\n\x0bLiteralType\x12\x0e\n\nUNKNOWN_LT\x10\x00\x12\n\n\x06STRING\x10\x01\x12\x08\n\x04LONG\x10\x02\x12\n\n\x06\x44OUBLE\x10\x03\x12\x0b\n\x07\x42OOLEAN\x10\x04\x12\r\n\tTIMESTAMP\x10\x05\x12\x06\n\x02ID\x10\x06\x12\x10\n\x0cSTRING_ARRAY\x10\x07\x12\x0e\n\nLONG_ARRAY\x10\x08\x12\x10\n\x0c\x44OUBLE_ARRAY\x10\t\x12\x11\n\rBOOLEAN_ARRAY\x10\n\x12\x0c\n\x08ID_ARRAY\x10\x0c\x12\x0f\n\x0bNULL_STRING\x10\r\x12\x0f\n\x0bNULL_NUMBER\x10\x0e\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.literal_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _LITERALTYPE._serialized_start=816
  _LITERALTYPE._serialized_end=1034
  _IDLITERAL._serialized_start=77
  _IDLITERAL._serialized_end=275
  _IDLITERAL_TYPE._serialized_start=228
  _IDLITERAL_TYPE._serialized_end=269
  _LITERAL._serialized_start=278
  _LITERAL._serialized_end=723
  _LITERALTYPEDESCRIPTION._serialized_start=725
  _LITERALTYPEDESCRIPTION._serialized_end=813
# @@protoc_insertion_point(module_scope)

protos/event/literal_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/metric.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import literal_pb2 as protos_dot_event_dot_literal__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19protos/event/metric.proto\x12\x0cprotos.event\x1a\x17protos/event/base.proto\x1a\x1dprotos/event/query_base.proto\x1a\x1aprotos/event/literal.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\x82\x01\n\x0eMetricSelector\x12\x33\n\x08\x66unction\x18\x01 \x01(\x0e\x32!.protos.event.AggregationFunction\x12,\n\nexpression\x18\x02 \x01(\x0b\x32\x18.protos.event.Expression\x12\r\n\x05\x61lias\x18\x03 \x01(\t\"\xfa\x01\n\x10MetricExpression\x12$\n\x06\x66ilter\x18\x01 \x01(\x0b\x32\x14.protos.event.Filter\x12*\n\x08group_by\x18\x02 \x03(\x0b\x32\x18.protos.event.Expression\x12/\n\tselectors\x18\x03 \x03(\x0b\x32\x1c.protos.event.MetricSelector\x12\x31\n\ris_timeseries\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\nresolution\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"L\n\rLabelMetadata\x12,\n\nexpression\x18\x01 \x01(\x0b\x32\x18.protos.event.Expression\x12\r\n\x05\x61lias\x18\x02 \x01(\t\"F\n\x0bTsDataPoint\x12\x11\n\ttimestamp\x18\x01 \x01(\x04\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.protos.event.Literal\"`\n\x04\x44\x61ta\x12\x32\n\x0ftimeseries_data\x18\x02 \x03(\x0b\x32\x19.protos.event.TsDataPoint\x12$\n\x05value\x18\x03 \x01(\x0b\x32\x15.protos.event.Literal\"D\n\x05Label\x12$\n\x05value\x18\x01 \x01(\x0b\x32\x15.protos.event.Literal\x12\x15\n\rdisplay_value\x18\x02 \x01(\t\"\xd5\x01\n\x0bLabeledData\x12\x13\n\x0blabel_group\x18\x03 \x01(\t\x12#\n\x06labels\x18\x01 \x03(\x0b\x32\x13.protos.event.Label\x12\x43\n\x0e\x61lias_data_map\x18\x02 \x03(\x0b\x32+.protos.event.LabeledData.AliasDataMapEntry\x1aG\n\x11\x41liasDataMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12!\n\x05value\x18\x02 \x01(\x0b\x32\x12.protos.event.Data:\x02\x38\x01\"\x9a\x03\n\x12MetricDataMetadata\x12\x34\n\x0flabels_metadata\x18\x01 \x03(\x0b\x32\x1b.protos.event.LabelMetadata\x12\x31\n\ris_timeseries\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x30\n\nresolution\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12+\n\ntime_range\x18\x05 \x01(\x0b\x32\x17.protos.event.TimeRange\x12_\n\x19metric_alias_selector_map\x18\x06 \x03(\x0b\x32<.protos.event.MetricDataMetadata.MetricAliasSelectorMapEntry\x1a[\n\x1bMetricAliasSelectorMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.protos.event.MetricSelector:\x02\x38\x01\"q\n\nMetricData\x12\x32\n\x08metadata\x18\x01 \x01(\x0b\x32 .protos.event.MetricDataMetadata\x12/\n\x0clabeled_data\x18\x02 \x03(\x0b\x32\x19.protos.event.LabeledData*\x83\x01\n\x13\x41ggregationFunction\x12\x0e\n\nUNKNOWN_MF\x10\x00\x12\t\n\x05\x43OUNT\x10\x01\x12\x07\n\x03SUM\x10\x02\x12\x07\n\x03\x41VG\x10\x03\x12\x07\n\x03MIN\x10\x04\x12\x07\n\x03MAX\x10\x05\x12\x0b\n\x07STD_DEV\x10\x06\x12\x0c\n\x08VARIANCE\x10\x07\x12\x12\n\x0e\x43OUNT_DISTINCT\x10\x08\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.metric_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _LABELEDDATA_ALIASDATAMAPENTRY._options = None
  _LABELEDDATA_ALIASDATAMAPENTRY._serialized_options = b'8\001'
  _METRICDATAMETADATA_METRICALIASSELECTORMAPENTRY._options = None
  _METRICDATAMETADATA_METRICALIASSELECTORMAPENTRY._serialized_options = b'8\001'
  _AGGREGATIONFUNCTION._serialized_start=1608
  _AGGREGATIONFUNCTION._serialized_end=1739
  _METRICSELECTOR._serialized_start=160
  _METRICSELECTOR._serialized_end=290
  _METRICEXPRESSION._serialized_start=293
  _METRICEXPRESSION._serialized_end=543
  _LABELMETADATA._serialized_start=545
  _LABELMETADATA._serialized_end=621
  _TSDATAPOINT._serialized_start=623
  _TSDATAPOINT._serialized_end=693
  _DATA._serialized_start=695
  _DATA._serialized_end=791
  _LABEL._serialized_start=793
  _LABEL._serialized_end=861
  _LABELEDDATA._serialized_start=864
  _LABELEDDATA._serialized_end=1077
  _LABELEDDATA_ALIASDATAMAPENTRY._serialized_start=1006
  _LABELEDDATA_ALIASDATAMAPENTRY._serialized_end=1077
  _METRICDATAMETADATA._serialized_start=1080
  _METRICDATAMETADATA._serialized_end=1490
  _METRICDATAMETADATA_METRICALIASSELECTORMAPENTRY._serialized_start=1399
  _METRICDATAMETADATA_METRICALIASSELECTORMAPENTRY._serialized_end=1490
  _METRICDATA._serialized_start=1492
  _METRICDATA._serialized_end=1605
# @@protoc_insertion_point(module_scope)

protos/event/metric_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/monitor.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aprotos/event/monitor.proto\x12\x0cprotos.event\x1a\x17protos/event/base.proto\x1a\x1egoogle/protobuf/wrappers.proto\"\xbc\x02\n\x0cMonitorStats\x12\x37\n\x11transaction_count\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12@\n\x1a\x66inished_transaction_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12;\n\x15transaction_avg_delay\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12@\n\x0bpercentiles\x18\x04 \x03(\x0b\x32+.protos.event.MonitorStats.PercentilesEntry\x1a\x32\n\x10PercentilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"*\n\x0eMonitorPartial\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xce\x01\n\x07Monitor\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x31\n\x11primary_event_key\x18\x04 \x01(\x0b\x32\x16.protos.event.EventKey\x12\x33\n\x13secondary_event_key\x18\x06 \x01(\x0b\x32\x16.protos.event.EventKey\x12\x12\n\ncreated_at\x18\x07 \x01(\x10\x12-\n\tis_active\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"f\n\x11MonitorDefinition\x12&\n\x07monitor\x18\x01 \x01(\x0b\x32\x15.protos.event.Monitor\x12)\n\x05stats\x18\x02 \x01(\x0b\x32\x1a.protos.event.MonitorStats\"\xc2\x01\n\x17MonitorEventTypeDetails\x12-\n\x07monitor\x18\x01 \x01(\x0b\x32\x1c.protos.event.MonitorPartial\x12:\n\x12primary_event_type\x18\x02 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12<\n\x14secondary_event_type\x18\x03 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\"\xa3\x01\n\x1bMonitorTransactionGetParams\x12\x17\n\x0ftransaction_ids\x18\x01 \x03(\x04\x12\x14\n\x0ctransactions\x18\x02 \x03(\t\x12U\n\x12transaction_status\x18\x03 \x01(\x0e\x32\x39.protos.event.MonitorTransaction.MonitorTransactionStatus\"\x87\x04\n\x12MonitorTransaction\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x13\n\x0btransaction\x18\x02 \x01(\t\x12-\n\x07monitor\x18\x03 \x01(\x0b\x32\x1c.protos.event.MonitorPartial\x12\x12\n\ncreated_at\x18\x04 \x01(\x10\x12I\n\x06status\x18\x05 \x01(\x0e\x32\x39.protos.event.MonitorTransaction.MonitorTransactionStatus\x12.\n\nhas_alerts\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12\x35\n\x0ftransaction_age\x18\x07 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x36\n\x10transaction_time\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\"L\n\x1bMonitorTransactionEventType\x12\x11\n\rUNKNOWN_MT_ET\x10\x00\x12\x0b\n\x07PRIMARY\x10\x01\x12\r\n\tSECONDARY\x10\x02\"U\n\x18MonitorTransactionStatus\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x14\n\x10PRIMARY_RECEIVED\x10\x01\x12\x16\n\x12SECONDARY_RECEIVED\x10\x02\"\x8c\x01\n\x17MonitorTransactionStats\x12\x36\n\x10transaction_time\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x39\n\x15primary_event_missing\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\x90\x01\n\x19MonitorTransactionDetails\x12=\n\x13monitor_transaction\x18\x01 \x01(\x0b\x32 .protos.event.MonitorTransaction\x12\x34\n\x05stats\x18\x02 \x01(\x0b\x32%.protos.event.MonitorTransactionStats\"\xa7\x01\n\x19MonitorTransactionPartial\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0btransaction\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12-\n\x07monitor\x18\x03 \x01(\x0b\x32\x1c.protos.event.MonitorPartial\"\xaa\x08\n\x0fUpdateMonitorOp\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .protos.event.UpdateMonitorOp.Op\x12N\n\x13update_monitor_name\x18\x02 \x01(\x0b\x32/.protos.event.UpdateMonitorOp.UpdateMonitorNameH\x00\x12[\n\x1aupdate_monitor_primary_key\x18\x03 \x01(\x0b\x32\x35.protos.event.UpdateMonitorOp.UpdateMonitorPrimaryKeyH\x00\x12_\n\x1cupdate_monitor_secondary_key\x18\x04 \x01(\x0b\x32\x37.protos.event.UpdateMonitorOp.UpdateMonitorSecondaryKeyH\x00\x12R\n\x15update_monitor_status\x18\x05 \x01(\x0b\x32\x31.protos.event.UpdateMonitorOp.UpdateMonitorStatusH\x00\x12]\n\x1bupdate_monitor_is_generated\x18\x06 \x01(\x0b\x32\x36.protos.event.UpdateMonitorOp.UpdateMonitorIsGeneratedH\x00\x1a?\n\x11UpdateMonitorName\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x1aL\n\x17UpdateMonitorPrimaryKey\x12\x31\n\x11primary_event_key\x18\x01 \x01(\x0b\x32\x16.protos.event.EventKey\x1aP\n\x19UpdateMonitorSecondaryKey\x12\x33\n\x13secondary_event_key\x18\x01 \x01(\x0b\x32\x16.protos.event.EventKey\x1a\x44\n\x13UpdateMonitorStatus\x12-\n\tis_active\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x1aL\n\x18UpdateMonitorIsGenerated\x12\x30\n\x0cis_generated\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\xa8\x01\n\x02Op\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x17\n\x13UPDATE_MONITOR_NAME\x10\x01\x12\x1e\n\x1aUPDATE_MONITOR_PRIMARY_KEY\x10\x02\x12 \n\x1cUPDATE_MONITOR_SECONDARY_KEY\x10\x03\x12\x19\n\x15UPDATE_MONITOR_STATUS\x10\x04\x12\x1f\n\x1bUPDATE_MONITOR_IS_GENERATED\x10\x05\x42\x08\n\x06updateb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.monitor_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _MONITORSTATS_PERCENTILESENTRY._options = None
  _MONITORSTATS_PERCENTILESENTRY._serialized_options = b'8\001'
  _MONITORSTATS._serialized_start=102
  _MONITORSTATS._serialized_end=418
  _MONITORSTATS_PERCENTILESENTRY._serialized_start=368
  _MONITORSTATS_PERCENTILESENTRY._serialized_end=418
  _MONITORPARTIAL._serialized_start=420
  _MONITORPARTIAL._serialized_end=462
  _MONITOR._serialized_start=465
  _MONITOR._serialized_end=671
  _MONITORDEFINITION._serialized_start=673
  _MONITORDEFINITION._serialized_end=775
  _MONITOREVENTTYPEDETAILS._serialized_start=778
  _MONITOREVENTTYPEDETAILS._serialized_end=972
  _MONITORTRANSACTIONGETPARAMS._serialized_start=975
  _MONITORTRANSACTIONGETPARAMS._serialized_end=1138
  _MONITORTRANSACTION._serialized_start=1141
  _MONITORTRANSACTION._serialized_end=1660
  _MONITORTRANSACTION_MONITORTRANSACTIONEVENTTYPE._serialized_start=1497
  _MONITORTRANSACTION_MONITORTRANSACTIONEVENTTYPE._serialized_end=1573
  _MONITORTRANSACTION_MONITORTRANSACTIONSTATUS._serialized_start=1575
  _MONITORTRANSACTION_MONITORTRANSACTIONSTATUS._serialized_end=1660
  _MONITORTRANSACTIONSTATS._serialized_start=1663
  _MONITORTRANSACTIONSTATS._serialized_end=1803
  _MONITORTRANSACTIONDETAILS._serialized_start=1806
  _MONITORTRANSACTIONDETAILS._serialized_end=1950
  _MONITORTRANSACTIONPARTIAL._serialized_start=1953
  _MONITORTRANSACTIONPARTIAL._serialized_end=2120
  _UPDATEMONITOROP._serialized_start=2123
  _UPDATEMONITOROP._serialized_end=3189
  _UPDATEMONITOROP_UPDATEMONITORNAME._serialized_start=2637
  _UPDATEMONITOROP_UPDATEMONITORNAME._serialized_end=2700
  _UPDATEMONITOROP_UPDATEMONITORPRIMARYKEY._serialized_start=2702
  _UPDATEMONITOROP_UPDATEMONITORPRIMARYKEY._serialized_end=2778
  _UPDATEMONITOROP_UPDATEMONITORSECONDARYKEY._serialized_start=2780
  _UPDATEMONITOROP_UPDATEMONITORSECONDARYKEY._serialized_end=2860
  _UPDATEMONITOROP_UPDATEMONITORSTATUS._serialized_start=2862
  _UPDATEMONITOROP_UPDATEMONITORSTATUS._serialized_end=2930
  _UPDATEMONITOROP_UPDATEMONITORISGENERATED._serialized_start=2932
  _UPDATEMONITOROP_UPDATEMONITORISGENERATED._serialized_end=3008
  _UPDATEMONITOROP_OP._serialized_start=3011
  _UPDATEMONITOROP_OP._serialized_end=3179
# @@protoc_insertion_point(module_scope)

protos/event/monitor_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/notification.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()




DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fprotos/event/notification.proto\x12\x0cprotos.event\")\n\x12SlackConfiguration\x12\x13\n\x0bwebhook_url\x18\x01 \x01(\t\"0\n\x12\x45mailConfiguration\x12\x1a\n\x12recipient_email_id\x18\x01 \x01(\t\"\x89\x02\n\x12NotificationConfig\x12\x39\n\x07\x63hannel\x18\x01 \x01(\x0e\x32(.protos.event.NotificationConfig.Channel\x12?\n\x13slack_configuration\x18\x02 \x01(\x0b\x32 .protos.event.SlackConfigurationH\x00\x12?\n\x13\x65mail_configuration\x18\x03 \x01(\x0b\x32 .protos.event.EmailConfigurationH\x00\",\n\x07\x43hannel\x12\x0b\n\x07UNKNOWN\x10\x00\x12\t\n\x05SLACK\x10\x01\x12\t\n\x05\x45MAIL\x10\x02\x42\x08\n\x06\x63onfigb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.notification_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _SLACKCONFIGURATION._serialized_start=49
  _SLACKCONFIGURATION._serialized_end=90
  _EMAILCONFIGURATION._serialized_start=92
  _EMAILCONFIGURATION._serialized_end=140
  _NOTIFICATIONCONFIG._serialized_start=143
  _NOTIFICATIONCONFIG._serialized_end=408
  _NOTIFICATIONCONFIG_CHANNEL._serialized_start=354
  _NOTIFICATIONCONFIG_CHANNEL._serialized_end=398
# @@protoc_insertion_point(module_scope)

protos/event/notification_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/options.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import trigger_pb2 as protos_dot_event_dot_trigger__pb2
from protos.event import notification_pb2 as protos_dot_event_dot_notification__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aprotos/event/options.proto\x12\x0cprotos.event\x1a\x17protos/event/base.proto\x1a\x1aprotos/event/trigger.proto\x1a\x1fprotos/event/notification.proto\"C\n\x0eMonitorOptions\x12\x31\n\x11\x65vent_key_options\x18\x01 \x03(\x0b\x32\x16.protos.event.EventKey\"\xd9\x02\n\rTriggerOption\x12:\n\x0ctrigger_type\x18\x01 \x01(\x0e\x32$.protos.event.TriggerDefinition.Type\x12Q\n$default_missing_event_trigger_config\x18\x02 \x01(\x0b\x32!.protos.event.MissingEventTriggerH\x00\x12X\n$default_delayed_event_trigger_config\x18\x03 \x01(\x0b\x32(.protos.event.DelayedEventTriggerOptionsH\x00\x12M\n\x1c\x62\x61sic_trigger_filter_options\x18\x04 \x01(\x0b\x32\'.protos.event.BasicTriggerFilterOptionsB\x10\n\x0e\x63onfig_options\"\xe3\x01\n\x12NotificationOption\x12\x39\n\x07\x63hannel\x18\x01 \x01(\x0e\x32(.protos.event.NotificationConfig.Channel\x12?\n\x13slack_configuration\x18\x02 \x01(\x0b\x32 .protos.event.SlackConfigurationH\x00\x12?\n\x13\x65mail_configuration\x18\x03 \x01(\x0b\x32 .protos.event.EmailConfigurationH\x00\x42\x10\n\x0e\x63onfig_options\"\xcf\x01\n\x15MonitorTriggerOptions\x12<\n\x17\x64\x65\x66\x61ult_trigger_options\x18\x01 \x03(\x0b\x32\x1b.protos.event.TriggerOption\x12>\n\x14notification_options\x18\x02 \x03(\x0b\x32 .protos.event.NotificationOption\x12\x38\n\x18\x65vent_key_filter_options\x18\x03 \x03(\x0b\x32\x16.protos.event.EventKey\"B\n\rEntityOptions\x12\x31\n\x11\x65vent_key_options\x18\x01 \x03(\x0b\x32\x16.protos.event.EventKeyb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.options_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _MONITOROPTIONS._serialized_start=130
  _MONITOROPTIONS._serialized_end=197
  _TRIGGEROPTION._serialized_start=200
  _TRIGGEROPTION._serialized_end=545
  _NOTIFICATIONOPTION._serialized_start=548
  _NOTIFICATIONOPTION._serialized_end=775
  _MONITORTRIGGEROPTIONS._serialized_start=778
  _MONITORTRIGGEROPTIONS._serialized_end=985
  _ENTITYOPTIONS._serialized_start=987
  _ENTITYOPTIONS._serialized_end=1053
# @@protoc_insertion_point(module_scope)

protos/event/options_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/panel.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
from protos.event import metric_pb2 as protos_dot_event_dot_metric__pb2
from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import entity_pb2 as protos_dot_event_dot_entity__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18protos/event/panel.proto\x12\x0cprotos.event\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x19protos/event/metric.proto\x1a\x17protos/event/base.proto\x1a\x1dprotos/event/query_base.proto\x1a\x19protos/event/entity.proto\x1a\x1aprotos/event/monitor.proto\"c\n\nCoordinate\x12(\n\x03row\x18\x01 \x01(\x0b\x32\x1b.google.protobuf.Int64Value\x12+\n\x06\x63olumn\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.Int64Value\"b\n\x10PanelCoordinates\x12\'\n\x05start\x18\x01 \x01(\x0b\x32\x18.protos.event.Coordinate\x12%\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x18.protos.event.Coordinate\"l\n\x0b\x46unnelPanel\x12\x16\n\x0e\x65vent_key_name\x18\x01 \x01(\t\x12\x16\n\x0e\x65vent_type_ids\x18\x02 \x03(\x04\x12\x17\n\x0f\x66ilter_key_name\x18\x03 \x01(\t\x12\x14\n\x0c\x66ilter_value\x18\x04 \x01(\t\"(\n\rWorkflowPanel\x12\x17\n\x0fworkflow_config\x18\x01 \x01(\t\"\xba\x06\n\x05\x43hart\x12&\n\x04type\x18\x01 \x01(\x0e\x32\x18.protos.event.Chart.Type\x12J\n\x17\x63hart_metric_expression\x18\x02 \x01(\x0b\x32).protos.event.Chart.ChartMetricExpression\x12\x42\n\x17primary_grouping_labels\x18\x03 \x03(\x0b\x32!.protos.event.AttributeIdentifier\x12:\n\x0flabels_sequence\x18\x04 \x03(\x0b\x32!.protos.event.AttributeIdentifier\x12\x42\n\x17\x64isplay_metrics_mapping\x18\x05 \x03(\x0b\x32!.protos.event.Chart.MetricMapping\x12\x38\n\x0elabels_mapping\x18\x06 \x03(\x0b\x32 .protos.event.Chart.LabelMapping\x1a{\n\x15\x43hartMetricExpression\x12&\n\x07\x63ontext\x18\x01 \x01(\x0e\x32\x15.protos.event.Context\x12:\n\x12metric_expressions\x18\x02 \x03(\x0b\x32\x1e.protos.event.MetricExpression\x1a+\n\x0cLabelMapping\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\x1a\x9b\x01\n\rMetricMapping\x12\x30\n\x02op\x18\x01 \x01(\x0e\x32$.protos.event.Chart.MetricMapping.Op\x12\x14\n\x0cmetric_label\x18\x02 \x01(\t\x12\x18\n\x10\x63omputed_metrics\x18\x03 \x03(\t\"(\n\x02Op\x12\x0e\n\nUNKNOWN_OP\x10\x00\x12\x07\n\x03SUM\x10\x01\x12\t\n\x05MATCH\x10\x02\"w\n\x04Type\x12\x0e\n\nUNKNOWN_CT\x10\x00\x12\x07\n\x03\x42\x41R\x10\x01\x12\x07\n\x03PIE\x10\x02\x12\x08\n\x04LINE\x10\x03\x12\t\n\x05TABLE\x10\x04\x12\t\n\x05VALUE\x10\x05\x12\x10\n\x0cSIMPLE_TABLE\x10\x06\x12\x11\n\rCOMPLEX_TABLE\x10\x07\x12\x08\n\x04\x41REA\x10\x08\"\xc2\x04\n\x10\x43omplexTableData\x12\x14\n\x0cmetric_label\x18\x01 \x01(\t\x12@\n\x04rows\x18\x02 \x03(\x0b\x32\x32.protos.event.ComplexTableData.ComplexTableDataRow\x1aR\n\x0e\x43omputedMetric\x12\x14\n\x0cmetric_label\x18\x01 \x01(\t\x12\x14\n\x0cmetric_value\x18\x02 \x01(\t\x12\x14\n\x0c\x66ilter_query\x18\x03 \x01(\t\x1a\xa3\x01\n\x17\x43omplexTableDataElement\x12\x15\n\relement_value\x18\x01 \x01(\t\x12\x12\n\nfill_color\x18\x02 \x01(\t\x12G\n\x10\x63omputed_metrics\x18\x03 \x03(\x0b\x32-.protos.event.ComplexTableData.ComputedMetric\x12\x14\n\x0c\x65lement_type\x18\x04 \x01(\t\x1a}\n\x16\x43omplexTableDataColumn\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12G\n\x07\x65lement\x18\x03 \x01(\x0b\x32\x36.protos.event.ComplexTableData.ComplexTableDataElement\x1a]\n\x13\x43omplexTableDataRow\x12\x46\n\x07\x63olumns\x18\x01 \x03(\x0b\x32\x35.protos.event.ComplexTableData.ComplexTableDataColumn\"*\n\x08LinkType\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08icon_url\x18\x02 \x01(\t\">\n\tPanelLink\x12$\n\x04type\x18\x01 \x01(\x0b\x32\x16.protos.event.LinkType\x12\x0b\n\x03url\x18\x02 \x01(\t\"f\n\rPanelMetaInfo\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12&\n\x05links\x18\x04 \x03(\x0b\x32\x17.protos.event.PanelLink\"\xbd\x02\n\tPanelData\x12\x33\n\x04type\x18\x01 \x01(\x0e\x32%.protos.event.PanelData.PanelDataType\x12\x12\n\x08markdown\x18\x02 \x01(\tH\x00\x12$\n\x05\x63hart\x18\x03 \x01(\x0b\x32\x13.protos.event.ChartH\x00\x12+\n\x06\x66unnel\x18\x04 \x01(\x0b\x32\x19.protos.event.FunnelPanelH\x00\x12/\n\x08workflow\x18\x05 \x01(\x0b\x32\x1b.protos.event.WorkflowPanelH\x00\"[\n\rPanelDataType\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0c\n\x08MARKDOWN\x10\x01\x12\t\n\x05\x43HART\x10\x02\x12\n\n\x06SEARCH\x10\x03\x12\n\n\x06\x46UNNEL\x10\x04\x12\x0c\n\x08WORKFLOW\x10\x05\x42\x06\n\x04\x64\x61ta\"\x95\x01\n\x07PanelV1\x12.\n\tmeta_info\x18\x01 \x01(\x0b\x32\x1b.protos.event.PanelMetaInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.protos.event.PanelData\x12\x33\n\x0b\x63oordinates\x18\x03 \x01(\x0b\x32\x1e.protos.event.PanelCoordinates\"j\n\x0b\x44\x61shboardV1\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncreated_at\x18\x02 \x01(\x10\x12%\n\x06panels\x18\x03 \x03(\x0b\x32\x15.protos.event.PanelV1\x12\x12\n\nstart_hour\x18\x04 \x01(\x04\"\x99\x01\n\x11\x45ntityFunnelPanel\x12\x31\n\x0c\x65ntity_panel\x18\x01 \x01(\x0b\x32\x1b.protos.event.EntityPartial\x12+\n\x05nodes\x18\x02 \x03(\x0b\x32\x1c.protos.event.MonitorPartial\x12$\n\x06\x66ilter\x18\x03 \x01(\x0b\x32\x14.protos.event.Filterb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.panel_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _COORDINATE._serialized_start=242
  _COORDINATE._serialized_end=341
  _PANELCOORDINATES._serialized_start=343
  _PANELCOORDINATES._serialized_end=441
  _FUNNELPANEL._serialized_start=443
  _FUNNELPANEL._serialized_end=551
  _WORKFLOWPANEL._serialized_start=553
  _WORKFLOWPANEL._serialized_end=593
  _CHART._serialized_start=596
  _CHART._serialized_end=1422
  _CHART_CHARTMETRICEXPRESSION._serialized_start=975
  _CHART_CHARTMETRICEXPRESSION._serialized_end=1098
  _CHART_LABELMAPPING._serialized_start=1100
  _CHART_LABELMAPPING._serialized_end=1143
  _CHART_METRICMAPPING._serialized_start=1146
  _CHART_METRICMAPPING._serialized_end=1301
  _CHART_METRICMAPPING_OP._serialized_start=1261
  _CHART_METRICMAPPING_OP._serialized_end=1301
  _CHART_TYPE._serialized_start=1303
  _CHART_TYPE._serialized_end=1422
  _COMPLEXTABLEDATA._serialized_start=1425
  _COMPLEXTABLEDATA._serialized_end=2003
  _COMPLEXTABLEDATA_COMPUTEDMETRIC._serialized_start=1533
  _COMPLEXTABLEDATA_COMPUTEDMETRIC._serialized_end=1615
  _COMPLEXTABLEDATA_COMPLEXTABLEDATAELEMENT._serialized_start=1618
  _COMPLEXTABLEDATA_COMPLEXTABLEDATAELEMENT._serialized_end=1781
  _COMPLEXTABLEDATA_COMPLEXTABLEDATACOLUMN._serialized_start=1783
  _COMPLEXTABLEDATA_COMPLEXTABLEDATACOLUMN._serialized_end=1908
  _COMPLEXTABLEDATA_COMPLEXTABLEDATAROW._serialized_start=1910
  _COMPLEXTABLEDATA_COMPLEXTABLEDATAROW._serialized_end=2003
  _LINKTYPE._serialized_start=2005
  _LINKTYPE._serialized_end=2047
  _PANELLINK._serialized_start=2049
  _PANELLINK._serialized_end=2111
  _PANELMETAINFO._serialized_start=2113
  _PANELMETAINFO._serialized_end=2215
  _PANELDATA._serialized_start=2218
  _PANELDATA._serialized_end=2535
  _PANELDATA_PANELDATATYPE._serialized_start=2436
  _PANELDATA_PANELDATATYPE._serialized_end=2527
  _PANELV1._serialized_start=2538
  _PANELV1._serialized_end=2687
  _DASHBOARDV1._serialized_start=2689
  _DASHBOARDV1._serialized_end=2795
  _ENTITYFUNNELPANEL._serialized_start=2798
  _ENTITYFUNNELPANEL._serialized_end=2951
# @@protoc_insertion_point(module_scope)

protos/event/panel_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/query_base.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import literal_pb2 as protos_dot_event_dot_literal__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dprotos/event/query_base.proto\x12\x0cprotos.event\x1a\x1aprotos/event/literal.proto\x1a\x1egoogle/protobuf/wrappers.proto\"b\n\rOpDescription\x12\x1c\n\x02op\x18\x01 \x01(\x0e\x32\x10.protos.event.Op\x12\r\n\x05label\x18\x02 \x01(\t\x12\x10\n\x08is_unary\x18\x03 \x01(\x08\x12\x12\n\nis_logical\x18\x04 \x01(\x08\"M\n\x05OpRhs\x12\x1c\n\x02op\x18\x01 \x01(\x0e\x32\x10.protos.event.Op\x12&\n\x03rhs\x18\x02 \x01(\x0e\x32\x19.protos.event.LiteralType\"X\n\tOpMapping\x12&\n\x03lhs\x18\x01 \x01(\x0e\x32\x19.protos.event.LiteralType\x12#\n\x06op_rhs\x18\x02 \x03(\x0b\x32\x13.protos.event.OpRhs\"I\n\x10\x43olumnIdentifier\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x04type\x18\x02 \x01(\x0e\x32\x19.protos.event.LiteralType\"Z\n\x13\x41ttributeIdentifier\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\'\n\x04type\x18\x03 \x01(\x0e\x32\x19.protos.event.LiteralType\"\\\n\x15\x41ttributeIdentifierV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\'\n\x04type\x18\x02 \x01(\x0e\x32\x19.protos.event.LiteralType\x12\x0c\n\x04path\x18\x03 \x03(\t\"\x87\x02\n\nExpression\x12;\n\x11\x63olumn_identifier\x18\x01 \x01(\x0b\x32\x1e.protos.event.ColumnIdentifierH\x00\x12\x41\n\x14\x61ttribute_identifier\x18\x02 \x01(\x0b\x32!.protos.event.AttributeIdentifierH\x00\x12\x46\n\x17\x61ttribute_identifier_v2\x18\x03 \x01(\x0b\x32#.protos.event.AttributeIdentifierV2H\x00\x12(\n\x07literal\x18\x04 \x01(\x0b\x32\x15.protos.event.LiteralH\x00\x42\x07\n\x05value\"\x9b\x01\n\x06\x46ilter\x12%\n\x03lhs\x18\x01 \x01(\x0b\x32\x18.protos.event.Expression\x12\x1c\n\x02op\x18\x02 \x01(\x0e\x32\x10.protos.event.Op\x12%\n\x03rhs\x18\x03 \x01(\x0b\x32\x18.protos.event.Expression\x12%\n\x07\x66ilters\x18\x04 \x03(\x0b\x32\x14.protos.event.Filter\"\xb1\x01\n\x0cQueryRequest\x12$\n\x06\x66ilter\x18\x01 \x01(\x0b\x32\x14.protos.event.Filter\x12+\n\tselection\x18\x02 \x03(\x0b\x32\x18.protos.event.Expression\x12\x1b\n\x13\x64istinct_selections\x18\t \x01(\x08\x12\x31\n\x08order_by\x18\n \x01(\x0b\x32\x1f.protos.event.OrderByExpression\"\xca\x01\n\x11OrderByExpression\x12,\n\nexpression\x18\x01 \x01(\x0b\x32\x18.protos.event.Expression\x12&\n\x05order\x18\x02 \x01(\x0e\x32\x17.protos.event.SortOrder\x12/\n\x0b\x61llow_nulls\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x12.\n\nnulls_last\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValue*\x8e\x01\n\x02Op\x12\x0e\n\nUNKNOWN_OP\x10\x00\x12\x06\n\x02\x45Q\x10\x01\x12\x07\n\x03NEQ\x10\x02\x12\x06\n\x02GT\x10\x03\x12\x06\n\x02LT\x10\x04\x12\x07\n\x03GTE\x10\x05\x12\x07\n\x03LTE\x10\x06\x12\x06\n\x02IN\x10\x07\x12\n\n\x06NOT_IN\x10\x08\x12\x0b\n\x07IS_NULL\x10\t\x12\n\n\x06\x45XISTS\x10\n\x12\x07\n\x03\x41ND\x10\x14\x12\x06\n\x02OR\x10\x15\x12\x07\n\x03NOT\x10\x16*\x1e\n\tSortOrder\x12\x07\n\x03\x41SC\x10\x00\x12\x08\n\x04\x44\x45SC\x10\x01\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.query_base_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _OP._serialized_start=1447
  _OP._serialized_end=1589
  _SORTORDER._serialized_start=1591
  _SORTORDER._serialized_end=1621
  _OPDESCRIPTION._serialized_start=107
  _OPDESCRIPTION._serialized_end=205
  _OPRHS._serialized_start=207
  _OPRHS._serialized_end=284
  _OPMAPPING._serialized_start=286
  _OPMAPPING._serialized_end=374
  _COLUMNIDENTIFIER._serialized_start=376
  _COLUMNIDENTIFIER._serialized_end=449
  _ATTRIBUTEIDENTIFIER._serialized_start=451
  _ATTRIBUTEIDENTIFIER._serialized_end=541
  _ATTRIBUTEIDENTIFIERV2._serialized_start=543
  _ATTRIBUTEIDENTIFIERV2._serialized_end=635
  _EXPRESSION._serialized_start=638
  _EXPRESSION._serialized_end=901
  _FILTER._serialized_start=904
  _FILTER._serialized_end=1059
  _QUERYREQUEST._serialized_start=1062
  _QUERYREQUEST._serialized_end=1239
  _ORDERBYEXPRESSION._serialized_start=1242
  _ORDERBYEXPRESSION._serialized_end=1444
# @@protoc_insertion_point(module_scope)

protos/event/query_base_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/schema.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()




DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19protos/event/schema.proto\x12\x0cprotos.event\"\xe7\x01\n\x05Value\x12\x16\n\x0cstring_value\x18\x01 \x01(\tH\x00\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x16\n\x0c\x64ouble_value\x18\x04 \x01(\x01H\x00\x12/\n\x0b\x61rray_value\x18\x05 \x01(\x0b\x32\x18.protos.event.ArrayValueH\x00\x12\x32\n\x0ckvlist_value\x18\x06 \x01(\x0b\x32\x1a.protos.event.KeyValueListH\x00\x12\x15\n\x0b\x62ytes_value\x18\x07 \x01(\x0cH\x00\x42\x07\n\x05value\"1\n\nArrayValue\x12#\n\x06values\x18\x01 \x03(\x0b\x32\x13.protos.event.Value\"6\n\x0cKeyValueList\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.protos.event.KeyValue\";\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.protos.event.Value\"V\n\x0eIngestionEvent\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x03kvs\x18\x02 \x03(\x0b\x32\x16.protos.event.KeyValue\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"E\n\x15IngestionEventPayload\x12,\n\x06\x65vents\x18\x01 \x03(\x0b\x32\x1c.protos.event.IngestionEvent\"Q\n\x1cIngestionEventPayloadRequest\x12\x31\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32#.protos.event.IngestionEventPayload\"D\n\x1dIngestionEventPayloadResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x04\x12\x14\n\x0c\x65rrorMessage\x18\x02 \x01(\t\"&\n\x16\x41WSKinesisEventPayload\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\t\"\x86\x01\n\'AWSKinesisIngestionStreamPayloadRequest\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x04\x12\x35\n\x07records\x18\x03 \x03(\x0b\x32$.protos.event.AWSKinesisEventPayload\"f\n(AWSKinesisIngestionStreamPayloadResponse\x12\x11\n\trequestId\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x04\x12\x14\n\x0c\x65rrorMessage\x18\x03 \x01(\tB\x10Z\x0e./protos/eventb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.schema_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  DESCRIPTOR._serialized_options = b'Z\016./protos/event'
  _VALUE._serialized_start=44
  _VALUE._serialized_end=275
  _ARRAYVALUE._serialized_start=277
  _ARRAYVALUE._serialized_end=326
  _KEYVALUELIST._serialized_start=328
  _KEYVALUELIST._serialized_end=382
  _KEYVALUE._serialized_start=384
  _KEYVALUE._serialized_end=443
  _INGESTIONEVENT._serialized_start=445
  _INGESTIONEVENT._serialized_end=531
  _INGESTIONEVENTPAYLOAD._serialized_start=533
  _INGESTIONEVENTPAYLOAD._serialized_end=602
  _INGESTIONEVENTPAYLOADREQUEST._serialized_start=604
  _INGESTIONEVENTPAYLOADREQUEST._serialized_end=685
  _INGESTIONEVENTPAYLOADRESPONSE._serialized_start=687
  _INGESTIONEVENTPAYLOADRESPONSE._serialized_end=755
  _AWSKINESISEVENTPAYLOAD._serialized_start=757
  _AWSKINESISEVENTPAYLOAD._serialized_end=795
  _AWSKINESISINGESTIONSTREAMPAYLOADREQUEST._serialized_start=798
  _AWSKINESISINGESTIONSTREAMPAYLOADREQUEST._serialized_end=932
  _AWSKINESISINGESTIONSTREAMPAYLOADRESPONSE._serialized_start=934
  _AWSKINESISINGESTIONSTREAMPAYLOADRESPONSE._serialized_end=1036
# @@protoc_insertion_point(module_scope)

protos/event/schema_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/search.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import base_pb2 as protos_dot_event_dot_base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19protos/event/search.proto\x12\x0cprotos.event\x1a\x17protos/event/base.proto\"M\n\x12\x46uzzySearchRequest\x12&\n\x07\x63ontext\x18\x01 \x01(\x0e\x32\x15.protos.event.Context\x12\x0f\n\x07pattern\x18\x02 \x01(\tb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.search_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _FUZZYSEARCHREQUEST._serialized_start=68
  _FUZZYSEARCHREQUEST._serialized_end=145
# @@protoc_insertion_point(module_scope)

protos/event/search_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/timeline.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2
from protos.event import alert_pb2 as protos_dot_event_dot_alert__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bprotos/event/timeline.proto\x12\x0cprotos.event\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/event/base.proto\x1a\x1aprotos/event/monitor.proto\x1a\x18protos/event/alert.proto\x1a\x1dprotos/event/query_base.proto\"\xfe\x03\n\x1c\x45ntityInstanceTimelineRecord\x12\x11\n\ttimestamp\x18\x01 \x01(\x10\x12\"\n\x05\x65vent\x18\x02 \x01(\x0b\x32\x13.protos.event.Event\x12u\n\x1etransaction_event_type_mapping\x18\x03 \x03(\x0b\x32M.protos.event.EntityInstanceTimelineRecord.MonitorTransactionEventTypeMapping\x12\x35\n\x06\x61lerts\x18\x04 \x03(\x0b\x32%.protos.event.AlertMonitorTransaction\x1a\xf8\x01\n\"MonitorTransactionEventTypeMapping\x12\x44\n\x13monitor_transaction\x18\x01 \x01(\x0b\x32\'.protos.event.MonitorTransactionPartial\x12\\\n\x16transaction_event_type\x18\x02 \x01(\x0e\x32<.protos.event.MonitorTransaction.MonitorTransactionEventType\x12.\n\nhas_alerts\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.BoolValueb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.timeline_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _ENTITYINSTANCETIMELINERECORD._serialized_start=188
  _ENTITYINSTANCETIMELINERECORD._serialized_end=698
  _ENTITYINSTANCETIMELINERECORD_MONITORTRANSACTIONEVENTTYPEMAPPING._serialized_start=450
  _ENTITYINSTANCETIMELINERECORD_MONITORTRANSACTIONEVENTTYPEMAPPING._serialized_end=698
# @@protoc_insertion_point(module_scope)

protos/event/timeline_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/event/trigger.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2
from protos.event import entity_pb2 as protos_dot_event_dot_entity__pb2
from protos.event import literal_pb2 as protos_dot_event_dot_literal__pb2
from protos.event import query_base_pb2 as protos_dot_event_dot_query__base__pb2
from protos.event import notification_pb2 as protos_dot_event_dot_notification__pb2
from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aprotos/event/trigger.proto\x12\x0cprotos.event\x1a\x1aprotos/event/monitor.proto\x1a\x19protos/event/entity.proto\x1a\x1aprotos/event/literal.proto\x1a\x1dprotos/event/query_base.proto\x1a\x1fprotos/event/notification.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\"W\n\x13MissingEventTrigger\x12@\n\x1atransaction_time_threshold\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\"\x97\x03\n\x13\x44\x65layedEventTrigger\x12@\n\x1atransaction_time_threshold\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x37\n\x11trigger_threshold\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x30\n\nresolution\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x34\n\x04type\x18\x04 \x01(\x0e\x32&.protos.event.DelayedEventTrigger.Type\x12@\n\x1cskip_unfinished_transactions\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"[\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x12\n\x0e\x44\x45LAYED_EVENTS\x10\x01\x12\x1e\n\x1aMISSING_AND_DELAYED_EVENTS\x10\x02\x12\x12\n\x0eMISSING_EVENTS\x10\x03\"\xff\x01\n\x1a\x44\x65layedEventTriggerOptions\x12@\n\x1atransaction_time_threshold\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x37\n\x11trigger_threshold\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.DoubleValue\x12\x30\n\nresolution\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x34\n\x04type\x18\x04 \x03(\x0e\x32&.protos.event.DelayedEventTrigger.Type\"x\n\x0cTriggerStats\x12\x35\n\x11last_triggered_at\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x0b\x61lert_count\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\"\x99\x03\n\x11TriggerDefinition\x12\x32\n\x04type\x18\x01 \x01(\x0e\x32$.protos.event.TriggerDefinition.Type\x12\x42\n\x15missing_event_trigger\x18\x02 \x01(\x0b\x32!.protos.event.MissingEventTriggerH\x00\x12\x42\n\x15\x64\x65layed_event_trigger\x18\x03 \x01(\x0b\x32!.protos.event.DelayedEventTriggerH\x00\x12?\n\x15primary_event_filters\x18\x04 \x03(\x0b\x32 .protos.event.BasicTriggerFilter\x12\x41\n\x17secondary_event_filters\x18\x05 \x03(\x0b\x32 .protos.event.BasicTriggerFilter\"9\n\x04Type\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x11\n\rMISSING_EVENT\x10\x01\x12\x11\n\rDELAYED_EVENT\x10\x02\x42\t\n\x07trigger\"\x8b\x02\n\x07Trigger\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12&\n\x07monitor\x18\x03 \x01(\x0b\x32\x15.protos.event.Monitor\x12/\n\x08priority\x18\x04 \x01(\x0e\x32\x1d.protos.event.TriggerPriority\x12\x33\n\ndefinition\x18\x05 \x01(\x0b\x32\x1f.protos.event.TriggerDefinition\x12)\n\x05stats\x18\x06 \x01(\x0b\x32\x1a.protos.event.TriggerStats\x12-\n\tis_active\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\"\x9e\x01\n\x0eTriggerSummary\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x32\n\x04type\x18\x03 \x01(\x0e\x32$.protos.event.TriggerDefinition.Type\x12>\n\x0fmonitor_details\x18\x04 \x01(\x0b\x32%.protos.event.MonitorEventTypeDetails\"p\n\x12\x42\x61sicTriggerFilter\x12\x14\n\x0c\x65vent_key_id\x18\x01 \x01(\x04\x12\x1c\n\x02op\x18\x02 \x01(\x0e\x32\x10.protos.event.Op\x12&\n\x07literal\x18\x03 \x01(\x0b\x32\x15.protos.event.Literal\"P\n\x19\x42\x61sicTriggerFilterOptions\x12\x33\n\x0eop_description\x18\x01 \x03(\x0b\x32\x1b.protos.event.OpDescription\"\x83\x01\n MonitorTriggerNotificationDetail\x12&\n\x07trigger\x18\x01 \x01(\x0b\x32\x15.protos.event.Trigger\x12\x37\n\rnotifications\x18\x02 \x03(\x0b\x32 .protos.event.NotificationConfig\"\x88\x01\n\x1f\x45ntityTriggerNotificationDetail\x12,\n\x07trigger\x18\x01 \x01(\x0b\x32\x1b.protos.event.EntityTrigger\x12\x37\n\rnotifications\x18\x02 \x03(\x0b\x32 .protos.event.NotificationConfig\"\xa1\x08\n\x0fUpdateTriggerOp\x12,\n\x02op\x18\x01 \x01(\x0e\x32 .protos.event.UpdateTriggerOp.Op\x12N\n\x13update_trigger_name\x18\x02 \x01(\x0b\x32/.protos.event.UpdateTriggerOp.UpdateTriggerNameH\x00\x12R\n\x15update_trigger_status\x18\x03 \x01(\x0b\x32\x31.protos.event.UpdateTriggerOp.UpdateTriggerStatusH\x00\x12V\n\x17update_trigger_priority\x18\x04 \x01(\x0b\x32\x33.protos.event.UpdateTriggerOp.UpdateTriggerPriorityH\x00\x12Z\n\x19update_trigger_definition\x18\x05 \x01(\x0b\x32\x35.protos.event.UpdateTriggerOp.UpdateTriggerDefinitionH\x00\x12`\n\x1cupdate_trigger_notifications\x18\x06 \x01(\x0b\x32\x38.protos.event.UpdateTriggerOp.UpdateTriggerNotificationsH\x00\x1a?\n\x11UpdateTriggerName\x12*\n\x04name\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x1a\x44\n\x13UpdateTriggerStatus\x12-\n\tis_active\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.BoolValue\x1aH\n\x15UpdateTriggerPriority\x12/\n\x08priority\x18\x01 \x01(\x0e\x32\x1d.protos.event.TriggerPriority\x1aN\n\x17UpdateTriggerDefinition\x12\x33\n\ndefinition\x18\x01 \x01(\x0b\x32\x1f.protos.event.TriggerDefinition\x1aU\n\x1aUpdateTriggerNotifications\x12\x37\n\rnotifications\x18\x01 \x03(\x0b\x32 .protos.event.NotificationConfig\"\xa3\x01\n\x02Op\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x17\n\x13UPDATE_TRIGGER_NAME\x10\x01\x12\x19\n\x15UPDATE_TRIGGER_STATUS\x10\x02\x12\x1b\n\x17UPDATE_TRIGGER_PRIORITY\x10\x03\x12\x1d\n\x19UPDATE_TRIGGER_DEFINITION\x10\x04\x12 \n\x1cUPDATE_TRIGGER_NOTIFICATIONS\x10\x05\x42\x08\n\x06update*R\n\x0fTriggerPriority\x12\r\n\tUNKNOWN_P\x10\x00\x12\x08\n\x04TP_0\x10\x01\x12\x08\n\x04TP_1\x10\x02\x12\x08\n\x04TP_2\x10\x03\x12\x08\n\x04TP_3\x10\x04\x12\x08\n\x04TP_4\x10\x05\x62\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.event.trigger_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _TRIGGERPRIORITY._serialized_start=3507
  _TRIGGERPRIORITY._serialized_end=3589
  _MISSINGEVENTTRIGGER._serialized_start=256
  _MISSINGEVENTTRIGGER._serialized_end=343
  _DELAYEDEVENTTRIGGER._serialized_start=346
  _DELAYEDEVENTTRIGGER._serialized_end=753
  _DELAYEDEVENTTRIGGER_TYPE._serialized_start=662
  _DELAYEDEVENTTRIGGER_TYPE._serialized_end=753
  _DELAYEDEVENTTRIGGEROPTIONS._serialized_start=756
  _DELAYEDEVENTTRIGGEROPTIONS._serialized_end=1011
  _TRIGGERSTATS._serialized_start=1013
  _TRIGGERSTATS._serialized_end=1133
  _TRIGGERDEFINITION._serialized_start=1136
  _TRIGGERDEFINITION._serialized_end=1545
  _TRIGGERDEFINITION_TYPE._serialized_start=1477
  _TRIGGERDEFINITION_TYPE._serialized_end=1534
  _TRIGGER._serialized_start=1548
  _TRIGGER._serialized_end=1815
  _TRIGGERSUMMARY._serialized_start=1818
  _TRIGGERSUMMARY._serialized_end=1976
  _BASICTRIGGERFILTER._serialized_start=1978
  _BASICTRIGGERFILTER._serialized_end=2090
  _BASICTRIGGERFILTEROPTIONS._serialized_start=2092
  _BASICTRIGGERFILTEROPTIONS._serialized_end=2172
  _MONITORTRIGGERNOTIFICATIONDETAIL._serialized_start=2175
  _MONITORTRIGGERNOTIFICATIONDETAIL._serialized_end=2306
  _ENTITYTRIGGERNOTIFICATIONDETAIL._serialized_start=2309
  _ENTITYTRIGGERNOTIFICATIONDETAIL._serialized_end=2445
  _UPDATETRIGGEROP._serialized_start=2448
  _UPDATETRIGGEROP._serialized_end=3505
  _UPDATETRIGGEROP_UPDATETRIGGERNAME._serialized_start=2955
  _UPDATETRIGGEROP_UPDATETRIGGERNAME._serialized_end=3018
  _UPDATETRIGGEROP_UPDATETRIGGERSTATUS._serialized_start=3020
  _UPDATETRIGGEROP_UPDATETRIGGERSTATUS._serialized_end=3088
  _UPDATETRIGGEROP_UPDATETRIGGERPRIORITY._serialized_start=3090
  _UPDATETRIGGEROP_UPDATETRIGGERPRIORITY._serialized_end=3162
  _UPDATETRIGGEROP_UPDATETRIGGERDEFINITION._serialized_start=3164
  _UPDATETRIGGEROP_UPDATETRIGGERDEFINITION._serialized_end=3242
  _UPDATETRIGGEROP_UPDATETRIGGERNOTIFICATIONS._serialized_start=3244
  _UPDATETRIGGEROP_UPDATETRIGGERNOTIFICATIONS._serialized_end=3329
  _UPDATETRIGGEROP_OP._serialized_start=3332
  _UPDATETRIGGEROP_OP._serialized_end=3495
# @@protoc_insertion_point(module_scope)

protos/event/trigger_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/base.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.event import schema_pb2 as protos_dot_event_dot_schema__pb2
from protos.event import base_pb2 as protos_dot_event_dot_base__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17protos/kafka/base.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x19protos/event/schema.proto\x1a\x17protos/event/base.proto\x1a\x1aprotos/event/monitor.proto\"\xe7\x01\n\x18\x44iscoveredEntityInstance\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x38\n\x12\x65ntity_instance_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12/\n\tentity_id\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12.\n\x08instance\x18\x04 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\xc8\x03\n\x17ProcessedIngestionEvent\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12(\n\x02id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x32\n\nevent_type\x18\x03 \x01(\x0b\x32\x1e.protos.event.EventTypePartial\x12\x44\n\x03kvs\x18\x04 \x03(\x0b\x32\x37.protos.kafka.ProcessedIngestionEvent.ProcessedKeyValue\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12\x35\n\x0c\x65vent_source\x18\x06 \x01(\x0e\x32\x1f.protos.event.Event.EventSource\x1a\x8c\x01\n\x11ProcessedKeyValue\x12(\n\x02id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12)\n\x03key\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\"\n\x05value\x18\x03 \x01(\x0b\x32\x13.protos.event.Value\"\xd2\x03\n\x1bProcessedMonitorTransaction\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x44\n\x13monitor_transaction\x18\x02 \x01(\x0b\x32\'.protos.event.MonitorTransactionPartial\x12]\n\x1amonitor_transaction_status\x18\x03 \x01(\x0e\x32\x39.protos.event.MonitorTransaction.MonitorTransactionStatus\x12>\n\x0fingestion_event\x18\x04 \x01(\x0b\x32%.protos.kafka.ProcessedIngestionEvent\x12\x64\n\x1emonitor_transaction_event_type\x18\x05 \x01(\x0e\x32<.protos.event.MonitorTransaction.MonitorTransactionEventType\x12\x36\n\x10transaction_time\x18\x06 \x01(\x0b\x32\x1c.google.protobuf.DoubleValueb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.base_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _DISCOVEREDENTITYINSTANCE._serialized_start=154
  _DISCOVEREDENTITYINSTANCE._serialized_end=385
  _PROCESSEDINGESTIONEVENT._serialized_start=388
  _PROCESSEDINGESTIONEVENT._serialized_end=844
  _PROCESSEDINGESTIONEVENT_PROCESSEDKEYVALUE._serialized_start=704
  _PROCESSEDINGESTIONEVENT_PROCESSEDKEYVALUE._serialized_end=844
  _PROCESSEDMONITORTRANSACTION._serialized_start=847
  _PROCESSEDMONITORTRANSACTION._serialized_end=1313
# @@protoc_insertion_point(module_scope)

protos/kafka/base_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/event_entity.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.kafka import base_pb2 as protos_dot_kafka_dot_base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fprotos/kafka/event_entity.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/kafka/base.proto\"\x17\n\x15\x45ventEntityPayloadKey\"\xce\x01\n\x17\x45ventEntityPayloadValue\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x34\n\x05\x65vent\x18\x02 \x01(\x0b\x32%.protos.kafka.ProcessedIngestionEvent\x12K\n\x1b\x64iscovered_entity_instances\x18\x03 \x03(\x0b\x32&.protos.kafka.DiscoveredEntityInstanceb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.event_entity_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _EVENTENTITYPAYLOADKEY._serialized_start=106
  _EVENTENTITYPAYLOADKEY._serialized_end=129
  _EVENTENTITYPAYLOADVALUE._serialized_start=132
  _EVENTENTITYPAYLOADVALUE._serialized_end=338
# @@protoc_insertion_point(module_scope)

protos/kafka/event_entity_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/processed_events.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.kafka import base_pb2 as protos_dot_kafka_dot_base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#protos/kafka/processed_events.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/kafka/base.proto\"\x1a\n\x18ProcessedEventPayloadKey\"\x84\x01\n\x1aProcessedEventPayloadValue\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x34\n\x05\x65vent\x18\x02 \x01(\x0b\x32%.protos.kafka.ProcessedIngestionEventb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.processed_events_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _PROCESSEDEVENTPAYLOADKEY._serialized_start=110
  _PROCESSEDEVENTPAYLOADKEY._serialized_end=136
  _PROCESSEDEVENTPAYLOADVALUE._serialized_start=139
  _PROCESSEDEVENTPAYLOADVALUE._serialized_end=271
# @@protoc_insertion_point(module_scope)

protos/kafka/processed_events_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/processed_monitor_transactions.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.kafka import base_pb2 as protos_dot_kafka_dot_base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n1protos/kafka/processed_monitor_transactions.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/kafka/base.proto\"\'\n%ProcessedMonitorTransactionPayloadKey\"\xad\x01\n\'ProcessedMonitorTransactionPayloadValue\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12P\n\x1dprocessed_monitor_transaction\x18\x02 \x01(\x0b\x32).protos.kafka.ProcessedMonitorTransactionb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.processed_monitor_transactions_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _PROCESSEDMONITORTRANSACTIONPAYLOADKEY._serialized_start=124
  _PROCESSEDMONITORTRANSACTIONPAYLOADKEY._serialized_end=163
  _PROCESSEDMONITORTRANSACTIONPAYLOADVALUE._serialized_start=166
  _PROCESSEDMONITORTRANSACTIONPAYLOADVALUE._serialized_end=339
# @@protoc_insertion_point(module_scope)

protos/kafka/processed_monitor_transactions_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/raw_events.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.event import schema_pb2 as protos_dot_event_dot_schema__pb2
from protos.event import base_pb2 as protos_dot_event_dot_base__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dprotos/kafka/raw_events.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x19protos/event/schema.proto\x1a\x17protos/event/base.proto\"\x14\n\x12RawEventPayloadKey\"\xac\x01\n\x14RawEventPayloadValue\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12+\n\x05\x65vent\x18\x02 \x01(\x0b\x32\x1c.protos.event.IngestionEvent\x12\x35\n\x0c\x65vent_source\x18\x03 \x01(\x0e\x32\x1f.protos.event.Event.EventSourceb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.raw_events_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _RAWEVENTPAYLOADKEY._serialized_start=131
  _RAWEVENTPAYLOADKEY._serialized_end=151
  _RAWEVENTPAYLOADVALUE._serialized_start=154
  _RAWEVENTPAYLOADVALUE._serialized_end=326
# @@protoc_insertion_point(module_scope)

protos/kafka/raw_events_pb2.py
DrDroidLab/kenobi
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: protos/kafka/raw_monitor_transactions.proto
"""Generated protocol buffer code."""
from google.protobuf.internal import builder as _builder
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()


from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2
from protos.kafka import base_pb2 as protos_dot_kafka_dot_base__pb2
from protos.event import monitor_pb2 as protos_dot_event_dot_monitor__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+protos/kafka/raw_monitor_transactions.proto\x12\x0cprotos.kafka\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17protos/kafka/base.proto\x1a\x1aprotos/event/monitor.proto\"\xb8\x01\n\x1fRawMonitorTransactionPayloadKey\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x30\n\nmonitor_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0btransaction\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\"\x9b\x03\n!RawMonitorTransactionPayloadValue\x12\x30\n\naccount_id\x18\x01 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x30\n\nmonitor_id\x18\x02 \x01(\x0b\x32\x1c.google.protobuf.UInt64Value\x12\x31\n\x0btransaction\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.StringValue\x12\x34\n\x05\x65vent\x18\x04 \x01(\x0b\x32%.protos.kafka.ProcessedIngestionEvent\x12\\\n\x16transaction_event_type\x18\x05 \x01(\x0e\x32<.protos.event.MonitorTransaction.MonitorTransactionEventType\x12K\n\x1b\x64iscovered_entity_instances\x18\x06 \x03(\x0b\x32&.protos.kafka.DiscoveredEntityInstanceb\x06proto3')

_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'protos.kafka.raw_monitor_transactions_pb2', globals())
if _descriptor._USE_C_DESCRIPTORS == False:

  DESCRIPTOR._options = None
  _RAWMONITORTRANSACTIONPAYLOADKEY._serialized_start=147
  _RAWMONITORTRANSACTIONPAYLOADKEY._serialized_end=331
  _RAWMONITORTRANSACTIONPAYLOADVALUE._serialized_start=334
  _RAWMONITORTRANSACTIONPAYLOADVALUE._serialized_end=745
# @@protoc_insertion_point(module_scope)

protos/kafka/raw_monitor_transactions_pb2.py
DrDroidLab/kenobi
from django.db.models import Aggregate, Func


class Percentile(Aggregate):
    function = "PERCENTILE_CONT"
    name = "Percentile"
    template = "%(function)s (%(percentile)s) WITHIN GROUP (ORDER BY %(expressions)s)"


class DistinctFunc(Func):
    template = "%(function)s(DISTINCT %(expressions)s)"

prototype/aggregates.py
DrDroidLab/kenobi
from rest_framework.authentication import SessionAuthentication


class CsrfExemptedSessionAuthentication(SessionAuthentication):
    """
    DRF SessionAuthentication is enforcing CSRF, which may be problematic.
    That's why we want to make sure we are exempting any kind of CSRF checks for APIs.
    """

    def enforce_csrf(self, request):
        return
prototype/auth.py
DrDroidLab/kenobi
"""
Django settings for prototype project.

Generated by 'django-admin startproject' using Django 4.1.4.

For more information on this file, see
https://docs.djangoproject.com/en/4.1/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.1/ref/settings/
"""
import os
from datetime import timedelta
from pathlib import Path

from environs import Env

env = Env()
env.read_env()

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env.str("DJANGO_SECRET_KEY", default='django-insecure-rij37)-oles^@_!o7elf$o-hrn5s%_d5&bw2t&a3b6-cngiqo4')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("DJANGO_DEBUG", default=True)
DATA_UPLOAD_MAX_MEMORY_SIZE = 10485760

ALLOWED_HOSTS = ['*']

# Application definition

LOCAL_APPS = [
    'accounts.apps.AccountsConfig',
    'event.apps.EventConfig',
    'connectors.apps.ConnectorsConfig',
    'management.apps.ManagementConfig',
    'prototype.apps.PrototypeConfig',
]

THIRD_PARTY_APPS = [
    'rest_framework',
    'rest_framework.authtoken',
    'dj_rest_auth',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'allauth.socialaccount.providers.google',
    'corsheaders',
    "djcelery_email"
]

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_prometheus',
    'django_celery_beat',
    'django.contrib.sites',
    'django.contrib.postgres',
    *LOCAL_APPS,
    *THIRD_PARTY_APPS,
]

MIDDLEWARE = [
    'django_prometheus.middleware.PrometheusBeforeMiddleware',
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'prototype.middleware.RequestThreadLocalMiddleware',
    'django_prometheus.middleware.PrometheusAfterMiddleware',
]

ROOT_URLCONF = 'prototype.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'prototype.wsgi.application'

_default_postgres_host = env.str('POSTGRES_HOST', default='127.0.0.1')

# Database
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': env.str('POSTGRES_DB', default='db'),
        'USER': env.str('POSTGRES_USER', default='user'),
        'PASSWORD': env.str('POSTGRES_PASSWORD', default='pass'),
        'HOST': _default_postgres_host,
        'PORT': env.str('POSTGRES_PORT', default='5432'),
    },
    'replica1': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': env.str('POSTGRES_DB', default='db'),
        'USER': env.str('POSTGRES_USER', default='user'),
        'PASSWORD': env.str('POSTGRES_PASSWORD', default='pass'),
        'HOST': env.str('REPLICA1_POSTGRES_HOST', default=_default_postgres_host),
        'PORT': env.str('POSTGRES_PORT', default='5432'),
    },
    'clickhouse': {
        'ENGINE': 'clickhouse_backend.backend',
        'HOST': env.str("CLICKHOUSE_HOST", default='localhost'),
        'PORT': env.str("CLICKHOUSE_PORT", default='9000'),
        'USER': env.str("CLICKHOUSE_USERNAME", default='default'),
        'PASSWORD': env.str("CLICKHOUSE_PASSWORD", default=''),
        'OPTIONS': {
            'secure': env.bool("CLICKHOUSE_SECURE", default=True),
            'settings': {
                'allow_experimental_object_type': 1
            }
        }
    }
}

DATABASE_ROUTERS = ['prototype.db.router.DbRouter']

# Celery Configuration Options
CELERY_BROKER_URL = env.str('CELERY_BROKER_URL', default='redis://localhost:6379/0')
CELERY_RESULT_BACKEND = env.str('CELERY_RESULT_BACKEND', default='redis://localhost:6379/0')
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_TIME_ZONE = 'UTC'

# Celery Beat Configuration Options
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'

# Redis Configuration
REDIS_URL = env.str('REDIS_URL', default='redis://localhost:6379')

# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

AUTH_USER_MODEL = "accounts.User"

# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/

STATIC_URL = 'static/'
STATIC_ROOT = os.path.join(BASE_DIR, "static")

# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'root': {
        'handlers': ['console'],
        'level': 'WARNING',
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'),
            'propagate': False,
        },
        # 'django.db.backends': {
        #     'level': 'DEBUG',
        #     'handlers': ['console'],
        # },
    },
}

CSRF_COOKIE_SECURE = False
CSRF_TRUSTED_ORIGINS = ['http://*', 'https://*']

REST_FRAMEWORK = {
    "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",),
    'DEFAULT_AUTHENTICATION_CLASSES': (
        # We have disabled csrf checks here.
        'prototype.auth.CsrfExemptedSessionAuthentication',
    ),
}

SITE_ID = 1

ACCOUNT_USER_MODEL_USERNAME_FIELD = None
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_USERNAME_REQUIRED = False
ACCOUNT_AUTHENTICATION_METHOD = 'email'
ACCOUNT_EMAIL_VERIFICATION = 'mandatory'
ACCOUNT_EMAIL_VERIFICATION_URL = '/confirm-email/{}/'
ACCOUNT_USER_DISPLAY = 'accounts.authentication.user_display'
ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'http'
ACCOUNT_ADAPTER = 'accounts.authentication.AccountAdapter'

REST_SESSION_LOGIN = True
REST_USE_JWT = True
REST_AUTH_SERIALIZERS = {
    'LOGIN_SERIALIZER': 'accounts.serializer.AccountLoginSerializer'
}
REST_AUTH_REGISTER_SERIALIZERS = {
    'REGISTER_SERIALIZER': 'accounts.serializer.AccountRegisterSerializer',
}
SIMPLE_JWT = {
    'ACCESS_TOKEN_LIFETIME': timedelta(hours=6),
    'REFRESH_TOKEN_LIFETIME': timedelta(days=7),
}
JWT_AUTH_REFRESH_COOKIE = 'jwt-refresh'

RESTRICT_EMAIL_DOMAIN = env.str("RESTRICT_EMAIL_DOMAIN", default='')

# For demo purposes only. Use a white list in the real world.
CORS_ORIGIN_ALLOW_ALL = True

LOGIN_URL = '/accounts/login/'
ALERT_PAGE_URL = '/alerts/{}'

# Email settings
EMAIL_USE_SITES = False
EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
CELERY_EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
CELERY_EMAIL_TASK_CONFIG = {
    'queue': 'notification',
    'rate_limit': '50/m',  # * CELERY_EMAIL_CHUNK_SIZE (default: 10)
}

# Alert settings
ALERT_USE_SITE = True
ALERT_SITE_HTTP_PROTOCOL = 'http'

DELAYED_EVENT_TRIGGER_RESOLUTION_WINDOW_LOWER_BOUND_IN_SEC = int(env.str(
    'DELAYED_EVENT_TRIGGER_RESOLUTION_WINDOW_LOWER_BOUND_IN_SEC', default='60'))

ENTITY_PROCESSING_ENABLED = env.bool("ENTITY_PROCESSING_ENABLED", default=True)

RAW_EVENT_PROCESSING = {
    'async': env.bool("KAFKA_PRODUCER_RAW_EVENT_ENABLED", default=False),
    'producer': 'raw_events'
}

PROCESSED_EVENT_PROCESSING = {
    'async': env.bool("KAFKA_PRODUCER_PROCESSED_EVENT_ENABLED", default=False),
    'producer': 'processed_events'
}

RAW_MONITOR_TRANSACTION_PROCESSING = {
    'async': env.bool("KAFKA_PRODUCER_RAW_MONITOR_TRANSACTION_ENABLED", default=False),
    'producer': 'raw_monitor_transactions'
}

PROCESSED_MONITOR_TRANSACTION_PROCESSING = {
    'async': env.bool("KAFKA_PRODUCER_PROCESSED_MONITOR_TRANSACTION_ENABLED", default=False),
    'producer': 'processed_monitor_transactions'
}

KAFKA_PRODUCER_CONFIG = {
    'raw_events': {
        'enabled': env.bool("KAFKA_PRODUCER_RAW_EVENT_ENABLED", default=False),
        'topic': env.str("KAFKA_RAW_EVENT_TOPIC", default='raw-events'),
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'acks': 1,
        }
    },
    'raw_monitor_transactions': {
        'enabled': env.bool("KAFKA_PRODUCER_RAW_MONITOR_TRANSACTION_ENABLED", default=False),
        'topic': env.str("KAFKA_RAW_MONITOR_TRANSACTION_TOPIC", default='raw-monitor-transactions'),
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'acks': 1,
        }
    },
    'processed_events': {
        'enabled': env.bool("KAFKA_PRODUCER_PROCESSED_EVENT_ENABLED", default=False),
        'topic': env.str("KAFKA_PROCESSED_EVENT_TOPIC", default='processed-events'),
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'acks': 1,
        }
    },
    'processed_monitor_transactions': {
        'enabled': env.bool("KAFKA_PRODUCER_PROCESSED_MONITOR_TRANSACTION_ENABLED", default=False),
        'topic': env.str("KAFKA_PROCESSED_MONITOR_TRANSACTION_TOPIC", default='processed-monitor-transactions'),
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'acks': 1,
        }
    },
}

KAFKA_CONSUMER_CONFIG = {
    'raw_events': {
        'enabled': env.bool("KAFKA_CONSUMER_RAW_EVENT_ENABLED", default=True),
        'topic': env.str("KAFKA_RAW_EVENT_TOPIC", default='raw-events'),
        'processor': 'event.processors.raw_events_processor.RawEventProcessor',
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'auto.offset.reset': 'earliest'
        }
    },
    'raw_monitor_transactions': {
        'enabled': env.bool("KAFKA_CONSUMER_RAW_MONITOR_TRANSACTION_ENABLED", default=True),
        'topic': env.str("KAFKA_RAW_MONITOR_TRANSACTION_TOPIC", default='raw-monitor-transactions'),
        'processor': 'event.processors.raw_monitor_transactions_processor.RawMonitorTransactionProcessor',
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'auto.offset.reset': 'earliest'
        }
    },
    'processed_events_clickhouse': {
        'enabled': env.bool("KAFKA_CONSUMER_PROCESSED_EVENTS_CLICKHOUSE_ENABLED", default=True),
        'topic': env.str("KAFKA_PROCESSED_EVENT_TOPIC", default='processed-events'),
        'processor': 'event.processors.processed_events_processor.ProcessedEventClickhouseIngestProcessor',
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'auto.offset.reset': 'earliest'
        }
    },
    'processed_monitor_transactions_clickhouse': {
        'enabled': env.bool("KAFKA_CONSUMER_PROCESSED_MONITOR_TRANSACTIONS_CLICKHOUSE_ENABLED", default=True),
        'topic': env.str("KAFKA_PROCESSED_MONITOR_TRANSACTIONS_TOPIC", default='processed-monitor-transactions'),
        'processor': 'event.processors.processed_monitor_transaction_processor.ProcessedMonitorTransactionClickhouseIngestProcessor',
        'config': {
            'bootstrap.servers': env.str("KAFKA_BOOTSTRAP_SERVERS", default='localhost:9092'),
            'auto.offset.reset': 'earliest'
        }
    }
}

def get_cache_backend(alias='default'):
    if env.str("CACHE_BACKEND", default='locmem') == 'redis':
        return {
            alias: {
                "BACKEND": "django_redis.cache.RedisCache",
                "LOCATION": env.str("REDIS_URL", default='redis://localhost:6379/1'),
                "OPTIONS": {
                    "CLIENT_CLASS": "django_redis.client.DefaultClient",
                }
            }
        }
    else:
        return {
            alias: {
                "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
            }
        }


CACHES = {
    **get_cache_backend('default'),
}

GLOBAL_EVENT_TYPE_CACHE = {
    'cache_key': env.str("EVENT_TYPE_CACHE_KEY", default='default'),
    'enabled': env.bool("EVENT_TYPE_CACHE_ENABLED", default=True),
}

GLOBAL_EVENT_KEY_CACHE = {
    'cache_key': env.str("EVENT_KEY_CACHE_KEY", default='default'),
    'enabled': env.bool("EVENT_KEY_CACHE_ENABLED", default=True),
}

GLOBAL_PANEL_CACHE = {
    'cache_key': env.str("PANEL_CACHE_KEY", default='default'),
    'enabled': env.bool("PANEL_CACHE_ENABLED", default=True),
}

GLOBAL_DASHBOARD_CACHE = {
    'cache_key': env.str("DASHBOARD_CACHE_KEY", default='default'),
    'enabled': env.bool("DASHBOARD_CACHE_ENABLED", default=True),
}

GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE = {
    'cache_key': env.str("GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE_KEY", default='default'),
    'enabled': env.bool("GLOBAL_EVENT_QUERY_SEARCH_REQUEST_CACHE_ENABLED", default=True),
}

GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE = {
    'cache_key': env.str("GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE_KEY", default='default'),
    'enabled': env.bool("GLOBAL_MONITOR_TRANSACTION_QUERY_SEARCH_REQUEST_CACHE_ENABLED", default=True),
}

RECAPTCHA_SECRET_KEY = env.str("RECAPTCHA_SECRET_KEY", default='6LfGFpgmAAAAAL-caY9EiWPECwnbfaVPrjAUmaAJ')

GLOBAL_ACCOUNT_API_TOKEN_CACHE = {
    'cache_key': env.str("ACCOUNT_API_TOKEN_CACHE_KEY", default='default'),
    'enabled': env.bool("ACCOUNT_API_TOKEN_CACHE_ENABLED", default=True),
}

GLOBAL_ACCOUNT_CACHE = {
    'cache_key': env.str("ACCOUNT_CACHE_KEY", default='default'),
    'enabled': env.bool("ACCOUNT_CACHE_ENABLED", default=True),
}

GLOBAL_EXPORT_CONTEXT_CACHE = {
    'cache_key': env.str("EXPORT_CONTEXT_CACHE_KEY", default='default'),
    'enabled': env.bool("EXPORT_CONTEXT_CACHE_ENABLED", default=True),
}

ACCOUNT_DAILY_EVENT_QUOTA = env.int("ACCOUNT_DAILY_EVENT_QUOTA", default=10000)

CLICKHOUSE_CLIENT_CONFIG = {
    'host': env.str("CLICKHOUSE_HOST", default='localhost'),
    'port': env.int("CLICKHOUSE_PORT", default=8443),
    'username': env.str("CLICKHOUSE_USERNAME", default='default'),
    'password': env.str("CLICKHOUSE_PASSWORD", default=''),
}

prototype/base_settings.py
DrDroidLab/kenobi
from __future__ import absolute_import, unicode_literals

import os

from celery import Celery, shared_task

# Set the default Django settings mode for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'prototype.base_settings')

app = Celery('prototype')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.conf.update(task_routes={'accounts.tasks.*': {'queue': 'account_management'}})
app.autodiscover_tasks()


@shared_task
def debug_task():
    print('Debug task executed')

prototype/celery.py
DrDroidLab/kenobi
import threading
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Optional


@dataclass
class DbSelect:
    use_default_db: Optional[bool] = None
    use_read_replicas: Optional[bool] = None
    selected_db: Optional[str] = None


class _DbSelectStorage(threading.local):
    def __init__(self):
        super().__init__()
        self.db_select = DbSelect()


_db_select_storage = _DbSelectStorage()


class DbSelectException(Exception):
    pass


class PresetUseReadReplicaDbException(DbSelectException):
    pass


class PresetUseDefaultDbException(DbSelectException):
    pass


@contextmanager
def use_default_db():
    old_db_select = _db_select_storage.db_select
    if old_db_select.use_read_replicas:
        raise PresetUseReadReplicaDbException('use_read_replicas is already set')
    if old_db_select.use_default_db:
        yield
    else:
        _db_select_storage.db_select = DbSelect(
            use_default_db=True,
            use_read_replicas=False,
            selected_db=None,
        )
        try:
            yield
        finally:
            _db_select_storage.db_select = old_db_select


@contextmanager
def use_read_replica_db():
    old_db_select = _db_select_storage.db_select
    if old_db_select.use_default_db:
        raise PresetUseDefaultDbException('use_default_db is already set')
    if old_db_select.use_read_replicas:
        yield
    else:
        _db_select_storage.db_select = DbSelect(
            use_default_db=False,
            use_read_replicas=True,
            selected_db=None,
        )
        try:
            yield
        finally:
            _db_select_storage.db_select = old_db_select


def get_db_select() -> DbSelect:
    return _db_select_storage.db_select

prototype/db/db.py
DrDroidLab/kenobi
from prototype.db.db import use_read_replica_db


def use_read_replica(func):
    def wrapper(*args, **kwargs):
        with use_read_replica_db():
            return func(*args, **kwargs)

    return wrapper

prototype/db/decorator.py
DrDroidLab/kenobi
import random
from typing import List

from prototype.db.db import DbSelect, get_db_select


class DbRouter:
    def __init__(self):
        self._default_db_key: str = 'default'
        self._clickhouse_db_key: str = 'clickhouse'
        self._replica_db_keys: List[str] = ['replica1']
        self._db_set = {'default', *self._replica_db_keys}

    def db_for_read(self, model, **hints):

        if model.__name__ in ['Events', 'MonitorTransactions']:
            return self._clickhouse_db_key

        db_select: DbSelect = get_db_select()
        if db_select.use_default_db:
            if not db_select.selected_db:
                db_select.selected_db = self._default_db_key

        if db_select.use_read_replicas:
            if not db_select.selected_db:
                db_select.selected_db = random.choice(self._replica_db_keys)

        return db_select.selected_db or self._default_db_key

    def db_for_write(self, model, **hints):
        """
        Writes always go to primary.
        """
        if model.__name__ in ['Events', 'MonitorTransactions']:
            return self._clickhouse_db_key

        return self._default_db_key

    def allow_relation(self, obj1, obj2, **hints):
        """
        Relations between objects are allowed if both objects are
        in the primary/replica pool.
        """
        if obj1._state.db in self._db_set and obj2._state.db in self._db_set:
            return True
        return None

    def allow_migrate(self, db, app_label, model_name=None, **hints):
        return db == self._default_db_key

prototype/db/router.py
DrDroidLab/kenobi
from django.db.models import Subquery, OuterRef, F, QuerySet, Exists

from event.models import AlertMonitorTransactionMapping
from prototype.aggregates import DistinctFunc
from prototype.utils.timerange import DateTimeRange, filter_dtr


def annotate_entity_stats(account, qs, dtr: DateTimeRange):
    qs = qs.annotate(
        new_instance_count=Subquery(
            filter_dtr(account.entityinstance_set.filter(
                entity=OuterRef('pk'),
                created_at__gt=dtr.time_geq
            ), dtr, 'created_at').annotate(
                count=DistinctFunc(F('id'), function='Count')
            ).values_list('count', flat=True)
        )
    )

    qs = qs.annotate(
        active_instance_count=Subquery(
            filter_dtr(account.entityinstanceeventmapping_set.filter(
                entity=OuterRef('pk'),
            ), dtr, 'event_timestamp').annotate(
                count=DistinctFunc(F('entity_instance'), function='Count')
            ).values_list("count", flat=True)
        )
    )

    qs = qs.annotate(
        event_count=Subquery(
            filter_dtr(account.entityinstanceeventmapping_set.filter(
                entity=OuterRef('pk')
            ), dtr, 'event_timestamp').annotate(
                count=DistinctFunc(F('event'), function='Count')
            ).values_list("count", flat=True)
        )
    )

    qs = qs.annotate(
        transaction_count=Subquery(
            filter_dtr(account.entityinstancemonitortransactionmapping_set.filter(
                entity=OuterRef('pk')
            ), dtr, 'created_at').annotate(
                count=DistinctFunc(F('monitor_transaction'), function='Count')
            ).values_list("count", flat=True)
        )
    )

    return qs


def annotate_entity_instance_stats(account, qs: QuerySet, dtr: DateTimeRange):
    qs = qs.annotate(
        event_count=Subquery(
            filter_dtr(account.entityinstanceeventmapping_set.filter(
                entity_instance=OuterRef('pk'),
            ), dtr, 'created_at').annotate(
                count=DistinctFunc(F('event'), function='Count')
            ).values_list("count", flat=True)
        )
    )

    # qs = qs.annotate(
    #     has_alerts=Exists(
    #         account.alertmonitortransactionmapping_set.filter(
    #             monitor_transaction__entityinstancemonitortransactionmapping__entity_instance=OuterRef('pk'),
    #         )
    #     )
    # )

    # qs = qs.annotate(
    #     alerts=account.alertentityinstancemapping_set.filter(
    #             entity_instance=OuterRef('pk'),
    #         ).values_list('alert_id')
    # )

    return qs


def annotate_entity_timeline(qs: QuerySet):
    qs = qs.annotate(
        event_type_id=F('event__event_type__id')
    ).annotate(
        event_type_name=F('event__event_type__name')
    ).annotate(
        monitor_transaction_id=F('event__monitortransactioneventmapping__monitor_transaction_id')
    ).annotate(
        transaction=F('event__monitortransactioneventmapping__monitor_transaction__transaction')
    ).annotate(
        monitor_name=F('event__monitortransactioneventmapping__monitor_transaction__monitor__name')
    ).annotate(
        monitor_id=F('event__monitortransactioneventmapping__monitor_transaction__monitor__id')
    ).annotate(
        transaction_event_type=F('event__monitortransactioneventmapping__type')
    ).annotate(
        event_event=F('event__event')
    ).annotate(
        monitor_transaction_has_alert=Exists(
            AlertMonitorTransactionMapping.objects.filter(
                monitor_transaction__in=OuterRef('event__monitortransactioneventmapping__monitor_transaction'),
                account=OuterRef('account'),
                created_at__gte=OuterRef('event_timestamp')
            )
        )
    ).values(
        'id',
        'event_id',
        'event_timestamp',
        'event_event',
        'event_type_id',
        'event_type_name',
        'monitor_transaction_id',
        'transaction',
        'monitor_name',
        'monitor_id',
        'transaction_event_type',
        'monitor_transaction_has_alert'
    )

    return qs

prototype/entity_utils.py
DrDroidLab/kenobi
from django import forms
from django.contrib.postgres.fields import ArrayField


class ChoiceArrayField(ArrayField):
    """
    A field that allows us to store an array of choices.

    Uses Django 1.9's postgres ArrayField
    and a MultipleChoiceField for its formfield.

    Usage:

        choices = ChoiceArrayField(models.CharField(max_length=...,
                                                    choices=(...,)),
                                   default=[...])
    """

    def __init__(self, coerce=lambda val: val, **kwargs):
        self._coerce = coerce
        super().__init__(**kwargs)

    def formfield(self, **kwargs):
        defaults = {
            'form_class': forms.TypedMultipleChoiceField,
            'choices': self.base_field.choices,
            'coerce': self._coerce,
        }
        defaults.update(kwargs)
        print(defaults)
        # Skip our parent's formfield implementation completely as we don't
        # care for it.
        # pylint:disable=bad-super-call
        return super(ArrayField, self).formfield(**defaults)

prototype/fields.py
DrDroidLab/kenobi
import time
import traceback
from typing import Dict

from confluent_kafka import Consumer, TIMESTAMP_NOT_AVAILABLE

from prototype.kafka.processor import Processor
from utils.import_utils import import_attribute
from utils.tracer import tracer


def cur_time_milli():
    return int(time.time() * 1000)


def delay_time_milli(prev_time):
    return cur_time_milli() - prev_time


def get_message_process_latency(kafka_timestamp_type, kafka_timestamp):
    if kafka_timestamp_type == TIMESTAMP_NOT_AVAILABLE:
        return None
    return cur_time_milli() - kafka_timestamp


def get_message_delay_time(kafka_timestamp_type, kafka_timestamp, process_time):
    if kafka_timestamp_type == TIMESTAMP_NOT_AVAILABLE:
        return None
    return process_time - kafka_timestamp


def get_message_produce_time(kafka_timestamp_type, kafka_timestamp):
    if kafka_timestamp_type == TIMESTAMP_NOT_AVAILABLE:
        return None
    return kafka_timestamp


class KafkaConsumer:
    def __init__(self, topic: str, config: Dict, processor_cls: str, batch_size: int = 100,
                 polling_timeout: float = 1.0):
        print('Initializing Kafka consumer')
        self._topic = topic
        self._client = Consumer(config)
        self._client.subscribe([topic])
        self._member_id = self._client.memberid()
        self._processor: Processor = import_attribute(processor_cls)()
        self._batch_size = batch_size
        self._polling_timeout = polling_timeout
        self._is_shutting_down = False
        self.log(
            f'Kafka consumer initialized with batch size {self._batch_size} and polling timeout {self._polling_timeout}'
        )
        self._trace_name = f'kafka_consumer.{topic}.consume_msgs'

    def log(self, message):
        if not self._member_id:
            self._member_id = self._client.memberid()
        print(f'Consumer{{{self._member_id}}}: {message}')

    def _consume_msgs(self, msgs):
        with tracer.trace(self._trace_name) as span:
            try:
                keys, values = [], []
                min_msg_produce_time = cur_time_milli()
                for msg in msgs:
                    if msg.error():
                        self.log(f"Consumer error: {msg.error()}")
                        continue
                    key, value = self._processor.transform(key=msg.key(), value=msg.value())
                    keys.append(key)
                    values.append(value)
                    msg_produce_time = get_message_produce_time(*msg.timestamp())
                    if msg_produce_time is not None and msg_produce_time < min_msg_produce_time:
                        min_msg_produce_time = msg_produce_time
                self._processor.process(keys, values)
                num_messages = len(msgs)
                min_msg_delay_time = delay_time_milli(min_msg_produce_time)
                span.set_tag('num_messages', num_messages)
                span.set_tag('min_msg_delay_time', min_msg_delay_time)

                self.log(f'Processed {len(msgs)} message with max delay: {min_msg_delay_time}')
            except Exception as ex:
                span.set_traceback()
                self.log(f'Error processing message: {traceback.format_exception(type(ex), ex, ex.__traceback__)}')

    def run(self):
        self.log(f'Running kafka consumer for {self._topic}')
        while not self._is_shutting_down:
            self.log('Polling on client')
            msgs = self._client.consume(num_messages=self._batch_size, timeout=self._polling_timeout)
            if msgs is None or len(msgs) == 0:
                continue

            self._consume_msgs(msgs)

        if self._is_shutting_down:
            self.log(f'Kafka consumer for {self._topic} is shutting down')
            self._client.close()

    def shutdown(self):
        self._is_shutting_down = True

prototype/kafka/consumer.py
DrDroidLab/kenobi
class ProcessorError(ValueError):
    pass


class KeyDeserializationError(ProcessorError):
    pass


class ValueDeserializationError(ProcessorError):
    pass


class Processor:
    key_cls = None
    value_cls = None

    def transform(self, key, value):
        try:
            proto_key = self.key_cls()
            proto_key.ParseFromString(key)
        except Exception as e:
            raise KeyDeserializationError(f'{e} error deserializing key to message type {self.key_cls}')

        try:
            proto_value = self.value_cls()
            proto_value.ParseFromString(value)
        except Exception as e:
            raise ValueDeserializationError(f'{e} error deserializing value to message type {self.value_cls}')

        return proto_key, proto_value

    def process(self, keys, values):
        pass


class ConsoleProcessor(Processor):
    def process(self, keys, values):
        for key, value in zip(keys, values):
            print(key, value)

prototype/kafka/processor.py
DrDroidLab/kenobi
from typing import Dict

from confluent_kafka import Producer
from django.conf import settings
from google.protobuf.message import Message


class KafkaProducer:
    def __init__(self, topic: str, config: Dict):
        self._client: Producer = Producer(config)
        self._topic: str = topic

    def shutdown(self):
        self._client.flush()
        return

    def process(self, key: Message, value: Message):
        self._client.produce(topic=self._topic, key=key.SerializeToString(), value=value.SerializeToString())
        return

    def flush(self):
        self._client.flush()
        return


def create_kafka_producers():
    kafka_producer_config = getattr(settings, 'KAFKA_PRODUCER_CONFIG', {})
    producers = {}
    for name, config in kafka_producer_config.items():
        if not config.get('enabled', False):
            continue

        producer = KafkaProducer(topic=config['topic'], config=config['config'])
        producers[name] = producer

    return producers


kafka_producers: Dict = create_kafka_producers()


def get_kafka_producer(name):
    return kafka_producers.get(name, None)

prototype/kafka/producer.py
DrDroidLab/kenobi
import signal
import sys
import traceback

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from prototype.kafka.consumer import KafkaConsumer


def signal_handler(consumer: KafkaConsumer):
    def fn(signum, frame):
        signame = signal.Signals(signum).name
        print(f'Signal handler called with signal {signame} ({signum})')
        consumer.shutdown()
        return

    return fn


class Command(BaseCommand):
    help = "Process kafka messages from topic"

    def add_arguments(self, parser):
        parser.add_argument(
            "--consumer",
            default=None,
            help=(
                "Consumer to start"
            ),
        )

        parser.add_argument(
            "--group",
            default='console-consumer-group',
            help=(
                "Consumer group for kafka consumer"
            ),
        )

        parser.add_argument(
            "--processor",
            default=None,
            help=(
                "Processor for the kafka consumer"
            ),
        )

        parser.add_argument(
            "--polling-timeout",
            default=0.5,
            type=float,
            help=(
                "Polling timeout for kafka consumer"
            ),
        )

        parser.add_argument(
            "--batch-size",
            default=200,
            type=int,
            help=(
                "Batch size for kafka consumer"
            ),
        )

    def handle(self, *args, **options):
        consumer = options['consumer']
        if not consumer:
            raise CommandError("A consumer needs to be defined which should be present in django settings")
        kafka_consumer_config = getattr(settings, 'KAFKA_CONSUMER_CONFIG', {})
        config = kafka_consumer_config.get(consumer, None)
        if not config:
            raise CommandError("Consumer config is not present in django settings")

        config['config']['group.id'] = options['group']

        processor = config['processor']
        if options.get('processor', None):
            processor = options['processor']
        try:
            consumer = KafkaConsumer(
                # group.instance.id -> This property needs to be added for static member ids
                topic=config['topic'],
                config=config['config'],
                processor_cls=processor,
                batch_size=options['batch_size'],
                polling_timeout=options['polling_timeout']
            )
        except Exception as ex:
            print(f"Error initializing KafkaConsumer: {traceback.format_exception(type(ex), ex, ex.__traceback__)}")
            sys.exit(1)

        handler = signal_handler(consumer)
        signal.signal(signal.SIGINT, handler)
        signal.signal(signal.SIGTERM, handler)
        while True:
            try:
                consumer.run()
                return
            except Exception as ex:
                print(f'Error running consumer: {traceback.format_exception(type(ex), ex, ex.__traceback__)}')

prototype/management/commands/run_kafka_consumer.py
DrDroidLab/kenobi
from prototype.threadlocal import set_current_request, reset_request_storage

try:
    from django.utils.deprecation import MiddlewareMixin
except ImportError:
    MiddlewareMixin = object


class RequestThreadLocalMiddleware(MiddlewareMixin):
    """Middleware that puts the request object in thread local storage."""

    def process_request(self, request):
        set_current_request(request)

    def process_response(self, request, response):
        reset_request_storage()
        return response

    def process_exception(self, request, exception):
        reset_request_storage()

prototype/middleware.py
DrDroidLab/kenobi
from dj_rest_auth.registration.serializers import RegisterSerializer
from django.conf import settings
from rest_framework import serializers


class DomainEmailRegisterSerializer(RegisterSerializer):
    username = None

    def validate_email(self, email):
        email: str = super().validate_email(email)
        if settings.RESTRICT_EMAIL_DOMAIN:
            if settings.RESTRICT_EMAIL_DOMAIN != email.split('@')[1]:
                raise serializers.ValidationError(
                    f'A email domain needs to be of format {settings.RESTRICT_EMAIL_DOMAIN}.',
                )

        return email


prototype/serializer.py
DrDroidLab/kenobi
import threading
from dataclasses import dataclass
from typing import Dict

from django.http import HttpRequest

_ORG_KEY = 'org'
_ACCOUNT_KEY = 'account'
_USER_KEY = 'user'


@dataclass
class _RequestStore:
    request: HttpRequest
    variables: Dict


class _RequestStorage(threading.local):
    def __init__(self):
        super().__init__()
        self.request_store = _RequestStore(None, {})

    def reset(self):
        self.request_store = _RequestStore(None, {})

    def add_request(self, request):
        self.request_store.request = request

    def get_request(self):
        return self.request_store.request

    def add_variable(self, key, val):
        if not self.request_store.request:
            raise RuntimeError(f'Trying to add variable {key} with no active request')
        self.request_store.variables[key] = val

    def get_variable(self, key):
        if key not in self.request_store.request:
            return None
        return self.request_store.variables[key]


_storage = _RequestStorage()


def set_current_request(request):
    _storage.add_request(request)


def reset_request_storage():
    _storage.reset()


def set_current_request_org(org):
    _storage.add_variable(_ORG_KEY, org)


def get_current_request():
    return _storage.get_request()


def get_current_request_org():
    return _storage.get_variable(_ORG_KEY)


def set_current_request_account(account):
    _storage.add_variable(_ACCOUNT_KEY, account)


def get_current_request_account():
    return _storage.get_variable(_ACCOUNT_KEY)


def set_current_request_user(user):
    _storage.add_variable(_USER_KEY, user)


def get_current_request_user():
    return _storage.get_variable(_USER_KEY)

prototype/threadlocal.py
DrDroidLab/kenobi
"""prototype URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include, re_path

from prototype import views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('e/', include('event.urls')),
    path('accounts/', include('accounts.urls')),
    path('connectors/', include('connectors.urls')),
    path('', include('django_prometheus.urls')),
    path('', views.index),
]

prototype/urls.py
DrDroidLab/kenobi
import functools
from typing import Dict

from allauth.account.utils import has_verified_email
from dj_rest_auth.jwt_auth import JWTCookieAuthentication
from django.contrib.auth.decorators import login_required
from django.http import HttpRequest, JsonResponse, HttpResponse
from django.views.decorators.csrf import csrf_exempt
from google.protobuf.wrappers_pb2 import BoolValue
from rest_framework.decorators import authentication_classes, api_view

from accounts.authentication import AccountApiTokenAuthentication, \
    AwsKinesisApiTokenAuthentication
from protos.event.api_pb2 import ErrorMessage, Message
from google.protobuf.message import Message as ProtoMessage
from prototype.threadlocal import get_current_request
from utils.error_utils import error_dict
from utils.proto_utils import json_to_proto, proto_to_dict


def skip_signal():
    def _skip_signal(signal_func):
        @functools.wraps(signal_func)
        def _decorator(sender, instance, **kwargs):
            if hasattr(instance, 'skip_signal'):
                return None
            return signal_func(sender, instance, **kwargs)

        return _decorator

    return _skip_signal


class ProtoJsonResponse(JsonResponse):
    def __init__(
            self,
            data,
            **kwargs,
    ):
        if not isinstance(data, ProtoMessage):
            raise TypeError(
                f"data must be an instance of google.protobuf.message.Message, not {type(data)}"
            )

        json_data: Dict = proto_to_dict(data)
        super().__init__(json_data, **kwargs)


def proto_schema_validator(request_schema):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(request: HttpRequest):
            body = request.body
            try:
                if body:
                    request_message = json_to_proto(body, request_schema)
                else:
                    request_message = request_schema()
            except Exception as e:
                return JsonResponse(error_dict('Error while deserializing the proto msg', e), status=400,
                                    content_type='application/json')

            try:
                response = func(request_message)
                if isinstance(response, ProtoMessage):
                    return JsonResponse(proto_to_dict(response), status=200, content_type='application/json')
                elif isinstance(response, dict):
                    return JsonResponse(response, status=200, content_type='application/json')
                elif isinstance(response, HttpResponse):
                    return response
            except Exception as e:
                return JsonResponse(error_dict('Error while processing the request', e), status=500,
                                    content_type='application/json')

        return wrapper

    return decorator


def web_api(request_schema):
    def decorator(func):
        @functools.wraps(func)
        @csrf_exempt
        @api_view(['POST'])
        @authentication_classes([JWTCookieAuthentication])
        @login_required
        @proto_schema_validator(request_schema)
        def wrapper(message):
            return func(message)

        return wrapper

    return decorator


def check_user_email_verified(func):
    @functools.wraps(func)
    def _wrapped_view(message):
        user = get_current_request().user
        if not has_verified_email(user):
            return ProtoJsonResponse(
                ErrorMessage(
                    error=BoolValue(value=True),
                    message=Message(title='User email not verified')
                ),
                status=200
            )
        return func(message)

    return _wrapped_view


def api_auth_check(func):
    @functools.wraps(func)
    def _wrapped_view(request, *args, **kwargs):
        if request.user.is_authenticated:
            return func(request, *args, **kwargs)
        return HttpResponse(status=401)

    return _wrapped_view


def account_data_api(request_schema):
    def decorator(func):
        @functools.wraps(func)
        @csrf_exempt
        @api_view(['POST'])
        @authentication_classes([AccountApiTokenAuthentication])
        @api_auth_check
        @proto_schema_validator(request_schema)
        def wrapper(message):
            return func(message)

        return wrapper

    return decorator


def aws_kinesis_data_api(request_schema):
    def decorator(func):
        @functools.wraps(func)
        @csrf_exempt
        @api_view(['POST'])
        @authentication_classes([AwsKinesisApiTokenAuthentication])
        @api_auth_check
        @proto_schema_validator(request_schema)
        def wrapper(message):
            return func(message)

        return wrapper

    return decorator

prototype/utils/decorators.py
DrDroidLab/kenobi
X_REQUEST_ORG = 'X-REQUEST-ORG'
X_REQUEST_ACCOUNT = 'X-REQUEST-ACCOUNT'
X_REQUEST_AWS_FIREHOSE = 'X-Amz-Firehose-Access-Key'

prototype/utils/headers.py
DrDroidLab/kenobi
from google.protobuf.wrappers_pb2 import UInt32Value, BoolValue

from protos.event.api_pb2 import Meta
from protos.event.base_pb2 import TimeRange, Page


def get_meta(tr: TimeRange = TimeRange(), page: Page = Page(), total_count: int = 0,
             show_inactive: BoolValue = BoolValue(value=False)):
    return Meta(time_range=tr, page=page, total_count=UInt32Value(value=total_count), show_inactive=show_inactive)

prototype/utils/meta.py
DrDroidLab/kenobi
from datetime import datetime

import pytz
from django.db.models import QuerySet

from protos.event.base_pb2 import Page


def filter_time_range(qs: QuerySet, tr, ts_field: str):
    ts_filter_map = {}
    # TODO(MG) : Add default tr if tr.time_geq not set
    if tr.time_geq > 0:
        ts_filter_map[f'{ts_field}__gte'] = datetime.utcfromtimestamp(tr.time_geq).replace(tzinfo=pytz.utc)
    if tr.time_lt > 0:
        ts_filter_map[f'{ts_field}__lt'] = datetime.utcfromtimestamp(tr.time_lt).replace(tzinfo=pytz.utc)

    return qs.filter(**ts_filter_map)


def filter_page(qs: QuerySet, p: Page):
    if not p:
        return qs[0:100]
    if not p.limit.value or p.limit.value > 100:
        p.limit.value = 100
    if p.offset.value:
        return qs[p.offset.value:p.offset.value + p.limit.value]

    p.offset.value = 0
    return qs[:p.limit.value]

def filter_page_for_sql(p: Page):
    filter_str = ""
    if not p:
        return "LIMIT 100 OFFSET 0"
    if not p.limit.value or p.limit.value > 100:
        filter_str = "LIMIT 100"
    if p.offset.value:
        return filter_str + " OFFSET {}".format(p.offset.value)

    p.offset.value = 0
    return "LIMIT {}".format(p.limit.value)

prototype/utils/queryset.py
DrDroidLab/kenobi
from dataclasses import dataclass
from datetime import datetime, timedelta

import pytz
from django.db.models import QuerySet

from protos.event.base_pb2 import TimeRange


@dataclass
class DateTimeRange:
    time_geq: datetime
    time_lt: datetime

    def to_tr(self):
        return TimeRange(time_geq=int(self.time_geq.timestamp()), time_lt=int(self.time_lt.timestamp()))

    def to_tr_str(self):
        return to_str(self.time_geq), to_str(self.time_lt)

    def to_tr_int(self):
        return int(self.time_geq.timestamp()), int(self.time_lt.timestamp())

    def get_prev_dtr(self, offset):
        offset_seconds = time_offset_to_seconds(offset)
        return DateTimeRange(time_geq=self.time_geq - timedelta(seconds=offset_seconds), time_lt=self.time_lt - timedelta(seconds=offset_seconds))



def to_dtr(tr: TimeRange = None, now=None, delta=timedelta(minutes=5)):
    if not now:
        now = datetime.now(tz=pytz.UTC)
    if not tr:
        return DateTimeRange(time_geq=(now - delta), time_lt=now)
    if tr.time_lt > 0:
        time_lt = datetime.utcfromtimestamp(tr.time_lt).replace(tzinfo=pytz.utc)
    else:
        time_lt = now

    if tr.time_geq > 0:
        time_geq = datetime.utcfromtimestamp(tr.time_geq).replace(tzinfo=pytz.utc)
    else:
        time_geq = time_lt - delta

    return DateTimeRange(time_geq=time_geq, time_lt=time_lt)


def filter_dtr(qs: QuerySet, dtr: DateTimeRange, ts_field: str):
    ts_filter_map = {
        f'{ts_field}__gte': dtr.time_geq,
        f'{ts_field}__lt': dtr.time_lt,
    }
    return qs.filter(**ts_filter_map)

def filter_str_dtr(qs: QuerySet, start_time: str, end_time: str , ts_field: str):
    ts_filter_map = {
        f'{ts_field}__gte':  start_time,
        f'{ts_field}__lt': end_time,
    }
    return qs.filter(**ts_filter_map)


def to_tr(dtr: DateTimeRange):
    return dtr.to_tr()

def to_str(t: datetime):
    return t.strftime('%Y-%m-%d %H:%M:%S')


def time_offset_to_seconds(offset_str):
    if not isinstance(offset_str, str):
        raise ValueError("Input must be a string.")

    # Define mapping for time units to seconds
    time_unit_map = {
        's': 1,
        'm': 60,
        'h': 3600,
        'd': 86400,
        'w': 604800,
        'mo': 2592000  # Assuming a month has 30 days for simplicity
    }

    # Parse the offset_str
    try:
        num = int(offset_str[:-1])
        unit = offset_str[-1]
    except (ValueError, IndexError):
        raise ValueError("Invalid time offset string format.")

    # Convert to seconds
    if unit in time_unit_map:
        return num * time_unit_map[unit]
    else:
        raise ValueError("Invalid time unit. Use 's', 'm', 'h', 'd', 'w', or 'mo'.")

prototype/utils/timerange.py
DrDroidLab/kenobi
import time

from urllib.parse import urlsplit

from django.contrib.sites.models import Site
from django.core.exceptions import ImproperlyConfigured
from django.conf import settings


def build_absolute_uri(request, location, protocol=None, enabled=False):
    """request.build_absolute_uri() helper
    like request.build_absolute_uri, but gracefully handling
    the case where request is None.
    """
    if not protocol:
        protocol = settings.SITE_DEFAULT_HTTP_PROTOCOL

    if request is None:
        if not enabled:
            raise ImproperlyConfigured("Passing `request=None` requires `sites` to be enabled")

        site = Site.objects.get_current()
        bits = urlsplit(location)
        if not (bits.scheme and bits.netloc):
            uri = "{protocol}://{domain}{url}".format(
                protocol=protocol,
                domain=site.domain,
                url=location,
            )
        else:
            uri = location
    else:
        uri = request.build_absolute_uri(location)
    # NOTE: We only force a protocol if we are instructed to do so
    # via the `protocol` parameter, or, if the default is set to
    # HTTPS. The latter keeps compatibility with the debatable use
    # case of running your site under both HTTP and HTTPS, where one
    # would want to make sure HTTPS links end up in password reset
    # mails even while they were initiated on an HTTP password reset
    # form.
    # (end NOTE)
    if protocol:
        uri = protocol + ":" + uri.partition(":")[2]
    return uri


def current_milli_time():
    return round(time.time() * 1000)

prototype/utils/utils.py
DrDroidLab/kenobi
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpRequest
from django.views.decorators.http import require_http_methods


@login_required()
@require_http_methods(['GET'])
def index(request: HttpRequest):
    # Process the events and metadata and put it into db
    return HttpResponse(f'Welcome to the world of workflows: {request.user}')

prototype/views.py
DrDroidLab/kenobi
"""
WSGI config for prototype project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'prototype.base_settings')

application = get_wsgi_application()

prototype/wsgi.py
DrDroidLab/kenobi
import requests
import time
import json
import random

import os


HOST_URL = os.environ.get("HOST_URL", None)

if HOST_URL is None:
    raise Exception("HOST_URL environment variable is not set")

ENDPOINT = HOST_URL + "/e/ingest/events/v1"
API_TOKEN = os.environ.get("API_TOKEN", None)
if API_TOKEN is None:
    raise Exception("API_TOKEN environment variable is not set")

def publish_event(e_name, e_payload):

    url = ENDPOINT
    headers = {
        "Content-Type": "application/json",
        "Authorization": "Bearer {}".format(API_TOKEN)
    }
    e_data = {"name": e_name, "payload": e_payload}
    try:
        resp = requests.post(url, data=json.dumps(e_data), headers=headers, timeout=1.0)
        if resp.status_code != 200:
            print("Error: {} - {}".format(resp.status_code, resp.text))
        else:
            print("Event published successfully")
    except Exception as e:
        print(str(e))

def payments_events():

    tnx_id_range = range(20000, 2500000)
    user_id_range = range(20000, 2500000)
    amount_range = range(0, 5000)
    bank_name_choices = ["AmEX", "Citi", "National Bank"]

    t_id = "a_" + str(random.choice(tnx_id_range))
    u_id = "u_" + str(random.choice(user_id_range))
    amount = random.choice(amount_range)
    bank_name = random.choice(bank_name_choices)
    time.sleep(0.1)
    publish_event(
        "payment_data_collected",
        {"payment_id": t_id, "amount": amount, "user_id": u_id},
    )
    time.sleep(0.1)
    publish_event(
        "bank_api_called",
        {"payment_id": t_id, "bank_name": bank_name, "channel_type": "online"},
    )
    tnx_status_p = random.choice(range(0, 100))
    if tnx_status_p <= 90:
        tnx_status_code = 200
        tnx_status = "success"
    elif (tnx_status_p > 90) & (tnx_status_p < 99):
        tnx_status_code = random.choice([400, 401, 404, 501])
        tnx_status = "fail"
    else:
        return None
    time.sleep(0.1)
    publish_event(
        "bank_response_receive",
        {
            "payment_id": t_id,
            "bank_name": bank_name,
            "response_status": tnx_status,
            "response_status_code": tnx_status_code,
            "user_id": u_id,
        },
    )
    if tnx_status == "fail":
        time.sleep(0.1)
        publish_event(
            "reverse_payment_initiated",
            {"payment_id": t_id, "user_id": u_id, "amount": amount},
        )
    else:
        time.sleep(0.1)
        publish_event(
            "ledger_updated",
            {"payment_id": t_id, "user_id": u_id, "ledger_status": "updated"},
        )
        time.sleep(0.1)
        publish_event(
            "invoice_generated",
            {"invoice_id": "inv9876", "payment_id": t_id, "user_id": u_id},
        )
        if tnx_status_p != 61:
            time.sleep(0.1)
            publish_event(
                "email_sent",
                {
                    "email_id": "user@drdroid.io",
                    "user_id": u_id,
                    "payment_id": t_id,
                },
            )
    return None

while 1:
    payments_events()
    time.sleep(30)
scripts/publish_events.py
DrDroidLab/kenobi
import traceback


def error_dict(message: str, ex: Exception):
    return {
        'message': message,
        'error': traceback.format_exception(type(ex), ex, ex.__traceback__)
    }
utils/error_utils.py
DrDroidLab/kenobi
def generate_choices(enum):
    return [(v, k) for k, v in enum.items()]
utils/model_utils.py
DrDroidLab/kenobi
from typing import Dict

from google.protobuf.json_format import MessageToJson, Parse, MessageToDict, ParseDict
from google.protobuf.message import Message


class ProtoException(ValueError):
    pass


def proto_to_json(obj: Message, use_snake_case: bool = True) -> str:
    if not obj:
        raise ProtoException('Trying to serialize None obj')
    try:
        text = MessageToJson(obj, preserving_proto_field_name=use_snake_case)
    except Exception as e:
        raise ProtoException(f'Error serializing proto message: {e}')
    return text


def proto_to_dict(obj: Message, use_snake_case: bool = True) -> Dict:
    if not obj:
        raise ProtoException('Trying to serialize None obj')
    try:
        message_dict = MessageToDict(obj, preserving_proto_field_name=use_snake_case)
    except Exception as e:
        raise ProtoException(f'Error converting proto message to dict: {e}')
    return message_dict


def json_to_proto(text: str, proto_clazz, ignore_unknown_fields=True) -> Message:
    if proto_clazz:
        msg = proto_clazz()
    else:
        raise ProtoException('No message class defined')

    try:
        msg = Parse(text, msg, ignore_unknown_fields)
    except Exception as e:
        raise ProtoException(f'Error while parsing text: {e}')
    return msg


def dict_to_proto(d: Dict, proto_clazz, ignore_unknown_fields=True) -> Message:
    if proto_clazz:
        msg = proto_clazz()
    else:
        raise ProtoException('No message class defined')

    try:
        msg = ParseDict(d, msg, ignore_unknown_fields)
    except Exception as e:
        raise ProtoException(f'Error while parsing text: {e}')
    return msg


def get_value(v):
    which_one_of = v.WhichOneof('value')
    if which_one_of in ('string_value', 'bool_value', 'int_value', 'double_value', 'byte_value'):
        return getattr(v, v.WhichOneof('value'))
    elif which_one_of == 'array_value':
        return [get_value(value) for value in getattr(v, v.WhichOneof('value')).values]
    elif which_one_of == 'kvlist_value':
        value_dict = {}
        for member in getattr(v, v.WhichOneof('value')).values:
            value_dict[member.key] = get_value(member.value)
        return value_dict
    return None

utils/proto_utils.py
DrDroidLab/kenobi
from functools import wraps


def singleton(orig_cls):
    orig_new = orig_cls.__new__
    instance = None

    @wraps(orig_cls.__new__)
    def __new__(cls, *args, **kwargs):
        nonlocal instance
        if instance is None:
            instance = orig_new(cls, *args, **kwargs)
        return instance

    orig_cls.__new__ = __new__
    return orig_cls


def singleton_function(func):
    instance = None

    def wrapper(*args, **kwargs):
        nonlocal instance
        if instance is None:
            instance = func(*args, **kwargs)
        return instance

    return wrapper

utils/singleton.py
DrDroidLab/kenobi
import json
from unittest import TestCase

from protos.test.test_pb2 import TestMessage
from utils.proto_utils import proto_to_json, json_to_proto


class TestProtoUtils(TestCase):
    def test_proto_to_json(self):
        tm = TestMessage(
            workflow_name='workflow_name',
            workflow_id=1000,
            test_nested_message=TestMessage.TestNestedMessage(
                string_field='string', bool_field=True
            )
        )

        snake_case = """
        {
          "workflow_name": "workflow_name",
          "workflow_id": "1000",
          "test_nested_message": {
            "string_field": "string",
            "bool_field": true
          }
        }
        """
        self.assertEqual(proto_to_json(tm), json.dumps(json.loads(snake_case), indent=2))

        camel_case = """
        {
          "workflowName": "workflow_name",
          "workflowId": "1000",
          "testNestedMessage": {
            "stringField": "string",
            "boolField": true
          }
        }
        """
        self.assertEqual(proto_to_json(tm, use_snake_case=False), json.dumps(json.loads(camel_case), indent=2))

    def test_json_to_proto(self):
        expected_out = tm = TestMessage(
            workflow_name='workflow_name',
            workflow_id=1000,
            test_nested_message=TestMessage.TestNestedMessage(
                string_field='string', bool_field=True
            )
        )

        snake_case = """
        {
          "workflow_name": "workflow_name",
          "workflow_id": "1000",
          "test_nested_message": {
            "string_field": "string",
            "bool_field": true
          }
        }
        """

        self.assertEqual(json_to_proto(snake_case, TestMessage), expected_out)

        camel_case = """
        {
          "workflowName": "workflow_name",
          "workflowId": "1000",
          "testNestedMessage": {
            "stringField": "string",
            "boolField": true
          }
        }
        """
        self.assertEqual(json_to_proto(camel_case, TestMessage), expected_out)


utils/test_proto_utils.py
DrDroidLab/kenobi
import datetime


def get_current_day_epoch():
    current_date = datetime.date.today()
    return int(current_date.strftime("%s"))

utils/time.py
DrDroidLab/kenobi
from decorator import contextmanager


class NoopSpan:
    def set_tag(self, *args, **kwargs):
        return

    def set_traceback(self):
        return


class NoopTracer:
    @contextmanager
    def trace(self, name):
        try:
            yield NoopSpan()
        finally:
            pass


tracer = NoopTracer()
try:
    from ddtrace import tracer as dd_tracer

    tracer = dd_tracer
except ImportError:
    pass

utils/tracer.py
DrDroidLab/kenobi