#
#  Author: Hari Sekhon
#  Date: Thu Nov 26 13:35:31 2020 +0000
#
#  vim:ts=2:sts=2:sw=2:et
#
#  https://github.com/HariSekhon/Kubernetes-configs
#
#  License: see accompanying Hari Sekhon LICENSE file
#
#  If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
#  https://www.linkedin.com/in/HariSekhon
#

# ============================================================================ #
#                     T e a m C i t y   C I   -   A g e n t
# ============================================================================ #

# StatefulSet maintains agent configurations (including authorization token), logs, plugins etc.

# See Also:
#
#   teamcity-agent-cloudprofile-deployment.yaml
#
#   https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/setup/teamcity-docker-compose.yml

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: teamcity-agent
  namespace: teamcity
  labels:
    app: teamcity-agent
spec:
  replicas: 3
  serviceName: teamcity-agent
  selector:
    matchLabels:
      app: teamcity-agent
  template:
    metadata:
      labels:
        app: teamcity-agent
    spec:
      priorityClassName: high-priority  # requires priorityclass.yaml
      affinity:
        # avoid preemption which can cause build failures
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: cloud.google.com/gke-preemptible
                    operator: DoesNotExist
                  - key: eks.amazonaws.com/capacityType
                    operator: NotIn
                    values:
                      - SPOT
                  - key: kubernetes.azure.com/scalesetpriority
                    operator: NotIn
                    values:
                      - spot
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                topologyKey: topology.kubernetes.io/zone
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - teamcity-agent
            - weight: 100
              podAffinityTerm:
                topologyKey: kubernetes.io/hostname
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - teamcity-agent
      securityContext:
        #runAsUser: 0
        # set mount point group to gid of teamcity process
        fsGroup: 1000
#      initContainers:
#        - name: init
#          image: busybox:latest
#          # XXX: it's either this or run teamcity-agent as root to access the socket
#          #      Perm/Group changes are not propagated to the host's /var/run/docker.sock but there
#          #      is still a serious risk of malicious builds running docker containers as root
#          #      you must trust your developers, and preferably segregate to a separate CI k8s cluster
#          command: ['/bin/chgrp', '1000', '/var/run/docker.sock']
#          #command: ['/bin/sh', '-c', 'chgrp 1000 /var/run/docker.sock && chmod 0660 /var/run/docker.sock']
#          volumeMounts:
#          - name: var-run-docker-sock
#            mountPath: /var/run/docker.sock
      containers:
        - name: teamcity-agent
          #image: jetbrains/teamcity-agent:2020.2.1
          # DOCKER_IN_DOCKER - must use linux-sudo image or run entire container as root since Docker must be started by root
          image: jetbrains/teamcity-agent:2020.2.1-linux-sudo
          ports:
            - containerPort: 9090
          securityContext:
            # required for DOCKER_IN_DOCKER
            privileged: true
          env:
            - name: AGENT_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: DOCKER_IN_DOCKER
              value: start
            - name: SERVER_URL
              value: http://teamcity-server.teamcity.svc.cluster.local:8111
          readinessProbe:
            tcpSocket:
              port: 9090
            initialDelaySeconds: 30
            successThreshold: 1
            failureThreshold: 3
            periodSeconds: 5
            timeoutSeconds: 5
          livenessProbe:
            tcpSocket:
              port: 9090
            initialDelaySeconds: 150  # takes ~ 1:20 - 2:30 for port to pass healthcheck
            successThreshold: 1
            failureThreshold: 10
            periodSeconds: 5
            timeoutSeconds: 5
          resources:
            limits:
              cpu: 2
              # TeamCity agent's 2 processes JVM heaps are -Xmx384m and -Xmx16m, plus overheads and wiggle room = 1Gi
              # + build processes which could take a couple GB each = 3Gi
              memory: 3Gi
            requests:
              cpu: 100m    # 6m resting
              memory: 1Gi  # 700-850Mi resting + pipeline progs
          volumeMounts:
            # not persisting system and work as they're just caches
            #  mountPath: /opt/buildagent/system
            #  mountPath: /opt/buildagent/work
            #
            # persist the buildAgent.properties containing the agent id and autosaved authorization token
            - name: teamcity-agent-conf
              mountPath: /data/teamcity_agent/conf
            # if we reschedule the pods on other nodes we still want continuity of logs and their history for use in the UI
            - name: teamcity-agent-logs
              mountPath: /opt/buildagent/logs
            - name: teamcity-agent-plugins
              mountPath: /opt/buildagent/plugins
            - name: teamcity-agent-tools
              mountPath: /opt/buildagent/tools
            # don't mount /var/run/docker.sock or you'll be using the underlying host's Docker instead of TeamCity's DOCKER_IN_DOCKER, which exposes too much of the underlying k8s but also you'll get errors like this from the underlying host when the docker wrapper tries to mount all the buildagent paths:
            # docker: Error response from daemon: error while creating mount source path '/opt/buildagent/temp/buildTmp': mkdir /opt/buildagent: read-only file system.
            #- name: var-run-docker-sock
            #  mountPath: /var/run/docker.sock
#      volumes:
#        - name: var-run-docker-sock
#          hostPath:
#            path: /var/run/docker.sock
#            type: Socket
  volumeClaimTemplates:
    - metadata:
        name: teamcity-agent-conf
      spec:
        storageClassName: gcp-standard-resizeable
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            # 36K in practice
            # gets a minimum of 1GB regardless on GCP
            storage: 50Mi
    - metadata:
        name: teamcity-agent-plugins
      spec:
        storageClassName: gcp-standard-resizeable
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            # uses 262M on my local cluster
            storage: 5Gi
    - metadata:
        name: teamcity-agent-logs
      spec:
        storageClassName: gcp-standard-resizeable
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            # after nearly a year as the team's only CI system for all releases the agent logs were rotated to be only 50-70MB - this may increase if we add more pipelines or do more iterations and deployments but this has plenty of room
            storage: 2Gi
    - metadata:
        name: teamcity-agent-tools
      spec:
        storageClassName: gcp-standard-resizeable
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            # uses 73M on my local cluster for things like ant, gant, jps
            storage: 5Gi
