Install Kube State Metrics on Kubernetes

We are going to deploy a kube-state-metrics service to generates metrics about the state of our Kubernetes objects. The metrics are exported on the HTTP endpoint /metrics on the listening port TCP 8080.

Pre-requisites

We are using our Kubernetes homelab to deploy kube-state-metrics.

Download Files from GitHub

Configuration files used in this article are hosted on GitHub. Clone the following repository:

$ git clone https://github.com/lisenet/kubernetes-homelab.git

TLDR; Install and Configure kube-state-metrics: All in One Go

Create a monitoring namespace:

$ kubectl create ns monitoring

Create everything with a single command:

$ kubectl apply -f kubernetes-homelab/kube-state-metrics/

Install and Configure kube-state-metrics: Step by Step

Step by step instructions. Note that this homelab project is under development, therefore please refer to GitHub for any source code changes.

Create a Namespace

Create a monitoring namespace:

$ kubectl create ns monitoring

Create Cluster Role and Service Account

$ kubectl apply -f kubernetes-homelab/kube-state-metrics/kube-state-metrics-cluster-role.yml

This is what the code looks like:

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kube-state-metrics
  labels:
    app: kube-state-metrics
rules:
  - verbs:
      - list
      - watch
    apiGroups:
      - certificates.k8s.io
    resources:
      - certificatesigningrequests
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - configmaps
  - verbs:
      - list
      - watch
    apiGroups:
      - batch
    resources:
      - cronjobs
  - verbs:
      - list
      - watch
    apiGroups:
      - extensions
      - apps
    resources:
      - daemonsets
  - verbs:
      - list
      - watch
    apiGroups:
      - extensions
      - apps
    resources:
      - deployments
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - endpoints
  - verbs:
      - list
      - watch
    apiGroups:
      - autoscaling
    resources:
      - horizontalpodautoscalers
  - verbs:
      - list
      - watch
    apiGroups:
      - extensions
      - networking.k8s.io
    resources:
      - ingresses
  - verbs:
      - list
      - watch
    apiGroups:
      - batch
    resources:
      - jobs
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - limitranges
  - verbs:
      - list
      - watch
    apiGroups:
      - admissionregistration.k8s.io
    resources:
      - mutatingwebhookconfigurations
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - namespaces
  - verbs:
      - list
      - watch
    apiGroups:
      - networking.k8s.io
    resources:
      - networkpolicies
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - nodes
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - persistentvolumeclaims
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - persistentvolumes
  - verbs:
      - list
      - watch
    apiGroups:
      - policy
    resources:
      - poddisruptionbudgets
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - pods
  - verbs:
      - list
      - watch
    apiGroups:
      - extensions
      - apps
    resources:
      - replicasets
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - replicationcontrollers
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - resourcequotas
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - secrets
  - verbs:
      - list
      - watch
    apiGroups:
      - ''
    resources:
      - services
  - verbs:
      - list
      - watch
    apiGroups:
      - apps
    resources:
      - statefulsets
  - verbs:
      - list
      - watch
    apiGroups:
      - storage.k8s.io
    resources:
      - storageclasses
  - verbs:
      - list
      - watch
    apiGroups:
      - admissionregistration.k8s.io
    resources:
      - validatingwebhookconfigurations
  - verbs:
      - list
      - watch
    apiGroups:
      - storage.k8s.io
    resources:
      - volumeattachments
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-state-metrics
  labels:
    app: kube-state-metrics
subjects:
  - kind: ServiceAccount
    name: kube-state-metrics
    namespace: monitoring
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-state-metrics
  namespace: monitoring
  labels:
    app: kube-state-metrics
secrets:
  - name: kube-state-metrics-token

Create a Deployment Configuration

$ kubectl apply -f kubernetes-homelab/kube-state-metrics/kube-state-metrics-deployment.yml

This is what the code looks like:

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-state-metrics
  namespace: monitoring
  labels:
    app: kube-state-metrics
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-state-metrics
  template:
    metadata:
      labels:
        app: kube-state-metrics
    spec:
      serviceAccountName: kube-state-metrics
      serviceAccount: kube-state-metrics
      securityContext:
        runAsUser: 65534
        runAsGroup: 65534
        fsGroup: 65534    
      containers:
        - name: kube-state-metrics
          image: 'quay.io/coreos/kube-state-metrics:v1.9.7'
          imagePullPolicy: IfNotPresent          
          ports:
            - containerPort: 8080
              protocol: TCP
          resources: {}
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 5
            timeoutSeconds: 5
            periodSeconds: 10
            successThreshold: 1
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 5
            timeoutSeconds: 5
            periodSeconds: 10
            successThreshold: 1
            failureThreshold: 3
      restartPolicy: Always
      terminationGracePeriodSeconds: 30

Create a Service

$ kubectl apply -f kubernetes-homelab/kube-state-metrics/kube-state-metrics-service.yml

This is what the code looks like:

---
apiVersion: v1
kind: Service
metadata:
  name: kube-state-metrics
  namespace: monitoring
  annotations:
    prometheus.io/scrape: 'true'
    prometheus.io/port:   '8080'
  labels:
    app: kube-state-metrics
spec:
  selector:
    app: kube-state-metrics
  type: ClusterIP  
  ports:
    - port: 8080
      targetPort: 8080

Check Monitoring Namespace

$ kubectl -n monitoring get all -l app=kube-state-metrics
NAME                                      READY   STATUS    RESTARTS   AGE
pod/kube-state-metrics-5f44d96774-2zcl4   1/1     Running   0          6d3h

NAME                         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
service/kube-state-metrics   ClusterIP   10.100.211.238   none          8080/TCP   6d3h

NAME                                 READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kube-state-metrics   1/1     1            1           6d3h

NAME                                            DESIRED   CURRENT   READY   AGE
replicaset.apps/kube-state-metrics-5f44d96774   1         1         1       6d3h