Hi Team, I have installed helm chart version 1.5.2 for SCK. After Installation , I found that few pods are getting crashloopbackoff with below error logs and the pods which are showing as running status does not show logs in splunk and in splunk getting this logs for these runnning pods. Crashloopbackoff Error logs ``` kubectl logs -n splunk-sck -f lv-splunk-logging-76l6d 2023-03-07 13:56:38 +0000 [info]: init supervisor logger path=nil rotate_age=nil rotate_size=nil 2023-03-07 13:56:38 +0000 [info]: parsing config file is succeeded path="/fluentd/etc/fluent.conf" 2023-03-07 13:56:38 +0000 [info]: gem 'fluentd' version '1.15.3' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-concat' version '2.4.0' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-jq' version '0.5.1' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-kubernetes_metadata_filter' version '3.1.0' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-prometheus' version '2.0.2' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-record-modifier' version '2.1.0' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-splunk-hec' version '1.3.1' 2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-systemd' version '1.0.2' 2023-03-07 13:56:38 +0000 [INFO]: Reading bearer token from /var/run/secrets/kubernetes.io/serviceaccount/token 2023-03-07 13:56:41 +0000 [error]: config error file="/fluentd/etc/fluent.conf" error_class=Fluent::ConfigError error="Invalid Kubernetes API v1 endpoint https://10.96.0.1:443/api: Timed out connecting to server" ``` splunk logs Demonset.yaml ``` kubectl get ds -n splunk-sck lv-splunk-logging -o yaml apiVersion: apps/v1 kind: DaemonSet metadata: annotations: deprecated.daemonset.template.generation: "1" meta.helm.sh/release-name: lv-splunk-connect meta.helm.sh/release-namespace: splunk-sck creationTimestamp: "2023-03-07T13:40:11Z" generation: 1 labels: app: splunk-kubernetes-logging app.kubernetes.io/managed-by: Helm chart: splunk-kubernetes-logging-1.5.2 engine: fluentd heritage: Helm release: lv-splunk-connect name: lv-splunk-logging namespace: splunk-sck resourceVersion: "390920101" selfLink: /apis/apps/v1/namespaces/splunk-sck/daemonsets/lv-splunk-logging uid: ed892500-8054-49c5-bc75-da098dbce325 spec: revisionHistoryLimit: 10 selector: matchLabels: app: splunk-kubernetes-logging release: lv-splunk-connect template: metadata: annotations: checksum/config: 6401fdcfd0a7ddd7c71e0b459aa342ebc61ed26afe237a64101f8369da6007a0 prometheus.io/port: "24231" prometheus.io/scrape: "true" creationTimestamp: null labels: app: splunk-kubernetes-logging release: lv-splunk-connect spec: containers: - env: - name: K8S_NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - name: MY_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: MY_POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: SPLUNK_HEC_TOKEN valueFrom: secretKeyRef: key: splunk_hec_token name: splunk-kubernetes-logging image: docker.io/splunk/fluentd-hec:1.3.1 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: /api/plugins.json port: 24220 scheme: HTTP initialDelaySeconds: 60 periodSeconds: 60 successThreshold: 1 timeoutSeconds: 1 name: splunk-fluentd-k8s-logs ports: - containerPort: 24231 name: metrics protocol: TCP - containerPort: 24220 name: monitor-agent protocol: TCP resources: requests: cpu: 100m memory: 200Mi securityContext: privileged: false runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/log name: varlog - mountPath: /var/log/pods name: varlogdest readOnly: true - mountPath: /var/log/journal name: journallogpath readOnly: true - mountPath: /fluentd/etc name: conf-configmap - mountPath: /fluentd/etc/splunk name: secrets readOnly: true dnsPolicy: ClusterFirst nodeSelector: beta.kubernetes.io/os: linux restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: lv-splunk-logging serviceAccountName: lv-splunk-logging terminationGracePeriodSeconds: 30 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master volumes: - hostPath: path: /var/log type: "" name: varlog - hostPath: path: /var/log/pods type: "" name: varlogdest - hostPath: path: /var/log/journal type: "" name: journallogpath - configMap: defaultMode: 420 name: lv-splunk-logging name: conf-configmap - name: secrets secret: defaultMode: 420 secretName: splunk-kubernetes-logging updateStrategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate status: currentNumberScheduled: 53 desiredNumberScheduled: 53 numberAvailable: 50 numberMisscheduled: 0 numberReady: 50 numberUnavailable: 3 observedGeneration: 1 updatedNumberScheduled: 53 ```
... View more