Hi Team,
I have installed helm chart version 1.5.2 for SCK. After Installation , I found that few pods are getting crashloopbackoff with below error logs and the pods which are showing as running status does not show logs in splunk and in splunk getting this logs for these runnning pods.
Crashloopbackoff Error logs
```
kubectl logs -n splunk-sck -f lv-splunk-logging-76l6d
2023-03-07 13:56:38 +0000 [info]: init supervisor logger path=nil rotate_age=nil rotate_size=nil
2023-03-07 13:56:38 +0000 [info]: parsing config file is succeeded path="/fluentd/etc/fluent.conf"
2023-03-07 13:56:38 +0000 [info]: gem 'fluentd' version '1.15.3'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-concat' version '2.4.0'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-jq' version '0.5.1'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-kubernetes_metadata_filter' version '3.1.0'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-prometheus' version '2.0.2'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-record-modifier' version '2.1.0'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-splunk-hec' version '1.3.1'
2023-03-07 13:56:38 +0000 [info]: gem 'fluent-plugin-systemd' version '1.0.2'
2023-03-07 13:56:38 +0000 [INFO]: Reading bearer token from /var/run/secrets/kubernetes.io/serviceaccount/token
2023-03-07 13:56:41 +0000 [error]: config error file="/fluentd/etc/fluent.conf" error_class=Fluent::ConfigError error="Invalid Kubernetes API v1 endpoint https://10.96.0.1:443/api: Timed out connecting to server"
```
splunk logs
Demonset.yaml
```
kubectl get ds -n splunk-sck lv-splunk-logging -o yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
deprecated.daemonset.template.generation: "1"
meta.helm.sh/release-name: lv-splunk-connect
meta.helm.sh/release-namespace: splunk-sck
creationTimestamp: "2023-03-07T13:40:11Z"
generation: 1
labels:
app: splunk-kubernetes-logging
app.kubernetes.io/managed-by: Helm
chart: splunk-kubernetes-logging-1.5.2
engine: fluentd
heritage: Helm
release: lv-splunk-connect
name: lv-splunk-logging
namespace: splunk-sck
resourceVersion: "390920101"
selfLink: /apis/apps/v1/namespaces/splunk-sck/daemonsets/lv-splunk-logging
uid: ed892500-8054-49c5-bc75-da098dbce325
spec:
revisionHistoryLimit: 10
selector:
matchLabels:
app: splunk-kubernetes-logging
release: lv-splunk-connect
template:
metadata:
annotations:
checksum/config: 6401fdcfd0a7ddd7c71e0b459aa342ebc61ed26afe237a64101f8369da6007a0
prometheus.io/port: "24231"
prometheus.io/scrape: "true"
creationTimestamp: null
labels:
app: splunk-kubernetes-logging
release: lv-splunk-connect
spec:
containers:
- env:
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: MY_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: SPLUNK_HEC_TOKEN
valueFrom:
secretKeyRef:
key: splunk_hec_token
name: splunk-kubernetes-logging
image: docker.io/splunk/fluentd-hec:1.3.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /api/plugins.json
port: 24220
scheme: HTTP
initialDelaySeconds: 60
periodSeconds: 60
successThreshold: 1
timeoutSeconds: 1
name: splunk-fluentd-k8s-logs
ports:
- containerPort: 24231
name: metrics
protocol: TCP
- containerPort: 24220
name: monitor-agent
protocol: TCP
resources:
requests:
cpu: 100m
memory: 200Mi
securityContext:
privileged: false
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/log
name: varlog
- mountPath: /var/log/pods
name: varlogdest
readOnly: true
- mountPath: /var/log/journal
name: journallogpath
readOnly: true
- mountPath: /fluentd/etc
name: conf-configmap
- mountPath: /fluentd/etc/splunk
name: secrets
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
beta.kubernetes.io/os: linux
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: lv-splunk-logging
serviceAccountName: lv-splunk-logging
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- hostPath:
path: /var/log
type: ""
name: varlog
- hostPath:
path: /var/log/pods
type: ""
name: varlogdest
- hostPath:
path: /var/log/journal
type: ""
name: journallogpath
- configMap:
defaultMode: 420
name: lv-splunk-logging
name: conf-configmap
- name: secrets
secret:
defaultMode: 420
secretName: splunk-kubernetes-logging
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
status:
currentNumberScheduled: 53
desiredNumberScheduled: 53
numberAvailable: 50
numberMisscheduled: 0
numberReady: 50
numberUnavailable: 3
observedGeneration: 1
updatedNumberScheduled: 53
```
