Hi folks,
So I'm working to migrate from the old Splunk Connect for Kubernetes log collector to the new Splunk OTEL Collector. I am getting the logs from pods, so I know that I have it partially configured correctly at least.
I'm not getting logs from /var/log/kubernetes/audit/ nor from /var/log/audit/ as I believe I have configured in the below values file.
I am not getting logs from the containers that begin with `audit*` to any index, let alone what I'd expect from the transform processor statement here:
set(resource.attributes["com.splunk.index"], Concat(["audit", resource.attributes["k8s.namespace.name"]], "-")) where IsMatch(resource.attributes["k8s.container.name"], "audit-.*")
The full values file is below, though I think the formatting looks better in the github gist
splunk-otel-collector:
clusterName: ${env:CLUSTER_NAME}
priorityClassName: "system-cluster-critical"
splunkPlatform:
# sets Splunk Platform as a destination. Use the /services/collector/event
# endpoint for proper extraction of fields.
endpoint: wheeeeee
token: "fake-placeholder-token"
index: "k8s" # should be able to replace with "" to dynamically set index as was done with SCK but this chart does not allow
logsEnabled: true
secret:
create: false
name: fake-credentials
validateSecret: false
logsCollection:
containers:
enabled: true
excludePaths:
- /var/log/containers/*fluent-bit*
- /var/log/containers/*speaker*
- /var/log/containers/*datadog*
- /var/log/containers/*collectd*
- /var/log/containers/*rook-ceph*
- /var/log/containers/*bird*
- /var/log/containers/*logdna*
- /var/log/containers/*6c6f616462616c2d*
- /var/log/containers/*lb-6c6f616462616c2d*
# extraOperators:
# - type: copy
# # Copy the name of the namespace associated with the log record.
# from: resource["k8s.namespace.name"]
# # Copy to the index key, so the record will be ingested under the index named after the k8s namespace.
# to: resource["com.splunk.index"]
extraFileLogs:
filelog/kube-audit: # sck logs go to audit-kube index, but got it in otel index for now.
include:
- /var/log/kubernetes/audit/kube-apiserver-audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-kube
com.splunk.sourcetype: kube:apiserver-audit
com.splunk.source: /var/log/kubernetes/audit/kube-apiserver-audit.log
filelog/linux-audit:
include:
- /var/log/audit/audit*.log
start_at: beginning
include_file_path: true
include_file_name: false
resource:
host.name: resource["k8s.node.name"]
com.splunk.index: audit-linux
com.splunk.sourcetype: linux:audit
com.splunk.source: /var/log/audit/audit.log # can't find these results for SCK yet
extraAttributes:
fromLabels:
- key: k8s.pod.labels.cluster.name
tag_name: cluster_name
from: pod
- key: k8s.namespace.labels.cluster.class
tag_name: cluster_class
from: namespace
- key: k8s.namespace.labels.cluster.env
from: namespace
- key: k8s.node.name
tag_name: host
from: node
agent:
enabled: true
config:
processors:
# add cluster metadata to each logged event
# these are pulled in as environment variables due to a limitation
# as helm is unable to use templating when specifying values.
attributes/cluster_name_filter:
actions:
- key: cluster_name
action: upsert
value: ${env:CLUSTER_NAME}
attributes/cluster_class_filter:
actions:
- key: cluster_class
action: upsert
value: ${env:CLUSTER_CLASS}
attributes/cluster_env_filter:
actions:
- key: cluster_env
action: upsert
value: ${env:CLUSTER_ENV}
transform/namespace_to_index:
error_mode: ignore
log_statements:
- context: log
statements:
- set(resource.attributes["com.splunk.index"], Concat(["audit", resource.attributes["k8s.namespace.name"]], "-")) where IsMatch(resource.attributes["k8s.container.name"], "audit-.*")
- set(resource.attributes["com.splunk.index"], resource.attributes["k8s.namespace.name"])
# attributes/namespace_filter:
# actions:
# - key: com.splunk.index
# action: upsert
# value: k8s.namespace.name
# - key: logindex
# action: delete
exporters:
debug:
verbosity: detailed
service:
pipelines:
logs:
processors:
- memory_limiter
- k8sattributes
- filter/logs
- batch
- resourcedetection
- resource
- resource/logs
- attributes/cluster_name_filter
- attributes/cluster_class_filter
- attributes/cluster_env_filter
- transform/namespace_to_index
# - attributes/namespace_filter
receivers:
kubeletstats:
metric_groups:
- node
- pod
- container
filelog:
include:
- /var/log/pods/*/*/*.log
- /var/log/kubernetes/audit/*.log
- /var/log/audit/audit*.log
start_at: beginning
include_file_name: false
include_file_path: true
operators:
# parse cri-o format
- type: regex_parser
id: parser-crio
regex:
'^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)
?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout_type: gotime
layout: '2006-01-02T15:04:05.999999999Z07:00'
# Parse CRI-Containerd format
- type: regex_parser
id: parser-containerd
regex:
'^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)
?(?P<log>.*)$'
output: extract_metadata_from_filepath
timestamp:
parse_from: attributes.time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
- type: copy
from: resource["k8s.namespace.name"]
to: resource["com.splunk.index"]
# Set Environment Variables to be set on every Pod in the DaemonSet
# Many of these are used as a work-around to include additional log metadata
# from what is available in `.Values` but inaccessible due to limitations of
# Helm.
extraEnvs:
- name: CLUSTER_NAME
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_NAME
- name: CLUSTER_CLASS
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_CLASS
- name: CLUSTER_ENV
valueFrom:
configMapKeyRef:
name: cluster-info
key: CLUSTER_ENV
# The container logs may actually be a series of symlinks. In order to read
# them, all directories need to be accessible by the logging pods. We use
# volumes and volume mounts to achieve that.
extraVolumes:
- name: containerdlogs
hostPath:
path: /var/lib/containerd/pod-logs
- name: podlogs
hostPath:
path: /var/log/pods
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: kubeauditlogs
hostPath:
path: /var/log/kubernetes/audit
- name: linuxauditlogs
hostPath:
path: /var/log/audit
extraVolumeMounts:
- name: containerdlogs
mountPath: /var/lib/containerd/pod-logs
readOnly: true
- name: podlogs
mountPath: /var/log/pods
readOnly: true
- name: varlogcontainers
mountPath: /var/log/containers
readOnly: true
- name: kubeauditlogs
mountPath: /var/log/kubernetes/audit
readOnly: true
- name: linuxauditlogs
mountPath: /var/log/audit
readOnly: true
resources:
limits:
cpu: 1
memory: 4Gi
requests:
cpu: 1
memory: 1Gi