Getting Data In
×

Join the Conversation

Without signing in, you're just watching from the sidelines. Sign in or Register to connect, share, and be part of the Splunk Community.
Ask a Question

Props.conf for JSON File

daniel333
Builder
‎02-21-2016 06:51 PM

All,

Having some trouble with a JSON file field extractions. It’s funny the only extraction I am getting is “PATH” and “HOME”, but nothing else.

here is my props.conf

KV_MODE = json
LINE_BREAKER = ([\n\r]+){
NO_BINARY_CHECK = 1
TRUNCATE = 0
SHOULD_LINEMERGE = false

here is the source. 

[
    {
        "Id": "7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b",
        "Created": "2016-02-16T02:05:02.34848574Z",
        "Path": "/helloworld",
        "Args": [],
        "State": {
            "Status": "exited",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": 2,
            "Error": "",
            "StartedAt": "2016-02-16T02:05:02.601829354Z",
            "FinishedAt": "2016-02-22T01:05:06.469761519Z"
        },
        "Image": "sha256:4fa84a96f0d641a79ad7574fd75eabee71e93095fb35af9c30e9b59e3269206d",
        "ResolvConfPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hostname",
        "HostsPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hosts",
        "LogPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b-json.log",
        "Name": "/amazing_chandrasekhar",
        "RestartCount": 0,
        "Driver": "devicemapper",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "default",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "ShmSize": 67108864,
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "KernelMemory": 0,
            "Memory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": -1,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null
        },
        "GraphDriver": {
            "Name": "devicemapper",
            "Data": {
                "DeviceId": "34",
                "DeviceName": "docker-8:3-138887304-c257e534edd9a476815162ad68ba12c38a29076de1bc0e7443ab6d3f2b01edd0",
                "DeviceSize": "10737418240"
            }
        },
        "Mounts": [],
        "Config": {
            "Hostname": "7284c626c33d",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "HOME=/",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "/helloworld"
            ],
            "Image": "adejonge/helloworld",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {},
            "StopSignal": "SIGTERM"
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "de93e0972ae55ac119ab75d2e68efd5a594c3e3b7ec7bd5a3c664ba237a8ac93",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "/var/run/docker/netns/de93e0972ae5",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "bridge": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": null,
                    "NetworkID": "68597bc306a458587665bbf4d8cd71465f49879a90ccabf016c08e4dc1a8fa9d",
                    "EndpointID": "",
                    "Gateway": "",
                    "IPAddress": "",
                    "IPPrefixLen": 0,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": ""
                }
            }
        }
    }
]



[
    {
        "Id": "7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b",
        "Created": "2016-02-16T02:05:02.34848574Z",
        "Path": "/helloworld",
        "Args": [],
        "State": {
            "Status": "exited",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": 2,
            "Error": "",
            "StartedAt": "2016-02-16T02:05:02.601829354Z",
            "FinishedAt": "2016-02-22T01:05:06.469761519Z"
        },
        "Image": "sha256:4fa84a96f0d641a79ad7574fd75eabee71e93095fb35af9c30e9b59e3269206d",
        "ResolvConfPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hostname",
        "HostsPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hosts",
        "LogPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b-json.log",
        "Name": "/amazing_chandrasekhar",
        "RestartCount": 0,
        "Driver": "devicemapper",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "default",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "ShmSize": 67108864,
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "KernelMemory": 0,
            "Memory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": -1,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null
        },
        "GraphDriver": {
            "Name": "devicemapper",
            "Data": {
                "DeviceId": "34",
                "DeviceName": "docker-8:3-138887304-c257e534edd9a476815162ad68ba12c38a29076de1bc0e7443ab6d3f2b01edd0",
                "DeviceSize": "10737418240"
            }
        },
        "Mounts": [],
        "Config": {
            "Hostname": "7284c626c33d",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "HOME=/",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "/helloworld"
            ],
            "Image": "adejonge/helloworld",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {},
            "StopSignal": "SIGTERM"
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "de93e0972ae55ac119ab75d2e68efd5a594c3e3b7ec7bd5a3c664ba237a8ac93",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "/var/run/docker/netns/de93e0972ae5",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "bridge": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": null,
                    "NetworkID": "68597bc306a458587665bbf4d8cd71465f49879a90ccabf016c08e4dc1a8fa9d",
                    "EndpointID": "",
                    "Gateway": "",
                    "IPAddress": "",
                    "IPPrefixLen": 0,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": ""
                }
            }
        }
    }
]
0 Karma
Reply

fdi01
Motivator
‎02-22-2016 05:11 AM

try like :

[you_index_name]
CHARSET = UTF-8
NO_BINARY_CHECK = 1
TIME_FORMAT = %a %b %d %H:%M:%S %z %Y
TIME_PREFIX = "__time":"
MAX_TIMESTAMP_LOOKAHEAD = 150
SHOULD_LINEMERGE = false
TZ = UTC
KV_MODE = json
0 Karma
Reply

jeffland
SplunkTrust
SplunkTrust
‎02-22-2016 04:49 AM

I don't notice anything wrong when using the standard json sourcetype settings. Why did you specify explicit line breaking options? If you have to, use this one:

\]([\s]+)\[

Yours doesn't match the data.
Have you confirmed standard timestamp extraction works with your props.conf?

0 Karma
Reply
Got questions? Get answers!

Join the Splunk Community Slack to learn, troubleshoot, and make connections with fellow Splunk practitioners in real time!

Meet up IRL or virtually!

Join Splunk User Groups to connect and learn in-person by region or remotely by topic or industry.

Get Updates on the Splunk Community!

Best Practices: Splunk auto adjust pipeline queue

When you enable autoAdjustQueue in Splunk, maxSize should be understood as the queue size Splunk starts with ...

Laser Bananas and Edge Hubs: Exploring Operational Technology (OT) Data Through a ...

  OT is a different environment to traditional IT and can have interesting challenges when interfacing the ...

Event Series: Mastering AI Tokenomics and Splunk Agent Observability

Beyond the Black Box: Correlating AI Performance and Tokenomics with Splunk Agent Observability   As ...