Getting Data In
×

Join the Conversation

Without signing in, you're just watching from the sidelines. Sign in or Register to connect, share, and be part of the Splunk Community.
Ask a Question

Props.conf for JSON File

daniel333
Builder
‎02-21-2016 06:51 PM

All,

Having some trouble with a JSON file field extractions. It’s funny the only extraction I am getting is “PATH” and “HOME”, but nothing else.

here is my props.conf

KV_MODE = json
LINE_BREAKER = ([\n\r]+){
NO_BINARY_CHECK = 1
TRUNCATE = 0
SHOULD_LINEMERGE = false

here is the source. 

[
    {
        "Id": "7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b",
        "Created": "2016-02-16T02:05:02.34848574Z",
        "Path": "/helloworld",
        "Args": [],
        "State": {
            "Status": "exited",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": 2,
            "Error": "",
            "StartedAt": "2016-02-16T02:05:02.601829354Z",
            "FinishedAt": "2016-02-22T01:05:06.469761519Z"
        },
        "Image": "sha256:4fa84a96f0d641a79ad7574fd75eabee71e93095fb35af9c30e9b59e3269206d",
        "ResolvConfPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hostname",
        "HostsPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hosts",
        "LogPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b-json.log",
        "Name": "/amazing_chandrasekhar",
        "RestartCount": 0,
        "Driver": "devicemapper",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "default",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "ShmSize": 67108864,
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "KernelMemory": 0,
            "Memory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": -1,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null
        },
        "GraphDriver": {
            "Name": "devicemapper",
            "Data": {
                "DeviceId": "34",
                "DeviceName": "docker-8:3-138887304-c257e534edd9a476815162ad68ba12c38a29076de1bc0e7443ab6d3f2b01edd0",
                "DeviceSize": "10737418240"
            }
        },
        "Mounts": [],
        "Config": {
            "Hostname": "7284c626c33d",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "HOME=/",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "/helloworld"
            ],
            "Image": "adejonge/helloworld",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {},
            "StopSignal": "SIGTERM"
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "de93e0972ae55ac119ab75d2e68efd5a594c3e3b7ec7bd5a3c664ba237a8ac93",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "/var/run/docker/netns/de93e0972ae5",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "bridge": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": null,
                    "NetworkID": "68597bc306a458587665bbf4d8cd71465f49879a90ccabf016c08e4dc1a8fa9d",
                    "EndpointID": "",
                    "Gateway": "",
                    "IPAddress": "",
                    "IPPrefixLen": 0,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": ""
                }
            }
        }
    }
]



[
    {
        "Id": "7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b",
        "Created": "2016-02-16T02:05:02.34848574Z",
        "Path": "/helloworld",
        "Args": [],
        "State": {
            "Status": "exited",
            "Running": false,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 0,
            "ExitCode": 2,
            "Error": "",
            "StartedAt": "2016-02-16T02:05:02.601829354Z",
            "FinishedAt": "2016-02-22T01:05:06.469761519Z"
        },
        "Image": "sha256:4fa84a96f0d641a79ad7574fd75eabee71e93095fb35af9c30e9b59e3269206d",
        "ResolvConfPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hostname",
        "HostsPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/hosts",
        "LogPath": "/var/lib/docker/containers/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b/7284c626c33d5b064452423f97ea300ff55c52336313d75a1cc06653b29e260b-json.log",
        "Name": "/amazing_chandrasekhar",
        "RestartCount": 0,
        "Driver": "devicemapper",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": null,
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "default",
            "PortBindings": {},
            "RestartPolicy": {
                "Name": "no",
                "MaximumRetryCount": 0
            },
            "VolumeDriver": "",
            "VolumesFrom": null,
            "CapAdd": null,
            "CapDrop": null,
            "Dns": [],
            "DnsOptions": [],
            "DnsSearch": [],
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "ShmSize": 67108864,
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": [],
            "KernelMemory": 0,
            "Memory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": -1,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null
        },
        "GraphDriver": {
            "Name": "devicemapper",
            "Data": {
                "DeviceId": "34",
                "DeviceName": "docker-8:3-138887304-c257e534edd9a476815162ad68ba12c38a29076de1bc0e7443ab6d3f2b01edd0",
                "DeviceSize": "10737418240"
            }
        },
        "Mounts": [],
        "Config": {
            "Hostname": "7284c626c33d",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "HOME=/",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
            ],
            "Cmd": [
                "/helloworld"
            ],
            "Image": "adejonge/helloworld",
            "Volumes": null,
            "WorkingDir": "",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {},
            "StopSignal": "SIGTERM"
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "de93e0972ae55ac119ab75d2e68efd5a594c3e3b7ec7bd5a3c664ba237a8ac93",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": null,
            "SandboxKey": "/var/run/docker/netns/de93e0972ae5",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "bridge": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": null,
                    "NetworkID": "68597bc306a458587665bbf4d8cd71465f49879a90ccabf016c08e4dc1a8fa9d",
                    "EndpointID": "",
                    "Gateway": "",
                    "IPAddress": "",
                    "IPPrefixLen": 0,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": ""
                }
            }
        }
    }
]
0 Karma
Reply

fdi01
Motivator
‎02-22-2016 05:11 AM

try like :

[you_index_name]
CHARSET = UTF-8
NO_BINARY_CHECK = 1
TIME_FORMAT = %a %b %d %H:%M:%S %z %Y
TIME_PREFIX = "__time":"
MAX_TIMESTAMP_LOOKAHEAD = 150
SHOULD_LINEMERGE = false
TZ = UTC
KV_MODE = json
0 Karma
Reply

jeffland
SplunkTrust
SplunkTrust
‎02-22-2016 04:49 AM

I don't notice anything wrong when using the standard json sourcetype settings. Why did you specify explicit line breaking options? If you have to, use this one:

\]([\s]+)\[

Yours doesn't match the data.
Have you confirmed standard timestamp extraction works with your props.conf?

0 Karma
Reply
Got questions? Get answers!

Join the Splunk Community Slack to learn, troubleshoot, and make connections with fellow Splunk practitioners in real time!

Meet up IRL or virtually!

Join Splunk User Groups to connect and learn in-person by region or remotely by topic or industry.

Get Updates on the Splunk Community!

Data Outage Monitoring and Reporting

We’ve all been there, right? Living our best life, working on our Splunk environment, when out of nowhere a ...

Manage Datasets and Connections in Catalog on Splunk Cloud Platform

Set Up Data Where You Use It Catalog already helps teams find and understand data across Splunk indexes, ...

Persistent Queue at TcpOut — One of Splunk's Most Practical Features

Splunk introduced persistent queueing at the tcpout layer as one of the most practical resilience features in ...