Getting Data In

extract fields from json array with multivalue and sub-array

Contributor

Here is my sample data

{
  "applications": [
    {
      "id": 2537302,
      "name": "addressdb_prod",
      "language": "dotnet",
      "health_status": "unknown",
      "reporting": true,
      "last_reported_at": "2017-08-03T10:41:52+00:00",
      "application_summary": {
        "response_time": 5.04,
        "throughput": 32.3,
        "error_rate": 0.0,
        "apdex_target": 0.5,
        "apdex_score": 1.0,
        "host_count": 2,
        "instance_count": 2
      },
      "settings": {
        "app_apdex_threshold": 0.5,
        "end_user_apdex_threshold": 7.0,
        "enable_real_user_monitoring": true,
        "use_server_side_config": false
      },
      "links": {
        "application_instances": [
          2536708,
          2561561
        ],
        "servers": [
          25610567,
          22686416
        ],
        "application_hosts": [
          2536699,
          2561553
        ]
      }
    },
    {
      "id": 2537304,
      "name": "addressdb_trn",
      "language": "dotnet",
      "health_status": "unknown",
      "reporting": true,
      "last_reported_at": "2017-08-03T10:41:47+00:00",
      "application_summary": {
        "response_time": 4.93,
        "throughput": 32.0,
        "error_rate": 0.0,
        "apdex_target": 0.5,
        "apdex_score": 1.0,
        "host_count": 2,
        "instance_count": 2
      },
      "settings": {
        "app_apdex_threshold": 0.5,
        "end_user_apdex_threshold": 7.0,
        "enable_real_user_monitoring": true,
        "use_server_side_config": false
      },
      "links": {
        "application_instances": [
          2536709,
          2561556
        ],
        "servers": [
          25610567,
          22686416
        ],
        "application_hosts": [
          2536699,
          2561553
        ]
      }
    },
    {
      "id": 2268424,
      "name": "All unnamed",
      "language": "dotnet",
      "health_status": "unknown",
      "reporting": true,
      "last_reported_at": "2017-08-03T10:41:52+00:00",
      "application_summary": {
        "response_time": 5.31,
        "throughput": 276.0,
        "error_rate": 0.0,
        "apdex_target": 0.5,
        "apdex_score": 1.0,
        "host_count": 7,
        "instance_count": 17
      },
      "settings": {
        "app_apdex_threshold": 0.5,
        "end_user_apdex_threshold": 7.0,
        "enable_real_user_monitoring": true,
        "use_server_side_config": false
      },
      "links": {
        "application_instances": [
          38654227,
          15703550,
          47088155,
          45753664,
          19172730,
          2561560,
          19335897,
          3249555,
          14322182,
          14322163,
          3249553,
          2536710,
          2561555,
          32941739,
          32941738
        ],
        "servers": [
          14652052,
          13742329,
          47086419,
          45753649,
          22686416,
          2512533,
          25610567
        ],
        "application_hosts": [
          19172728,
          2545902,
          47088154,
          45753663,
          2561553,
          2474911,
          2492596,
          2536699
        ]
      }
    },
    {
      "id": 2569403,
      "name": "allaboutme_dev",
      "language": "dotnet",
      "health_status": "gray",
      "reporting": false,
      "settings": {
        "app_apdex_threshold": 0.5,
        "end_user_apdex_threshold": 7.0,
        "enable_real_user_monitoring": true,
        "use_server_side_config": false
      },
      "links": {
        "application_instances": [],
        "servers": [],
        "application_hosts": []
      }
    },
    {
      "id": 17088365,
      "name": "api_cop_dev_emos",
      "language": "dotnet",
      "health_status": "gray",
      "reporting": false,
      "settings": {
        "app_apdex_threshold": 0.5,
        "end_user_apdex_threshold": 7.0,
        "enable_real_user_monitoring": true,
        "use_server_side_config": false
      },
      "links": {
        "application_instances": [],
        "servers": [],
        "application_hosts": []
      }
    }
    ],
  "links": {
    "application.servers": "/v2/servers?ids={server_ids}",
    "application.server": "/v2/servers/{server_id}",
    "application.application_hosts": "/v2/application/{application_id}/hosts?ids={host_ids}",
    "application.application_host": "/v2/application/{application_id}/hosts/{host_id}",
    "application.application_instances": "/v2/application/{application_id}/instances?ids={instance_ids}",
    "application.application_instance": "/v2/application/{application_id}/instances/{instance_id}",
    "application.alert_policy": "/v2/alert_policies/{alert_policy_id}"
  }
}

I tried to follow the link
https://answers.splunk.com/answers/289520/how-to-split-a-json-array-into-multiple-events-wit.html

with configs in my props.conf file

[json_no_timestamp:nrc]
SHOULD_LINEMERGE=false
LINE_BREAKER=((?<!"),|[\r\n]+)
SEDCMD-remove_prefix=s/{"applications":\[//g
SEDCMD-remove_suffix=s/\]}//g

However, it does not seem to work. I assume, most likely, the regex is not correct. Venturing spath options as well.

Any help will be appreciated.

0 Karma

Path Finder

assuming you want to break before "id": try this

LINE_BREAKER=\{\s+\"id\"\:

And maybe try to start first without the SEDCMD-removeprefix and SEDCMD-removesuffix.

0 Karma

Contributor

Thanks for your prompt response, christianhuber. I tried what you suggested, it does not work. Apparently, it parses every single line as an event. That is not the intention.

An event should start at
{
"id":

and finish at before the start of next
{
"id:"

0 Karma

SplunkTrust
SplunkTrust

Your props.conf has an error:

SHOULD_LINEMERGE=true

That may not be all that is wrong, but it definitely should be true instead of false.

0 Karma

Contributor

SHOULD_LINEMERGE=true does not make any difference. Still every single is parsed as a single event.

0 Karma

Path Finder

Hi,

try to see at the documentation http://docs.splunk.com/Documentation/Splunk/6.6.2/Data/Configureeventlinebreaking. I use for my json sources the following configuration.
[source::jsoninput]
KV
MODE = json
TRUNCATE = 0
BREAKONLYBEFOREDATE = false
LINE
BREAKER = **** your Line_Breaker ****

According to the Documentation this is not really correct, as it says with shouldlinemerge=true (which is default, so you don't have to add it) you should use BREAKONLYBEFORE instead of LINEBREAKER. Anyway for me it works. With the configuration above.

The Truncate is only to asure that splunk doesn't truncate larger json arrays.

0 Karma