topic Re: Json data not extracting properly in Splunk Cloud Platform

Json data not extracting properly

chandrasekhar46 — Wed, 17 Jul 2024 10:17:58 GMT

i am trying to upload json file using UI in Splunk cloud and applying settings for parsing as below but data is coming as a single event

[custom_json_sourcetype]
INDEXED_EXTRACTIONS = json
SHOULD_LINEMERGE = false
KV_MODE = json
LINE_BREAKER = },\s*{

please advise correct settings to apply under sourcetypes in web when uploading

here is the data:

{

"sourcetype": "testoracle_sourcetype",

"data": {

"cdb_tbs_check": [

{

"check_error": "",

"check_name": "cdb_tbs_check",

"check_status": "OK",

"current_use_mb": "1355",

"percent_used": "2",

"tablespace_name": "SYSTEM",

"total_physical_all_mb": "65536"

{

"check_error": "",

"check_name": "cdb_tbs_check",

"check_status": "OK",

"current_use_mb": "23596",

"percent_used": "36",

"tablespace_name": "SYSAUX",

"total_physical_all_mb": "65536"

{

"check_error": "",

"check_name": "cdb_tbs_check",

"check_status": "OK",

"current_use_mb": "29",

"percent_used": "0",

"tablespace_name": "UNDOTBS1",

"total_physical_all_mb": "65536"

Re: Json data not extracting properly

chandrasekhar46 — Wed, 17 Jul 2024 10:19:21 GMT

EXAMPLE DATA:

{ "sourcetype": "testoracle_sourcetype", "data": { "cdb_tbs_check": [ { "check_error": "", "check_name": "cdb_tbs_check", "check_status": "OK", "current_use_mb": "1355", "percent_used": "2", "tablespace_name": "SYSTEM", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "cdb_tbs_check", "check_status": "OK", "current_use_mb": "23596", "percent_used": "36", "tablespace_name": "SYSAUX", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "cdb_tbs_check", "check_status": "OK", "current_use_mb": "29", "percent_used": "0", "tablespace_name": "UNDOTBS1", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "cdb_tbs_check", "check_status": "OK", "current_use_mb": "4", "percent_used": "0", "tablespace_name": "USERS", "total_physical_all_mb": "65536" } ], "fra_check": [ { "check_error": "", "check_name": "fra_check", "check_status": "OK", "flash_in_gb": "40", "flash_reclaimable_gb": "0", "flash_used_in_gb": "1.5", "percent_of_space_used": "3.74" } ], "global_parameters": { "check_error": "", "check_name": "General_parameters", "check_status": "OK", "database_major_version": "19", "database_minor_version": "0", "database_name": "C2N48617", "database_version": "19.0.0.0.0", "host_name": "flosclnrhv03.pharma.aventis.com", "instance_name": "C2N48617", "script_version": "1.0" }, "pdb_tbs_check": [ { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "76", "pdb_name": "O1S48633", "percent_used": "0", "tablespace_name": "UNDOTBS1", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "5", "pdb_name": "O1S48633", "percent_used": "0", "tablespace_name": "TOOLS", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "21", "pdb_name": "O1NN2467", "percent_used": "0", "tablespace_name": "UNDOTBS1", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "627", "pdb_name": "O1NN2467", "percent_used": "1", "tablespace_name": "SYSAUX", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "784", "pdb_name": "O1S48633", "percent_used": "1", "tablespace_name": "SYSTEM", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "1547", "pdb_name": "O1NN8944", "percent_used": "2", "tablespace_name": "SYSAUX", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "1149", "pdb_name": "O1S48633", "percent_used": "2", "tablespace_name": "USERS", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "58", "pdb_name": "O1NN8944", "percent_used": "0", "tablespace_name": "UNDOTBS1", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "7804", "pdb_name": "O1S48633", "percent_used": "12", "tablespace_name": "SYSAUX", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "1176", "pdb_name": "O1NN8944", "percent_used": "4", "tablespace_name": "USERS", "total_physical_all_mb": "32767" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "378", "pdb_name": "O1NN8944", "percent_used": "1", "tablespace_name": "INDX", "total_physical_all_mb": "32767" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "705", "pdb_name": "O1NN8944", "percent_used": "1", "tablespace_name": "SYSTEM", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "623", "pdb_name": "O1NN2467", "percent_used": "1", "tablespace_name": "SYSTEM", "total_physical_all_mb": "65536" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "3", "pdb_name": "O1S48633", "percent_used": "0", "tablespace_name": "AUDIT_TBS", "total_physical_all_mb": "8192" }, { "check_error": "", "check_name": "pdb_tbs_check", "check_status": "OK", "current_use_mb": "128", "pdb_name": "O1S48633", "percent_used": "0", "tablespace_name": "USRINDEX", "total_physical_all_mb": "65536" } ], "processes": { "check_error": "", "check_name": "processes", "check_status": "OK", "process_current_value": "294", "process_limit": "1000", "process_percent": "29.4" }, "queue_mem_check": [ { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "AQ$_Q_PIWORKTASK_TAB_E", "queue_owner": "LIVE2459_VAL", "queue_sharable_mem": "4072" }, { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "AQ$_Q_TASKREPORTWORKTASK_TAB_E", "queue_owner": "LIVE2459_VAL", "queue_sharable_mem": "4072" }, { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "AQ$_Q_LABELWORKTASK_TAB_E", "queue_owner": "LIVE2459_VAL", "queue_sharable_mem": "4072" }, { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "AQ$_Q_PIPROCESS_TAB_E", "queue_owner": "LIVE2459_VAL", "queue_sharable_mem": "4072" }, { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "AQ$_ALERT_QT_E", "queue_owner": "SYS", "queue_sharable_mem": "4032" }, { "check_error": "", "check_name": "queue_mem_check", "check_status": "OK", "queue_name": "ALERT_QUE", "queue_owner": "SYS", "queue_sharable_mem": "0" } ], "script_version": "1.0", "sessions": { "check_error": "", "check_name": "sessions", "check_status": "OK", "sessions_current_value": "293", "sessions_limit": "1536", "sessions_percent": "19.08" } } }

Re: Json data not extracting properly

richgalloway — Wed, 17 Jul 2024 12:32:57 GMT

Don't use both INDEXED_EXTRACTIONS = JSON and KV_MODE=json in the same stanza or the fields will be extracted twice.

The LINE_BREAKER setting requires a capture group.

Try these settings

[custom_json_sourcetype] SHOULD_LINEMERGE = false KV_MODE = json LINE_BREAKER = }(,\s*){

Re: Json data not extracting properly

chandrasekhar46 — Thu, 18 Jul 2024 06:00:23 GMT

i have applied but data is events are getting merged in online please check attachments

sorry i have modified the json file and here is it what will the sourcetype settings

[
{
"sourcetype": "testoracle_sourcetype",
"check_name": "cdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "1355",
"percent_used": "2",
"tablespace_name": "SYSTEM",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "cdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "23596",
"percent_used": "36",
"tablespace_name": "SYSAUX",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "cdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "29",
"percent_used": "0",
"tablespace_name": "UNDOTBS1",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "cdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "4",
"percent_used": "0",
"tablespace_name": "USERS",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "fra_check",
"check_error": "",
"check_status": "OK",
"flash_in_gb": "40",
"flash_reclaimable_gb": "0",
"flash_used_in_gb": "1.5",
"percent_of_space_used": "3.74",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "General_parameters",
"check_error": "",
"check_status": "OK",
"database_major_version": "19",
"database_minor_version": "0",
"database_name": "C2N48617",
"database_version": "19.0.0.0.0",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617",
"script_version": "1.0"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "76",
"pdb_name": "O1S48633",
"percent_used": "0",
"tablespace_name": "UNDOTBS1",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "5",
"pdb_name": "O1S48633",
"percent_used": "0",
"tablespace_name": "TOOLS",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "21",
"pdb_name": "O1NN2467",
"percent_used": "0",
"tablespace_name": "UNDOTBS1",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "627",
"pdb_name": "O1NN2467",
"percent_used": "1",
"tablespace_name": "SYSAUX",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "784",
"pdb_name": "O1S48633",
"percent_used": "1",
"tablespace_name": "SYSTEM",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "1547",
"pdb_name": "O1NN8944",
"percent_used": "2",
"tablespace_name": "SYSAUX",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "1149",
"pdb_name": "O1S48633",
"percent_used": "2",
"tablespace_name": "USERS",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "58",
"pdb_name": "O1NN8944",
"percent_used": "0",
"tablespace_name": "UNDOTBS1",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
},
{
"sourcetype": "testoracle_sourcetype",
"check_name": "pdb_tbs_check",
"check_error": "",
"check_status": "OK",
"current_use_mb": "7804",
"pdb_name": "O1S48633",
"percent_used": "12",
"tablespace_name": "SYSAUX",
"total_physical_all_mb": "65536",
"database_name": "C2N48617",
"host_name": "flosclnrhv03.pharma.aventis.com",
"instance_name": "C2N48617"
}
]

Re: Json data not extracting properly

richgalloway — Thu, 18 Jul 2024 12:16:43 GMT

Try this variation on the settings. It should better account for newlines.

[custom_json_sourcetype] SHOULD_LINEMERGE = false KV_MODE = json LINE_BREAKER = }(,[\S\s]*){

Re: Json data not extracting properly

PickleRick — Thu, 18 Jul 2024 12:32:00 GMT

With this data you will have some "bad events" - while you might be able to extract the structures from the middle but you will have some dangling "headers" or "footers". I'd suggest you pass this through some external filter extracting the contents based on structure, not just breaking with regex.