topic Props for Splitting JSON data as a separate event in Getting Data In

Props for Splitting JSON data as a separate event

Kothandapanin — Thu, 18 Mar 2021 14:29:23 GMT

Here is the JSON data and looking for Props settings for splitting the event based on "Level:4" as the correlation ID mentioned is a unique transaction.

Esisting Props:

[mscs:azure:eventhub:vad01_apim_qa]
SHOULD_LINEMERGE = false
INDEXED_EXTRACTIONS = JSON
KV_MODE = none
TRUNCATE = 0

JSON Data:

{
"body":{
"records":[
{
"Level":4,
"isRequestSuccess":true,
"time":"2021-03-18T12:20:48.7984746Z",
"operationName":"Microsoft.ApiManagement/GatewayLogs",
"category":"GatewayLogs",
"durationMs":35,
"callerIpAddress":"10.244.53.200",
"correlationId":"8c5bb044-db88-47fd-9fc6-997959ac1aae",
"location":"East US 2",
"properties":{
"method":"GET",
"url":"https://easyapiqa-corp-azure.staples.com/nephos/gp-api-tierc-store-locator/v1/storelocator/storeSearch?limit=10&radius=50&storeNumber=0193&locale=en_US",
"backendResponseCode":200,
"responseCode":200,
"responseSize":2680,
"cache":"none",
"backendTime":30,
"requestSize":1878,
"apiId":"nephos",
"operationId":"get-call",
"productId":"nephos-pni",
"apimSubscriptionId":"PNI",
"clientTime":4,
"clientProtocol":"HTTP/1.1",
"backendProtocol":"HTTP/1.1",
"apiRevision":"3",
"clientTlsVersion":"1.2",
"responseHeaders":{
"X-B3-TraceId":"f884152944db8c96"
},
"backendMethod":"GET",
"backendUrl":"https://nephos-qa-open.staples.com/gp-api-tierc-store-locator/v1/storelocator/storeSearch?limit=10&radius=50&storeNumber=0193&locale=en_US",
"requestHeaders":{
"X-Forwarded-For":"199.10.28.20,204.2.136.181,23.215.131.118:42751",
"X-Azure-Client-Id":"567e713e-e278-4cdd-a90a-653e1de87e0d",
"Ocp-Apim-Subscription-Key":"431e8630d74b45eca5921fec2f21f08f"
},
"backendResponseHeaders":{
"X-B3-TraceId":"f884152944db8c96"
},
"backendRequestHeaders":{
"X-Forwarded-For":"199.10.28.20,204.2.136.181,23.215.131.118:42751",
"X-Azure-Client-Id":"567e713e-e278-4cdd-a90a-653e1de87e0d",
"Ocp-Apim-Subscription-Key":"431e8630d74b45eca5921fec2f21f08f"
},
"responseBody":"{\"staplesURL\":\"//www.staples.com\",\"results\":{\"status\":\"SUCCESS\",\"stores\":[{\"storeNumber\":\"0193\",\"storeTitle\":\"Staples 0193 - Store 0193 - 217 Broadway (Ves\",\"latitude\":40.7116,\"longitude\":-74.0087,\"storeDivision\":\"V1\",\"storeRegion\":\"R03\",\"emailCopyCenter\":\"print.marketing0193@Staples.com\",\"address\":{\"addressLine1\":\"217 Broadway (Vesey Street)\",\"city\":\"New York\",\"state\":\"NY\",\"zipcode\":\"10007\",\"country\":\"USA\",\"phoneNumber\":\"2123469624\",\"faxNumber\":\"2123469633\"},\"featureVOs\":[{\"featureName\":\"ISP\",\"featureLabel\":\"Buy online. Pickup in store\",\"featureTooltip\":\"We will have your online order ready at your local store within one hour. See an associate for details.\"},{\"featureName\":\"TS\",\"featureLabel\":\"Technology Services\",\"featureTooltip\":\"Complete technology solutions for your home or office from our certified techs.\"},{\"featureName\":\"F1\",\"featureLabel\":\"Computer Workstation\",\"featureTooltip\":\"Private rental areas with Microsoft® Office, Internet access, and printing capabilities.\"},{\"featureName\":\"STS\",\"featureLabel\":\"Ship to Store\",\"featureTooltip\":\"Free shipping to your local Staples® store when you order online.\"},{\"featureName\":\"CPC\",\"featureLabel\":\"Print & Marketing Services\",\"featureTooltip\":\"Print services ranging from presentations & business cards to signs & banners with expert advice from our Certified Print Pros.\"},{\"featureName\":\"UPS\",\"featureLabel\":\"UPS® Prepaid Drop-off\",\"featureTooltip\":\"Drop off any UPS® prepaid package for shipment.\"},{\"featureName\":\"MPR\",\"featureLabel\":\"Mobile Printing\",\"featureTooltip\":\"Ability to accept documents submitted through a web enabled mobile device.\"},{\"featureName\":\"FSU\",\"featureLabel\":\"Full-service UPS® Shipping\",\"featureTooltip\":\"Easy shipping with UPS and up to 5% back in Staples Rewards. All Staples stores accept UPS prepaid drop-off packages\"}],\"workingHourVOs\":[{\"day\":\"THU\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"TUS\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"WED\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SAT\",\"openTime\":\"0900\",\"closeTime\":\"1800\"},{\"day\":\"FRI\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SUN\",\"openTime\":\"1100\",\"closeTime\":\"1800\"},{\"day\":\"MON\",\"openTime\":\"0800\",\"closeTime\":\"1900\"}]}],\"count\":1}}",
"backendResponseBody":"{\"staplesURL\":\"//www.staples.com\",\"results\":{\"status\":\"SUCCESS\",\"stores\":[{\"storeNumber\":\"0193\",\"storeTitle\":\"Staples 0193 - Store 0193 - 217 Broadway (Ves\",\"latitude\":40.7116,\"longitude\":-74.0087,\"storeDivision\":\"V1\",\"storeRegion\":\"R03\",\"emailCopyCenter\":\"print.marketing0193@Staples.com\",\"address\":{\"addressLine1\":\"217 Broadway (Vesey Street)\",\"city\":\"New York\",\"state\":\"NY\",\"zipcode\":\"10007\",\"country\":\"USA\",\"phoneNumber\":\"2123469624\",\"faxNumber\":\"2123469633\"},\"featureVOs\":[{\"featureName\":\"ISP\",\"featureLabel\":\"Buy online. Pickup in store\",\"featureTooltip\":\"We will have your online order ready at your local store within one hour. See an associate for details.\"},{\"featureName\":\"TS\",\"featureLabel\":\"Technology Services\",\"featureTooltip\":\"Complete technology solutions for your home or office from our certified techs.\"},{\"featureName\":\"F1\",\"featureLabel\":\"Computer Workstation\",\"featureTooltip\":\"Private rental areas with Microsoft® Office, Internet access, and printing capabilities.\"},{\"featureName\":\"STS\",\"featureLabel\":\"Ship to Store\",\"featureTooltip\":\"Free shipping to your local Staples® store when you order online.\"},{\"featureName\":\"CPC\",\"featureLabel\":\"Print & Marketing Services\",\"featureTooltip\":\"Print services ranging from presentations & business cards to signs & banners with expert advice from our Certified Print Pros.\"},{\"featureName\":\"UPS\",\"featureLabel\":\"UPS® Prepaid Drop-off\",\"featureTooltip\":\"Drop off any UPS® prepaid package for shipment.\"},{\"featureName\":\"MPR\",\"featureLabel\":\"Mobile Printing\",\"featureTooltip\":\"Ability to accept documents submitted through a web enabled mobile device.\"},{\"featureName\":\"FSU\",\"featureLabel\":\"Full-service UPS® Shipping\",\"featureTooltip\":\"Easy shipping with UPS and up to 5% back in Staples Rewards. All Staples stores accept UPS prepaid drop-off packages\"}],\"workingHourVOs\":[{\"day\":\"THU\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"TUS\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"WED\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SAT\",\"openTime\":\"0900\",\"closeTime\":\"1800\"},{\"day\":\"FRI\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SUN\",\"openTime\":\"1100\",\"closeTime\":\"1800\"},{\"day\":\"MON\",\"openTime\":\"0800\",\"closeTime\":\"1900\"}]}],\"count\":1}}"
},
"resourceId":"/SUBSCRIPTIONS/B00FC482-62DC-49BF-BBD6-9B4CE971B3DB/RESOURCEGROUPS/VAD01_APIM_QE_0/PROVIDERS/MICROSOFT.APIMANAGEMENT/SERVICE/VAD01QEAPIM01"
},
{
"Level":4,
"isRequestSuccess":true,
"time":"2021-03-18T12:20:50.8832104Z",
"operationName":"Microsoft.ApiManagement/GatewayLogs",
"category":"GatewayLogs",
"durationMs":29,
"callerIpAddress":"10.244.53.200",
"correlationId":"a4f739b5-6502-4725-8fe1-31bddb1bade1",
"location":"East US 2",
"properties":{
"method":"GET",
"url":"https://easyapiqa-corp-azure.staples.com/nephos/gp-api-tierc-store-locator/v1/storelocator/storeSearch?limit=10&radius=50&storeNumber=0193&locale=en_US",
"backendResponseCode":200,
"responseCode":200,
"responseSize":2680,
"cache":"none",
"backendTime":27,
"requestSize":1878,
"apiId":"nephos",
"operationId":"get-call",
"productId":"nephos-pni",
"apimSubscriptionId":"PNI",
"clientTime":1,
"clientProtocol":"HTTP/1.1",
"backendProtocol":"HTTP/1.1",
"apiRevision":"3",
"clientTlsVersion":"1.2",
"responseHeaders":{
"X-B3-TraceId":"43bde31fa5060147"
},
"backendMethod":"GET",
"backendUrl":"https://nephos-qa-open.staples.com/gp-api-tierc-store-locator/v1/storelocator/storeSearch?limit=10&radius=50&storeNumber=0193&locale=en_US",
"requestHeaders":{
"X-Forwarded-For":"199.10.28.20,63.217.232.36,23.215.131.118:42751",
"X-Azure-Client-Id":"567e713e-e278-4cdd-a90a-653e1de87e0d",
"Ocp-Apim-Subscription-Key":"431e8630d74b45eca5921fec2f21f08f"
},
"backendResponseHeaders":{
"X-B3-TraceId":"43bde31fa5060147"
},
"backendRequestHeaders":{
"X-Forwarded-For":"199.10.28.20,63.217.232.36,23.215.131.118:42751",
"X-Azure-Client-Id":"567e713e-e278-4cdd-a90a-653e1de87e0d",
"Ocp-Apim-Subscription-Key":"431e8630d74b45eca5921fec2f21f08f"
},
"responseBody":"{\"staplesURL\":\"//www.staples.com\",\"results\":{\"status\":\"SUCCESS\",\"stores\":[{\"storeNumber\":\"0193\",\"storeTitle\":\"Staples 0193 - Store 0193 - 217 Broadway (Ves\",\"latitude\":40.7116,\"longitude\":-74.0087,\"storeDivision\":\"V1\",\"storeRegion\":\"R03\",\"emailCopyCenter\":\"print.marketing0193@Staples.com\",\"address\":{\"addressLine1\":\"217 Broadway (Vesey Street)\",\"city\":\"New York\",\"state\":\"NY\",\"zipcode\":\"10007\",\"country\":\"USA\",\"phoneNumber\":\"2123469624\",\"faxNumber\":\"2123469633\"},\"featureVOs\":[{\"featureName\":\"ISP\",\"featureLabel\":\"Buy online. Pickup in store\",\"featureTooltip\":\"We will have your online order ready at your local store within one hour. See an associate for details.\"},{\"featureName\":\"TS\",\"featureLabel\":\"Technology Services\",\"featureTooltip\":\"Complete technology solutions for your home or office from our certified techs.\"},{\"featureName\":\"F1\",\"featureLabel\":\"Computer Workstation\",\"featureTooltip\":\"Private rental areas with Microsoft® Office, Internet access, and printing capabilities.\"},{\"featureName\":\"STS\",\"featureLabel\":\"Ship to Store\",\"featureTooltip\":\"Free shipping to your local Staples® store when you order online.\"},{\"featureName\":\"CPC\",\"featureLabel\":\"Print & Marketing Services\",\"featureTooltip\":\"Print services ranging from presentations & business cards to signs & banners with expert advice from our Certified Print Pros.\"},{\"featureName\":\"UPS\",\"featureLabel\":\"UPS® Prepaid Drop-off\",\"featureTooltip\":\"Drop off any UPS® prepaid package for shipment.\"},{\"featureName\":\"MPR\",\"featureLabel\":\"Mobile Printing\",\"featureTooltip\":\"Ability to accept documents submitted through a web enabled mobile device.\"},{\"featureName\":\"FSU\",\"featureLabel\":\"Full-service UPS® Shipping\",\"featureTooltip\":\"Easy shipping with UPS and up to 5% back in Staples Rewards. All Staples stores accept UPS prepaid drop-off packages\"}],\"workingHourVOs\":[{\"day\":\"THU\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"TUS\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"WED\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SAT\",\"openTime\":\"0900\",\"closeTime\":\"1800\"},{\"day\":\"FRI\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SUN\",\"openTime\":\"1100\",\"closeTime\":\"1800\"},{\"day\":\"MON\",\"openTime\":\"0800\",\"closeTime\":\"1900\"}]}],\"count\":1}}",
"backendResponseBody":"{\"staplesURL\":\"//www.staples.com\",\"results\":{\"status\":\"SUCCESS\",\"stores\":[{\"storeNumber\":\"0193\",\"storeTitle\":\"Staples 0193 - Store 0193 - 217 Broadway (Ves\",\"latitude\":40.7116,\"longitude\":-74.0087,\"storeDivision\":\"V1\",\"storeRegion\":\"R03\",\"emailCopyCenter\":\"print.marketing0193@Staples.com\",\"address\":{\"addressLine1\":\"217 Broadway (Vesey Street)\",\"city\":\"New York\",\"state\":\"NY\",\"zipcode\":\"10007\",\"country\":\"USA\",\"phoneNumber\":\"2123469624\",\"faxNumber\":\"2123469633\"},\"featureVOs\":[{\"featureName\":\"ISP\",\"featureLabel\":\"Buy online. Pickup in store\",\"featureTooltip\":\"We will have your online order ready at your local store within one hour. See an associate for details.\"},{\"featureName\":\"TS\",\"featureLabel\":\"Technology Services\",\"featureTooltip\":\"Complete technology solutions for your home or office from our certified techs.\"},{\"featureName\":\"F1\",\"featureLabel\":\"Computer Workstation\",\"featureTooltip\":\"Private rental areas with Microsoft® Office, Internet access, and printing capabilities.\"},{\"featureName\":\"STS\",\"featureLabel\":\"Ship to Store\",\"featureTooltip\":\"Free shipping to your local Staples® store when you order online.\"},{\"featureName\":\"CPC\",\"featureLabel\":\"Print & Marketing Services\",\"featureTooltip\":\"Print services ranging from presentations & business cards to signs & banners with expert advice from our Certified Print Pros.\"},{\"featureName\":\"UPS\",\"featureLabel\":\"UPS® Prepaid Drop-off\",\"featureTooltip\":\"Drop off any UPS® prepaid package for shipment.\"},{\"featureName\":\"MPR\",\"featureLabel\":\"Mobile Printing\",\"featureTooltip\":\"Ability to accept documents submitted through a web enabled mobile device.\"},{\"featureName\":\"FSU\",\"featureLabel\":\"Full-service UPS® Shipping\",\"featureTooltip\":\"Easy shipping with UPS and up to 5% back in Staples Rewards. All Staples stores accept UPS prepaid drop-off packages\"}],\"workingHourVOs\":[{\"day\":\"THU\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"TUS\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"WED\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SAT\",\"openTime\":\"0900\",\"closeTime\":\"1800\"},{\"day\":\"FRI\",\"openTime\":\"0800\",\"closeTime\":\"1900\"},{\"day\":\"SUN\",\"openTime\":\"1100\",\"closeTime\":\"1800\"},{\"day\":\"MON\",\"openTime\":\"0800\",\"closeTime\":\"1900\"}]}],\"count\":1}}"
},
"resourceId":"/SUBSCRIPTIONS/B00FC482-62DC-49BF-BBD6-9B4CE971B3DB/RESOURCEGROUPS/VAD01_APIM_QE_0/PROVIDERS/MICROSOFT.APIMANAGEMENT/SERVICE/VAD01QEAPIM01"
}
]
},
"x-opt-sequence-number":16240,
"x-opt-offset":"236247275192",
"x-opt-enqueued-time":1616070109930
}

Re: Props for Splitting JSON data as a separate event

scelikok — Thu, 18 Mar 2021 19:58:00 GMT

Hi @Kothandapanin,

Please try below props;

[mscs:azure:eventhub:vad01_apim_qa] SHOULD_LINEMERGE=true LINE_BREAKER=(?:(\,)?([\r\n]+)\{([\r\n]+)\"Level\":4)|((?:[\r\n]+)\][^$]+) NO_BINARY_CHECK=true KV_MODE=json TRUNCATE = 0

Re: Props for Splitting JSON data as a separate event

Kothandapanin — Wed, 24 Mar 2021 07:03:54 GMT

Tried as per below and doesn't seems working as expected. Do we need to try with SEDCMD and break the events before ingesting?

Re: Props for Splitting JSON data as a separate event

Kothandapanin — Wed, 24 Mar 2021 07:56:11 GMT

@scelikok,

Kindly assist

Re: Props for Splitting JSON data as a separate event

scelikok — Wed, 24 Mar 2021 08:02:45 GMT

You don't need to cut before ingestion. Actually I tested the same way you did and copied the config to you.

There should be something different with your sample data or some other setting. Can you please send the full config for your sourcetype by clicking "Copy to clipboard" button on your screenshot?

Re: Props for Splitting JSON data as a separate event

Kothandapanin — Fri, 26 Mar 2021 10:09:11 GMT

Here is the data for parsing:

{"body":{"records": [{ "Level": 4, "isRequestSuccess": false, "time": "2021-03-24T06:36:00.9756941Z", "operationName": "Microsoft.ApiManagement/GatewayLogs", "category": "GatewayLogs", "durationMs": 0, "callerIpAddress": "10.244.53.203", "correlationId": "e1788e72-4a8b-4350-bae4-1a7909076961", "location": "East US 2", "properties": {"method":"GET","url":"https://easyapiqa-corp-azure.staples.com/sbd/cre/akamai/test.html","responseCode":404,"responseSize":198,"cache":"none","apiId":"sabillto","clientProtocol":"HTTP/1.1","apiRevision":"1","clientTlsVersion":"1.2","lastError":{"source":"configuration","reason":"OperationNotFound","message":"Unable to match incoming request to an operation.","section":"backend"},"requestHeaders":{"X-Forwarded-For":"127.0.0.1,23.213.54.244,23.215.131.118:38211"},"responseBody":"{ \"statusCode\": 404, \"message\": \"Resource not found\" }"}, "resourceId": "/SUBSCRIPTIONS/B00FC482-62DC-49BF-BBD6-9B4CE971B3DB/RESOURCEGROUPS/VAD01_APIM_QE_0/PROVIDERS/MICROSOFT.APIMANAGEMENT/SERVICE/VAD01QEAPIM01"},{ "Level": 4, "isRequestSuccess": false, "time": "2021-03-24T06:36:01.0956958Z", "operationName": "Microsoft.ApiManagement/GatewayLogs", "category": "GatewayLogs", "durationMs": 0, "callerIpAddress": "10.244.53.200", "correlationId": "53c90103-aa38-4161-b287-122d25505942", "location": "East US 2", "properties": {"method":"GET","url":"https://easyapiqa-corp-azure.staples.com/sbd/cre/akamai/test.html","responseCode":404,"responseSize":198,"cache":"none","apiId":"sabillto","clientProtocol":"HTTP/1.1","apiRevision":"1","clientTlsVersion":"1.2","lastError":{"source":"configuration","reason":"OperationNotFound","message":"Unable to match incoming request to an operation.","section":"backend"},"requestHeaders":{"X-Forwarded-For":"127.0.0.1,23.213.54.244,23.48.94.47:37018"},"responseBody":"{ \"statusCode\": 404, \"message\": \"Resource not found\" }"}, "resourceId": "/SUBSCRIPTIONS/B00FC482-62DC-49BF-BBD6-9B4CE971B3DB/RESOURCEGROUPS/VAD01_APIM_QE_0/PROVIDERS/MICROSOFT.APIMANAGEMENT/SERVICE/VAD01QEAPIM01"},{ "Level": 4, "isRequestSuccess": true, "time": "2021-03-24T06:36:00.6722185Z", "operationName": "Microsoft.ApiManagement/GatewayLogs", "category": "GatewayLogs", "durationMs": 461, "callerIpAddress": "10.244.53.200", "correlationId": "bc8d6392-89dc-4edc-909b-f9b6c284064c", "location": "East US 2", "properties": {"method":"GET","url":"https://easyapiqa-corp-azure.staples.com/staples-preferred/pcam/order-bot/orders?maxRecords=25&beginDate=2021-03-24T12%3a01%3a00.133-05%3a00","backendResponseCode":200,"responseCode":200,"responseSize":852,"cache":"none","backendTime":460,"requestSize":1058,"apiId":"staples-preferred","operationId":"get","productId":"orderbot","apimSubscriptionId":"Orderbot","clientProtocol":"HTTP/1.1","backendProtocol":"HTTP/1.1","apiRevision":"1","clientTlsVersion":"1.2","backendMethod":"GET","backendUrl":"https://thor-qa.staples.com/pcam/order-bot/orders?maxRecords=25&beginDate=2021-03-24T12%3a01%3a00.133-05%3a00","requestHeaders":{"X-Forwarded-For":"216.117.4.105,23.213.54.244,23.48.94.47:37012","X-Azure-Client-Id":"2bdb7baf-18f8-4f46-971c-e60e7f309b0e","Ocp-Apim-Subscription-Key":"1647fc49aa2b47c29b0d57fa9a52b995"},"backendRequestHeaders":{"X-Forwarded-For":"216.117.4.105,23.213.54.244,23.48.94.47:37012","X-Azure-Client-Id":"2bdb7baf-18f8-4f46-971c-e60e7f309b0e","Ocp-Apim-Subscription-Key":"1647fc49aa2b47c29b0d57fa9a52b995"},"responseBody":"{\"token\":null,\"total_order_count\":0,\"orders\":null}","backendResponseBody":"{\"token\":null,\"total_order_count\":0,\"orders\":null}"}, "resourceId": "/SUBSCRIPTIONS/B00FC482-62DC-49BF-BBD6-9B4CE971B3DB/RESOURCEGROUPS/VAD01_APIM_QE_0/PROVIDERS/MICROSOFT.APIMANAGEMENT/SERVICE/VAD01QEAPIM01"}]},"x-opt-sequence-number":20208,"x-opt-offset":"292078407528","x-opt-enqueued-time":1616567869049}

Re: Props for Splitting JSON data as a separate event

Kothandapanin — Tue, 30 Mar 2021 12:35:08 GMT

@scelikok , I have attached the log file.

Could you please check assist if there is any change in the props

Re: Props for Splitting JSON data as a separate event

Kothandapanin — Thu, 15 Apr 2021 08:17:40 GMT

@scelikok, any assistance would be greatly appreciated.