Hi In the end this was very tricky, but the below code is what I used. Please note this was for the full example. I would suggest reading this answer as well as there are lots of hints and tricks that have to be applied to get it to work - To be honest its a lot of work - but it does work. https://community.splunk.com/t5/Splunk-Search/What-exactly-are-the-rules-requirements-for-using-quot-tstats/m-p/319801 | mstats append=t prestats=t min("mx.replica.status") min("mx.process.resources.status") WHERE "index"="metrics_test" AND mx.env=http://mx20267vm:15000 span=10s BY service.name replica.name service.type
| eval threshold = ""
| eval pid=""
| eval cmd=""
| eval host.name=""
| eval component.name=""
| mstats append=t prestats=t min("mx.process.threads") min("mx.process.memory.usage") min("mx.process.file_descriptors") min("mx.process.cpu.utilization") min("mx.process.up.time") avg("mx.process.creation.time") WHERE "index"="metrics_test" AND mx.env=http://mx20267vm:15000 span=10s BY pid cmd service.type host.name service.name replica.name component.name threshold
| rename service.name as service_name
| rename replica.name as replica_name
| rename "service.type" as service_type
| stats min("mx.process.resources.status") as Resources min("mx.process.up.time") as upTime avg("mx.process.creation.time") as creationTime min("mx.replica.status") as Replica min("mx.process.threads") as nbOfThreads min("mx.process.memory.usage") as memoryCons min("mx.process.file_descriptors") as nbOfOpenFiles min("mx.process.cpu.utilization") as cpuPerc by _time pid cmd service_type host.name service_name replica_name component.name threshold
| eval T_NbOfThreads=if(isnull(nbOfThreads),"",threshold)
| eval T_MemoryCons=if(isnull(memoryCons),"",threshold)
| eval T_NbOfOpenFiles=if(isnull(nbOfOpenFiles),"",threshold)
| eval T_CpuPerc=if(isnull(cpuPerc),"",threshold)
| eval Process_Name=((service_name . " # ") . replica_name)
| sort 0 - _time Process_Name
| streamstats last(Replica) as Replica
| streamstats last(Resources) as Resources
| where cmd !=""
| stats values(Resources) as Resources values(Replica) as Replica values(cpuPerc) as cpuPerc values(nbOfThreads) as nbOfThreads values(memoryCons) as memoryCons values(nbOfOpenFiles) as nbOfOpenFiles values(upTime) as upTime values(creationTime) as creationTime values(T_NbOfOpenFiles) as T_NbOfOpenFiles values(T_MemoryCons) as T_MemoryCons values(T_CpuPerc) as T_CpuPerc values(T_NbOfThreads) as T_NbOfThreads by _time pid cmd Process_Name service_type host.name service_name replica_name component.name
| eval Status=(Resources * Replica)
| eval Status=if((Status == 4),2,if((Status == 0),0,1))
| eval Replica=case((Process_Name == "xmlserver # xmlserver"),"2",(Process_Name == "zookeeper # zookeeper"),"2",(Process_Name == "fileserver # fileserver"),"2",true(),Replica)
| dedup _time pid
| sort 0 - _time pid
| table _time Resources Replica pid cmd service_type host.name service_name replica_name component.name cpuPerc nbOfThreads memoryCons nbOfOpenFiles upTime creationTime
... View more