Hi Splunk Community, I am pretty new to using Splunk for reporting purposes. Below are my use case :   Every month, I am required to generate report to calculate monthly response time for each action report of that is requesting to our service our services.  However, calculating the response time is not straight forward as in the report, we wanted to calculate NetResponseTime, where ResponseTime - MOMDuration (External API call) - EMCDuration (External API call).  By getting the NetResponseTime, this will only contains the internal related processing time. All logs is able to be correlated by cid. Currently, i am able to come up with the query and it is tested working accurately provided the subsearch limit is not reached, however, as the logs are increasing (number of logs/ per month), there is a concern where i noticed the subsearch is auto-finalized and it will be truncating due to some subsearch limits(i.e subsearch only returns 50k rows). Would be helpful if someone is able to provide me some guidance on how can I refractor my query to use multi-search(with proper grouping event from multiple sources and performing computation within each sources) instead of subsearch. Below is the query(it will be quite long): --------------------This is the main search: Get The Response Time for each request------------------------------------------ index=someindex sourcetype= DiagnosticsLog host=DiagServer |eval ActionEnum= if(like(CsUriStem,"%StampTransactions%"),2,if(like(CsUriStem,"%/kiosk/api/Transactions/Stamp%"),2,if(like(CsUriStem,"%/kiosk/api/Transactions/Search/Passport%"),1,if(like(CsUriStem,"%/kiosk/api/Refund/emcpayAccount/%"),3,if(like(CsUriStem,"%/kiosk/api/Refund/emcpayAccountConfirmation%"),4,null()))))) |eval CsUriStem = if(like(CsUriStem,"%/kiosk/api/Refund/emcpayAccount/%"),(mvindex(split(CsUriStem,"/kiosk/api/Refund/emcpayAccount/"),0))+"/kiosk/api/Refund/emcpayAccount/",CsUriStem) |eval DateTime_Unix=strptime(DateTime,"%Y-%m-%d %H:%M:%S.%7N") |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |eval nEnd=relative_time(nEnd,"-8h") |where DateTime_Unix>=nStart AND DateTime_Unix<nEnd |table LogId CsUriQuery CsUriStem DateTime DateTime_Unix SComputerName SPort SiteName TimeTaken TransferTime ActionEnum |join type=left CsUriQuery [search index=someindex sourcetype= DiagnosticsLog host=DiagServer |eval DateTime_Unix=strptime(DateTime,"%Y-%m-%d %H:%M:%S.%7N") |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |eval nEnd=relative_time(nEnd,"-8h") |where DateTime_Unix>=nStart AND DateTime_Unix<nEnd |eventstats values(CsUriQuery) by cid |eval SearchPassportEndDateTime_Unix = if(like(CsUriStem,"%/kiosk/api/Transactions/Search/Passport%"),DateTime_Unix+(TimeTaken/1000),null()) |eval SearchPassportEndDateTime = strftime(SearchPassportEndDateTime_Unix,"%Y-%m-%d %H:%M:%S.%5N") |table CsUriQuery SearchPassportEndDateTime SearchPassportEndDateTime_Unix |where isnotnull(SearchPassportEndDateTime_Unix)] |eval TimeTakenMilli=TimeTaken/1000 |eval TimeTakenNew=if(like(CsUriStem,"%RetrieveTransactions%") AND isnotnull(SearchPassportEndDateTime),if(DateTime_Unix>SearchPassportEndDateTime_Unix,TimeTaken,(DateTime_Unix+TimeTakenMilli-SearchPassportEndDateTime_Unix)*1000),TimeTaken) |table LogId CsUriQuery CsUriStem DateTime SComputerName SPort SiteName TimeTaken TimeTakenNew TransferTime ActionEnum |rename CsUriQuery as cid --------------------This is the main search------------------------------------------ |join type=left cid,ActionEnum --------------------This is the sub search:Computing MOMDuration (External API call)------------------------------------------ [search index=someindex Application=ExternalValidation (host=ValidationServer1 OR host=ValidationServer2) (Event="MomGateway_GetEP_Begin" OR Event="MomGateway_GetEP_End" ) |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |where (Event="MomGateway_GetEP_Begin" OR Event="MomGateway_GetEP_End" ) |rex field=_raw "(?<epdate>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |transaction cid startswith=(Event="MomGateway_GetEP_Begin") endswith=(Event="MomGateway_GetEP_End" ) mvlist=epdate |eval MOM_start_time=mvindex(epdate,0) |eval MOM_end_time=mvindex(epdate,1) |eval MOM_end_timeUnix=strptime(MOM_end_time,"%Y-%m-%d %H:%M:%S.%5N") |eval MOM_start_timeUnix=strptime(MOM_start_time,"%Y-%m-%d %H:%M:%S.%5N") |eval MOM_request_type = "EP" |eval differences = MOM_end_timeUnix-MOM_start_timeUnix |where MOM_start_timeUnix>=nStart AND MOM_start_timeUnix<nEnd |union [search index=someindex Application ="ExternalValidation" (host=ValidationServer1 OR host=ValidationServer2) (Event="MomGateway_GetWP_Begin" OR Event="MomGateway_GetWP_End" ) |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") | where (Event="MomGateway_GetWP_Begin" OR Event="MomGateway_GetWP_End" ) |rex field=_raw "(?<wpdate>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |transaction cid startswith=( Event="MomGateway_GetWP_Begin") endswith=(Event="MomGateway_GetWP_End") mvlist=wpdate |eval MOM_start_time=mvindex(wpdate,0) |eval MOM_end_time=mvindex(wpdate,1) |eval MOM_end_timeUnix=strptime(MOM_end_time,"%Y-%m-%d %H:%M:%S.%5N") |eval MOM_start_timeUnix=strptime(MOM_start_time,"%Y-%m-%d %H:%M:%S.%5N") |eval MOM_request_type = "WP" ```8. Calculate the differences between start/end time for each transactions``` |eval differences = MOM_end_timeUnix-MOM_start_timeUnix |where MOM_start_timeUnix>=nStart AND MOM_start_timeUnix<nEnd ] |sort 0 cid MOM_start_time |streamstats current=f window=0 global=f min(MOM_start_time) as MinTime max(MOM_end_time) as MaxTime min(MOM_start_timeUnix) as MinTimeUnix max(MOM_end_timeUnix) as MaxTimeUnix by cid |eval overlapped=if(MOM_start_timeUnix<= MaxTimeUnix ,1,0) |eval NetMOMDuration=if(overlapped>0,if(MOM_end_timeUnix>MaxTimeUnix,MOM_end_timeUnix-MaxTimeUnix,0),differences) |join type=inner cid [search index=someindex (host=ValidationServer1 OR host=ValidationServer2) (*TransactionsController_SearchByPassport_Begin* OR *TransactionsController_Stamp_Begin* OR *TransactionsController_SearchByPassport_End* OR *TransactionsController_Stamp_End*) |where Application="ApiFacadeKiosk" |rex field=_raw "(?<date>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |eval date_Unix=strptime(date,"%Y-%m-%d %H:%M:%S.%5N") |eval startTime_SearchPassport = if(like(_raw,"%TransactionsController_SearchByPassport_Begin%"),date,null()) |eval startTime_Stamping = if(like(_raw,"%TransactionsController_Stamp_Begin%"),date,null()) |eval endTime_SearchPassport = if(like(_raw,"%TransactionsController_SearchByPassport_End%"),date,null()) |eval endTime_Stamping = if(like(_raw,"%TransactionsController_Stamp_End%"),date,null()) |where date_Unix>=nStart AND date_Unix<nEnd |stats values(startTime_SearchPassport) AS startTime_SearchPassport values(startTime_Stamping) AS startTime_Stamping values(endTime_SearchPassport) AS endTime_SearchPassport values(endTime_Stamping) AS endTime_Stamping by cid ] |eval startTime_Stamping_Filled=if(isnull(startTime_Stamping),null(),strptime(startTime_Stamping,"%Y-%m-%d %H:%M:%S.%5N")) |eval startTime_SearchPassport_Filled=if(isnull(startTime_SearchPassport),null(),strptime(startTime_SearchPassport,"%Y-%m-%d %H:%M:%S.%5N")) |eval endTime_Stamping_Filled=if(isnull(endTime_Stamping),null(),strptime(endTime_Stamping,"%Y-%m-%d %H:%M:%S.%5N")) |eval endTime_SearchPassport_Filled=if(isnull(endTime_SearchPassport),null(),strptime(endTime_SearchPassport,"%Y-%m-%d %H:%M:%S.%5N")) |eval ActionType=if(isnull(startTime_Stamping_Filled),"SearchPassport",if(startTime_Stamping_Filled<MOM_start_timeUnix,"Stamping","SearchPassport")) |eventstats sum(NetMOMDuration) as TotalMOMDurationByAction count(cid) As TotalMOMRequest by cid ActionType |eval ActionEnum= if(like(ActionType,"SearchPassport"),1,2) |table cid TotalMOMDurationByAction ActionType TotalMOMRequest ActionEnum |dedup cid TotalMOMDurationByAction ActionType TotalMOMRequest ActionEnum] --------------------This is the sub search:Computing MOMDuration (External API call)------------------------------------------ |join type=left cid,ActionEnum --------------------This is the sub search:Computing EMCDuration (External API call)------------------------------------------ [search index=someindex Application=RefundControl (host=ValidationServer1 OR host=ValidationServer2) (Event="PostFirstemcpay_Begin" OR Event="PostFirstemcpay_End" ) |where (Event="PostFirstemcpay_Begin" OR Event="PostFirstemcpay_End" ) |rex field=_raw "(?<emcpayfirstdate>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |transaction cid startswith=( Event=PostFirstemcpay_Begin) endswith=(PostFirstemcpay_End) mvlist=emcpayfirstdate |eval emcpay_start_time=mvindex(emcpayfirstdate,0) |eval emcpay_end_time=mvindex(emcpayfirstdate,1) |eval emcpay_end_timeUnix=strptime(emcpay_end_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_start_timeUnix=strptime(emcpay_start_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_request_type = "emcpayAccount" |eval NetemcpayDuration = emcpay_end_timeUnix-emcpay_start_timeUnix |where emcpay_start_timeUnix>=nStart AND emcpay_start_timeUnix<nEnd |sort 0 cid emcpay_start_time |eventstats max(emcpay_start_timeUnix) As Maxemcpay_start_timeUnix by cid |where emcpay_start_timeUnix = Maxemcpay_start_timeUnix |table cid emcpay_start_time emcpay_end_time overlapped NetemcpayDuration emcpay_request_type |union [search index=someindex Application=RefundControl (host=ValidationServer1 OR host=ValidationServer2) (Event="PostSecondemcpay_Begin" OR Event="PostSecondemcpay_End" ) |where (Event="PostSecondemcpay_Begin" OR Event="PostSecondemcpay_End" ) |rex field=_raw "(?<emcpayseconddate>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |transaction cid startswith=( Event=PostSecondemcpay_Begin) endswith=(Event=PostSecondemcpay_End) mvlist=emcpayseconddate |eval emcpay_start_time=mvindex(emcpayseconddate,0) |eval emcpay_end_time=mvindex(emcpayseconddate,1) |eval emcpay_end_timeUnix=strptime(emcpay_end_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_start_timeUnix=strptime(emcpay_start_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_request_type = "emcpayConfirm" |eval NetemcpayDuration = emcpay_end_timeUnix-emcpay_start_timeUnix |where emcpay_start_timeUnix>=nStart AND emcpay_start_timeUnix<nEnd |sort 0 cid emcpay_start_time |eventstats max(emcpay_start_timeUnix) As Maxemcpay_start_timeUnix by cid |where emcpay_start_timeUnix = Maxemcpay_start_timeUnix |table cid emcpay_start_time emcpay_end_time overlapped NetemcpayDuration emcpay_request_type] |union [search index=someindex Application=RefundControl (host=ValidationServer1 OR host=ValidationServer2) (Event="Writeemcpay_Begin" OR Event="Writeemcpay_End" ) |where (Event="NcsWriteGateway_WriteNcsemcpay_Begin" OR Event="NcsWriteGateway_WriteNcsemcpay_End" ) |rex field=_raw "(?<emcpaythirddate>\d\d\d\d-\w+-\d\d\s+\d\d:\d\d:\d\d\.\d+)" |eval nStart=relative_time(now(),"-1mon@mon") |eval nStart=relative_time(nStart,"-8h") |eval nEnd=relative_time(now(),"@mon") |transaction cid startswith=( Event=Writeemcpay_Begin) endswith=(Event=Writeemcpay_End) mvlist=emcpaythirddate |eval emcpay_start_time=mvindex(emcpaythirddate,0) |eval emcpay_end_time=mvindex(emcpaythirddate,1) |eval emcpay_end_timeUnix=strptime(emcpay_end_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_start_timeUnix=strptime(emcpay_start_time,"%Y-%m-%d %H:%M:%S.%5N") |eval emcpay_request_type = "emcpayRefund" |eval NetemcpayDuration = emcpay_end_timeUnix-emcpay_start_timeUnix |where emcpay_start_timeUnix>=nStart AND emcpay_start_timeUnix<nEnd |sort 0 cid emcpay_start_time |eventstats max(emcpay_start_timeUnix) As Maxemcpay_start_timeUnix by cid |where emcpay_start_timeUnix = Maxemcpay_start_timeUnix |table cid emcpay_start_time emcpay_end_time overlapped NetemcpayDuration emcpay_request_type] |eventstats sum(NetemcpayDuration) as TotalemcpayDurationByAction count(cid) As TotalemcpayRequest by cid emcpay_request_type |eval ActionEnum= if(like(emcpay_request_type,"emcpayAccount"),3,if(like(emcpay_request_type,"emcpayConfirm"),4,if(like(emcpay_request_type,"emcpayRefund"),2,null()))) |table cid TotalemcpayDurationByAction emcpay_request_type TotalemcpayRequest ActionEnum |dedup cid TotalemcpayDurationByAction emcpay_request_type TotalemcpayRequest ActionEnum] --------------------This is the sub search:Computing EMCDuration (External API call)------------------------------------------ |eval startTime_Stamping_Filled=if(isnull(startTime_Stamping),null(),strptime(startTime_Stamping,"%Y-%m-%d %H:%M:%S.%5N")) |eval startTime_SearchPassport_Filled=if(isnull(startTime_SearchPassport),null(),strptime(startTime_SearchPassport,"%Y-%m-%d %H:%M:%S.%5N")) |eval endTime_Stamping_Filled=if(isnull(endTime_Stamping),null(),strptime(endTime_Stamping,"%Y-%m-%d %H:%M:%S.%5N")) |eval endTime_SearchPassport_Filled=if(isnull(endTime_SearchPassport),null(),strptime(endTime_SearchPassport,"%Y-%m-%d %H:%M:%S.%5N")) |eval TotalemcpayDurationByAction=if(isnull(TotalemcpayDurationByAction),0,TotalemcpayDurationByAction*1000) |eval TotalMOMDurationByAction=if(isnull(TotalMOMDurationByAction),0,TotalMOMDurationByAction*1000) |eval NetResponseTime = TimeTakenNew-TotalMOMDurationByAction-TotalemcpayDurationByAction |table LogId cid CsUriStem DateTime SComputerName SPort SiteName TimeTaken TimeTakenNew TransferTime ActionType TotalMOMDurationByAction TotalemcpayDurationByAction NetResponseTime ... View more