Why is my SPL number of results different when appended?

[xnx_1012]

Hello I have this SPL which returns like 40 000 records when run alone however when it's appended to another SPL which is similar except with different Report ID and monitored commands. The record of this SPL shrinks down from 40 000 to 16k, what causes this weird problem?

`comment(Standard Users)`
(index=* source=/var/log/secure* AND TERM(sudo) AND (TERM(bin) OR TERM(sbin))  AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *bin/rpm*-*i* OR *bin/rpm*-*e* OR *bin/*tar*x*) AND COMMAND!="*egrep*") OR
    (index="*" source=/var/log/audit/audit.log* addr!=? acct=* res=success*
[search index=* source=/var/log/secure* AND TERM(sudo)  AND (TERM(bin) OR TERM(sbin)) AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *bin/rpm*-*i* OR *bin/rpm*-*e* OR *bin/*tar*x*)  AND COMMAND!="*egrep*"
| regex _raw != ".*bin\/grep|.*bin\/man|.*bin\/which"
| regex _raw!= ".*user NOT in sudoers.*"
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
| stats  latest(_time) as latest earliest(_time) as mod_time
| eval earliest= relative_time(mod_time, "-24h@s")
| fields earliest latest])
| regex _raw != ".*bin\/grep|.*bin\/man|.*bin\/which"
| regex _raw!= ".*user NOT in sudoers.*"
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]|type.*addr.*success"
| rename acct as Users
| rex field=_raw "(?<=sudo:)\s*(?P<Users>[[:alnum:]]\S*[[:alnum:]])\s*(?=\:).*(?<=COMMAND\=)(?P<command>.*)"
| eval "Command/Events" = replace(command,"^(\/bin\/|\/sbin\/)","")
| eval Users = if(match(Users,"(?<=[[:alnum:]])\@[[:alnum:]]\S*[[:alnum:]]"),
    replace(Users,"(?<=[[:alnum:]])\@[[:alnum:]]\S*[[:alnum:]]",""),
    if(match(Users,"[[:alnum:]]+\\\(?=[[:alnum:]]\S*[[:alnum:]])"),
    replace(Users,"[[:alnum:]]+\\\(?=[[:alnum:]]\S*[[:alnum:]])","")
    ,Users))

| eval Time = if(match(_raw,"(?<=sudo:)\s*[[:alnum:]]\S*[[:alnum:]]\s*(?=\:).*(?<=COMMAND\=)*") ,strftime(_time, "%Y-%d-%m %H:%M:%S"),null()), Date = strftime(_time, "%Y-%d-%m")
| eval "Report ID" = "ABLR-008"
| eval "Agency HF" = if(isnull(agencyhf),"",agencyhf)
| stats list(Time) as Time list("Command/Events") as "Command/Events" latest(addr) as "IP Address" by Users Date host index "Report ID" "Agency HF"
| where 'Command/Events' !=""
| eval counter=mvrange(0,mvcount(Time))
| streamstats count as sessions
| stats list(*) as * by sessions counter
| foreach Time "Command/Events"  [ eval <<FIELD>> = mvindex('<<FIELD>>', counter)]
| fields - counter sessions
| rename index as Agency, host as Hostname
| where isnotnull('IP Address') OR Users!="root"
| fields "Report ID" Time Agency Command/Events Hostname Users "IP Address" "Agency HF"

`comment(root doing sudo)`
| union
[search index=* source=/var/log/secure* AND TERM(sudo)  AND (TERM(bin) OR TERM(sbin))  AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *bin/rpm*-*i* OR *bin/rpm*-*e* OR *bin/*tar*x*)  AND COMMAND!="*egrep*" AND " root"
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
| rex field=_raw "(?<=sudo:)\s*(?P<Users>[[:alnum:]]\S*[[:alnum:]])\s*(?=\:).*(?<=COMMAND\=)(?P<command>.*)"
| eval "Command/Events" = replace(command,"^(\/bin\/|\/sbin\/)","")
| eval Users = "root"
| eval Time = strftime(_time, "%Y-%d-%m %H:%M:%S"), Date = strftime(_time, "%Y-%d-%m")
| eval "Agency HF" = if(isnull(agencyhf),"",agencyhf)
| eval "Report ID" = "ABLR-008"
| eval "Command/Events" = replace(command,"^(\/bin\/|\/sbin\/)","")
| rename  host as Hostname, index as Agency
| table "Report ID" Time Date Users "Command/Events" Hostname Agency "Agency HF"
| join type=left Date Hostname
[search (index=* source=/var/log/secure* AND TERM(sudo) AND  "session opened for user root"
[ search index=* source=/var/log/secure*  AND TERM(sudo)  AND (TERM(bin) OR TERM(sbin))  AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *bin/rpm*-*i* OR *bin/rpm*-*e* OR *bin/*tar*x*)  AND COMMAND!="*egrep*" AND " root"
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
        | stats latest(_time) as latest earliest(_time) as mod_time
        | eval earliest= relative_time(mod_time, "-24h@s")
        | fields earliest latest])
         OR
(index="*" source=/var/log/audit/audit.log* addr!=? res=success* acct=*   
[ search index=* source=/var/log/secure* AND TERM(sudo)  AND (TERM(bin) OR TERM(sbin))  AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *bin/rpm*-*i* OR *bin/rpm*-*e* OR *bin/*tar*x*)  AND COMMAND!="*egrep*" AND " root"
        | regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
        | stats latest(_time) as latest earliest(_time) as mod_time
        | eval earliest= relative_time(mod_time, "-24h@s")
        | fields earliest latest])

| regex _raw != ".*LOGIN.*"
| eval Time = if(match(_raw,"(?<=user)\s*[[:alnum:]]\S*[[:alnum:]].*(?<=by)\s*[[:alnum:]]\S*[[:alnum:]](?=\Suid)") ,strftime(_time, "%Y-%d-%m %H:%M:%S"),null()), Date = strftime(_time, "%Y-%d-%m")
| stats list(Time) as Time  values(addr) as ip by  Date host
| where Time !=""
| rename host as Hostname
| stats values(ip) as "IP Address"  by Date Hostname]
| fillnull "IP Address" value="localhost"
| fillnull Hostname value="N.A"
| fields "Report ID"  Time   Agency Command/Events Hostname Users "IP Address" "Agency HF"]

| union
[ search index=* source="/root/.bash_history" AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *rpm*-*i* OR *rpm*-*e* OR *tar*x*)
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
| rex field=_raw "(?P<command>.*)"
| eval Users = "root"
| eval Time = strftime(_time, "%Y-%d-%m %H:%M:%S"), Date = strftime(_time, "%Y-%d-%m")
| eval "Agency HF" = if(isnull(agencyhf),"",agencyhf)
| eval "Report ID" = "ABLR-008"
| rename command as "Command/Events", host as Hostname, index as Agency
| fields "Report ID" Time Date Users "Command/Events" Hostname Agency "Agency HF"
| join type=left Date Hostname
[search (index=* source=/var/log/secure* AND TERM(sudo) AND  "session opened for user root"
[ search index=* source="/root/.bash_history" AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *rpm*-*i* OR *rpm*-*e* OR *tar*x*)
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
        | stats latest(_time) as latest earliest(_time) as mod_time
        | eval earliest= relative_time(mod_time, "-24h@s")
        | fields earliest latest])
         OR
(index="*" source=/var/log/audit/audit.log* addr!=? res=success* acct=*   
[ search index=* source="/root/.bash_history"  AND ((TERM(yum) AND (TERM(install) OR TERM(remove))) OR *rpm*-*i* OR *rpm*-*e* OR *tar*x*)
| regex _raw = ".*rpm -[ivhe]+|.*yum.*|.*tar\s+(-|)[xvzjf]+"
        | stats latest(_time) as latest earliest(_time) as mod_time
        | eval earliest= relative_time(mod_time, "-24h@s")
        | fields earliest latest])

| regex _raw != ".*LOGIN.*"
| eval Time = if(match(_raw,"(?<=user)\s*[[:alnum:]]\S*[[:alnum:]].*(?<=by)\s*[[:alnum:]]\S*[[:alnum:]](?=\Suid)") ,strftime(_time, "%Y-%d-%m %H:%M:%S"),null()), Date = strftime(_time, "%Y-%d-%m")
| stats list(Time) as Time  values(addr) as ip by  Date host
| where Time !=""
| rename host as Hostname
| stats values(ip) as "IP Address"  by Date Hostname]
| fillnull "IP Address" value="localhost"
| fillnull Hostname value="N.A"
| fields "Report ID"  Time   Agency Command/Events Hostname Users "IP Address" "Agency HF"
]

| dedup Time Command/Events
| table "Report ID" Time Agency Command/Events Hostname Users "IP Address" "Agency HF"
| fillnull "IP Address" value="localhost"
| sort 0 -Time

[skoelpin · Accepted solution]

You're hitting a sub searching limit which is truncating your results. Easy fix would be to forego subsearching and use an OR up top to search across multiple indexes and return them into a single large bucket of results

https://docs.splunk.com/Documentation/Splunk/8.0.3/Search/Aboutsubsearches#Subsearch_performance_con...