Unpredictable data volume in Splunk indexes
<input type="radio" token="time_field">
<label>Splunk data volume alerts:</label>
<choice value="20">Yesterday</choice>
<choice value="140">Last 7 days</choice>
<choice value="600">Last 30 days</choice>
<default>20</default>
<initialValue>20</initialValue>
</input>
<panel>
<html>
<h1>Information:</h1>
<div>
All <font color="#d93f3c">critical</font> alerts are monitored by <a href="https://splunk.analytics.vodafone.com/en-US/app/analytics/alert?s=%2FservicesNS%2Fnobody%2Fanalytics%2Fsaved%2Fsearches%2FSplunk%2520Alert%2520-%2520Detected%2520unpredicted%2520data%2520volume%2520in%2520Splunk%2520indexes" target="_blank">Splunk Alert - Detected unpredicted data volume in Splunk indexes</a> and sent to Operational Intelligence Team.
</div>
<div>
Please also visit <a href="https://splunk.analytics.vodafone.com/en-US/app/analytics/admin_traffic_forecasts_teams_products" target="_blank">Traffic forecasts by teams/products</a> dashboard for more details.
</div>
</html>
</panel>
<panel>
<title>Number of indexes with data volume alerts</title>
<single>
<search>
<query>index=splunk_internal_db source=splunk_internals_daily_load
[ search index=splunk_internal_db source=splunk_internals_daily_load
| stats sum(usage) as usageMB by idx
| sort - usageMB
| head 20 | table idx ]
| bucket _time span=1d
| stats sum(eval(usage/1024)) as usage by _time, idx
| streamstats window=400 current=false mean(usage) as median_number by idx
| eval absDev=(abs(usage-median_number))
| streamstats window=400 current=false mean(absDev) as medianAbsDev by idx
| eval lowerBound=abs(median_number-medianAbsDev*3), upperBound=(median_number+medianAbsDev*3)
| eval upperBound=if((median_number/upperBound*100 > 85) OR (median_number/upperBound*100 < 75), median_number+median_number*0.15, upperBound)
| eval lowerBound=if(lowerBound/median_number*100 > 95, median_number*0.90, lowerBound)
| eval isOutlier=if(usage < lowerBound OR usage > upperBound, 1, 0)
| eval priority=case(
isOutlier=1 AND usage>50 AND usage>upperBound, "1. critical",
isOutlier=1 AND usage>20 AND usage<50 AND usage>lowerBound, "2. warning",
isOutlier=1, "3. low"
)
| stats max(isOutlier) as isOutlier, max(usage) as data_volume_GB, values(priority) as priority by _time, idx
| tail $time_field$
| stats dc(idx) as number by priority
| where priority="1. critical"
| table number
-60d@d
@d
block
["0xd93f3c","0xd93f3c"]
[0]
critical
1
index=splunk_internal_db source=splunk_internals_daily_load
[ search index=splunk_internal_db source=splunk_internals_daily_load
| stats sum(usage) as usageMB by idx
| sort - usageMB
| head 20 | table idx ]
| bucket _time span=1d
| stats sum(eval(usage/1024)) as usage by _time, idx
| streamstats window=400 current=false mean(usage) as median_number by idx
| eval absDev=(abs(usage-median_number))
| streamstats window=400 current=false mean(absDev) as medianAbsDev by idx
| eval lowerBound=abs(median_number-medianAbsDev*3), upperBound=(median_number+medianAbsDev*3)
| eval upperBound=if((median_number/upperBound*100 > 85) OR (median_number/upperBound*100 < 75), median_number+median_number*0.15, upperBound)
| eval lowerBound=if(lowerBound/median_number*100 > 95, median_number*0.90, lowerBound)
| eval isOutlier=if(usage < lowerBound OR usage > upperBound, 1, 0)
| eval priority=case(
isOutlier=1 AND usage>50 AND usage>upperBound, "1. critical",
isOutlier=1 AND usage>20 AND usage<50 AND usage>lowerBound, "2. warning",
isOutlier=1, "3. low"
)
| stats max(isOutlier) as isOutlier, max(usage) as data_volume_GB, values(priority) as priority by _time, idx
| tail $time_field$
| stats dc(idx) as number by priority
| where priority="2. warning"
| fillnull value=0 number
| table number
-60d@d
@d
block
["0xf7bc38","0xf7bc38"]
[0]
warning
1
index=splunk_internal_db source=splunk_internals_daily_load
[ search index=splunk_internal_db source=splunk_internals_daily_load
| stats sum(usage) as usageMB by idx
| sort - usageMB
| head 20 | table idx ]
| bucket _time span=1d
| stats sum(eval(usage/1024)) as usage by _time, idx
| streamstats window=400 current=false mean(usage) as median_number by idx
| eval absDev=(abs(usage-median_number))
| streamstats window=400 current=false mean(absDev) as medianAbsDev by idx
| eval lowerBound=abs(median_number-medianAbsDev*3), upperBound=(median_number+medianAbsDev*3)
| eval upperBound=if((median_number/upperBound*100 > 85) OR (median_number/upperBound*100 < 75), median_number+median_number*0.15, upperBound)
| eval lowerBound=if(lowerBound/median_number*100 > 95, median_number*0.90, lowerBound)
| eval isOutlier=if(usage < lowerBound OR usage > upperBound, 1, 0)
| eval priority=case(
isOutlier=1 AND usage>50 AND usage>upperBound, "1. critical",
isOutlier=1 AND usage>20 AND usage<50 AND usage>lowerBound, "2. warning",
isOutlier=1, "3. low"
)
| stats max(isOutlier) as isOutlier, max(usage) as data_volume_GB, values(priority) as priority by _time, idx
| tail $time_field$
| stats dc(idx) as number by priority
| where priority="3. low"
| table number
-60d@d
@d
block
["0x6db7c6","0x6db7c6"]
[0]
low
1
<panel>
<table>
<title>Data volume alerts for Top 20 indexes (click for details)</title>
<search>
<query>index=splunk_internal_db source=splunk_internals_daily_load
[ search index=splunk_internal_db source=splunk_internals_daily_load
| stats sum(usage) as usageMB by idx
| sort - usageMB
| head 20 | table idx ]
| bucket _time span=1d
| stats sum(eval(usage/1024)) as usage by _time, idx
| streamstats window=400 current=false mean(usage) as median_number by idx
| eval absDev=(abs(usage-median_number))
| streamstats window=400 current=false mean(absDev) as medianAbsDev by idx
| eval lowerBound=abs(median_number-medianAbsDev*3), upperBound=(median_number+medianAbsDev*3)
| eval upperBound=if((median_number/upperBound*100 > 85) OR (median_number/upperBound*100 < 75), median_number+median_number*0.15, upperBound)
| eval lowerBound=if(lowerBound/median_number*100 > 95, median_number*0.90, lowerBound)
| eval isOutlier=if(usage < lowerBound OR usage > upperBound, 1, 0)
| eval priority=case(
isOutlier=1 AND usage>50 AND usage>upperBound, "1. critical",
isOutlier=1 AND usage>20 AND usage<50 AND usage>lowerBound, "2. warning",
isOutlier=1, "3. low"
)
| stats max(isOutlier) as isOutlier, max(usage) as data_volume_GB, values(priority) as priority by _time, idx
| tail $time_field$
| search isOutlier>0
| chart count over idx by priority
| sort - "1. critical", "2. warning", "3.low"
-60d@d
@d
1
100
none
cell
false
false
false
true
<!-- Use set to specify the new token to be created.
Use any token from the page or from the click event to produce the value needed. -->
$row.idx|n$
<!-- If we also set the form.sourcetype the input will get updated too
$row.sourcetype$ -->
<panel depends="$index_token$">
<viz type="Splunk_ML_Toolkit.OutliersViz">
<title>Outlier detection for index=$index_token$ in last 60 days</title>
<search>
<query>index=splunk_internal_db source=splunk_internals_daily_load idx=$index_token$
| bucket _time span=1d
| stats sum(eval(usage/1024)) as usage by _time, idx
| streamstats window=20 current=false mean(usage) as median_number by idx
| eval absDev=(abs(usage-median_number))
| streamstats window=20 current=false mean(absDev) as medianAbsDev by idx
| eval lowerBound=abs(median_number-medianAbsDev*3), upperBound=(median_number+medianAbsDev*3)
| eval upperBound=if((median_number/upperBound*100 > 85) OR (median_number/upperBound*100 < 75), median_number+median_number*0.15, upperBound)
| eval lowerBound=if(lowerBound/median_number*100 > 95, median_number*0.90, lowerBound)
| eval isOutlier=if(usage < lowerBound OR usage > upperBound, 1, 0)
| table _time, usage, lowerBound, upperBound, median_number, isOutlier
| rename usage as "data volume [GB]"
-60d@d
@d
1
true
<panel>
<chart>
<title>Daily volume by sourcetype for index=$index_token$ in last 10 days</title>
<search>
<query>index=_internal tag=LS source=*license_usage.log type=Usage idx=$index_token$ st=*
| bucket _time span=1d
| stats sum(b) as "usage" by _time, st
| eval usage=round(usage/1024/1024/1024,2)
| timechart limit=30 span=1d max(usage) as usage by st
-10d@d
@d
collapsed
GB/day
visible
line
469
progressbar
... View more