Search Lag Root Cause(s): The number of extremely lagged searches (1) over the last hour exceeded the red threshold (1) on this Splunk instance Last 50 related messages: 06-12-2020 10:15:28.204 -0400 INFO SavedSplunker - Scheduler Health Report recording a extremely lagged search="Splunk Web Login Attempts" with lag=267 search_period=60 06-11-2020 23:03:00.663 -0400 INFO SavedSplunker - Scheduler Health Report recording a extremely lagged search="Splunk Web Login Attempts" with lag=2100 search_period=60 06-11-2020 22:17:54.510 -0400 INFO SavedSplunker - Scheduler Health Report recording a extremely lagged search="Splunk Web Login Attempts" with lag=354 search_period=60 06-11-2020 18:39:31.208 -0400 INFO SavedSplunker - Scheduler Health Report recording a extremely lagged search="Splunk Web Login Attempts" with lag=1770 search_period=60 06-11-2020 17:09:09.800 -0400 INFO SavedSplunker - Scheduler Health Report recording a extremely lagged search="Splunk Web Login Attempts" with lag=189 search_period=60 06-11-2020 16:15:55.517 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=107.691, result_count=1, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.12", suppressed=1, fired=0, skipped=1, action_time_ms=44, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:50.575 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=102.711, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.11", suppressed=2, fired=0, skipped=2, action_time_ms=52, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:45.572 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=97.714, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.10", suppressed=2, fired=0, skipped=2, action_time_ms=48, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:40.578 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=92.709, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.9", suppressed=2, fired=0, skipped=2, action_time_ms=55, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:35.575 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=87.719, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.8", suppressed=2, fired=0, skipped=2, action_time_ms=43, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:30.520 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=82.709, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.7", suppressed=2, fired=0, skipped=2, action_time_ms=30, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:25.550 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=77.703, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.6", suppressed=2, fired=0, skipped=2, action_time_ms=41, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:20.579 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=72.702, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.5", suppressed=2, fired=0, skipped=2, action_time_ms=67, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:15.563 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=67.707, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.4", suppressed=2, fired=0, skipped=2, action_time_ms=47, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:10.567 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=62.706, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.3", suppressed=2, fired=0, skipped=2, action_time_ms=48, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:05.565 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=57.705, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.2", suppressed=2, fired=0, skipped=2, action_time_ms=47, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:15:00.518 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=52.681, result_count=2, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.1", suppressed=2, fired=0, skipped=2, action_time_ms=62, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 16:14:55.518 -0400 INFO SavedSplunker - savedsearch_id="nobody;search;Splunk Web Login Attempts", search_type="", user="kmill78", app="search", savedsearch_name="Splunk Web Login Attempts", priority=default, status=success, digest_mode=0, scheduled_time=1591906445, window_time=0, dispatch_time=1591906447, run_time=47.676, result_count=1, alert_actions="", sid="rt_scheduler__kmill78__search__RMD52fa94ba1191f811b_at_1591906445_1.0", suppressed=0, fired=1, skipped=0, action_time_ms=50, thread_id="AlertNotifierWorker-0", message="", workload_pool="" 06-11-2020 14:02:46.137 -0400 INFO SavedSplunker - savedsearch_id="nobody;splunk_monitoring_console;DMC Asset - Build Standalone Asset Table", search_type="scheduled", user="nobody", app="splunk_monitoring_console", savedsearch_name="DMC Asset - Build Standalone Asset Table", priority=default, status=success, digest_mode=1, scheduled_time=1591898534, window_time=0, dispatch_time=1591898565, run_time=0.252, result_count=4, alert_actions="populate_lookup", sid="scheduler__nobody_c3BsdW5rX21vbml0b3JpbmdfY29uc29sZQ__RMD54740dfff07b17ef1_at_1591898534_0", suppressed=0, thread_id="AlertNotifierWorker-0", workload_pool="" 06-11-2020 14:02:45.291 -0400 INFO SavedSplunker - DCSS: completed reading history for continuous scheduled searches
... View more