I want to start of by saying I am extremely new to splunk, so please bear with me, I'm not sure at all if I'm on the right track so please feel let me know if I need to try something else.
I have two Cisco ASA5506s are used as firewalls. Searching for either of their hostnames only yields results for about 17 days or so. So if today is the 1st day, it will overwrite the 17th day to record tomorrows logs. Since all I was doing was just trying to get a total view of how many total entries it's pulling from all indexes I wasn't sure which index could be the reason why it's not logging past 17 days. Poking around I found that the _syslog and _metrics indexes both only had logs 14-15 days old. So that lead me to modify the indexes.conf file, however this did not help log the firewalls past 17 days. What else should I be looking for? These devices see millions of hits daily, so that could possibly be contribiting to this as well.
Previous: Indexes.conf
[default]
serviceSubtaskTimingPeriod = 30
serviceInactiveIndexesPeriod = 60
enableRealtimeSearch = true
timePeriodInSecBeforeTsidxReduction = 604800
serviceMetaPeriod = 25
defaultDatabase = main
rotatePeriodInSecs = 60
rtRouterThreads = 0
enableTsidxReduction = false
maxHotIdleSecs = 0
bucketRebuildMemoryHint = auto
suspendHotRollByDeleteQuery = false
maxHotSpanSecs = 7776000
suppressBannerList =
maxBucketSizeCacheEntries = 0
hotBucketTimeRefreshInterval = 10
maxHotBuckets = 3
processTrackerServiceInterval = 1
maxDataSize = auto
maxRunningProcessGroups = 8
minRawFileSyncSecs = disable
enableDataIntegrityControl = false
minStreamGroupQueueSize = 2000
maxMetaEntries = 1000000
throttleCheckPeriod = 15
tstatsHomePath = volume:_splunk_summaries\$_index_name\datamodel_summary
tsidxReductionCheckPeriodInSec = 600
maxBloomBackfillBucketAge = 30d
datatype = event
syncMeta = true
partialServiceMetaPeriod = 0
frozenTimePeriodInSecs = 188697600
maxGlobalDataSizeMB = 0
quarantinePastSecs = 77760000
compressRawdata = true
coldToFrozenScript =
coldPath.maxDataSizeMB = 0
enableOnlineBucketRepair = true
repFactor = 0
rtRouterQueueSize = 10000
maxTimeUnreplicatedWithAcks = 60
assureUTF8 = false
maxTimeUnreplicatedNoAcks = 300
rawChunkSizeBytes = 131072
memPoolMB = auto
homePath.maxDataSizeMB = 0
warmToColdScript =
maxWarmDBCount = 300
minHotIdleSecsBeforeForceRoll = auto
coldToFrozenDir =
maxTotalDataSizeMB = 500000
maxConcurrentOptimizes = 6
maxRunningProcessGroupsLowPriority = 1
streamingTargetTsidxSyncPeriodMsec = 5000
journalCompression = gzip
quarantineFutureSecs = 2592000
splitByIndexKeys =
sync = 0
serviceOnlyAsNeeded = true
[_audit]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\audit\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\audit\thaweddb
tstatsHomePath = volume:_splunk_summaries\audit\datamodel_summary
homePath = $SPLUNK_DB\audit\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 5120
rtRouterQueueSize =
[_internal]
bucketRebuildMemoryHint = 0
syncMeta = 1
maxHotSpanSecs = 432000
compressRawdata = 1
coldPath = $SPLUNK_DB\_internaldb\colddb
minHotIdleSecsBeforeForceRoll = 0
maxDataSize = 1000
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_internaldb\thaweddb
tstatsHomePath = volume:_splunk_summaries\_internaldb\datamodel_summary
homePath = $SPLUNK_DB\_internaldb\db
rtRouterThreads =
enableTsidxReduction = 0
maxTotalDataSizeMB = 25600
frozenTimePeriodInSecs = 188697600
rtRouterQueueSize =
[_introspection]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\_introspection\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_introspection\thaweddb
homePath = $SPLUNK_DB\_introspection\db
rtRouterThreads =
maxDataSize = 1024
maxTotalDataSizeMB = 5120
frozenTimePeriodInSecs = 1209600
rtRouterQueueSize =
[_telemetry]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\_telemetry\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_telemetry\thaweddb
homePath = $SPLUNK_DB\_telemetry\db
rtRouterThreads =
maxDataSize = 256
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 63072000
rtRouterQueueSize =
[_thefishbucket]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\fishbucket\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\fishbucket\thaweddb
tstatsHomePath = volume:_splunk_summaries\fishbucket\datamodel_summary
homePath = $SPLUNK_DB\fishbucket\db
rtRouterThreads =
maxDataSize = 500
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 188697600
rtRouterQueueSize =
[history]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\historydb\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\historydb\thaweddb
tstatsHomePath = volume:_splunk_summaries\historydb\datamodel_summary
homePath = $SPLUNK_DB\historydb\db
rtRouterThreads =
maxDataSize = 10
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 604800
rtRouterQueueSize =
[main]
enableOnlineBucketRepair = 1
bucketRebuildMemoryHint = 0
syncMeta = 1
minHotIdleSecsBeforeForceRoll = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\defaultdb\colddb
maxHotBuckets = 10
maxDataSize = auto_high_volume
maxConcurrentOptimizes = 6
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\defaultdb\thaweddb
tstatsHomePath = volume:_splunk_summaries\defaultdb\datamodel_summary
homePath = $SPLUNK_DB\defaultdb\db
rtRouterThreads =
enableTsidxReduction = 0
maxHotIdleSecs = 86400
maxTotalDataSizeMB = 10240
rtRouterQueueSize =
[splunklogger]
disabled = true
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\splunklogger\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\splunklogger\thaweddb
homePath = $SPLUNK_DB\splunklogger\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 500
rtRouterQueueSize =
[summary]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\summarydb\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\summarydb\thaweddb
tstatsHomePath = volume:_splunk_summaries\summarydb\datamodel_summary
homePath = $SPLUNK_DB\summarydb\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 500
rtRouterQueueSize =
[volume:_splunk_summaries]
path = $SPLUNK_DB
Modified indexes.conf:
[default]
serviceSubtaskTimingPeriod = 30
serviceInactiveIndexesPeriod = 60
enableRealtimeSearch = true
timePeriodInSecBeforeTsidxReduction = 604800
serviceMetaPeriod = 25
defaultDatabase = main
rotatePeriodInSecs = 60
rtRouterThreads = 0
enableTsidxReduction = false
maxHotIdleSecs = 0
bucketRebuildMemoryHint = auto
suspendHotRollByDeleteQuery = false
maxHotSpanSecs = 7776000
suppressBannerList =
maxBucketSizeCacheEntries = 0
hotBucketTimeRefreshInterval = 10
maxHotBuckets = 3
processTrackerServiceInterval = 1
maxDataSize = auto
maxRunningProcessGroups = 8
minRawFileSyncSecs = disable
enableDataIntegrityControl = false
minStreamGroupQueueSize = 2000
maxMetaEntries = 1000000
throttleCheckPeriod = 15
tstatsHomePath = volume:_splunk_summaries\$_index_name\datamodel_summary
tsidxReductionCheckPeriodInSec = 600
maxBloomBackfillBucketAge = 30d
datatype = event
syncMeta = true
partialServiceMetaPeriod = 0
frozenTimePeriodInSecs = 188697600
maxGlobalDataSizeMB = 0
quarantinePastSecs = 77760000
compressRawdata = true
coldToFrozenScript =
coldPath.maxDataSizeMB = 0
enableOnlineBucketRepair = true
repFactor = 0
rtRouterQueueSize = 10000
maxTimeUnreplicatedWithAcks = 60
assureUTF8 = false
maxTimeUnreplicatedNoAcks = 300
rawChunkSizeBytes = 131072
memPoolMB = auto
homePath.maxDataSizeMB = 0
warmToColdScript =
maxWarmDBCount = 300
minHotIdleSecsBeforeForceRoll = auto
coldToFrozenDir =
maxTotalDataSizeMB = 500000
maxConcurrentOptimizes = 6
maxRunningProcessGroupsLowPriority = 1
streamingTargetTsidxSyncPeriodMsec = 5000
journalCompression = gzip
quarantineFutureSecs = 2592000
splitByIndexKeys =
sync = 0
serviceOnlyAsNeeded = true
[_audit]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\audit\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\audit\thaweddb
tstatsHomePath = volume:_splunk_summaries\audit\datamodel_summary
homePath = $SPLUNK_DB\audit\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 5120
rtRouterQueueSize =
[_internal]
bucketRebuildMemoryHint = 0
syncMeta = 1
maxHotSpanSecs = 432000
compressRawdata = 1
coldPath = $SPLUNK_DB\_internaldb\colddb
minHotIdleSecsBeforeForceRoll = 0
maxDataSize = 1000
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_internaldb\thaweddb
tstatsHomePath = volume:_splunk_summaries\_internaldb\datamodel_summary
homePath = $SPLUNK_DB\_internaldb\db
rtRouterThreads =
enableTsidxReduction = 0
maxTotalDataSizeMB = 51200
frozenTimePeriodInSecs = 188697600
rtRouterQueueSize =
archiver.enableDataArchive = 0
metric.enableFloatingPointCompression = 1
selfStorageThreads =
tsidxWritingLevel =
[_introspection]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\_introspection\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_introspection\thaweddb
homePath = $SPLUNK_DB\_introspection\db
rtRouterThreads =
maxDataSize = 1024
maxTotalDataSizeMB = 5120
frozenTimePeriodInSecs = 1209600
rtRouterQueueSize =
[_telemetry]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\_telemetry\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_telemetry\thaweddb
homePath = $SPLUNK_DB\_telemetry\db
rtRouterThreads =
maxDataSize = 256
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 63072000
rtRouterQueueSize =
[_thefishbucket]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\fishbucket\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\fishbucket\thaweddb
tstatsHomePath = volume:_splunk_summaries\fishbucket\datamodel_summary
homePath = $SPLUNK_DB\fishbucket\db
rtRouterThreads =
maxDataSize = 500
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 188697600
rtRouterQueueSize =
[history]
bucketRebuildMemoryHint = 0
syncMeta = 1
compressRawdata = 1
coldPath = $SPLUNK_DB\historydb\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\historydb\thaweddb
tstatsHomePath = volume:_splunk_summaries\historydb\datamodel_summary
homePath = $SPLUNK_DB\historydb\db
rtRouterThreads =
maxDataSize = 10
maxTotalDataSizeMB = 500
frozenTimePeriodInSecs = 604800
rtRouterQueueSize =
[main]
enableOnlineBucketRepair = 1
bucketRebuildMemoryHint = 0
syncMeta = 1
minHotIdleSecsBeforeForceRoll = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\defaultdb\colddb
maxHotBuckets = 10
maxDataSize = auto_high_volume
maxConcurrentOptimizes = 6
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\defaultdb\thaweddb
tstatsHomePath = volume:_splunk_summaries\defaultdb\datamodel_summary
homePath = $SPLUNK_DB\defaultdb\db
rtRouterThreads =
enableTsidxReduction = 0
maxHotIdleSecs = 86400
maxTotalDataSizeMB = 10240
rtRouterQueueSize =
[splunklogger]
disabled = true
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\splunklogger\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\splunklogger\thaweddb
homePath = $SPLUNK_DB\splunklogger\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 500
rtRouterQueueSize =
[_syslog]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\_syslog\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_syslog\thaweddb
tstatsHomePath = volume:_splunk_summaries\_syslog\datamodel_summary
homePath = $SPLUNK_DB\_syslog\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 10240
frozenTimePeriodInSecs = 7776000
rtRouterQueueSize =
[_metrics]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\_metrics\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\_metrics\thaweddb
tstatsHomePath = volume:_splunk_summaries\_metrics\datamodel_summary
homePath = $SPLUNK_DB\_metrics\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 10240
frozenTimePeriodInSecs = 7776000
rtRouterQueueSize =
[summary]
bucketRebuildMemoryHint = 0
compressRawdata = 1
coldPath = $SPLUNK_DB\summarydb\colddb
minHotIdleSecsBeforeForceRoll = 0
enableTsidxReduction = 0
enableOnlineBucketRepair = 1
suspendHotRollByDeleteQuery = 0
enableDataIntegrityControl = 0
thawedPath = $SPLUNK_DB\summarydb\thaweddb
tstatsHomePath = volume:_splunk_summaries\summarydb\datamodel_summary
homePath = $SPLUNK_DB\summarydb\db
rtRouterThreads =
syncMeta = 1
maxTotalDataSizeMB = 500
rtRouterQueueSize =
[volume:_splunk_summaries]
path = $SPLUNK_DB
... View more