Things have improved a lot thanks to tsidxWritingLevel enhancements. If you set tsidxWritingLevel=4, the maximum available today, and all your buckets have been already written with this level you can achieve a compress ratio of 5.35:1 This means 55 TB of raw logs will occupy around 10 TB (tsidx + raw) on disk. At least this is what we have in our deployment. This number can vary depending on the type of data you are ingesting. Here the query I used, running All Time, starting from the one present in the Monitoring Console >> Indexing >> Index and Volumes >> Index Detail: Instance | rest splunk_server=<oneOfYourIndexers> /services/data/indexes datatype=all
| join type=outer title [
| rest splunk_server=<oneOfYourIndexers> /services/data/indexes-extended datatype=all
]
| `dmc_exclude_indexes`
| eval warm_bucket_size = coalesce('bucket_dirs.home.warm_bucket_size', 'bucket_dirs.home.size')
| eval cold_bucket_size = coalesce('bucket_dirs.cold.bucket_size', 'bucket_dirs.cold.size')
| eval hot_bucket_size = if(isnotnull(cold_bucket_size), total_size - cold_bucket_size - warm_bucket_size, total_size - warm_bucket_size)
| eval thawed_bucket_size = coalesce('bucket_dirs.thawed.bucket_size', 'bucket_dirs.thawed.size')
| eval warm_bucket_size_gb = coalesce(round(warm_bucket_size / 1024, 2), 0.00)
| eval hot_bucket_size_gb = coalesce(round(hot_bucket_size / 1024, 2), 0.00)
| eval cold_bucket_size_gb = coalesce(round(cold_bucket_size / 1024, 2), 0.00)
| eval thawed_bucket_size_gb = coalesce(round(thawed_bucket_size / 1024, 2), 0.00)
| eval warm_bucket_count = coalesce('bucket_dirs.home.warm_bucket_count', 0)
| eval hot_bucket_count = coalesce('bucket_dirs.home.hot_bucket_count', 0)
| eval cold_bucket_count = coalesce('bucket_dirs.cold.bucket_count', 0)
| eval thawed_bucket_count = coalesce('bucket_dirs.thawed.bucket_count', 0)
| eval home_event_count = coalesce('bucket_dirs.home.event_count', 0)
| eval cold_event_count = coalesce('bucket_dirs.cold.event_count', 0)
| eval thawed_event_count = coalesce('bucket_dirs.thawed.event_count', 0)
| eval home_bucket_size_gb = coalesce(round((warm_bucket_size + hot_bucket_size) / 1024, 2), 0.00)
| eval homeBucketMaxSizeGB = coalesce(round('homePath.maxDataSizeMB' / 1024, 2), 0.00)
| eval home_bucket_capacity_gb = if(homeBucketMaxSizeGB > 0, homeBucketMaxSizeGB, "unlimited")
| eval home_bucket_usage_gb = home_bucket_size_gb." / ".home_bucket_capacity_gb
| eval cold_bucket_capacity_gb = coalesce(round('coldPath.maxDataSizeMB' / 1024, 2), 0.00)
| eval cold_bucket_capacity_gb = if(cold_bucket_capacity_gb > 0, cold_bucket_capacity_gb, "unlimited")
| eval cold_bucket_usage_gb = cold_bucket_size_gb." / ".cold_bucket_capacity_gb
| eval currentDBSizeGB = round(currentDBSizeMB / 1024, 2)
| eval maxTotalDataSizeGB = if(maxTotalDataSizeMB > 0, round(maxTotalDataSizeMB / 1024, 2), "unlimited")
| eval disk_usage_gb = currentDBSizeGB." / ".maxTotalDataSizeGB
| eval currentTimePeriodDay = coalesce(round((now() - strptime(minTime,"%Y-%m-%dT%H:%M:%S%z")) / 86400, 0), 0)
| eval frozenTimePeriodDay = coalesce(round(frozenTimePeriodInSecs / 86400, 0), 0)
| eval frozenTimePeriodDay = if(frozenTimePeriodDay > 0, frozenTimePeriodDay, "unlimited")
| eval freeze_period_viz_day = currentTimePeriodDay." / ".frozenTimePeriodDay
| eval total_bucket_count = toString(coalesce(total_bucket_count, 0), "commas")
| eval totalEventCount = toString(coalesce(totalEventCount, 0), "commas")
| eval total_raw_size_gb = round(total_raw_size / 1024, 2)
| eval avg_bucket_size_gb = round(currentDBSizeGB / total_bucket_count, 2)
| eval compress_ratio = round(total_raw_size_gb / currentDBSizeGB, 2)." : 1"
| fields title, datatype
currentDBSizeGB, totalEventCount, total_bucket_count, avg_bucket_size_gb,
total_raw_size_gb, compress_ratio, minTime, maxTime
freeze_period_viz_day, disk_usage_gb, home_bucket_usage_gb, cold_bucket_usage_gb,
hot_bucket_size_gb, warm_bucket_size_gb, cold_bucket_size_gb, thawed_bucket_size_gb,
hot_bucket_count, warm_bucket_count, cold_bucket_count, thawed_bucket_count,
home_event_count, cold_event_count, thawed_event_count,
homePath, homePath_expanded, coldPath, coldPath_expanded, thawedPath, thawedPath_expanded, summaryHomePath_expanded, tstatsHomePath, tstatsHomePath_expanded,
maxTotalDataSizeMB, frozenTimePeriodInSecs, homePath.maxDataSizeMB, coldPath.maxDataSizeMB,
maxDataSize, maxHotBuckets, maxWarmDBCount | search title=* | table title currentDBSizeGB total_raw_size_gb compress_ratio | where isnotnull(total_raw_size_gb) | where isnotnull(compress_ratio)
| stats sum(currentDBSizeGB) as currentDBSizeGB, sum(total_raw_size_gb) as total_raw_size_gb | eval compress_ratio = round(total_raw_size_gb / currentDBSizeGB, 2)." : 1"
... View more