Virtual index time setting not effective

[mikechu]

Hi

Our data is stored in the following directories. Each directory contains 1 day of data.

s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/event_date=2015-10-27/

We set up our virtual index as follow:

Time capturing regex=s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/event_date=(\d+)-(\d+)-(\d+)/
Time Format=yyyyMMdd
Time Adjustment=0second(s)
Time Range=1day(s)
Time Zone=Default System Timezone

When we query this index with a time range (ex: Today), Hunk looks for all the data from all directories. The final result is correct (only today data is shown). However, we thought Hunk will figure out the source value and only look at the directory for "today" data. If we specify the source manually (ex: source=s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/event_date=2015-10-27/*), the query runs a lot faster.

[rdagan_splunk]

Try this:

[retail-device-app-analytics]
 vix.input.1.et.format = yyyyMMdd
 vix.input.1.et.regex = .*?/event_date=(\d+)-(\d+)-(\d+)/.*
 vix.input.1.lt.format = yyyyMMdd
 vix.input.1.lt.offset = 86400
 vix.input.1.lt.regex =.*?/event_date=(\d+)-(\d+)-(\d+)/.*
 vix.input.1.path = s3n://sra-event/retailDevice/prod/appAnalytics/...
 vix.provider = sra-rms

[rdagan_splunk]

Can you please send the file /opt/splunk/etc/apps/search/local/indexes.conf ?

[mikechu]

Thx.

[retail-device-app-analytics]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://sra-event/retailDevice/prod/appAnalytics/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://sra-event/retailDevice/prod/appAnalytics/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://sra-event/retailDevice/prod/appAnalytics/...
vix.provider = sra-rms

[retail-device-app-compliance]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://sra-event/retailDevice/prod/appCompliance/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://sra-event/retailDevice/prod/appCompliance/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://sra-event/retailDevice/prod/appCompliance/...
vix.provider = sra-rms

[provider:sra-rms]
vix.command.arg.3 = $SPLUNK_HOME/bin/jars/SplunkMR-s6.0-hy2.0.jar
vix.env.HADOOP_HOME = /opt/hadoop/apache/hadoop-2.4.0
vix.env.JAVA_HOME = /opt/java/latest/
vix.family = hadoop
vix.fs.default.name = hdfs://ip-172-31-35-19.us-west-2.compute.internal:9000
vix.mapreduce.framework.name = yarn
vix.mapreduce.jobhistory.address = ip-172-31-35-19.us-west-2.compute.internal:10020
vix.splunk.emr.cluster.ami.version = 3.9.0
vix.splunk.emr.cluster.date.creation = 1443709072
vix.splunk.emr.cluster.date.ready = 1443709335
vix.splunk.emr.cluster.hadoop.version = 2.4.0
vix.splunk.emr.cluster.id = j-KQADNCLW7WD
vix.splunk.emr.cluster.instance.group.core.id = ig-2SVVB6HXIEZEY
vix.splunk.emr.cluster.instance.group.core.instance.type = c3.8xlarge
vix.splunk.emr.cluster.instance.group.core.size = 1
vix.splunk.emr.cluster.instance.group.master.id = ig-1JPD70MV0UIKJ
vix.splunk.emr.cluster.instance.group.master.instance.type = m3.xlarge
vix.splunk.emr.cluster.instance.group.master.size = 1
vix.splunk.emr.cluster.master.external = ec2-52-89-25-131.us-west-2.compute.amazonaws.com
vix.splunk.emr.cluster.master.internal = ip-172-31-35-19.us-west-2.compute.internal
vix.splunk.emr.cluster.name = sra-rms
vix.splunk.emr.cluster.region = us-west-2
vix.splunk.emr.cluster.state = WAITING
vix.splunk.home.hdfs = /user/hunk/working-dir/
vix.yarn.resourcemanager.address = ip-172-31-35-19.us-west-2.compute.internal:9022
vix.yarn.resourcemanager.scheduler.address = ip-172-31-35-19.us-west-2.compute.internal:9024
vix.splunk.emr.cluster.latest.connection.check = 1446475334
vix.splunk.emr.cluster.latest.connection.success = 1446475334
vix.splunk.emr.cluster.instance.group.task.id = ig-QE7JS0IWGLQZ
vix.splunk.emr.cluster.instance.group.task.instance.type = m3.2xlarge
vix.splunk.emr.cluster.instance.group.task.size = 7

[preprod-retail-device-app-analytics]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/appAnalytics/...
vix.provider = sra-rms
vix.input.1.et.offset = 0

[preprod-retail-device-app-compliance]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/appCompliance/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/appCompliance/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/appCompliance/...
vix.provider = sra-rms

[retail-device-app-analytics-session]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://sra-event/retailDevice/prod/appAnalyticsSession/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://sra-event/retailDevice/prod/appAnalyticsSession/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://sra-event/retailDevice/prod/appAnalyticsSession/...
vix.provider = sra-rms

[retail-device-app-analytics-application]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://sra-event/retailDevice/prod/appAnalyticsApplication/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://sra-event/retailDevice/prod/appAnalyticsApplication/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://sra-event/retailDevice/prod/appAnalyticsApplication/...
vix.provider = sra-rms

[preprod-retail-device-app-analytics-application]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplication/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplication/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplication/...
vix.provider = sra-rms

[preprod-retail-device-app-analytics-session]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsSession/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsSession/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsSession/...
vix.provider = sra-rms

[preprod-rcs-api-request]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/prod/consolidated/apiRequest/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/prod/consolidated/apiRequest/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/prod/consolidated/apiRequest/...
vix.provider = sra-rms

[preprod-consumer-device-response-report-analytics-20-collected-info]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-consumer-event/cep/prod/consolidated/responseReportAnalytics20CollectedInfo/event_date=(\d+)-(\d+)-(\d+)                                                                  /
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-consumer-event/cep/prod/consolidated/responseReportAnalytics20CollectedInfo/event_date=(\d+)-(\d+)-(\d+)                                                                  /
vix.input.1.path = s3n://rcs-consumer-event/cep/prod/consolidated/responseReportAnalytics20CollectedInfo/...
vix.provider = sra-rms

[preprod-consumer-device-response-report-analytics-20-event-info]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/responseReportAnalytics20EventInfo/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/responseReportAnalytics20EventInfo/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/responseReportAnalytics20EventInfo/...
vix.provider = sra-rms

[preprod-consumer-device-request-reactivation]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/requestReactivation/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/requestReactivation/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/requestReactivation/...
vix.provider = sra-rms

[preprod-retail-device-app-analytics-screen]
vix.input.1.et.format = yyyyMMdd
vix.input.1.et.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplicationScreen/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.lt.format = yyyyMMdd
vix.input.1.lt.offset = 86400
vix.input.1.lt.regex = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplicationScreen/event_date=(\d+)-(\d+)-(\d+)/
vix.input.1.path = s3n://rcs-cms-event/cep/prod/consolidated/appAnalyticsApplicationScreen/...
vix.provider = sra-rms