datetime.xml with european default

dshakespeare_sp — Mon, 07 Jul 2014 19:47:12 GMT

I have a difficult time stamp extraction in a source file. The date and time are split over two lines. I have tried many variations on TIME_FORMAT, TIME_PREFIX etc but with little joy.

Leaving TIME_FORMAT, TIME_PREFIX blank and letting Splunk figure it out by itself is the closest I have got but unfortunatly splunk returns the date in US format mm/dd/yyyy. Is there a way to make datetime.xml file default to a Eurpean style date?

Re: datetime.xml with european default

dart — Mon, 07 Jul 2014 20:23:25 GMT

Sure, create a new datetime.xml in a new location, like $SPLUNK_HOME/etc/system/local/datetime.xml:

{noformat}

<datetime>

<define name="_year" extract="year">
    <text><![CDATA[(20\d\d|19\d\d|[901]\d(?!\d))]]></text>
</define>

<define name="_month" extract="month">
    <text><![CDATA[(0?[1-9]|1[012])(?!:)]]></text>
</define>

<define name="_litmonth"  extract="litmonth">
     <text><![CDATA[(?<![\d\w])(jan|\x{3127}\x{6708}|feb|\x{4E8C}\x{6708}|mar|\x{4E09}\x{6708}|apr|\x{56DB}\x{6708}|may|\x{4E94}\x{6708}|jun|\x{516D}\x{6708}|jul|\x{4E03}\x{6708}|aug|\x{516B}\x{6708}|sep|\x{4E5D}\x{6708}|oct|\x{5341}\x{6708}|nov|\x{5341}\x{3127}\x{6708}|dec|\x{5341}\x{4E8C}\x{6708})[a-z,\.;]*]]></text>
</define>

<define name="_allmonth" extract="litmonth, month">
    <text><![CDATA[(?:]]></text>
        <use name="_litmonth"/>
    <text><![CDATA[|]]></text>
        <use name="_month"/>
    <text><![CDATA[)]]></text>
</define>

<define name="_day"  extract="day">
    <text><![CDATA[(0?[1-9]|[12]\d|3[01])]]></text> 
</define>

<define name="_usday" extract="day">
    <use name="_day"/>
    <text><![CDATA[(?:st|nd|rd|th|[,\.;])?]]></text>
</define>

<define name="_hour" extract="hour">
    <text><![CDATA[([01]?[1-9]|[012][0-3])(?!\d)]]></text>
</define>

<define name="_minute" extract="minute">
    <text><![CDATA[([0-6]\d)(?!\d)]]></text>
</define>

<define name="_second" extract="second">
    <text><![CDATA[([0-6]\d)(?!\d)]]></text>
</define>

<define name="_zone" extract="zone">
     <text><![CDATA[((?:(?:UT|UTC|GMT(?![+-])|CET|CEST|CETDST|MET|MEST|METDST|MEZ|MESZ|EET|EEST|EETDST|WET|WEST|WETDST|MSK|MSD|IST|JST|KST|HKT|AST|ADT|EST|EDT|CST|CDT|MST|MDT|PST|PDT|CAST|CADT|EAST|EADT|WAST|WADT|Z)|(?:GMT)?[+-]\d\d?:?(?:\d\d)?)(?!\w))?]]></text>
</define>

<define name="_ampm" extract="ampm">
    <text><![CDATA[([ap]m|[\x{4E0A}\x{4E0B}]\x{5348})?]]></text>
</define>

<define name="_time" extract="hour, minute, second, subsecond, ampm, zone">
    <text><![CDATA[(?<!\d)]]></text>
        <use name="_hour"/>
    <text><![CDATA[:]]></text>
        <use name="_minute"/>
    <text><![CDATA[:]]></text>
        <use name="_second"/> 
    <text><![CDATA[(?:(?: \d{4})?[:,\.](\d+))? {0,2}]]></text>
        <use name="_ampm"/>
    <text><![CDATA[ {0,2}]]></text>
        <use name="_zone"/>
    <text><![CDATA[(?!:\d)]]></text>
</define>

<define name="_hmtime" extract="hour, minute, ampm">
    <text><![CDATA[(?<!\d)]]></text>
        <use name="_hour"/>
    <text><![CDATA[:]]></text>
        <use name="_minute"/>
    <text><![CDATA[(?: ([ap]m|[\x{4E0A}\x{4E0B}]\x{5348}))?(?!:[:\d])]]></text>
</define>


<define name="_dottime" extract="hour, minute, second, subsecond, zone">
    <text><![CDATA[(?<![\d\.])([01]\d|2[0-3])\.]]></text>
        <use name="_minute"/>
    <text><![CDATA[(?:\.?]]></text>
        <use name="_second"/>
    <text><![CDATA[(?:[:,]\d+)?(?:\.(\d\d\d\d+))?) {0,2}]]></text>
        <use name="_zone"/>
    <text><![CDATA[(?![0-9\.])]]></text>
</define>

<define name="_combdatetime" extract="year, month, day, hour, minute, second, subsecond">
        <!-- ... 20060502-000002 GMT ... -->
    <text><![CDATA[(?<![\d\.])(20\d\d)(0\d|1[012])([012]\d|3[01])[.-]?([01]\d|2[0123])([0-6]\d)([0-6]\d)(?:\.?(\d+))?]]>\s*</text>
        <use name="_zone"/> 
</define>

<define name="_combdatetime2" extract="year, ignored_sep, month, day, hour, minute, second, zone">
        <!-- ... 2007-3-22 0:0:2 GMT ...' -->
        <!-- ... 2007/3/22 0:0:2 GMT ...' -->
        <text><![CDATA[(?<![\d\.])(20\d\d)([-/])([01]?\d)\2([012]?\d|3[01])\s+([012]?\d):([0-6]?\d):([0-6]?\d)]]>\s*</text>
        <use name="_zone"/> 
</define>



<define name="_usdate" extract="litmonth, month, ignored_sep, day, zone, ignored_sep2, year">
     <text><![CDATA[(?<!\w|\d[:\.\-])]]></text>
     <use name="_allmonth"/> 
         <text><![CDATA[([/\- ]) {0,2}]]></text>
         <use name="_day"/>
         <text><![CDATA[(?!:) {0,2}(?:\d\d:\d\d:\d\d(?:[\.\,]\d+)? {0,2}]]></text>
     <use name="_zone"/>
         <text><![CDATA[)?((?:\3|,) {0,2}]]></text>
         <use name="_year"/> 
         <text><![CDATA[)?(?!/|\w|\.\d)]]></text>
</define>

<!-- Jan 21, 09.  allows spaces with litmonth only -->
<define name="_usdate1" extract="litmonth, ignored_sep, day, zone, ignored_sep2, year">
     <text><![CDATA[(?<!\w|\d[:\.\-])]]></text>
     <use name="_litmonth"/> 
         <text><![CDATA[([/\- ]) {0,2}]]></text>
         <use name="_day"/>
         <text><![CDATA[(?!:) {0,2}(?:\d\d:\d\d:\d\d(?:[\.\,]\d+)? {0,2}]]></text>
     <use name="_zone"/>
         <text><![CDATA[)?((?:\2|,) {0,2}]]></text>
         <use name="_year"/> 
         <text><![CDATA[)?(?!/|\w|\.\d)]]></text>
</define>

<!-- 10/21/09. doesn't allow spaces (e.g. 10 21 09) with numeric month -->
<define name="_usdate2" extract="month, ignored_sep, day, zone, ignored_sep2, year">
     <text><![CDATA[(?<!\w|\d[:\.\-])]]></text>
     <use name="_month"/> 
         <text><![CDATA[([/\-])]]></text>
         <use name="_day"/>
         <text><![CDATA[(?!:)(?:\d\d:\d\d:\d\d(?:[\.\,]\d+)? {0,2}]]></text>
     <use name="_zone"/>
         <text><![CDATA[)?((?:\2)]]></text>
         <use name="_year"/> 
         <text><![CDATA[)?(?!/|\w|\.\d)]]></text>
</define>


<define name="_isodate" extract="year, ignored_sep, litmonth, month, day">
        <text><![CDATA[(?<![\w\d])]]></text>
        <use name="_year"/>
    <text><![CDATA[([\./\- ])]]></text>
        <use name="_allmonth"/>
    <text><![CDATA[(?!\d)(?:[\./\- ] {0,2})?]]></text>
        <use name="_day"/>
        <text><![CDATA[(?!/)(?:(?=T)|(?!\w)(?!\.\d))]]></text>
</define>

<!-- eurodate format.  period/dot delim separated out to eurodate2 -->
<define name="_eurodate1" extract="day, ignored_sep, litmonth, month, year">
        <text><![CDATA[(?<![\w\.])]]></text>
        <use name="_usday"/> 
    <text><![CDATA[([\- /]) {0,2}]]></text>
        <use name="_allmonth"/>
        <text><![CDATA[\2 {0,2}]]></text>
        <use name="_year"/>
        <text><![CDATA[(?![\w\.])]]></text>
</define>

<!-- just period/dot delimiter.  do not allow any spaces after dots (e.g. "version 5.4. 10" -->
<define name="_eurodate2" extract="day, litmonth, month, year">
        <text><![CDATA[(?<![\w\.])]]></text>
        <use name="_usday"/> 
    <text><![CDATA[\.]]></text>
        <use name="_allmonth"/>
        <text><![CDATA[\.]]></text>
        <use name="_year"/>
        <text><![CDATA[(?![\w\.])]]></text>
</define>


<define name="_bareurlitdate" extract="day, litmonth, year">
    <text><![CDATA[(\d\d?)\|\|(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\|\|(20\d\d)]]></text>
</define>

<define name="_orddate" extract="year, ord">
    <text><![CDATA[\s([01]\d)([0123]\d\d)\s]]></text>
</define>

<!-- due to high number of false positive matches, this format is
     limited to special cases.  either at the start of a line or in
     filename matches only, by prefixing with a "source::" -->

<!-- don't allow multiple spaces after mashed date.  indicates number in column -->
<define name="_masheddate" extract="year, month, day">
    <text><![CDATA[(?:^|source::).*?(?<!\d|\d\.|-)(?:20)?([901]\d)(0\d|1[012])([012]\d|3[01])(?!\d|-| {2,})]]></text>
</define>
<define name="_masheddate2" extract="month, day, year">
    <text><![CDATA[(?:^|source::).*?(?<!\d|\d\.)(0\d|1[012])([012]\d|3[01])(?:20)?([901]\d)(?!\d| {2,})]]></text>
</define>

<define name="_utcepoch" extract="utcepoch, subsecond">
        <!-- update regex before '2017' -->
    <text><![CDATA[((?<=^|[\s#,"=\(\[\|\{])(?:1[012345]|9)\d{8}|^@[\da-fA-F]{16,24})(?:\.?(\d{1,6}))?(?![\d\(])]]></text>
</define>

<timePatterns>
      <use name="_time"/>
      <use name="_hmtime"/>
      <use name="_hmtime"/>
      <use name="_dottime"/>
      <use name="_combdatetime"/>
      <use name="_utcepoch"/>
      <use name="_combdatetime2"/>
</timePatterns>
<datePatterns>
      <use name="_isodate"/>
      <use name="_eurodate1"/>
      <use name="_eurodate2"/>
      <use name="_usdate1"/>
      <use name="_usdate2"/> 
      <use name="_bareurlitdate"/> 
      <use name="_orddate"/>
      <use name="_combdatetime"/>
      <use name="_masheddate"/>
      <use name="_masheddate2"/>
      <use name="_combdatetime2"/>
</datePatterns>

</datetime>

{noformat}
Then wire it up to your sourcetype with some props:

## props.conf
[my_euro_sourcetype]
DATETIME_CONFIG = /etc/system/local/datetime.xml

and restart Splunk!

Re: datetime.xml with european default

dshakespeare_sp — Tue, 08 Jul 2014 07:05:54 GMT

Thank you DART 🙂

Re: datetime.xml with european default

dshakespeare_sp — Fri, 13 Feb 2015 09:53:24 GMT

Note the props.conf is a example. You will need to add to a local/props.conf for your specific sourcetype, source or host

Re: datetime.xml with european default

nitesh218ss — Tue, 12 May 2015 05:40:04 GMT

Hi in my log event and filename date is not present i want give a fix date to log so what is do ?

Re: datetime.xml with european default

dshakespeare_sp — Mon, 18 Jul 2016 13:52:03 GMT

link text

topic Re: datetime.xml with european default in Getting Data In

datetime.xml with european default

Re: datetime.xml with european default

Re: datetime.xml with european default

Re: datetime.xml with european default

Re: datetime.xml with european default

Re: datetime.xml with european default