Hi gajananh999, you can use the below rex :\s(?<NoProcessed>\d+)\s(?<Category>[^\.]+)\.\s(?<NoofSuccess>\d+)\sSuccess,\s(?<NoOfFailed>\d)\sFailed\. Here Category tells Emails Processed. Reports or Faxes Processed. ... View more

any work around on this ?? ... View more

Hi royimad, Please find the modified script , # Copyright (C) 2005-2012 Splunk Inc. All Rights Reserved. Version 4.0 import re,sys,time,logging,splunk.Intersplunk, splunk.mining.dcutils as dcu import smtplib, sys, StringIO, base64, os, socket, urllib, urllib2, urlparse, ssl from xml.sax import saxutils import lxml.etree as et import traceback import cStringIO, csv from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.base import MIMEBase from email.mime.application import MIMEApplication from email.header import Header import splunk.search as search import splunk.entity as entity import splunk.clilib.cli_common as cli_common from splunk.util import setSSLWrapProtocol from splunk import mergeHostPath, SplunkdConnectionException from splunk.rest import simpleRequest import splunk.models.saved_search as saved_search PDF_REPORT_SERVER_TIMEOUT = 600 PDFGEN_SIMPLE_REQUEST_TIMEOUT = 3600 importanceMap = { "highest": "1 (Highest)", "high" : "2 (High)", "low" : "4 (Low)", "lowest" : "5 (Lowest)", "1" : "1 (Highest)", "2" : "2 (High)", "4" : "4 (Low)", "5" : "5 (Lowest)" } logger = dcu.getLogger() charset = "UTF-8" class PDFException(Exception): pass def getCredentials(sessionKey, namespace): try: ent = entity.getEntity('admin/alert_actions', 'email', namespace=namespace, owner='nobody', sessionKey=sessionKey) if 'auth_username' in ent and 'clear_password' in ent: return ent['auth_username'], ent['clear_password'] except Exception, e: logger.error("Could not get email credentials from splunk, using no credentials. Error: %s" % (str(e))) return '', '' def getJobMessages(searchid, sessionKey): try: # try to get the job status immediately without retry job = search.getJob(searchid, sessionKey=sessionKey, message_level='warn', status_fetch_timeout=0) return job.messages except Exception, e: logger.error("Could not get job status for searchId=%s, Error: %s" % (searchid, str(e))) return {} def toBool(strVal): if strVal == None: return False lStrVal = strVal.lower() if lStrVal == "true" or lStrVal == "t" or lStrVal == "1" or lStrVal == "yes" or lStrVal == "y" : return True return False def isASCII(str): for i in str: if ord(i) > 127: return False return True def escape(str, plainTextMode): if plainTextMode: return str return saxutils.escape(str) def renderJobMessages(messages, plainTextMode): result = '' for k,v in messages.items(): result += 'Message Level: ' + k.upper() + '\n' i = 1 for m in v: result += str(i) + '. ' + escape(m, plainTextMode) + '\n' i += 1 result + '\n' if len(result) > 0: result = '\n-- Search generated the following messages -- \n' + result return result def unquote(val): if val is not None and len(val) > 1 and val.startswith('"') and val.endswith('"'): return val[1:-1] return val def getarg(argvals, name, defaultVal=None): return unquote(argvals.get(name, defaultVal)) EMAIL_DELIM = re.compile('\s*[,;]\s*') def mail(argvals, settings, bodytext=''): serverURL = getarg(argvals, "server", "localhost") sender = getarg(argvals, "from", "splunk") to = getarg(argvals, "to" , None) cc = getarg(argvals, "cc" , None) bcc = getarg(argvals, "bcc", None) subject = getarg(argvals, "subject" , "Splunk Results") format = getarg(argvals, "format" , "html") importance = getarg(argvals, "priority", None) inline = getarg(argvals, "inline" , "true").lower() plainText = format != "html" sendresults = toBool(getarg(argvals, "sendresults" , "false")) sendpdf = toBool(getarg(argvals, "sendpdf" , "false")) pdfview = getarg(argvals, "pdfview" , "") searchid = getarg(argvals, "searchid" , None) if not searchid: for key in argvals: if "sid_" in key: searchid = searchid or {} searchid[key] = argvals[key] use_ssl = toBool(getarg(argvals, "use_ssl" , "false")) use_tls = toBool(getarg(argvals, "use_tls" , "false")) username = getarg(argvals, "username" , "") password = getarg(argvals, "password" , "") sessionKey = settings.get('sessionKey', None) paperSize = getarg(argvals, "papersize", "letter") paperOrientation = getarg(argvals, "paperorientation", "portrait") logger.info("sendemail:mail sendPDF = %s, pdfview = %s, searchid = %s" % (str(sendpdf), str(pdfview), str(searchid))) # fetch credentials from the endpoint if none are supplied or password is encrypted if (len(username) == 0 and len(password) == 0) or password.startswith('$1$') : namespace = settings.get("namespace", None) username, password = getCredentials(sessionKey, namespace) # use the Header object to specify UTF-8 msg headers, such as subject, to, cc etc message = MIMEMultipart() if isASCII(subject): message['Subject'] = subject else: message['Subject'] = Header(subject, charset) recipients = [] if to: message['To'] = to recipients.extend(EMAIL_DELIM.split(to)) if sender: message['From'] = sender if cc: for addr in EMAIL_DELIM.split(cc): message['Cc'] = addr recipients.append(addr) if bcc: for addr in EMAIL_DELIM.split(bcc): message['Bcc'] = addr recipients.append(addr) # Clear leading / trailing whitespace from recipients recipients = [r.strip() for r in recipients] if importance: # look up better name val = importanceMap.get(importance.lower(), None) # unknown value, use value user supplied if val == None: val = importance message['X-Priority'] = val intro = '' ssName = getarg(argvals, "ssname", None) # write out a condensed body if we are just delivering a PDF snapshot # of a view/URI if pdfview: intro += 'Scheduled view delivery.\n\nA PDF snapshot has been generated for the view: %s.\n\n' % pdfview else: intro += 'Hi,\n\nPFA the Splunk Report.\n\n Regards,\n Unix Team \n' bodyformat = "html" if plainText: bodyformat = "plain" ####################################################################################### # create the body of the email and attach or inline results if required. Make sure to # # adhere to the requested format and proper tag balancing. # ####################################################################################### body = StringIO.StringIO() pdf = None errorLines = [] namespace=settings['namespace'] owner=settings['owner'] if sendpdf: pdfService = "none" import splunk.pdf.availability as pdf_availability pdfService = pdf_availability.which_pdf_service(sessionKey=sessionKey, viewId=pdfview, namespace=namespace, owner=owner) logger.info("sendemail pdfService = %s" % pdfService) try: if pdfService is "pdfgen": # will raise an Exception on error pdf = generatePDF(serverURL, subject, searchid, settings, pdfview, ssName, paperSize, paperOrientation) elif pdfService is "deprecated": # will raise an Exception on error pdf = generatePDF_deprecated(getarg(argvals, "sslink", None), subject, searchid, settings, pdfview, paperSize, paperOrientation) except Exception, e: errorLines.append("An error occurred while generating a PDF of this report. Please see python.log for details.") logger.error("An error occurred while generating a PDF of this report: %s" % e) if sendresults and toBool(settings.get('truncated')): intro += '\nNOTE: Search results in this email might have been truncated. Please visit the search job page to view the full resultset\n' if (searchid != None) and (type(searchid) is not dict) and (len(searchid) > 0): intro += renderJobMessages(getJobMessages(searchid, sessionKey), plainText) if not plainText: body.write("<HTML><BODY>\n") intro = intro.replace("\n", "\n<BR> \n") + "<BR><BR>\n" body.write(intro) if toBool(inline) or inline == "none" or not sendresults: # inline the results if required to if inline != "none" and sendresults: body.write("\n\n") body.write(bodytext) if errorLines: if plainText: body.write("\n\n") body.write("\n".join(errorLines)) else: body.write("<BR><BR> \n\n") body.write("<BR>\n".join([saxutils.escape(err) for err in errorLines])) # correctly close the html if we're not in plaintext mode if not plainText: body.write("</BODY></HTML>") message.attach(MIMEText(body.getvalue(), bodyformat, _charset=charset)) elif not pdf: attachStr = "\n\nPlease do no reply to this mail.\n\n"; if errorLines: attachStr += "\n\n" if plainText: attachStr += "\n".join(errorLines) else: attachStr += "\n".join([saxutils.escape(err) for err in errorLines]) if not plainText: attachStr = attachStr.replace("\n", "\n<BR>\n") + "</BODY></HTML>" body.write(attachStr) message.attach(MIMEText(body.getvalue(), bodyformat, _charset=charset)) # now attach the results as a separate file filename = "Mojito-report.csv" mimetype = "text" subtype = "csv" attachment = MIMEBase(mimetype, subtype) attachment.set_payload(bodytext) attachment.add_header('Content-Disposition', 'attachment', filename=filename) message.attach(attachment) if pdf: message.attach(pdf) mail_log_msg = 'Sending email. subject="%s", results_link="%s", recipients="%s"' % ( subject, getarg(argvals, "sslink", None), str(recipients) ) try: # make sure the sender is a valid email address if sender.find("@") == -1: sender = sender + '@' + socket.gethostname() if sender.endswith("@"): sender = sender + 'localhost' # send the mail if not use_ssl: smtp = smtplib.SMTP(serverURL) else: smtp = smtplib.SMTP_SSL(serverURL) if use_tls: smtp.starttls() if len(username) > 0: smtp.login(username, password) smtp.sendmail(sender, recipients, message.as_string()) smtp.quit() #log an info message only if eveything passes logger.info(mail_log_msg) except Exception, e: #else log the same message at an error level logger.error(mail_log_msg) raise def numsort(x, y): if y[1] > x[1]: return -1 elif x[1] > y[1]: return 1 else: return 0 # sort columns from shortest to largest def getSortedColumns(results, width_sort_columns): if len(results) == 0: return [] columnMaxLens = {} for result in results: for k,v in result.items(): # ignore attributes that start with "_" if k.startswith("_") and k!="_raw" and k!="_time": continue newLen = len(str(v)) oldMax = columnMaxLens.get(k, -1) #initialize the column width to the length of header (name) if oldMax == -1: columnMaxLens[k] = oldMax = len(k) if newLen > oldMax: columnMaxLens[k] = newLen colsAndCounts = [] # sort columns iff asked to if width_sort_columns: colsAndCounts = columnMaxLens.items() colsAndCounts.sort(numsort) else: for k,v in results[0].items(): if k in columnMaxLens: colsAndCounts.append([k, columnMaxLens[k]]) return colsAndCounts def pad(count): padding = "" for i in range(0, count): padding += ' ' return padding def generateTextResults(results, width_sort_columns): columnMaxLens = getSortedColumns(results, width_sort_columns) text = "" space = " "*4 # output column names for col, maxlen in columnMaxLens: val = col padsize = maxlen - len(val) text += val + pad(padsize) + space text += "\n" + "-"*len(text) + "\n" # output each result's values for result in results: for col, maxlen in columnMaxLens: val = result.get(col, "") padsize = maxlen - len(val) # left justify ALL the columns text += val + pad(padsize) + space text += "\n" return text def generateHTMLResults(results): text = "<table border=1>\n<tr>" if len(results) != 0: cols = [] for k,v in results[0].items(): # ignore attributes that start with "_" if k.startswith("_") and k!="_raw" and k!="_time": continue cols.append(k) # output column names for col in cols: text += "<th>" + col + "</th>" text += "</tr>\n" # output each result's values for result in results: text += "<tr valign=top>" for col in cols: val = result.get(col, "") escval = saxutils.escape(val) text += "<td><pre>" + escval + "</pre></td>" text += "</tr>\n" text += "</table>" return text def esc(val): return val.replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t") def generateCSVResults(results): if len(results) == 0: return '' header = [] s = cStringIO.StringIO() w = csv.writer(s) if "_time" in results[0] : header.append("_time") if "_raw" in results[0] : header.append("_raw") # for backwards compatibility remove all internal fields except _raw and _time for k in results[0].keys(): if k.startswith("_") : continue header.append(k) w.writerow(header) # output each result's values for result in results: row = [esc(result.get(col,"")) for col in header] w.writerow(row) return s.getvalue() def generateRawResults(results): str = splunk.Intersplunk.rawresultsToString(results) if(len(str) == 0): str = "The results contain no '_raw' field. Please choose another result emailing format (csv, plain or html)." return str def renderTime(results): for result in results: if "_time" in result: try: result["_time"] = time.ctime(float(result["_time"])) except: pass def sendEmail(results, settings): keywords, argvals = splunk.Intersplunk.getKeywordsAndOptions() if not ('graceful' in argvals): argvals['graceful'] = 0 if getarg(argvals, "to") == None: return dcu.getErrorResults(results, argvals['graceful'], "missing required argument: to. Please specify at least on email recipient as: \"to=address@example.com\"") if 'subject' in argvals and '_ScheduledView__' in argvals['subject']: argvals['subject'] = argvals['subject'].replace('_ScheduledView__', '') if len(results) == 0: msgText = "No results." else: format = getarg(argvals, "format", "html").lower() # always attach in CSV format if not toBool(getarg(argvals, "inline", "true") ): format = "csv" else: # if inlining results render _time to something user readable renderTime(results) # see if we need to sort fields by widht (in text mode only) width_sort_columns = toBool(argvals.get("width_sort_columns", "true")) if format == "raw": msgText = generateRawResults(results) elif format == "html": msgText = generateHTMLResults(results) elif format =="csv": msgText = generateCSVResults(results) else: msgText = generateTextResults(results, width_sort_columns) try: # if not toBool(getarg(argvals, "sendresults", "false") ): # msgText='' mail(argvals, settings, msgText) except Exception, e: #import traceback #stack = traceback.format_exc() results = dcu.getErrorResults(results, argvals['graceful'], str(e) + ' while sending mail to: ' + getarg(argvals, "to")) return results def generatePDF(serverURL, subject, sid, settings, pdfViewID, ssName, paperSize, paperOrientation): """ Reach out and retrieve a PDF copy of the search results if possible and return the MIME attachment """ sessionKey = settings.get('sessionKey', None) owner = settings.get('owner', 'nobody') if not sessionKey: raise PDFException("Can't attach PDF - sessionKey unavailable") # build up filename to use with attachments pdfViewID_filename = pdfViewID and pdfViewID.strip(' .:;|><\'"') datestamp = time.strftime('%Y-%m-%d') if pdfViewID_filename: filename = '%s-%s.pdf' % (pdfViewID_filename[:50], datestamp) # strip control characters, forward & backslash filename = re.sub(r'[\x00-\x1f\x7f/\\]+', '-', filename) if isinstance(filename, unicode): filename = ('utf-8', '', filename.encode('utf-8')) else: filename = 'splunk-report-%s.pdf' % datestamp # build up parameters to the PDF server parameters = {} parameters['namespace'] = settings["namespace"] parameters['owner'] = owner if pdfViewID: parameters['input-dashboard'] = pdfViewID else: if ssName: parameters['input-report'] = ssName else: raise PDFException("Can't attach PDF - ssName and pdfViewID unavailable") if sid: if type(sid) is dict: for sidKey in sid: parameters[sidKey] = sid[sidKey] else: parameters['sid'] = sid if paperSize and len(paperSize) > 0: if paperOrientation and paperOrientation != "portrait": parameters['paper-size'] = "%s-%s" % (paperSize, paperOrientation) else: parameters['paper-size'] = paperSize # determine if we should set an effective dispatch "now" time for this job scheduledJobEffectiveTime = getEffectiveTimeOfScheduledJob(settings.get("sid", "")) logger.info("sendemail:mail effectiveTime=%s" % scheduledJobEffectiveTime) if scheduledJobEffectiveTime != None: parameters['now'] = scheduledJobEffectiveTime try: # Ensure compatibility with systems with supportSSLV3Only=tru setSSLWrapProtocol(ssl.PROTOCOL_SSLv3) #not sure we need this now that we are using simpleRequest instead of urlopen response, content = simpleRequest("pdfgen/render", sessionKey = sessionKey, getargs = parameters, timeout = PDFGEN_SIMPLE_REQUEST_TIMEOUT) except splunk.SplunkdConnectionException, e: raise PDFException("Failed to fetch PDF (SplunkdConnectionException): %s" % str(e)) except Exception, e: raise PDFException("Failed to fetch PDF (Exception type=%s): %s" % (str(type(e)), str(e))) if response['status']!='200': raise PDFException("Failed to fetch PDF (status = %s): %s" % (str(response['status']), str(content))) if response['content-type']!='application/pdf': raise PDFException("Failed to fetch PDF (content-type = %s): %s" % (str(response['content-type']), str(content))) mpart = MIMEApplication(content, 'pdf') mpart.add_header('content-disposition', 'attachment', filename=filename) logger.info('Generated PDF for email') return mpart def getEffectiveTimeOfScheduledJob(scheduledJobSid): """ parse out the effective time from the sid of a scheduled job if no effective time specified, then return None scheduledJobSid is of form: scheduler__<owner>__<namespace>_<hash>_at_<epoch seconds>_<mS> """ scheduledJobSidParts = scheduledJobSid.split("_") effectiveTime = None if "scheduler" in scheduledJobSidParts and len(scheduledJobSidParts) > 4 and scheduledJobSidParts[-3] == "at": secondsStr = scheduledJobSidParts[-2] try: effectiveTime = int(secondsStr) except: pass return effectiveTime def generatePDF_deprecated(sslink, subject, searchid, settings, pdfview, paperSize, paperOrientation): """ DEPRECATED generate a PDF using the old PDF server app Reach out and retrieve a PDF copy of the search results if possible and return the MIME attachment """ sessionKey = settings.get('sessionKey', None) owner = settings.get('owner', 'nobody') logger.info('sslink=%s searchid=%s settings=%s' % (sslink, searchid, settings)) if not (sslink and sessionKey): raise PDFException("Can't attach PDF - either ssLink or sessionKey unavailable") # send the report request to the appserver running on the host serving the content ss_scheme, ss_netloc, ss_path, ss_query, ss_fragment = urlparse.urlsplit(sslink) # Find the root prefix if the appserver is mounted on a prefix other than / prefix = ss_path[:ss_path.index('/app/')] server = "%s://%s%s/en-US/report/" % (ss_scheme, ss_netloc, prefix) pdfviewfn = pdfview and pdfview.strip(' .:;|><\'"') datestamp = time.strftime('%Y-%m-%d') if pdfviewfn: filename = '%s-%s.pdf' % (pdfviewfn[:50], datestamp) # strip control characters, forward & backslash filename = re.sub(r'[\x00-\x1f\x7f/\\]+', '-', filename) if isinstance(filename, unicode): filename = ('utf-8', '', filename.encode('utf-8')) else: filename = 'splunk-report-%s.pdf' % datestamp if pdfview: app = ss_path[len(prefix):].split('/')[2] target = "%s://%s%s/app/%s/%s" % (ss_scheme, ss_netloc, prefix, app, pdfview) else: # assume that no doc fragments are used here if ss_query: target = sslink + '&media=print' else: target = sslink + '?media=print' try: logger.info("sendemail opening PDF request to appserver at %s" % server) # Ensure compatibility with systems with supportSSLV3Only=tru setSSLWrapProtocol(ssl.PROTOCOL_SSLv3) response = urllib2.urlopen(server, urllib.urlencode({ 'request_path' : target, 'session_key' : sessionKey, 'owner': owner, 'title' : subject, 'papersize' : paperSize, 'orientation' : paperOrientation }), PDF_REPORT_SERVER_TIMEOUT) except urllib2.HTTPError, e: msg = e.fp.read().strip() if msg and msg[0]=='>': raise PDFException("Failed to generate PDF: %s" % msg[1:]) else: raise PDFException("Failed to contact appserver at %s: %s" % (server, e)) except Exception, e: raise PDFException("Failed to fetch PDF from appserver at %s: %s" % (server, e)) headers = response.info() #logger.debug('Response headers: %s' % headers) if headers['Content-Type']!='application/pdf': logger.error("Didn't receive PDF from Report Server") raise PDFException("Didn't receive PDF from Report Server") data = response.read() mpart = MIMEApplication(data, 'pdf') mpart.add_header('content-disposition', 'attachment', filename=filename) logger.info('Generated PDF for email') return mpart results, dummyresults, settings = splunk.Intersplunk.getOrganizedResults() results = sendEmail(results, settings) splunk.Intersplunk.outputResults(results) ... View more

Hi ilove275, brackets inside the rex field name cause the syntax issue.changing the field name count(domain) to domain_count would help u solving the issue. rex field=_raw "count\(domain\)=(?<domain_count>.*)" Thanks. ... View more

For internal logs in this particular server i have setup any forwarding setup Mus. as in my outputs.conf i defined default group as A_indexers , i expect them to go to A indexders. and in inputs.conf i have used _TCP_ROUTING parmater to forward the data to the desired indexers. and there are no props.conf or transforms.conf settings involved for forwarding .. not sure why data is being sent to C_indexers instead of A_indexers. ... View more

yes..i am not relying on any tokens ..but then i dont want the search to run on the form load.. i need the search to run only on user intervention.....how can i acheive this ?? ... View more

its is multiline..i have 2 lines of data cotinously with 2 empty lines space b/w them.. ... View more

Hi Kristian... thanks for ur update.. this even didnt work on my data 😞 ... View more

Hi theouhuios , You can use the following in ur transforms.conf . [xml-extr] REGEX=name=\"HOST_NAME\">(? [^<]*)< MV_ADD = true Hope this helps ! ... View more

Hi splunker12er, If there only three specfic hosts as you say to can use the eval statment like this in your search query [your search ] | eval NEWHOSTNAME=case(hostname==AU_DC,"Australia",hostname==IN_DC,"India",hostname==SD_DC,"Singapore") or if the no of hostnames are more you can use lookup .pls refer http://docs.splunk.com/Documentation/Splunk/6.0.2/SearchReference/Lookup Hope this helps ! ... View more

Hi Juniormint, authention.conf generally have the external configuration entites like LDAP settings,mapping of LDAP roles to Splunk roles kind of information. If you simply need SPLUNK roles to have access to ur apps , you can user-prefs.conf configuraiton in your app local folder. // user-prefs.conf # # map default app for ipub roles [role_mydefinedrole] default_namespace = myapp ... View more

have u tried blacklist =(576|538|540) ... View more

Wat do you mean by stopping few hosts . you mean stopping Stopping syslog data from some unknown hosts ?? you can do that pretty well using transforms.conf configuration i.e applying filtering on the hostips ... View more

ok..in the case a simple eval statement would do the trick for u. [your search] | accum n as sum | eval sum = if(sum<0,0,sum) | table n,sum this will not let sum below 0 🙂 ... View more

Hi Sushma6, Probably u need this .. host=SCAWTR813 source="E:\SPLUNK\FTPlogs _HYDservers\APP04\ifs20140310.xml" AND "filesize" OR "transtime" | transaction keepevicted=true _time | rex "filesize>(? [^<])<" | rex"transtime>(? [^<] )<" | eval MyField = filesize + "/" + transtime | table _time,MyField ... View more

Hi nikhil, can u pls try the following ?? TIME_PREFIX = \" TIME_FORMAT = %d-%b-%y %H:%M:%S %p ... View more

Oracle PL/SQL ... View more

Thanks for ur reply..even this didnt work .. 😞 ... View more