Splunk Search

Remove useless header HTML events

Madhu02splunk
New Member

Hey Splunkers!

Could someone please help me to remove useless header HTML events before it gets indexed into splunk.
There are 300 events we need to remove and indexed actual events.
I have already setup FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER in props.conf.

Below is the event I wanted to remove.

<html>
<style>
h1 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
h2 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: bold; color: navy;}
tr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000;}
td { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000; border: 0 solid dimgray; border-top-width: 1pt; border-right-width: xpt;vertical-align:text-top;}
hr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
body { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
table { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
td.navy {color: navy;}
tr.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
td.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
</style>
<script type="text/javascript">
<!--
function JSTrim(p_strToBeTrimmed)
{
var vChar
var vLength
var i
var vFirstNotSpace
var vLastNotSpace

vLength = p_strToBeTrimmed.length
for (i = 0; i < vLength;i++)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vFirstNotSpace = i
i = vLength
}
}
for (i = vLength-1 ; i>=0;i--)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vLastNotSpace = i
i = -1
}
}
return p_strToBeTrimmed.substring(vFirstNotSpace,vLastNotSpace+1);
}


function toggle(f_level, f_thread, f_method, f_message, f_login, f_IP){

mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(3);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;

var levels = "XXX";
if(f_level != "ERR"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
}
}
}

// go over all the row and show/hide them
for (i=1;i<numOfRows;i++) {
var tdarr = trArray.item(i).getElementsByTagName("td");
thread = tdarr.item(2).childNodes.item(0).data;
login = tdarr.item(3).childNodes.item(0).data;
IP = tdarr.item(4).childNodes.item(0).data;
logLevel = tdarr.item(5).childNodes.item(0).data;
method = tdarr.item(6).childNodes.item(0).data;
message = tdarr.item(7).childNodes.item(0).data;

logLevel = JSTrim(logLevel);

if((levels.search(XXXX) !=-1) &&
(thread.search(XXXX) !=-1) &&
(login.search(XXXX) !=-1) &&
(IP.search(XXXX) !=-1) &&
(method.search(XXXX) !=-1) &&
(message.search(XXXX) !=-1)){
trArray.item(i).style.display="inline";
}else{
trArray.item(i).style.display="none";
}
}
}

function clearFilter(){
document.filterForm.level.selectedIndex = 0;
document.filterForm.thread.value="";
document.filterForm.Method.value="";
document.filterForm.Message.value="";
showAll();
}
function showAll(){
mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(1);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;
for (i=1;i<numOfRows;i++) {
trArray.item(i).style.display="inline";
}
}
function filter(){
var w = document.filterForm.level.selectedIndex;
var XXXX = document.filterForm.level.options[w].text;
var XXXX = document.filterForm.thread.value;
var XXXX = document.filterForm.Method.value;
var XXXX = document.filterForm.Message.value;
var XXXX = document.filterForm.Login.value;
varXXXX = document.filterForm.IP.value;
toggle(logLevel,JSTrim(XXXX),JSTrim(XXXX),JSTrim(XXXX), JSTrim(XXXX), JSTrim(XXXX));
}
--></script>
<body bgcolor="XXXXXX">
<a href="xxxxxxxxxxxxxxx >Go to previous log</a>
<h2>xxxxxxxxxxxx</h2><table>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
</table>
<h2>Java Properties</h2>
<table cellSpacing="0" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr><td width="30%"><b>OS</b></td><td>&nbsp;</td></tr>
<tr><td>os.name</td><td>XXXXX</td></tr>
<tr><td>os.version</td><td>XXX</td></tr>
<tr><td>os.arch</td><td>XXX</td></tr>
<tr><td>os.home</td><td>XXX</td></tr>
<tr><td width="30%"><b>XX</b></td><td>&nbsp;</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXXXXX</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXX</td></tr>
<tr><td width="30%"><b>XXX</b></td><td>&XXXX;</td></tr>
<tr><td>user.name</td><td>XXXXXX</td></tr>
<tr><td>user.home</td><td>XX\</td></tr>
<tr><td>user.dir</td><td>XXXXXXXXXXXXXXXXXX</td></tr>
<tr><td>user.language</td><td>en</td></tr>
<tr><td width="30%"><b>Java</b></td><td>&nbsp;</td></tr>
<tr><td>java.vm.vendor</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.version</td><td>XXXX</td></tr>
<tr><td>java.vm.version</td><td>XXXXXXX</td></tr>
<tr><td>java.home</td><td>XXXXXXXXXXXX\java\jre</td></tr>
<tr><td>java.class.path</td><td>../wrapper/wrapper.jar;../server/lib/annotations;../server/lib/ext;../server/lib/jetty-ajp-7.5.4.v20111024.jar;../server/lib/jetty-all-7.5.4.v20111024-javadoc.jar;../server/lib/jetty-annotations-7.5.4.v20111024.jar;../server/lib/jetty-client-7.5.4.v20111024.jar;../server/lib/jetty-continuation-7.5.4.v20111024.jar;../server/lib/jetty-deploy-7.5.4.v20111024.jar;../server/lib/jetty-http-7.5.4.v20111024.jar;../server/lib/jetty-io-7.5.4.v20111024.jar;../server/lib/jetty-jmx-7.5.4.v20111024.jar;../server/lib/jetty-jndi-7.5.4.v20111024.jar;../server/lib/jetty-overlay-deployer-7.5.4.v20111024.jar;../server/lib/jetty-plus-7.5.4.v20111024.jar;../server/lib/jetty-policy-7.5.4.v20111024.jar;../server/lib/jetty-rewrite-7.5.4.v20111024.jar;../server/lib/jetty-security-7.5.4.v20111024.jar;../server/lib/jetty-server-7.5.4.v20111024.jar;../server/lib/jetty-servlet-7.5.4.v20111024.jar;../server/lib/jetty-servlets-7.5.4.v20111024.jar;../server/lib/jetty-util-7.5.4.v20111024.jar;../server/lib/jetty-webapp-7.5.4.v20111024.jar;../server/lib/jetty-websocket-7.5.4.v20111024.jar;../server/lib/jetty-xml-7.5.4.v20111024.jar;../server/lib/jndi;../server/lib/jsp;../server/lib/jta;../server/lib/launcher-11.50.9999-GA-SNAPSHOT.jar;../server/lib/lxxxxxxxxxxxxxx;../server/lib/launcher-11.51.9999-SNAPSHOT.jar;../server/lib/launcher-sources.jar;../server/lib/launcher.jar;../server/lib/monitor;../server/lib/policy;../server/lib/servlet-api-2.5.jar;../server/lib/annotations/javax.annotation_1.0.0.v20100513-0750.jar;../server/lib/annotations/org.objectweb.asm_3.1.0.v200803061910.jar;../server/lib/ext/.donotdelete;../server/lib/jndi/javax.activation_1.1.0.v201005080500.jar;../server/lib/jndi/javax.mail.glassfish_1.4.1.v201005082020.jar;../server/lib/jsp/com.sun.el_1.0.0.v201004190952.jar;../server/lib/jsp/ecj-3.6.jar;../server/lib/jsp/javax.el_2.1.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp.jstl_1.2.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp_2.1.0.v201004190952.jar;../server/lib/jsp/jetty-jsp-2.1-7.5.4.v20111024.jar;../server/lib/jsp/jsp-impl-2.1.3-b10.jar;../server/lib/jsp/org.apache.taglibs.standard.glassfish_1.2.0.v201004190952.jar;../server/lib/jta/javax.transaction_1.1.1.v201004190952.jar;../server/lib/monitor/jetty-monitor-7.5.4.v20111024.jar</td></tr>
<tr><td>java.specification.version</td><td>XXX</td></tr>
<tr><td>java.specification.vendor</td><td>XXXXX</td></tr>
<tr><td>java.specification.name</td><td>XXXXXX</td></tr>
<tr><td>java.vendor.url</td><td>http://XXXXXX/</td></tr>
<tr><td>java.vm.specification.version</td><td>XXXX</td></tr>
<tr><td>java.vm.specification.vendor</td><td>XXXXXXXX</td></tr>
<tr><td>java.vm.specification.name</td><td>XXXXXXXX</td></tr>
<tr><td>java.class.version</td><td>XXXXXXXX</td></tr>
<tr><td>java.library.path</td><td>../XXXX</td></tr>
<tr><td>java.io.tmpdir</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.compiler</td><td>XXXX</td></tr>
<tr><td>java.ext.dirs</td><td>XXXXXXXXXXX\java\jre\lib\ext;XXXXXXXXXXX</td></tr>
<tr><td width="30%"><b>Other</b></td><td>&nbsp;</td></tr>
<tr><td>Total memory</td><td>8984MB</td></tr>
<tr><td>Free memory</td><td>8121MB</td></tr>
<tr><td>Max memory to be used</td><td>1984MB</td></tr>
<tr><td>Available Processors</td><td>8</td></tr>
<tr><td>Using config file</td><td>XXXX</td></tr>
</table>

<form NAME ="filterForm">
<TABLE>
<tr class ="filter"></TD><B>XXXXXX</B><TD></TR>
<TR class ="filter">
<TD class ="filter">XXXXX:</TD>
<TD class ="filter"><XXXXXXXXXX> </TD>
<TD class ="filter">XXXXXX</TD>
<TD class ="filter"><XXXXXXXXXX="level">
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
</SELECT>
</TD>
</TR>
<TR class ="filter">
<TD class ="filter">Login:</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXX</TD>
<TD class ="filter"><INPUT NAME=XXXX SIZE=XXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXX TYPE=TEXT VALUE=""></TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><XXXXXXXXX SIZE=XX TYPE=TEXT VALUE=""></TD>
<TD class ="filter"></TD>
<TD class ="filter"><BUTTON name="filterB" type="button" onClick="filter()" > Filter </BUTTON>
<BUTTON name="clearDilterB" type="button" onClick="clearFilter()">Clear Filter</BUTTON>
</TD>
</TR>

</TABLE>
</FORM>
<table width="100%" cellPadding="8" cellSpacing="1" align="right" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr bgcolor="XXXX">
<td width="7%" style="color: Orange"><b>xxxxxx</td>
<td width="7%" style="color: Orange">xxxxxxxxxx</td>
<td width="18%" style="color: Orange"><b>xxxxxxxxxx</b></td>
<td width="8%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="7%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="5%" style="color: Orange"><b>xxxxxxx</b></td>
<td width="18%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="30%" style="color: Orange"><b>xxxxxxxxxx</b></td>
</tr>

<tr ><td>Actual event starts from here</td><td>


Any solution would be appreciated.

-Madhu 

0 Karma

richgalloway
SplunkTrust
SplunkTrust

The FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER properties only apply to CSV files and have nothing to do with removing events.

It's not clear if you want to remove the entire event or just the HTML.

To remove the entire event you'll need a transform.

To remove just the HTML you can use SEDCMD.

In either case, you will need a regular expression (regex) that matches the part you wish to discard.

---
If this reply helps you, Karma would be appreciated.
0 Karma
Get Updates on the Splunk Community!

Welcome to the Future of Data Search & Exploration

You have more data coming at you than ever before. Over the next five years, the total amount of digital data ...

What’s new on Splunk Lantern in August

This month’s Splunk Lantern update gives you the low-down on all of the articles we’ve published over the past ...

This Week's Community Digest - Splunk Community Happenings [8.3.22]

Get the latest news and updates from the Splunk Community here! News From Splunk Answers ✍️ Splunk Answers is ...