Splunk Search

Remove useless header HTML events

Madhu02splunk
New Member

Hey Splunkers!

Could someone please help me to remove useless header HTML events before it gets indexed into splunk.
There are 300 events we need to remove and indexed actual events.
I have already setup FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER in props.conf.

Below is the event I wanted to remove.

<html>
<style>
h1 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
h2 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: bold; color: navy;}
tr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000;}
td { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000; border: 0 solid dimgray; border-top-width: 1pt; border-right-width: xpt;vertical-align:text-top;}
hr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
body { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
table { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
td.navy {color: navy;}
tr.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
td.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
</style>
<script type="text/javascript">
<!--
function JSTrim(p_strToBeTrimmed)
{
var vChar
var vLength
var i
var vFirstNotSpace
var vLastNotSpace

vLength = p_strToBeTrimmed.length
for (i = 0; i < vLength;i++)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vFirstNotSpace = i
i = vLength
}
}
for (i = vLength-1 ; i>=0;i--)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vLastNotSpace = i
i = -1
}
}
return p_strToBeTrimmed.substring(vFirstNotSpace,vLastNotSpace+1);
}


function toggle(f_level, f_thread, f_method, f_message, f_login, f_IP){

mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(3);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;

var levels = "XXX";
if(f_level != "ERR"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
}
}
}

// go over all the row and show/hide them
for (i=1;i<numOfRows;i++) {
var tdarr = trArray.item(i).getElementsByTagName("td");
thread = tdarr.item(2).childNodes.item(0).data;
login = tdarr.item(3).childNodes.item(0).data;
IP = tdarr.item(4).childNodes.item(0).data;
logLevel = tdarr.item(5).childNodes.item(0).data;
method = tdarr.item(6).childNodes.item(0).data;
message = tdarr.item(7).childNodes.item(0).data;

logLevel = JSTrim(logLevel);

if((levels.search(XXXX) !=-1) &&
(thread.search(XXXX) !=-1) &&
(login.search(XXXX) !=-1) &&
(IP.search(XXXX) !=-1) &&
(method.search(XXXX) !=-1) &&
(message.search(XXXX) !=-1)){
trArray.item(i).style.display="inline";
}else{
trArray.item(i).style.display="none";
}
}
}

function clearFilter(){
document.filterForm.level.selectedIndex = 0;
document.filterForm.thread.value="";
document.filterForm.Method.value="";
document.filterForm.Message.value="";
showAll();
}
function showAll(){
mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(1);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;
for (i=1;i<numOfRows;i++) {
trArray.item(i).style.display="inline";
}
}
function filter(){
var w = document.filterForm.level.selectedIndex;
var XXXX = document.filterForm.level.options[w].text;
var XXXX = document.filterForm.thread.value;
var XXXX = document.filterForm.Method.value;
var XXXX = document.filterForm.Message.value;
var XXXX = document.filterForm.Login.value;
varXXXX = document.filterForm.IP.value;
toggle(logLevel,JSTrim(XXXX),JSTrim(XXXX),JSTrim(XXXX), JSTrim(XXXX), JSTrim(XXXX));
}
--></script>
<body bgcolor="XXXXXX">
<a href="xxxxxxxxxxxxxxx >Go to previous log</a>
<h2>xxxxxxxxxxxx</h2><table>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
</table>
<h2>Java Properties</h2>
<table cellSpacing="0" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr><td width="30%"><b>OS</b></td><td>&nbsp;</td></tr>
<tr><td>os.name</td><td>XXXXX</td></tr>
<tr><td>os.version</td><td>XXX</td></tr>
<tr><td>os.arch</td><td>XXX</td></tr>
<tr><td>os.home</td><td>XXX</td></tr>
<tr><td width="30%"><b>XX</b></td><td>&nbsp;</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXXXXX</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXX</td></tr>
<tr><td width="30%"><b>XXX</b></td><td>&XXXX;</td></tr>
<tr><td>user.name</td><td>XXXXXX</td></tr>
<tr><td>user.home</td><td>XX\</td></tr>
<tr><td>user.dir</td><td>XXXXXXXXXXXXXXXXXX</td></tr>
<tr><td>user.language</td><td>en</td></tr>
<tr><td width="30%"><b>Java</b></td><td>&nbsp;</td></tr>
<tr><td>java.vm.vendor</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.version</td><td>XXXX</td></tr>
<tr><td>java.vm.version</td><td>XXXXXXX</td></tr>
<tr><td>java.home</td><td>XXXXXXXXXXXX\java\jre</td></tr>
<tr><td>java.class.path</td><td>../wrapper/wrapper.jar;../server/lib/annotations;../server/lib/ext;../server/lib/jetty-ajp-7.5.4.v20111024.jar;../server/lib/jetty-all-7.5.4.v20111024-javadoc.jar;../server/lib/jetty-annotations-7.5.4.v20111024.jar;../server/lib/jetty-client-7.5.4.v20111024.jar;../server/lib/jetty-continuation-7.5.4.v20111024.jar;../server/lib/jetty-deploy-7.5.4.v20111024.jar;../server/lib/jetty-http-7.5.4.v20111024.jar;../server/lib/jetty-io-7.5.4.v20111024.jar;../server/lib/jetty-jmx-7.5.4.v20111024.jar;../server/lib/jetty-jndi-7.5.4.v20111024.jar;../server/lib/jetty-overlay-deployer-7.5.4.v20111024.jar;../server/lib/jetty-plus-7.5.4.v20111024.jar;../server/lib/jetty-policy-7.5.4.v20111024.jar;../server/lib/jetty-rewrite-7.5.4.v20111024.jar;../server/lib/jetty-security-7.5.4.v20111024.jar;../server/lib/jetty-server-7.5.4.v20111024.jar;../server/lib/jetty-servlet-7.5.4.v20111024.jar;../server/lib/jetty-servlets-7.5.4.v20111024.jar;../server/lib/jetty-util-7.5.4.v20111024.jar;../server/lib/jetty-webapp-7.5.4.v20111024.jar;../server/lib/jetty-websocket-7.5.4.v20111024.jar;../server/lib/jetty-xml-7.5.4.v20111024.jar;../server/lib/jndi;../server/lib/jsp;../server/lib/jta;../server/lib/launcher-11.50.9999-GA-SNAPSHOT.jar;../server/lib/lxxxxxxxxxxxxxx;../server/lib/launcher-11.51.9999-SNAPSHOT.jar;../server/lib/launcher-sources.jar;../server/lib/launcher.jar;../server/lib/monitor;../server/lib/policy;../server/lib/servlet-api-2.5.jar;../server/lib/annotations/javax.annotation_1.0.0.v20100513-0750.jar;../server/lib/annotations/org.objectweb.asm_3.1.0.v200803061910.jar;../server/lib/ext/.donotdelete;../server/lib/jndi/javax.activation_1.1.0.v201005080500.jar;../server/lib/jndi/javax.mail.glassfish_1.4.1.v201005082020.jar;../server/lib/jsp/com.sun.el_1.0.0.v201004190952.jar;../server/lib/jsp/ecj-3.6.jar;../server/lib/jsp/javax.el_2.1.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp.jstl_1.2.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp_2.1.0.v201004190952.jar;../server/lib/jsp/jetty-jsp-2.1-7.5.4.v20111024.jar;../server/lib/jsp/jsp-impl-2.1.3-b10.jar;../server/lib/jsp/org.apache.taglibs.standard.glassfish_1.2.0.v201004190952.jar;../server/lib/jta/javax.transaction_1.1.1.v201004190952.jar;../server/lib/monitor/jetty-monitor-7.5.4.v20111024.jar</td></tr>
<tr><td>java.specification.version</td><td>XXX</td></tr>
<tr><td>java.specification.vendor</td><td>XXXXX</td></tr>
<tr><td>java.specification.name</td><td>XXXXXX</td></tr>
<tr><td>java.vendor.url</td><td>http://XXXXXX/</td></tr>
<tr><td>java.vm.specification.version</td><td>XXXX</td></tr>
<tr><td>java.vm.specification.vendor</td><td>XXXXXXXX</td></tr>
<tr><td>java.vm.specification.name</td><td>XXXXXXXX</td></tr>
<tr><td>java.class.version</td><td>XXXXXXXX</td></tr>
<tr><td>java.library.path</td><td>../XXXX</td></tr>
<tr><td>java.io.tmpdir</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.compiler</td><td>XXXX</td></tr>
<tr><td>java.ext.dirs</td><td>XXXXXXXXXXX\java\jre\lib\ext;XXXXXXXXXXX</td></tr>
<tr><td width="30%"><b>Other</b></td><td>&nbsp;</td></tr>
<tr><td>Total memory</td><td>8984MB</td></tr>
<tr><td>Free memory</td><td>8121MB</td></tr>
<tr><td>Max memory to be used</td><td>1984MB</td></tr>
<tr><td>Available Processors</td><td>8</td></tr>
<tr><td>Using config file</td><td>XXXX</td></tr>
</table>

<form NAME ="filterForm">
<TABLE>
<tr class ="filter"></TD><B>XXXXXX</B><TD></TR>
<TR class ="filter">
<TD class ="filter">XXXXX:</TD>
<TD class ="filter"><XXXXXXXXXX> </TD>
<TD class ="filter">XXXXXX</TD>
<TD class ="filter"><XXXXXXXXXX="level">
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
</SELECT>
</TD>
</TR>
<TR class ="filter">
<TD class ="filter">Login:</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXX</TD>
<TD class ="filter"><INPUT NAME=XXXX SIZE=XXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXX TYPE=TEXT VALUE=""></TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><XXXXXXXXX SIZE=XX TYPE=TEXT VALUE=""></TD>
<TD class ="filter"></TD>
<TD class ="filter"><BUTTON name="filterB" type="button" onClick="filter()" > Filter </BUTTON>
<BUTTON name="clearDilterB" type="button" onClick="clearFilter()">Clear Filter</BUTTON>
</TD>
</TR>

</TABLE>
</FORM>
<table width="100%" cellPadding="8" cellSpacing="1" align="right" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr bgcolor="XXXX">
<td width="7%" style="color: Orange"><b>xxxxxx</td>
<td width="7%" style="color: Orange">xxxxxxxxxx</td>
<td width="18%" style="color: Orange"><b>xxxxxxxxxx</b></td>
<td width="8%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="7%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="5%" style="color: Orange"><b>xxxxxxx</b></td>
<td width="18%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="30%" style="color: Orange"><b>xxxxxxxxxx</b></td>
</tr>

<tr ><td>Actual event starts from here</td><td>


Any solution would be appreciated.

-Madhu 

0 Karma

richgalloway
SplunkTrust
SplunkTrust

The FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER properties only apply to CSV files and have nothing to do with removing events.

It's not clear if you want to remove the entire event or just the HTML.

To remove the entire event you'll need a transform.

To remove just the HTML you can use SEDCMD.

In either case, you will need a regular expression (regex) that matches the part you wish to discard.

---
If this reply helps you, an upvote would be appreciated.
0 Karma
Get Updates on the Splunk Community!

Improve Your Security Posture

Watch NowImprove Your Security PostureCustomers are at the center of everything we do at Splunk and security ...

Maximize the Value from Microsoft Defender with Splunk

 Watch NowJoin Splunk and Sens Consulting for this Security Edition Tech TalkWho should attend:  Security ...

This Week's Community Digest - Splunk Community Happenings [6.27.22]

Get the latest news and updates from the Splunk Community here! News From Splunk Answers ✍️ Splunk Answers is ...