Remove useless header HTML events

Madhu02splunk · ‎08-24-2020

Hey Splunkers!

Could someone please help me to remove useless header HTML events before it gets indexed into splunk.
There are 300 events we need to remove and indexed actual events.
I have already setup FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER in props.conf.

Below is the event I wanted to remove.

<html>
<style>
h1 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
h2 { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: bold; color: navy;}
tr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000;}
td { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: #000000; border: 0 solid dimgray; border-top-width: 1pt; border-right-width: xpt;vertical-align:text-top;}
hr { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: xxpx; font-weight: normal; color: navy;}
body { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
table { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
td.navy {color: navy;}
tr.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000;}
td.filter { font-family: xxxx, Arial, Helvetica, sans-serif; font-size: 12px; font-weight: normal; color: #000000; border: 0 solid dimgray;}
</style>
<script type="text/javascript">
<!--
function JSTrim(p_strToBeTrimmed)
{
var vChar
var vLength
var i
var vFirstNotSpace
var vLastNotSpace

vLength = p_strToBeTrimmed.length
for (i = 0; i < vLength;i++)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vFirstNotSpace = i
i = vLength
}
}
for (i = vLength-1 ; i>=0;i--)
{
vChar = p_strToBeTrimmed.charAt(i)
if (vChar != " ")
{
vLastNotSpace = i
i = -1
}
}
return p_strToBeTrimmed.substring(vFirstNotSpace,vLastNotSpace+1);
}


function toggle(f_level, f_thread, f_method, f_message, f_login, f_IP){

mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(3);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;

var levels = "XXX";
if(f_level != "ERR"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
if(f_level != "XXX"){
levels+="XXX";
}
}
}

// go over all the row and show/hide them
for (i=1;i<numOfRows;i++) {
var tdarr = trArray.item(i).getElementsByTagName("td");
thread = tdarr.item(2).childNodes.item(0).data;
login = tdarr.item(3).childNodes.item(0).data;
IP = tdarr.item(4).childNodes.item(0).data;
logLevel = tdarr.item(5).childNodes.item(0).data;
method = tdarr.item(6).childNodes.item(0).data;
message = tdarr.item(7).childNodes.item(0).data;

logLevel = JSTrim(logLevel);

if((levels.search(XXXX) !=-1) &&
(thread.search(XXXX) !=-1) &&
(login.search(XXXX) !=-1) &&
(IP.search(XXXX) !=-1) &&
(method.search(XXXX) !=-1) &&
(message.search(XXXX) !=-1)){
trArray.item(i).style.display="inline";
}else{
trArray.item(i).style.display="none";
}
}
}

function clearFilter(){
document.filterForm.level.selectedIndex = 0;
document.filterForm.thread.value="";
document.filterForm.Method.value="";
document.filterForm.Message.value="";
showAll();
}
function showAll(){
mybody=document.getElementsByTagName("body").item(0);
mytable= mybody.getElementsByTagName("table").item(1);
mytablebody=mytable.getElementsByTagName("tbody").item(0);
trArray = mytablebody.getElementsByTagName("tr");
numOfRows =mytablebody.getElementsByTagName("tr").length;
for (i=1;i<numOfRows;i++) {
trArray.item(i).style.display="inline";
}
}
function filter(){
var w = document.filterForm.level.selectedIndex;
var XXXX = document.filterForm.level.options[w].text;
var XXXX = document.filterForm.thread.value;
var XXXX = document.filterForm.Method.value;
var XXXX = document.filterForm.Message.value;
var XXXX = document.filterForm.Login.value;
varXXXX = document.filterForm.IP.value;
toggle(logLevel,JSTrim(XXXX),JSTrim(XXXX),JSTrim(XXXX), JSTrim(XXXX), JSTrim(XXXX));
}
--></script>
<body bgcolor="XXXXXX">
<a href="xxxxxxxxxxxxxxx >Go to previous log</a>
<h2>xxxxxxxxxxxx</h2><table>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxx</td><td class ="filter">XXXXX</td></tr>
<tr><td class ="filter">xxxxxxxxx</td><td class ="filter">XXXX</td></tr>
<tr><td class ="filter">xxxxxxxxxx</td><td class ="filter">1XXXXXX</td></tr>
</table>
<h2>Java Properties</h2>
<table cellSpacing="0" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr><td width="30%"><b>OS</b></td><td>&nbsp;</td></tr>
<tr><td>os.name</td><td>XXXXX</td></tr>
<tr><td>os.version</td><td>XXX</td></tr>
<tr><td>os.arch</td><td>XXX</td></tr>
<tr><td>os.home</td><td>XXX</td></tr>
<tr><td width="30%"><b>XX</b></td><td>&nbsp;</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXXXXX</td></tr>
<tr><td>xxxxxxxxxx</td><td>XXXXX</td></tr>
<tr><td width="30%"><b>XXX</b></td><td>&XXXX;</td></tr>
<tr><td>user.name</td><td>XXXXXX</td></tr>
<tr><td>user.home</td><td>XX\</td></tr>
<tr><td>user.dir</td><td>XXXXXXXXXXXXXXXXXX</td></tr>
<tr><td>user.language</td><td>en</td></tr>
<tr><td width="30%"><b>Java</b></td><td>&nbsp;</td></tr>
<tr><td>java.vm.vendor</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.version</td><td>XXXX</td></tr>
<tr><td>java.vm.version</td><td>XXXXXXX</td></tr>
<tr><td>java.home</td><td>XXXXXXXXXXXX\java\jre</td></tr>
<tr><td>java.class.path</td><td>../wrapper/wrapper.jar;../server/lib/annotations;../server/lib/ext;../server/lib/jetty-ajp-7.5.4.v20111024.jar;../server/lib/jetty-all-7.5.4.v20111024-javadoc.jar;../server/lib/jetty-annotations-7.5.4.v20111024.jar;../server/lib/jetty-client-7.5.4.v20111024.jar;../server/lib/jetty-continuation-7.5.4.v20111024.jar;../server/lib/jetty-deploy-7.5.4.v20111024.jar;../server/lib/jetty-http-7.5.4.v20111024.jar;../server/lib/jetty-io-7.5.4.v20111024.jar;../server/lib/jetty-jmx-7.5.4.v20111024.jar;../server/lib/jetty-jndi-7.5.4.v20111024.jar;../server/lib/jetty-overlay-deployer-7.5.4.v20111024.jar;../server/lib/jetty-plus-7.5.4.v20111024.jar;../server/lib/jetty-policy-7.5.4.v20111024.jar;../server/lib/jetty-rewrite-7.5.4.v20111024.jar;../server/lib/jetty-security-7.5.4.v20111024.jar;../server/lib/jetty-server-7.5.4.v20111024.jar;../server/lib/jetty-servlet-7.5.4.v20111024.jar;../server/lib/jetty-servlets-7.5.4.v20111024.jar;../server/lib/jetty-util-7.5.4.v20111024.jar;../server/lib/jetty-webapp-7.5.4.v20111024.jar;../server/lib/jetty-websocket-7.5.4.v20111024.jar;../server/lib/jetty-xml-7.5.4.v20111024.jar;../server/lib/jndi;../server/lib/jsp;../server/lib/jta;../server/lib/launcher-11.50.9999-GA-SNAPSHOT.jar;../server/lib/lxxxxxxxxxxxxxx;../server/lib/launcher-11.51.9999-SNAPSHOT.jar;../server/lib/launcher-sources.jar;../server/lib/launcher.jar;../server/lib/monitor;../server/lib/policy;../server/lib/servlet-api-2.5.jar;../server/lib/annotations/javax.annotation_1.0.0.v20100513-0750.jar;../server/lib/annotations/org.objectweb.asm_3.1.0.v200803061910.jar;../server/lib/ext/.donotdelete;../server/lib/jndi/javax.activation_1.1.0.v201005080500.jar;../server/lib/jndi/javax.mail.glassfish_1.4.1.v201005082020.jar;../server/lib/jsp/com.sun.el_1.0.0.v201004190952.jar;../server/lib/jsp/ecj-3.6.jar;../server/lib/jsp/javax.el_2.1.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp.jstl_1.2.0.v201004190952.jar;../server/lib/jsp/javax.servlet.jsp_2.1.0.v201004190952.jar;../server/lib/jsp/jetty-jsp-2.1-7.5.4.v20111024.jar;../server/lib/jsp/jsp-impl-2.1.3-b10.jar;../server/lib/jsp/org.apache.taglibs.standard.glassfish_1.2.0.v201004190952.jar;../server/lib/jta/javax.transaction_1.1.1.v201004190952.jar;../server/lib/monitor/jetty-monitor-7.5.4.v20111024.jar</td></tr>
<tr><td>java.specification.version</td><td>XXX</td></tr>
<tr><td>java.specification.vendor</td><td>XXXXX</td></tr>
<tr><td>java.specification.name</td><td>XXXXXX</td></tr>
<tr><td>java.vendor.url</td><td>http://XXXXXX/</td></tr>
<tr><td>java.vm.specification.version</td><td>XXXX</td></tr>
<tr><td>java.vm.specification.vendor</td><td>XXXXXXXX</td></tr>
<tr><td>java.vm.specification.name</td><td>XXXXXXXX</td></tr>
<tr><td>java.class.version</td><td>XXXXXXXX</td></tr>
<tr><td>java.library.path</td><td>../XXXX</td></tr>
<tr><td>java.io.tmpdir</td><td>XXXXXXXXXXXX</td></tr>
<tr><td>java.compiler</td><td>XXXX</td></tr>
<tr><td>java.ext.dirs</td><td>XXXXXXXXXXX\java\jre\lib\ext;XXXXXXXXXXX</td></tr>
<tr><td width="30%"><b>Other</b></td><td>&nbsp;</td></tr>
<tr><td>Total memory</td><td>8984MB</td></tr>
<tr><td>Free memory</td><td>8121MB</td></tr>
<tr><td>Max memory to be used</td><td>1984MB</td></tr>
<tr><td>Available Processors</td><td>8</td></tr>
<tr><td>Using config file</td><td>XXXX</td></tr>
</table>

<form NAME ="filterForm">
<TABLE>
<tr class ="filter"></TD><B>XXXXXX</B><TD></TR>
<TR class ="filter">
<TD class ="filter">XXXXX:</TD>
<TD class ="filter"><XXXXXXXXXX> </TD>
<TD class ="filter">XXXXXX</TD>
<TD class ="filter"><XXXXXXXXXX="level">
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
<OPTION VALUE="XXXX">XXXX</OPTION>
</SELECT>
</TD>
</TR>
<TR class ="filter">
<TD class ="filter">Login:</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXX</TD>
<TD class ="filter"><INPUT NAME=XXXX SIZE=XXX TYPE=TEXT VALUE=""> </TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><INPUT NAME="XXXXX" SIZE=XXX TYPE=TEXT VALUE=""></TD>
</TR>
<TR class ="filter">
<TD class ="filter">XXXXXXXXX</TD>
<TD class ="filter"><XXXXXXXXX SIZE=XX TYPE=TEXT VALUE=""></TD>
<TD class ="filter"></TD>
<TD class ="filter"><BUTTON name="filterB" type="button" onClick="filter()" > Filter </BUTTON>
<BUTTON name="clearDilterB" type="button" onClick="clearFilter()">Clear Filter</BUTTON>
</TD>
</TR>

</TABLE>
</FORM>
<table width="100%" cellPadding="8" cellSpacing="1" align="right" style="table-layout:fixed;word-break:break-all;border-width:1.5pt">
<tr bgcolor="XXXX">
<td width="7%" style="color: Orange"><b>xxxxxx</td>
<td width="7%" style="color: Orange">xxxxxxxxxx</td>
<td width="18%" style="color: Orange"><b>xxxxxxxxxx</b></td>
<td width="8%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="7%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="5%" style="color: Orange"><b>xxxxxxx</b></td>
<td width="18%" style="color: Orange"><b>xxxxxxxxx</b></td>
<td width="30%" style="color: Orange"><b>xxxxxxxxxx</b></td>
</tr>

<tr ><td>Actual event starts from here</td><td>

Any solution would be appreciated.

-Madhu

richgalloway · ‎08-24-2020

The FIELD_HEADER_REGEX and HEADER_FIELD_LINE_NUMBER properties only apply to CSV files and have nothing to do with removing events.

It's not clear if you want to remove the entire event or just the HTML.

To remove the entire event you'll need a transform.

To remove just the HTML you can use SEDCMD.

In either case, you will need a regular expression (regex) that matches the part you wish to discard.

---
If this reply helps you, Karma would be appreciated.

Remove useless header HTML events

What the End of Support for Splunk Add-on Builder Means for You

Solve, Learn, Repeat: New Puzzle Channel Now Live

Building Reliable Asset and Identity Frameworks in Splunk ES

Are you a member of the Splunk Community?

Remove useless header HTML events

What the End of Support for Splunk Add-on Builder Means for You

Solve, Learn, Repeat: New Puzzle Channel Now Live

Building Reliable Asset and Identity Frameworks in Splunk ES