Monitoring of Java Virtual Machines with JMX - - Issue getting this to work on the forwarders

[rachaelcrook89]

I have followed the steps defined in this Splunk answers but have been unable to get JMX data working for the majority of our servers. I say majority because 2 out of 20 servers are working. They are all using the same config, but differen't pid numbers, so it's possible there is some environment dependency, that's undocumented, missing.

https://answers.splunk.com/answers/210216/is-it-possible-to-move-the-monitoring-of-java-virt.html

-Here's the contents of my config file.

<jmxpoller>
<formatter className="com.dtdsoftware.splunk.formatter.TokenizedMBeanNameQuotesStrippedFormatter" />

<!-- Connect to a local JVM by PID obtained from the output of a custom command -->
<jmxserver pid="1234" jvmDescription="JVM Description">

<mbean domain="java.lang" properties="type=Threading">

<!-- simple attributes -->
<attribute name="TotalStartedThreadCount" outputname="started"/>
<attribute name="ThreadCpuTimeSupported" outputname="cpuTimeSupported"/>
<attribute name="ThreadCpuTimeEnabled" outputname="cpuTimeEnabled"/>
<attribute name="ThreadCount" outputname="count"/>
<attribute name="ThreadContentionMonitoringSupported" outputname="cmSupported"/>
<attribute name="ThreadContentionMonitoringEnabled" outputname="cmEnabled"/>
<attribute name="SynchronizerUsageSupported" outputname="synchSupported"/>
<attribute name="PeakThreadCount" outputname="peak"/>
<attribute name="ObjectMonitorUsageSupported" outputname="omuSupported"/>
<attribute name="DaemonThreadCount" outputname="daemonCount"/>
<attribute name="CurrentThreadUserTime" outputname="currentUserTime"/>
<attribute name="CurrentThreadCpuTimeSupported" outputname="cpuTimeSupported"/>
<attribute name="CurrentThreadCpuTime" outputname="currentCPUTime"/>

</mbean>


<mbean domain="java.lang" properties="type=Runtime">

<!-- tabular attributes -->
<attribute name="SystemProperties:user.timezone:value" outputname="timezone"/>
<attribute name="SystemProperties:java.vm.vendor:value" outputname="jvmVendor"/>

<!-- simple attributes -->
<attribute name="VmVersion" outputname="jvmVersion"/>
<attribute name="VmVendor" outputname="jvmVendor"/>
<attribute name="VmName" outputname="jvmName"/>
<attribute name="Uptime" outputname="jvmUptime"/>
<attribute name="StartTime" outputname="startedAt"/>
<attribute name="Name" outputname="jvmProcessName"/>

<!-- simple array attribute -->
<attribute name="InputArguments" outputname="inputArgs"/>

</mbean>


<mbean domain="java.lang" properties="type=OperatingSystem">

<!-- simple attributes -->
<attribute name="Version" outputname="version"/>
<attribute name="TotalSwapSpaceSize" outputname="swapSpace"/>
<attribute name="TotalPhysicalMemorySize" outputname="physMem"/>
<attribute name="SystemLoadAverage" outputname="sysLoad"/>
<attribute name="ProcessCpuTime" outputname="cpuTime"/>
<attribute name="OpenFileDescriptorCount" outputname="openFD"/>
<attribute name="Name" outputname="name"/>
<attribute name="MaxFileDescriptorCount" outputname="maxFD"/>
<attribute name="FreeSwapSpaceSize" outputname="freeSwap"/>
<attribute name="FreePhysicalMemorySize" outputname="freePhysMem"/>
<attribute name="CommittedVirtualMemorySize" outputname="committed"/>
<attribute name="AvailableProcessors" outputname="processors"/>
<attribute name="Arch" outputname="arch"/>


</mbean>

<mbean domain="java.lang" properties="type=Compilation">

<!-- simple attributes -->
<attribute name="TotalCompilationTime" outputname="time"/>

</mbean>

<mbean domain="java.lang" properties="type=ClassLoading">

<!-- simple attributes -->
<attribute name="TotalLoadedClassCount" outputname="total"/>
<attribute name="LoadedClassCount" outputname="current"/>
<attribute name="UnloadedClassCount" outputname="unloaded"/>

</mbean>


<mbean domain="java.lang" properties="type=Memory">

<!-- composite attributes -->
<attribute name="NonHeapMemoryUsage:used" outputname="nonHeapUsed"/>
<attribute name="NonHeapMemoryUsage:max" outputname="nonHeapMax"/>
<attribute name="NonHeapMemoryUsage:init" outputname="nonHeapInit"/>
<attribute name="NonHeapMemoryUsage:committed" outputname="nonHeapCommitted"/>

<attribute name="HeapMemoryUsage:used" outputname="heapUsed"/>
<attribute name="HeapMemoryUsage:max" outputname="heapMax"/>
<attribute name="HeapMemoryUsage:init" outputname="heapInit"/>
<attribute name="HeapMemoryUsage:committed" outputname="heapCommitted"/>

<attribute name="ObjectPendingFinalizationCount" outputname="objectsPending"/>

</mbean>


<mbean domain="java.lang" properties="type=MemoryPool,name=*">

<attribute name="Name" outputname="name"/>
<attribute name="Type" outputname="memoryType"/>
<attribute name="MemoryManagerNames" outputname="memManagerNames"/>

<attribute name="PeakUsage:used" outputname="peakUsageUsed"/>
<attribute name="PeakUsage:max" outputname="peakUsageMax"/>
<attribute name="PeakUsage:committed" outputname="peakUsageCommitted"/>
<attribute name="PeakUsage:init" outputname="peakUsageInit"/>

<!-- composite attributes -->
<attribute name="CollectionUsage:used" outputname="colUsageUsed"/>
<attribute name="CollectionUsage:max" outputname="colUsageMax"/>
<attribute name="CollectionUsage:committed" outputname="colUsageCommitted"/>
<attribute name="CollectionUsage:init" outputname="colUsageInit"/>

<!-- composite attributes -->
<attribute name="Usage:used" outputname="usageUsed"/>
<attribute name="Usage:max" outputname="usageMax"/>
<attribute name="Usage:committed" outputname="usageCommitted"/>
<attribute name="Usage:init" outputname="usageInit"/>


</mbean>


<mbean domain="java.lang" properties="type=GarbageCollector,name=*">

<!-- simple array attributes -->
<attribute name="Name" outputname="gcName"/>
<attribute name="MemoryPoolNames" outputname="memPools"/>

<!-- simple attributes -->
<attribute name="CollectionTime" outputname="colTime"/>
<attribute name="CollectionCount" outputname="colCount"/>

<!-- composite-simple attributes -->
<attribute name="LastGcInfo:startTime" outputname="gCStart"/>
<attribute name="LastGcInfo:endTime" outputname="gCEnd"/>
<attribute name="LastGcInfo:id" outputname="gCID"/>
<attribute name="LastGcInfo:duration" outputname="gCDuration"/>
<attribute name="LastGcInfo:GcThreadCount" outputname="gCThreadCount"/>

<!-- composite-tabular-composite-simple attributes --> 

<!-- 
Detailed GC Info for the various heap and non-heap memory pools.

I've included an example of extracting "Eden Space" & "Survivor Space"  attributes for HOTSPOT JVM with default GC setup, its just a copy paste exercise for the other pools

Perm Gen
Perm Gen [shared-ro]
Perm Gen [shared-rw]
Eden Space
Survivor Space
Tenured Gen

etc...


Based on which GC collector and policy you use and which JVM, these memory pool names may differ.


<attribute name="LastGcInfo:memoryUsageBeforeGc:Eden Space:value:used" outputname="eden_beforeGCUsed"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Eden Space:value:max" outputname="eden_beforeGCMax"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Eden Space:value:init" outputname="eden_beforeGCInit"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Eden Space:value:committed" outputname="eden_beforeGCCommitted"/>

<attribute name="LastGcInfo:memoryUsageAfterGc:Eden Space:value:used" outputname="eden_afterGCUsed"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Eden Space:value:max" outputname="eden_afterGCMax"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Eden Space:value:init" outputname="eden_afterGCInit"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Eden Space:value:committed" outputname="eden_afterGCCommitted"/>

<attribute name="LastGcInfo:memoryUsageBeforeGc:Survivor Space:value:used" outputname="survivor_beforeGCUsed"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Survivor Space:value:max" outputname="survivor_beforeGCMax"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Survivor Space:value:init" outputname="survivor_beforeGCInit"/>
<attribute name="LastGcInfo:memoryUsageBeforeGc:Survivor Space:value:committed" outputname="survivor_beforeGCCommitted"/>

<attribute name="LastGcInfo:memoryUsageAfterGc:Survivor Space:value:used" outputname="survivor_afterGCUsed"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Survivor Space:value:max" outputname="survivor_afterGCMax"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Survivor Space:value:init" outputname="survivor_afterGCInit"/>
<attribute name="LastGcInfo:memoryUsageAfterGc:Survivor Space:value:committed" outputname="survivor_afterGCCommitted"/>
-->

</mbean>

</jmxserver>


</jmxpoller>

Inputs

[jmx://JVM-Name]
config_file = config-JVM-Name.xml
polling_frequency = 60
sourcetype = jmx
index = custom-index_name
disabled = 0
interval = 30
_TCP_ROUTING = routinggroup
crcSalt = crcsalt2

Then here's the errors it throws.
jmx.log

2017-10-06 11:01:46,425 - com.splunk.modinput.ModularInput -5181309 [main] INFO - stanza count:3
2017-10-06 11:01:56,503 - com.splunk.modinput.ModularInput -5191387 [main] INFO - stanza count:3
2017-10-06 11:02:00,690 - org.exolab.castor.mapping.Mapping -5195574 [Thread-3] INFO - Loading mapping descriptors from jar:file:/D:/program%20files/SplunkUniversalForwarder/etc/apps/jmx_app_name/bin/lib/jmxmodinput.jar!/mapping.xml
2017-10-06 11:02:00,690 - org.exolab.castor.mapping.Mapping -5195574 [Thread-4] INFO - Loading mapping descriptors from jar:file:/D:/program%20files/SplunkUniversalForwarder/etc/apps/jmx_app_name/bin/lib/jmxmodinput.jar!/mapping.xml
2017-10-06 11:02:00,690 - org.exolab.castor.mapping.Mapping -5195574 [Thread-2] INFO - Loading mapping descriptors from jar:file:/D:/program%20files/SplunkUniversalForwarder/etc/apps/jmx_app_name/bin/lib/jmxmodinput.jar!/mapping.xml
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-4] ERROR - cannot create JMXServiceURL for server with description JVM Description - no providers installed
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-3] ERROR - cannot create JMXServiceURL for server with description JVM Description - no providers installed
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-4] INFO - 0 servers found in stanza jmx://JVM-Name
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-3] INFO - 0 servers found in stanza jmx://JVM-Name
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-2] ERROR - cannot create JMXServiceURL for server with description JVM Description - no providers installed
2017-10-06 11:02:00,690 - com.splunk.modinput.ModularInput -5195574 [Thread-2] INFO - 0 servers found in stanza jmx://jmx://JVM-Name

splunkd.log

10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py"" Traceback (most recent call last):
10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py"" File "D:\Program Files\SplunkUniversalForwarder\etc\apps\us_ssloansvc_qa_jmx4\bin\jmx.py", line 143, in
10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py"" monitor_tasks(process, token)
10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\us_ssloansvc_qa_jmx4\bin\jmx.py"" File "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py", line 103, in monitor_tasks
10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py"" if update_inputs:
10-06-2017 15:42:00.019 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\jmx_app_name\bin\jmx.py"" NameError: global name 'update_inputs' is not defined

[Damien_Dallimor · Accepted solution]

ERROR - cannot create JMXServiceURL
for server with description JVM
Description - no providers installed
2017-10-06 11:02:00,690 -
com.splunk.modinput.ModularInput -51

Perhaps this info from the docs can help you out

[akocak]

can you send link for doc?

[rachaelcrook89]

Thanks for this info! After adding the attach.dll file to the jdk_home/bin location on our windows servers, I was now able to get the jmx logs to send to Splunk. I had to add it under the jdk/bin in addition to the jre/bin location.

I'm still seeing a lot of errors in the jmx and splunkd logs, even though JMX data is sending.

JMX.log

2017-10-19 08:56:50,243 - com.splunk.modinput.ModularInput -1788 [Thread-3] ERROR - Exception@checkConnector, e=
java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: 10.0.0.1(IP Replaced); nested exception is:
java.net.ConnectException: Connection refused: connect]
at javax.management.remote.rmi.RMIConnector.connect(Unknown Source)
at javax.management.remote.JMXConnectorFactory.connect(Unknown Source)
at javax.management.remote.JMXConnectorFactory.connect(Unknown Source)
at com.splunk.jmx.ServerTask.connect(Unknown Source)
at com.splunk.jmx.ServerTask.checkConnector(Unknown Source)
at com.splunk.jmx.Scheduler.run(Unknown Source)
Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: 10.0.0.1; nested exception is:
java.net.ConnectException: Connection refused: connect]
at com.sun.jndi.rmi.registry.RegistryContext.lookup(Unknown Source)
at com.sun.jndi.toolkit.url.GenericURLContext.lookup(Unknown Source)
at javax.naming.InitialContext.lookup(Unknown Source)
at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(Unknown Source)
at javax.management.remote.rmi.RMIConnector.findRMIServer(Unknown Source)
... 6 more
Caused by: java.rmi.ConnectException: Connection refused to host: 10.254.184.159; nested exception is:
java.net.ConnectException: Connection refused: connect
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(Unknown Source)
at sun.rmi.transport.tcp.TCPChannel.createConnection(Unknown Source)
at sun.rmi.transport.tcp.TCPChannel.newConnection(Unknown Source)
at sun.rmi.server.UnicastRef.newCall(Unknown Source)
at sun.rmi.registry.RegistryImpl_Stub.lookup(Unknown Source)
... 11 more
Caused by: java.net.ConnectException: Connection refused: connect
at java.net.DualStackPlainSocketImpl.connect0(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(Unknown Source)
at java.net.AbstractPlainSocketImpl.doConnect(Unknown Source)
at java.net.AbstractPlainSocketImpl.connectToAddress(Unknown Source)
at java.net.AbstractPlainSocketImpl.connect(Unknown Source)
at java.net.PlainSocketImpl.connect(Unknown Source)
at java.net.SocksSocketImpl.connect(Unknown Source)
at java.net.Socket.connect(Unknown Source)
at java.net.Socket.connect(Unknown Source)
at java.net.Socket.(Unknown Source)
at java.net.Socket.(Unknown Source)
at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(Unknown Source)
at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(Unknown Source)
... 16 more

This error repeats in the splunkd.log file about 4 times every 10 seconds.

10-19-2017 09:03:12.626 -0500 ERROR ExecProcessor - message from "python "D:\Program Files\SplunkUniversalForwarder\etc\apps\us_ssloansvc_dev_jmx\bin\jmx.py"" [Fatal Error] :-1:-1: Premature end of file.

[Damien_Dallimor]

The docs are :

1) on splunkbase https://splunkbase.splunk.com/app/668/#/details
2) included in the app also