How to Monitor MomgoDB Replica set using Splunk?

[Amit79]

Hello All,

I need to monitor MongoDB Replica set for its status.

For this I have to run rs.status command in admin DB for MongoDB, this will give me JSON output and i need to look for status for replica set in that out and trigger the alert.

Appreciate any pointers on this and if someone could take a look at below code provide the feedback that will be helpful, this one is for triggering the alert based on condition, I am trying to use case for this.

index =XXXX
| eval rs_status=case(status == "Primary", "OK", status =="ARBITER", "OK", status == "SECONDARY", "OK", status == "STARTUP", "KO", status == "RECOVERING", "KO" status == "STARTUP2", "KO", status == "UNKNOWN", "KO", status == "DOWN", "KO", status == "ROLLBACK", "KO", status == "REMOVED", "KO")
| sort - _time
| where status="KO"

Let me know if you see any issues here.

Regards

Amit

[Amit79]

Updating my SPL to below

index=XXX source="XXX"
| eval status=spath(_raw,members{}.state)
| eval rs_status=case(status == "Primary", "OK", status =="ARBITER", "OK", status == "SECONDARY", "OK", status == "STARTUP", "KO", status == "RECOVERING", "KO" status == "STARTUP2", "KO", status == "UNKNOWN", "KO", status == "DOWN", "KO", status == "ROLLBACK", "KO", status == "REMOVED", "KO")
| sort - _time
| where rs_status="KO"

below is the JSON format

{
"set" : "replset",
"date" : ISODate("2020-03-05T05:24:45.567Z"),
"myState" : 1,
"term" : NumberLong(3),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"votingMembersCount" : 3, // Available starting in v4.4
"writableVotingMembersCount" : 3, // Available starting in v4.4
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1583385878, 1),
"t" : NumberLong(3)
},
"lastCommittedWallTime" : ISODate("2020-03-05T05:24:38.122Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1583385878, 1),
"t" : NumberLong(3)
},
"readConcernMajorityWallTime" : ISODate("2020-03-05T05:24:38.122Z"),
"appliedOpTime" : {
"ts" : Timestamp(1583385878, 1),
"t" : NumberLong(3)
},
"durableOpTime" : {
"ts" : Timestamp(1583385878, 1),
"t" : NumberLong(3)
},
"lastAppliedWallTime" : ISODate("2020-03-05T05:24:38.122Z"),
"lastDurableWallTime" : ISODate("2020-03-05T05:24:38.122Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1583385868, 2),
"electionCandidateMetrics" : {
"lastElectionReason" : "stepUpRequestSkipDryRun",
"lastElectionDate" : ISODate("2020-03-05T05:24:28.061Z"),
"electionTerm" : NumberLong(3),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1583385864, 1),
"t" : NumberLong(2)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1583385864, 1),
"t" : NumberLong(2)
},
"numVotesNeeded" : 2,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"priorPrimaryMemberId" : 1,
"numCatchUpOps" : NumberLong(0),
"newTermStartDate" : ISODate("2020-03-05T05:24:28.118Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2020-03-05T05:24:28.228Z")
},
"electionParticipantMetrics" : {
"votedForCandidate" : true,
"electionTerm" : NumberLong(2),
"lastVoteDate" : ISODate("2020-03-05T05:22:33.306Z"),
"electionCandidateMemberId" : 1,
"voteReason" : "",
"lastAppliedOpTimeAtElection" : {
"ts" : Timestamp(1583385748, 1),
"t" : NumberLong(1)
},
"maxAppliedOpTimeInSet" : {
"ts" : Timestamp(1583385748, 1),
"t" : NumberLong(1)
},
"priorityAtElection" : 1
},
"members" : [
{
"_id" : 0,
"name" : "m1.example.net:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 269,
"optime" : {
"ts" : Timestamp(1583385878, 1),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2020-03-05T05:24:38Z"),
"lastAppliedWallTime": ISODate("2020-03-05T05:24:38Z"),
"lastDurableWallTime": ISODate("2020-03-05T05:24:38Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1583385868, 1),
"electionDate" : ISODate("2020-03-05T05:24:28Z"),
"configVersion" : 1,
"configTerm" : 0,
"self" : true,
"lastHeartbeatMessage" : ""
},