I'm having an issue trying to get external_lookup.py to work to lookup from a specific dns server. I'm not sure what I'm doing wrong. I hope someone with better python knowledge than me can have a good idea. First I installed the dnspython module. Then I created the following script as a stand-alone and works like a champ.
ipinaddy = '173.252.110.27'
def newlookup(ipin):
try:
import dns.resolver
import string
my_resolver = dns.resolver.Resolver()
my_resolver.nameservers = ['8.8.8.8'] #set name servers seperated by comma
ipaddy = ipin
ipaddy = ipaddy.split('.')
ipaddy.reverse()
ipaddy = string.join(ipaddy, '.')
ipaddy = ipaddy + '.in-addr.arpa'
hostname = str(my_resolver.query(ipaddy,"PTR")[0])
return hostname
except:
return ''
result = newlookup(ipinaddy)
print result
The script above works and looks up from the google open dns and returns the facebook dns record. The next phase was to implement into external_lookup.py which is below. However it fails to lookup. I think I have something wrong with the csv part but I'm not sure. All I want it to do is return a dns record with IP input and be used like the original dnslookup in splunk. I would appreciate some help from someone that can script python better than me (I just started python scripting two days ago).
#!/usr/bin/env python
import csv
import sys
import commands
import socket
def rlookup(ipaddy):
try:
import dns.resolver
import string
my_resolver = dns.resolver.Resolver()
my_resolver.nameservers = ['8.8.8.8']
ipaddy = ipaddy.split('.')
ipaddy.reverse()
ipaddy = string.join(ipaddy, '.')
ipaddy = ipaddy + '.in-addr.arpa'
hostname = str(my_resolver.query(ipaddy,"PTR")[0])
return hostname
except:
return ''
# Given a host, find the ip
def lookup(host):
try:
hostname, aliaslist, ipaddrlist = socket.gethostbyname_ex(host)
return ipaddrlist
except:
return []
def main():
if len(sys.argv) != 3:
print "Usage: python external_lookup.py [host field] [ip field]"
sys.exit(1)
hostfield = sys.argv[1]
ipfield = sys.argv[2]
infile = sys.stdin
outfile = sys.stdout
r = csv.DictReader(infile)
header = r.fieldnames
w = csv.DictWriter(outfile, fieldnames=r.fieldnames)
w.writeheader()
for result in r:
if result[ipfield]:
# only ip was provided, add host
result[hostfield] = rlookup(result[ipfield])
if result[hostfield]:
w.writerow(result)
main()
Hi Apezuela,
Do you want to change the field names for clienthost or clientip into you own field? I'm just trying to understand ur goal.
Pryzrak
I want add a new field with the resolved ip address/hostname. Now, this script return a statistic table and i cannot do something more (other commands like chart,...).
This custom commands returns a table, is there any way to add new field in the place of a table?
I want use this command to add a new field in my search with the hosname or ipaddress.
Well, apparently had errors in constructing the "input / output" to splunk.
So I decided to embed the code of the function "rlookup ()" in the file dnslookup.py
Even adding the parameter dns server that can be sent from the splunk search. The code was me:
import csv
import sys
import commands
import socket
import dns.resolver
import string
import re
from socket import gethostbyaddr
from socket import gethostbyname
def rlookup(ipaddy,ipdns):
ipaddy = str(ipaddy)
try:
my_resolver = dns.resolver.Resolver()
my_resolver.nameservers = [ipdns] #seperate nameservers by ',' example: '8.8.8.8','8.8.8.6','etc'
ipaddy = ipaddy.split('.')
ipaddy.reverse()
ipaddy = string.join(ipaddy, '.')
ipaddy = ipaddy + '.in-addr.arpa'
hostname = str(my_resolver.query(ipaddy,"PTR")[0])
return hostname
except:
return ''
def main():
try:
debug=False
#debug=True
usage="usage: dnslookup <reverse|forward> <input field> <output field>"
if len(sys.argv)>=4:
method=sys.argv[1]
inputfield=sys.argv[2]
outputfield=sys.argv[3]
dnsserver=sys.argv[4] #Adding the argument dns server
else:
print(usage)
sys.exit(1)
#consume the extraneous info that splunk sends through
while True:
line = sys.stdin.readline()
if not line.strip(): break
if debug:
fo=open("/tmp/moo.log",'w')
first=True
r = csv.reader(sys.stdin)
for row in r:
if first:
headers=row
if debug:
headers_string=str(headers)
fo.write("headers_string: " + headers_string + "\n")
fo.write("index of inputfield (" + inputfield + "): " + str(headers.index(inputfield)) + "\n")
headers.insert(headers.index(inputfield)+1, outputfield)
if debug:
headers_string=str(headers)
fo.write("appended headers_string: " + headers_string + "\n")
csv.writer(sys.stdout).writerow(headers)
first=False
else:
if debug:
row_string=str(row)
fo.write("row: " + row_string + "\n")
fo.write("address to resolve: " + row[headers.index(inputfield)] + "\n")
try:
if method == "reverse":
# Condition to resolve IP addresses only, and omit values that are not IP. Requires import re
myipinput = row[headers.index(inputfield)]
is_valid = re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", myipinput)
if is_valid:
# Calling the function rlookup including dns server
fqdn=rlookup(row[headers.index(inputfield)],dnsserver)
else :
fqdn=myipinput
elif method == "forward":
fqdn=gethostbyname(row[headers.index(inputfield)])
else :
fqdn=usage
except:
fqdn=row[headers.index(inputfield)]
if debug:
fo.write("gethostby... result: " + fqdn + "\n")
fo.write("\n")
row.insert(headers.index(inputfield)+1, fqdn)
if debug:
row_string=str(row)
fo.write("row: " + row_string + "\n")
fo.write("address to resolve: " + row[headers.index(inputfield)] + "\n")
csv.writer(sys.stdout).writerow(row)
if debug:
fo.close()
except:
import traceback
stack = traceback.format_exc()
if debug:
fo=open("/tmp/moo.log",'a')
fo.write(stack)
fo.close()
results = splunk.Intersplunk.generateErrorResults("Error : Traceback: " + str(stack))
if __name__ == '__main__':
main()
Now I make them search like this: host=172.17.18.19 src_ip=* | dnslookup reverse src_ip hostname "172.24.24.24"
Where hostname is the output alias resolved.
And "172.24.24.24" is any dns server
To whom it may be useful.
John thank you very much for your post, I was a great help, and I will serve my splunk is multiclient and need to resolve different DNS servers.
Regards!!
Jorge
Bottom line is your resolver.py and all other dns mod .py files needs to be exactly in the below path:
/opt/splunk/lib/python2.7/site-packages/dns
It seems to me that your dns mod is just in the wrong place.
Also if you test with the standalone, it will need the dns mod in the below path for your system libraries:
/usr/local/lib/python2.7/dist-packages/dns
Substitute "python2.7" in the path for your correct version for system. Mine is python3.4. So my path would be:
/usr/local/lib/python3.4/dist-packages/dns
Additionally you commented that doing the steps in the file: /opt/splunk/etc/system/default/transforms.conf
Not recognize the command sending me the error: Unknown search command 'ezlookup'.
I have been doing currently editing the file
/opt/splunk/etc/apps/search/default/commands.conf
[ezlookup]
filename=ezlookup.py
run_in_preview = true
Thanks pryzrak
I have done the test.py and if I can get the result by giving the parameters, so I see the dns code works perfectly.
Then where would the problem would be on the line that I showed you:
File "/usr/lib64/python2.6/csv.py", line 90, in fieldnames
self._fieldnames = self.reader.next()
I think it's because it calls for: "/usr/lib64/python2.6/csv.py"
Instead of calling: /opt/splunk/lib/python2.7/csv.py
Do you think that is? How could I do to correct it?
:|
Hey,
The ezlookup.py will not run correctly from CLI even with it working correctly even if you use the arguments. With it's normal function, it doesn't output to CLI. It outputs to a table that Splunk needs to import. I'm not surprised with your errors that you got. But the output that you got from just running "./ezlookup.py" without arguments is the normal output to CLI. Now we just need to troubleshoot to see why your dns mod is not running.
At the very beginning of this post, there is a standalone script to test the dns.resolver python mod. That one will run correctly from CLI and output to CLI. Create a "test.py" in "/opt/splunk/etc/system/bin" with that script in the same location as your ezlookup.py. It doesn't need arguements when you run it. Just run the script to see if it outputs the "FaceBook" hostname. If it does then your problem is not the dns mod and probably an error in your transform.conf.
hi
I could solve the above problem, I found a theme folder permissions and directories. So download dnspython, installed with setup.py
And I moved the dns folder to "/opt/splunk/lib/python2.7/site-packages/dns"
Run ./ezlookup.py
And I get "Usage: python ezlookup.py [host field] [ip field]"
It's okay.
I try with: ./ezlookup.py google.com 200.48.225.130
And all the while charging runs, and cancel "ctrl + C" get
^CTraceback (most recent call last):
File "./ezlookup.py", line 77, in <module>
main()
File "./ezlookup.py", line 54, in main
header = r.fieldnames
File "/usr/lib64/python2.6/csv.py", line 90, in fieldnames
self._fieldnames = self.reader.next()
KeyboardInterrupt
What could be happening here, now if I missed
:(
Check to make sure your dns python mod is installed to:
/opt/splunk/lib/python2.7/site-packages/dns
and not:
/opt/splunk/lib/python2.7/site-packages/dns/dns
It depends on how you extracted the python mod.
Other things: try and execute the ezlookup.py from cli without splunk. You'll get a more specific error than the one in splunk will give you (Splunk errors can be very vague).
Running ezlookup.py I get:
Traceback (most recent call last):
File "./ezlookup.py", line 8, in <module>
import dns.resolver
ImportError: No module named dns.resolver
It is in the correct path:
drwxr-xr-x 3 splunk splunk 4096 Sep 11 21:14 /opt/splunk/lib/python2.7/site-packages/dns
your syntax is wrong. Check out the transforms.conf and you'll see why.
Your syntax should be:
search | ezlookup [clienthost] [clientip]
You configure ezlookup.py with the dnsserver of your choice and you don't enter as an argument.
There is two ways to use it. For IP resolution and Host resolution. Below are examples:
search | ezlookup clientip as src_ip #this will return hostname based on ip address input
search | ezlookup clienthost as host #this will return ip address based on host input
Let me know if this helps,
Pryzrak
On the other hand, I thought I could send the value of the DNS Server.
You can modify the code for this? Indeed I consider very important. Look, I tried to modify the dnslookup.py, but I see that uses the function gethostbyaddr(), which does not allow DNS parameter 😞
Hi
My search is: host="172.16.10.80" src_ip=172.12.23.224 | head 1 | ezlookup clientip as src_ip
And return error: External search command 'ezlookup' returned error code 1. Script output = "Usage: python ezlookup.py [host field] [ip field] "
My transforms in system es:
[ezlookup]
external_cmd = ezlookup.py clienthost clientip
fields_list = clienthost clientip
I might be missing?
Ok I ended up answering my own question. Getting dnslookup to resolve with an outside name server such as 8.8.8.8, requires the DNSpython mod to be installed. However it needs to be be installed under the Splunk python lib. The dnspython mod can be obtained from: www.dnspython.org. You don't have to use the setup.py. But if you use it (like I did) then just follow the instructions below:
If you used the setup.py then copy the contents of the folder below:
/usr/local/lib/python2.7/dist-packages/dns
Or just copy the dns/ folder from the zip/tar.gz
and placed it in:
/opt/splunk/lib/python2.7/site-packages/dns
Then create a new python script called ezlookup.py based off of external_lookup.py under the following folder:
/opt/splunk/etc/system/bin
ezlookup.py is below:
#!/usr/bin/env python
import csv
import sys
import commands
import socket
import dns.resolver
import string
""" Edited from external_lookup.py by Pryzrak.
Allows DNS queries from external nameserver of your choice.
"""
def rlookup(ipaddy):
ipaddy = str(ipaddy)
try:
my_resolver = dns.resolver.Resolver()
my_resolver.nameservers = ['8.8.8.8'] #seperate nameservers by ',' example: '8.8.8.8','8.8.8.6','etc'
ipaddy = ipaddy.split('.')
ipaddy.reverse()
ipaddy = string.join(ipaddy, '.')
ipaddy = ipaddy + '.in-addr.arpa'
hostname = str(my_resolver.query(ipaddy,"PTR")[0])
return hostname
except:
return ''
# Given a host, find the ip
def lookup(host):
try:
#hostname, aliaslist, ipaddrlist = socket.gethostbyname_ex(host)
my_resolver = dns.resolver.Resolver()
my_resolver.nameservers = ['8.8.8.8'] #seperate nameservers by ',' example: '8.8.8.8','8.8.8.6','etc'
rdata = ''
rawanswer = my_resolver.query(host, 'a')
for rdata in rawanswer:
ipaddrlist = rdata.address
return ipaddrlist
except:
return []
def main():
if len(sys.argv) != 3:
print "Usage: python ezlookup.py [host field] [ip field]"
sys.exit(1)
hostfield = sys.argv[1]
ipfield = sys.argv[2]
infile = sys.stdin
outfile = sys.stdout
r = csv.DictReader(infile)
header = r.fieldnames
w = csv.DictWriter(outfile, fieldnames=r.fieldnames)
w.writeheader()
for result in r:
# Perform the lookup or reverse lookup if necessary
if result[hostfield] and result[ipfield]:
# both fields were provided, just pass it along
w.writerow(result)
elif result[hostfield]:
# only host was provided, add ip
ip = lookup(result[hostfield])
result[ipfield] = ip
w.writerow(result)
elif result[ipfield]:
# only ip was provided, add host
result[hostfield] = rlookup(result[ipfield])
if result[hostfield]:
w.writerow(result)
main()
Then edit the following .conf to make a the new ezlookup command:
/opt/splunk/etc/syystem/default/transforms.conf
Add the following to transforms.conf:
# EZ external lookup
[ezlookup]
external_cmd = ezlookup.py clienthost clientip
fields_list = clienthost clientip
And finished. NO Splunk restart or system reboot is required.
DNS resolves with the open google nameserver 8.8.8.8 or your preferred nameserver.
If anyone has any questions, please shoot me a message. v/r, Pryzrak.
Hi, thanks for the contribution
I followed all the steps and do the search: ... | ezlookup src_ip dnsserver
I generates the error: External search command 'ezlookup' returned error code 1. Script output = "splunkVersion:6.1.1 "
How could I do?