I'm fetching some data from API via a python script and passing it to Splunk. it's is not paring the JSON format. I've tested my output with JSON parser with no error. If I set the source type to some custom I'm receiving events as a text. but when I'm putting source type as _json it is giving line breaking error expected : \
Below is the python script. I'm using json.dumps also while printing. Now I'm writing to the file and fetching with monitor.
# This sript is fetching data from virustotal api and passing to splunk.
# checkpointing is enabled to drop duplicate events
import json,requests,sys,time,os
from datetime import datetime
proxies = { 'https': 'http://security-proxy.emea.svc.corpintra.net:3128' }
url = "https://www.virustotal.com/api/v3/intelligence/hunting_notifications"
params = { 'limit' : 40,
'count_limit' : 10000
}
headers = {
"Accept": "application/json",
"x-apikey": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
}
current_time = datetime.now()
file_path = f'/opt/splunk/etc/apps/infy_ta_virustotal_livehunt_validation/bin/data/'
complete_name = file_path + f'livehunt_{time.strftime("%Y_%m_%d_%H_%M_%S")}'
keys_filename = f'/opt/splunk/etc/apps/infy_ta_virustotal_livehunt_validation/bin/keys.txt'
def write_new_keys_in_file(keys_filename, keys_to_be_indexed):
try:
with open(keys_filename, 'w') as file:
for key in keys_to_be_indexed:
file.write(str(key))
file.write('\n')
except Exception as e:
print(e)
def get_indexed_key(keys_filename):
try:
with open(keys_filename, 'r') as file:
indexed_keys = file.read().splitlines()
return indexed_keys
except Exception as e:
with open(keys_filename, 'w') as file:
indexed_keys = []
return indexed_keys
def get_json_data(url, headers, params, proxies):
try:
response = requests.get(url = url, headers=headers,params = params, proxies=proxies).json()
return response
except Exception as e:
print(e)
sys.exit(1)
def write_to_file(complete_name, data):
with open(complete_name, 'a') as f:
json.dump(data, f)
f.write('\n')
def stream_to_splunk(json_response,indexed_keys, complete_name):
try:
keys_to_be_indexed = []
events_to_be_indexed = []
for item in json_response['data']:
keys_to_be_indexed.append(item['id'])
if item['id'] not in indexed_keys:
write_to_file(complete_name = complete_name, data = item)
events_to_be_indexed.append(item)
print(json.dumps(events_to_be_indexed, indent = 4, sort_keys = True)) if len(events_to_be_indexed) else 1==1
return keys_to_be_indexed
except Exception as e:
print(e)
def main():
try:
json_response = get_json_data(url = url, headers = headers, params = params, proxies = proxies)
indexed_keys = get_indexed_key(keys_filename = keys_filename)
keys_to_be_indexed = stream_to_splunk(json_response = json_response, indexed_keys = indexed_keys, complete_name = complete_name)
write_new_keys_in_file(keys_filename = keys_filename, keys_to_be_indexed = keys_to_be_indexed)
except Exception as e:
print(e)
if __name__ == "__main__":
main()
... View more