We are trying to ingest JSON data to Splunk Ingest Processor. Sometimes JSON data is getting ingested properly and many times its not getting ingested. Below is the script we are running for ingesting the JSON records. We couldn't figure out what the issue could be. Please do the needful.
Also, we are able to see the data in Searching and Reporting but DM is not picking up.
#!/usr/bin/env python3
import json
import requests
from datetime import datetime, timezone
# === Splunk HEC configuration ===
HEC_URL = "https://XXXX/services/collector/event"
HEC_TOKEN = "XXX" # replace if needed
INDEX = "XXXX"
SOURCETYPE = "XXX"
SOURCE = "XXX"
HEADERS = {
"Authorization": f"Splunk {HEC_TOKEN}",
"Content-Type": "application/json; charset=utf-8",
}
# Example data_array – add your real fields here
data_array = [
{
"vendor": "Gigamon",
"version": "6.6.00",
"generator": "gs_apps_appInst18_ec2fc7c2-7a46-dc49-834d-3a7424cef6b1",
# ... other fields ...
},
# more events...
]
def send_events(events):
# Current time in UTC
now_utc = datetime.now(timezone.utc)
# # ts in UTC (human-readable)
ts_str = now_utc.strftime("%a %b %d %H:%M:%S %Y")
event_data["ts"] = ts_str
## Epoch (UTC) for that instant
event_time = now_utc.timestamp()
print("UTC datetime :", now_utc.isoformat())
print("UTC ts :", ts_str) print("Epoch :", event_time)
payload = {
"time": event_time, # epoch for UTC-8 instant
"index": INDEX,
"sourcetype": SOURCETYPE,
"source": SOURCE,
"event": event_data,
}
try:
resp = requests.post(
HEC_URL,
headers=HEADERS,
data=json.dumps(payload),
timeout=10,
)
except requests.RequestException as e:
print(f"Request error: {e}")
continue
if resp.status_code != 200:
print(f"Failed (HTTP {resp.status_code}): {resp.text}")
else:
print(f"Sent OK (time={event_time}, ts={ts_str})")
if __name__ == "__main__":
send_events(data_array)
