1. TA-MS_O365_Reporting/bin/input_module_ms_o365_message_trace.py version : 1.2.4 185 max_date = max([max_date, this_date_received])
186 #max_date = max([max_date, this_date_received,end_date])
187 if start_date==max_date:
188 helper.log_debug("stanza=%s,_Splunk_ probably one event found and max_date will be set with end_date to avoid loop" % (stanza_name))
189 max_date=end_date
190
191 event = helper.new_event( 2. Added while loop to catch up latest events with single call. and also added stanza to each debug log for easy troubleshooting. def get_events_continuous(helper, ew):
global_account = helper.get_arg("office_365_account")
global_microsoft_office_365_username = global_account["username"]
global_microsoft_office_365_password = global_account["password"]
query_window_size = int(helper.get_arg("query_window_size"))
delay_throttle = int(helper.get_arg("delay_throttle"))
interval = int(helper.get_arg("interval"))
check_point_key = "%s_obj_checkpoint" % helper.get_input_stanza_names()
messages = None
stanza_name = helper.get_input_stanza_names()
while True:
start_date = get_start_date(helper, check_point_key)
end_date = start_date + datetime.timedelta(minutes=query_window_size)
helper.log_debug("stanza=%s,_Splunk_ Start date: %s, End date: %s" % (stanza_name,start_date, end_date))
utc_now = datetime.datetime.utcnow()
if end_date > utc_now - datetime.timedelta(minutes=delay_throttle):
helper.log_debug("stanza=%s,end_date is greater than the specified delay throttle=%s (in minutes) [start_date=%s end_date=%s utc_now=%s utc_now-throttled=%s] Skipping..." % (stanza_name,delay_throttle,start_date, end_date, utc_now,utc_now - datetime.timedelta(minutes=delay_throttle)))
return
microsoft_trace_url = "https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace?$filter=StartDate eq datetime'%sZ' and EndDate eq datetime'%sZ'" % (start_date.isoformat(), end_date.isoformat())
message_response = get_messages(helper, microsoft_trace_url, global_microsoft_office_365_username, global_microsoft_office_365_password)
messages = message_response['value'] or None
if messages is None:
# Since no message were retrieved during this poll, move the query window forward by the amount of seconds in the interval.
max_date = start_date + datetime.timedelta(seconds=interval)
helper.log_debug("stanza=%s,_Splunk_ no messages returned. Setting max date to %s" % (stanza_name,max_date))
checkpoint_data = {}
checkpoint_data["max_date"] = str(max_date)
helper.save_check_point(check_point_key, checkpoint_data)
return
max_date = start_date
helper.log_debug("stanza=%s,_Splunk_ max date before getting message: %s" % (stanza_name,str(max_date)))
while messages:
for message in messages:
# According to https://msdn.microsoft.com/en-us/library/office/jj984335.aspx
# The StartDate and EndDate fields do not provide useful information in the report results...
# Sometimes popping "StartDate" fails because of unknown issue. So to avoid an unexpected error, Try/Except method is used here.
try:
message.pop("StartDate")
message.pop("EndDate")
except Exception as e:
helper.log_error("stanza=%s,_Splunk_ Message Pop error: %s" % (stanza_name,str(e)))
this_date_received = dateutil.parser.parse(message["Received"])
max_date = max([max_date, this_date_received])
#max_date = max([max_date, this_date_received,end_date])
if start_date==max_date:
helper.log_debug("stanza=%s,_Splunk_ probably one event found and max_date will be set with end_date to avoid loop" % (stanza_name))
max_date=end_date
event = helper.new_event(
source=helper.get_input_type(),
index=helper.get_output_index(),
sourcetype=helper.get_sourcetype(),
data=json.dumps(message))
ew.write_event(event)
sys.stdout.flush()
messages = None
# Check point the largest date seen during the query
checkpoint_data = {}
checkpoint_data["max_date"] = str(max_date)
helper.log_debug("stanza=%s,_Splunk_ max date after getting messages: %s" % (stanza_name,str(max_date)))
helper.save_check_point(check_point_key, checkpoint_data)
nextLink = None
if ('@odata.nextLink' in message_response):
nextLink = message_response['@odata.nextLink']
if ('odata.nextLink' in message_response):
nextLink = message_response['odata.nextLink']
if nextLink is not None:
nextLink = get_url(nextLink)
helper.log_debug("stanza=%s,_Splunk_ nextLink URL (@odata.nextLink): %s" % (stanza_name,nextLink))
# This should never happen, but just in case...
if not is_https(nextLink):
raise ValueError("stanza=%s,nextLink scheme is not HTTPS. nextLink URL: %s" % (stanza_name,nextLink))
message_response = get_messages(helper, nextLink, global_microsoft_office_365_username, global_microsoft_office_365_password)
messages = message_response['value'] or None
... View more