All Apps and Add-ons

Microsoft Office 365 Reporting Mail Add-on for Splunk issues identified

thambisetty
Super Champion
  1. Issue if API response returns only one message tracking event. In this scenario, the start date and end date for the next API call would be same as previous call and it becomes like a loop, hence the script was unable to make next start time and end time
  2. if script has some issue and couldn't collect data. lets say 48 hours then to catch up new logs, the script has to make 48 calls (each call collect 1 hours of messages). if the interval of script is 5 minutes then to catch up latest events, the script will take at least 48 calls * 5 minutes = 240 minutes(around 4 hours).
————————————
If this helps, give a like below.
Labels (1)
0 Karma

thambisetty
Super Champion

1.  TA-MS_O365_Reporting/bin/input_module_ms_o365_message_trace.py
version : 1.2.4

185                 max_date = max([max_date, this_date_received])
186                 #max_date = max([max_date, this_date_received,end_date])
187                 if start_date==max_date:
188                     helper.log_debug("stanza=%s,_Splunk_ probably one event found and max_date will be set with end_date to avoid loop" % (stanza_name))
189                     max_date=end_date
190
191                 event = helper.new_event(

 2. Added while loop to catch up latest events with single call. and also added stanza to each debug log for easy troubleshooting.

def get_events_continuous(helper, ew):
    global_account = helper.get_arg("office_365_account")
    global_microsoft_office_365_username = global_account["username"]
    global_microsoft_office_365_password = global_account["password"]
    query_window_size = int(helper.get_arg("query_window_size"))
    delay_throttle = int(helper.get_arg("delay_throttle"))
    interval = int(helper.get_arg("interval"))
    check_point_key = "%s_obj_checkpoint" % helper.get_input_stanza_names()
    messages = None

    stanza_name = helper.get_input_stanza_names()


    while True:
        start_date = get_start_date(helper, check_point_key)
        end_date = start_date + datetime.timedelta(minutes=query_window_size)
        helper.log_debug("stanza=%s,_Splunk_ Start date: %s, End date: %s" % (stanza_name,start_date, end_date))
        utc_now = datetime.datetime.utcnow()

        if end_date > utc_now - datetime.timedelta(minutes=delay_throttle):
            helper.log_debug("stanza=%s,end_date is greater than the specified delay throttle=%s (in minutes) [start_date=%s end_date=%s utc_now=%s utc_now-throttled=%s] Skipping..." % (stanza_name,delay_throttle,start_date, end_date, utc_now,utc_now - datetime.timedelta(minutes=delay_throttle)))
            return

        microsoft_trace_url = "https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace?$filter=StartDate eq datetime'%sZ' and EndDate eq datetime'%sZ'" % (start_date.isoformat(), end_date.isoformat())

        message_response = get_messages(helper, microsoft_trace_url, global_microsoft_office_365_username, global_microsoft_office_365_password)
        messages = message_response['value'] or None

        if messages is None:
            # Since no message were retrieved during this poll, move the query window forward by the amount of seconds in the interval.
            max_date = start_date + datetime.timedelta(seconds=interval)
            helper.log_debug("stanza=%s,_Splunk_ no messages returned.  Setting max date to %s" % (stanza_name,max_date))
            checkpoint_data = {}
            checkpoint_data["max_date"] = str(max_date)
            helper.save_check_point(check_point_key, checkpoint_data)
            return

        max_date = start_date
        helper.log_debug("stanza=%s,_Splunk_ max date before getting message: %s" % (stanza_name,str(max_date)))

        while messages:
            for message in messages:

                # According to https://msdn.microsoft.com/en-us/library/office/jj984335.aspx
                # The StartDate and EndDate fields do not provide useful information in the report results...
                # Sometimes popping "StartDate" fails because of unknown issue. So to avoid an unexpected error, Try/Except method is used here.
                try:
                    message.pop("StartDate")
                    message.pop("EndDate")
                except Exception as e:
                    helper.log_error("stanza=%s,_Splunk_ Message Pop error: %s" % (stanza_name,str(e)))

                this_date_received = dateutil.parser.parse(message["Received"])
                max_date = max([max_date, this_date_received])
                #max_date = max([max_date, this_date_received,end_date])
                if start_date==max_date:
                    helper.log_debug("stanza=%s,_Splunk_ probably one event found and max_date will be set with end_date to avoid loop" % (stanza_name))
                    max_date=end_date

                event = helper.new_event(
                source=helper.get_input_type(),
                index=helper.get_output_index(),
                sourcetype=helper.get_sourcetype(),
                data=json.dumps(message))
                ew.write_event(event)

            sys.stdout.flush()
            messages = None

            # Check point the largest date seen during the query
            checkpoint_data = {}
            checkpoint_data["max_date"] = str(max_date)
            helper.log_debug("stanza=%s,_Splunk_ max date after getting messages: %s" % (stanza_name,str(max_date)))
            helper.save_check_point(check_point_key, checkpoint_data)

            nextLink = None
            if ('@odata.nextLink' in message_response):
                nextLink = message_response['@odata.nextLink']

            if ('odata.nextLink' in message_response):
                nextLink = message_response['odata.nextLink']

            if nextLink is not None:
                nextLink = get_url(nextLink)
                helper.log_debug("stanza=%s,_Splunk_ nextLink URL (@odata.nextLink): %s" % (stanza_name,nextLink))

                # This should never happen, but just in case...
                if not is_https(nextLink):
                    raise ValueError("stanza=%s,nextLink scheme is not HTTPS. nextLink URL: %s" % (stanza_name,nextLink))

                message_response = get_messages(helper, nextLink, global_microsoft_office_365_username, global_microsoft_office_365_password)
                messages = message_response['value'] or None

  

————————————
If this helps, give a like below.