Hi All,  I need one help. I have created a savedsearch that writes data to metrics index.  Timerange : -2m to -1m scheduled to run every 2 mins.  index=*test sourcetype=aws:test log_processed.dmc.tenantId!=ALL_TENANTS host IN (“test1” , “test2” , “test”3, test4 ,test5) k8.pod_name=“*testpod*” | eval _raw=json_extract_exact(_raw,"log_processed.dmc") | spath | eval container_name='k8.container_name',container_image='k8.container_image', container_hash='k8.container_hash', namespace_name='k8.namespace_name', pod_name='k8.pod_name', pod_id='k8.pod_id', docker_id='k8.docker_id', MetricName = 'log_processed.dmc.metricName' , MetricValue = tonumber('log_processed.dmc.value'), _time = strptime('log_processed.timestamp', "%Y-%m-%d %H:%M:%S.%3N"), metric_name:{MetricName}=MetricValue, tenantId='log_processed.dmc.tenantId' | table _time host log_type tenantId namespace_name container_name container_image container_hash pod_name pod_id docker_id metric_name:* | mcollect index=tmetrics There is also a datamodel with different dataset based on different host filter. Example below : index=*test sourcetype=aws:test log_processed.dmc.tenantId!=ALL_TENANTS host=test1 k8.pod_name=“*testpod*” | eval _raw=json_extract_exact(_raw,"log_processed.mdc") index=*test sourcetype=aws:test log_processed.dmc.tenantId!=ALL_TENANTS host=test1 k8.pod_name=“*testpod*” | eval _raw=json_extract_exact(_raw,"log_processed.mdc") The issue happening is the stats count in new metrics index is not matching the actual dataset or original query stats count . Based on how savedsearch actually runs, have I written the savedsearch properly ? Sometimes data gets indexed late, keeping that in mind just not to loose any data I have made _time to be _indextime on which scheduled search runs (if my understanding is right here).  Please help me.  Regards, PNV ... View more

Hi All,   I am trying to create a modular input in splunk cloud that gets splunk observability metadata. Input has fields : realm, token and object_type. I can see the input form under "Settings -> Data Inputs" with all these fields and is fine.   I have created another script that masks token. But when I upgrade the app with this script and restmap.conf, the modular input is getting disappered from "Settings" --> "Data Inputs" .   splunk_observability.py ---> this is to create modular input schema. observability_object_helper.py --> This is helper script that makes ARI call. observability_admin_TA_rh_account.py --> This creates restmodel and encrypts token input field.   Directory Structure :  App Folder Name --> Splunk_Observability_Metadata metadata --> default.meta bin --> import_declare_test.py, splunk_observability.py,observability_object_helper.py,observability_admin_TA_rh_account.py README --> inputs.conf.spec local --> inputs.conf, app.conf lib --> required splunklib splunk_observability.py import import_declare_test import sys,os import json from splunklib import modularinput as smi sys.path.insert(0, os.path.dirname(__file__)) from observability_object_helper import stream_events, validate_input class SPLUNK_OBSERVABILITY(smi.Script): def __init__(self): super(SPLUNK_OBSERVABILITY, self).__init__() def get_scheme(self): scheme = smi.Scheme("Splunk Observability Metadata Input") scheme.use_external_validation = False scheme.use_single_instance = False scheme.description = "Modular Input" scheme.streaming_mode_xml = True scheme.add_argument( smi.Argument( 'realm', required_on_create=True ) ) scheme.add_argument( smi.Argument( 'name', required_on_create=True, description="Name should not contain whitespaces", ) ) scheme.add_argument( smi.Argument( 'token', required_on_create=True, description="Add API Key required to connect to Splunk Observability Cloud", ) ) scheme.add_argument( smi.Argument( 'object_type', required_on_create=True ) ) return scheme def validate_input(self, definition: smi.ValidationDefinition): return validate_input(definition) def stream_events(self, inputs: smi.InputDefinition, ew: smi.EventWriter): return stream_events(inputs, ew) if __name__ == "__main__": sys.exit(SPLUNK_OBSERVABILITY().run(sys.argv)) observability_object_helper.py import json import logging import time import requests # import import_declare_test from solnlib import conf_manager, log, credentials from splunklib import modularinput as smi ADDON_NAME = "splunk_observability" def get_key_name(input_name: str) -> str: # `input_name` is a string like "example://<input_name>". return input_name.split("/")[-1] def logger_for_input(input_name: str) -> logging.Logger: return log.Logs().get_logger(f"{ADDON_NAME.lower()}_{input_name}") def splunk_observability_get_endpoint(type, realm): BASE_URL = f"https://api.{realm}.signalfx.com" ENDPOINT = "" types = { "chart": f"{BASE_URL}/v2/chart", "dashboard": f"{BASE_URL}/v2/dashboard", "detector": f"{BASE_URL}/v2/detector", "heartbeat": f"{BASE_URL}/v2/detector", "synthetic": f"{BASE_URL}/v2/synthetics/tests", } for type_key in types: if type.lower() == type_key.lower(): ENDPOINT = types.get(type_key) return ENDPOINT def splunk_observability_get_sourcetype(type): sourcetypes = { "chart": "observability:chart_api:json", "dashboard": "observability:dashboard_api:json", "detector": "observability:detector_api:json", "synthetic": "observability:synthetic_api:json", "token": "observability:token_api:json", } for type_key in sourcetypes: if type.lower() == type_key.lower(): return sourcetypes.get(type_key) def splunk_observability_get_objects(type, realm, token, logger): TOKEN = token ENDPOINT_URL = splunk_observability_get_endpoint(type, realm) limit = 200 offset = 0 pagenation = True headers = {"Content-Type": "application/json", "X-SF-TOKEN": TOKEN} processStart = time.time() objects = [] while pagenation: params = {"limit": limit, "offset": offset} try: response = requests.get(ENDPOINT_URL, headers=headers, params=params) response.raise_for_status() except requests.exceptions.RequestException as e: log.log_exception(logger, e, "RequestError", msg_before="Error fetching data:") return [] data = response.json() if isinstance(data, list): results = data elif isinstance(data, dict): results = data.get("results", []) else: logger.error("Unexpected response format") objects.extend(results) logger.info(f"pagenating {type} result 'length': {len(results)} , offset: {offset}, limit {limit}") if len(results) < limit: pagenation = False # too many objects to query, splunk will max out at 10,000 elif (offset >= 10000-limit): pagenation = False logger.warn("Cannot ingest more than 10,000 objects") else: offset += limit count = offset+len(results) timeTakenProcess = str(round((time.time() - processStart) * 1000)) log.log_event(logger, {"message": f"{type} ingest finished", "time_taken": f"{timeTakenProcess}ms", "ingested": count}) return objects def splunk_observability_get_objects_synthetics(type, realm, token, logger): processStart = time.time() BASE_URL = f"https://api.{realm}.signalfx.com" ENDPOINT_URL = f"{BASE_URL}/v2/synthetics/tests" page = 1 pagenating = True headers = {"Content-Type": "application/json", "X-SF-TOKEN": token} synthetics_objects = [] while pagenating: params = {"perPage": 100, "page": page} try: response = requests.get(ENDPOINT_URL, headers=headers, params=params) response.raise_for_status() except requests.exceptions.RequestException as e: log.log_exception(logger, e, "RequestError", msg_before="Error fetching synthetic data:") return [] data = response.json() tests = data["tests"] for test in tests: synthetic = {"id": test["id"], "type": test["type"]} SYNTHETIC_TYPE = synthetic["type"] SYNTHETIC_ID = synthetic["id"] detail_url = f"{BASE_URL}/v2/synthetics/tests/{SYNTHETIC_TYPE}/{SYNTHETIC_ID}" if type=="synthetic_detailed": try: detail_response = requests.get(detail_url, headers=headers) detail_response.raise_for_status() synthetics_objects.append(detail_response.json()) except requests.exceptions.RequestException as e: log.log_exception(logger, e, "RequestError", msg_before=f"Error fetching synthetic details for ID: {SYNTHETIC_ID}") else: synthetics_objects.append(test) pagenating = data.get("nextPageLink") is not None page += 1 timeTakenProcess = str(round((time.time() - processStart) * 1000)) log.log_event(logger, {"message": "synthetic ingest finished", "time_taken": f"{timeTakenProcess}ms", "ingested": len(synthetics_objects)}) return synthetics_objects def validate_input(definition: smi.ValidationDefinition): return False def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter): for input_name, input_item in inputs.inputs.items(): normalized_input_name = input_name.split("/")[-1] logger = logger_for_input(normalized_input_name) try: observability_type = input_item.get("object_type") observability_token = input_item.get("token") observability_realm = input_item.get("realm") log.modular_input_start(logger, normalized_input_name) if observability_type.lower() == "synthetic": objects = splunk_observability_get_objects_synthetics(observability_type, observability_realm, observability_token, logger) else: objects = splunk_observability_get_objects(observability_type, observability_realm, observability_token, logger) # source_type = splunk_observability_get_sourcetype(observability_type) for obj in objects: logger.debug(f"DEBUG EVENT {observability_type} :{json.dumps(obj)}") event_writer.write_event( smi.Event( data=json.dumps(obj, ensure_ascii=False, default=str), index=input_item.get("index"), sourcetype=input_item.get("sourcetype"), ) ) log.events_ingested(logger, input_name, sourcetype, len(objects), input_item.get("index")) log.modular_input_end(logger, normalized_input_name) except Exception as e: log.log_exception(logger, e, "IngestionError", msg_before="Error processing input:") ​ observability_admin_TA_rh_account.py from splunktaucclib.rest_handler.endpoint import ( field, validator, RestModel, DataInputModel, ) from splunktaucclib.rest_handler import admin_external, util import logging util.remove_http_proxy_env_vars() fields = [ field.RestField('name', required=True, encrypted=False), field.RestField('realm', required=True, encrypted=False), field.RestField('token', required=True, encrypted=True), field.RestField('interval', required=True, encrypted=False, default="300"), ] model = RestModel(fields, name='splunk_observability') endpoint = DataInputModel(model, input_type='splunk_observability') if __name__ == '__main__': logging.getLogger().addHandler(logging.NullHandler()) admin_external.handle(endpoint) restmap.conf [endpoint:admin/input/splunk_observability] match = splunk_observability python.version = python3 handlerfile = observability_admin_TA_rh_account.py Please help me to resolve this issue. Thanks, PNV ... View more

Hi All, I have a lookup that contains set of email ids and associated accounts. Example :  Account ID OWNER_EMAIL 34234234 test1@gmail.com; test2@gmail.com 123234234 test3@gmail.com;test4@gmail.com <logic> | eval email_list = split(OWNER_EMAIL, ";") | stats values(email_list) as email_list values(ENVIRONMENT) as ENVIRONMENT values(category) as EVENT_CATEGORY values(EVENT_TYPE) as EVENT_TYPE values(REGION) as Region values(AFFECTED_RESOURCE_ARNS) as AFFECTED_RESOURCE_ARNS. I have configured $result.email_list$ in alert action - email.to setting. Email is getting sent successfully but all of the result together is sent to email recepient. Result : Account ID  Email_list Environment Category Type Region Arns Description 34234234 test1@gmail.com; test2@gmail.com Development test_cat1 Event1 global testarn1 testdescr1 123234234 test3@gmail.com;test4@gmail.com Production test_cat2 Event2 global testarn2 testdescr2 When alert is triggered, separate email should go to test1@gmail.com; test2@gmail.com with both of them in to field  with email body containing only first row and another email should go to test3@gmail.com;test4@gmail.com with  both of them in to field with email body containing only second row. Please help how to achieve this. Regards, PNV ... View more

Hi All, I have scheduled a splunk report to run at 11 AM IST everyday (cron schedule : 0 11 * * *). Search Head time zone is IST. But when I checked savedsearch , the next scheduled time  is shown as 4:30 PM IST  though advance settings has 0 11 * * *. Until yesterday it was fine . But suddenly seeing this issue. Does anyone has idea what is going wrong ? Regards, Poojitha NV ... View more

Hi All, I need help in knowing below. There is a field named lvl, which is of type=string.  Raw Data :    { "time": "2025-03-10T06:20:29", "corr": "3hgewhrger2346324632434gjhf", "dpnt": "test.dpmt", "appn": "test - appn", "lvl": "Warn", "mod": "test.mod", "tid": "171", "oper": "SetTestContext", "rslt": "Succeeded", "msg": "test msg", "inst": "test inst", "x-trace-id": "Root=1-65325bhg-test3;Sampled=1" }   Though lvl is of type string, if I try | search lvl="Warn" or lvl=Warn, it renders no result. Instead if I do  | spath  lvl and then | search lvl="Warn" or  lvl=Warn it is showing result. Whereas for other fields like dpnt which is again of type string, it is working fine with | search dpnt="test.dpmt".  I understand spath works on structured data format like json and xml but not getting what is happening in this case. Why is lvl string field not working as expected ? Please can anyone shade some light on this.  Thanks, PNV ... View more

Hi All, I have a dropdown multi-select created using dashboard studio with default value set as "All".  This All is nothing but the static value set under menu configuration. Label - "All" Value - * Query used :  index=test sourcetype="billing_test" productcode="testcode"  | fields account_id account_name cluster namespace pod cost  | search account_id IN ($account_id$) AND clustername IN ($cluster$) AND account_name IN ($account_name$) | stats count by namespace But when I click on this multi-select dropdown it is loading another "All" as value together with the default value I have set. Example Screenshot :      Full xml code "visualizations": {}, "dataSources": { "ds_1sGu0DN2": { "type": "ds.search", "options": { "query": "index=test sourcetype=\"billing_test\" productcode=\"testcode\"| fields account_id account_name cluster namespace pod cost" }, "name": "Base search" }, "ds_fURg97Gu": { "type": "ds.chain", "options": { "extend": "ds_1sGu0DN2", "query": "| search account_id IN ($account_id$) AND eks_clustername IN ($cluster$) AND account_name IN ($account_name$)| stats count by namespace" }, "name": "Namespacefilter" } }, "defaults": { "dataSources": { "ds.search": { "options": { "queryParameters": { "latest": "$global_time.latest$", "earliest": "$global_time.earliest$" } } } } }, "inputs": { "input_global_trp": { "type": "input.timerange", "options": { "token": "global_time", "defaultValue": "-7d@h,now" }, "title": "Global Time Range" }, "input_jHd4pV3L": { "options": { "items": [ { "label": "All", "value": "*" } ], "defaultValue": [ "All" ], "token": "account_id" }, "title": "Namespace", "type": "input.multiselect", "dataSources": { "primary": "ds_fURg97Gu" }, "context": {} } }, "layout": { "options": {}, "globalInputs": [ "input_global_trp", "input_jHd4pV3L" ], "tabs": { "items": [ { "layoutId": "layout_1", "label": "New tab" } ] }, "layoutDefinitions": { "layout_1": { "type": "grid", "structure": [], "options": { "width": 1440, "height": 960 } } } }, "description": "", "title": "Test Dashboard" } Please can anyone of you help me to know what is going wrong. Thanks , NVP ... View more

Hi , Please can anyone of you let me know what is the latest  sub-version of splunk 9.3.?  Regards, Poojitha NV ... View more