[ROB-1014] Prometheus auto discovery (#375)

Author: Avi-Robusta

helm/holmes/values.yaml

@@ -40,6 +40,8 @@ toolsets:
     enabled: true
   internet:
     enabled: true
+  prometheus/metrics:
+    enabled: true
 
 
 resources:

holmes/plugins/toolsets/prometheus/prometheus.py

@@ -20,7 +20,7 @@
 from requests import RequestException
 
 from urllib.parse import urljoin
-
+from holmes.plugins.toolsets.service_discovery import PrometheusDiscovery
 from holmes.plugins.toolsets.utils import (
     STANDARD_END_DATETIME_TOOL_PARAM_DESCRIPTION,
     standard_start_datetime_tool_param_description,
@@ -36,7 +36,7 @@
 
 class PrometheusConfig(BaseModel):
     # URL is optional because it can be set with an env var
-    prometheus_url: Union[str, None]
+    prometheus_url: Optional[str]
     healthcheck: str = "-/healthy"
     # Setting to None will remove the time window from the request for labels
     metrics_labels_time_window_hrs: Union[int, None] = 48
@@ -734,25 +734,35 @@ def _reload_llm_instructions(self):
         self._load_llm_instructions(jinja_template=f"file://{template_file_path}")
 
     def prerequisites_callable(self, config: dict[str, Any]) -> Tuple[bool, str]:
-        if not config and not os.environ.get("PROMETHEUS_URL", None):
-            return (
-                False,
-                "Prometheus is misconfigured. prometheus_url is required but missing",
-            )
-        elif not config and os.environ.get("PROMETHEUS_URL", None):
-            self.config = PrometheusConfig(
-                prometheus_url=os.environ.get("PROMETHEUS_URL"),
-                headers=add_prometheus_auth(
-                    os.environ.get("PROMETHEUS_AUTH_HEADER", None)
-                ),
-            )
-            self._reload_llm_instructions()
-            return True, ""
-        else:
+        if config:
             self.config = PrometheusConfig(**config)
             self._reload_llm_instructions()
             return self._is_healthy()
 
+        prometheus_url = os.environ.get("PROMETHEUS_URL")
+        if not prometheus_url:
+            prometheus_url = self.auto_detect_prometheus_url()
+            if not prometheus_url:
+                return (
+                    False,
+                    "Unable to auto-detect prometheus. Define prometheus_url in the configuration for tool prometheus/metrics",
+                )
+
+        self.config = PrometheusConfig(
+            prometheus_url=prometheus_url,
+            headers=add_prometheus_auth(os.environ.get("PROMETHEUS_AUTH_HEADER")),
+        )
+        logging.warning(f"Prometheus auto discovered at url {prometheus_url}")
+        self._reload_llm_instructions()
+        return True, ""
+
+    def auto_detect_prometheus_url(self) -> Optional[str]:
+        url: Optional[str] = PrometheusDiscovery.find_prometheus_url()
+        if not url:
+            url = PrometheusDiscovery.find_vm_url()
+
+        return url
+
     def _is_healthy(self) -> Tuple[bool, str]:
         if (
             not hasattr(self, "config")

holmes/plugins/toolsets/service_discovery.py

@@ -0,0 +1,91 @@
+import logging
+from kubernetes import client
+from kubernetes.client import V1ServiceList
+from kubernetes.client.models.v1_service import V1Service
+import os
+from typing import List, Optional
+from kubernetes import config
+
+CLUSTER_DOMAIN = os.environ.get("CLUSTER_DOMAIN", "cluster.local")
+
+try:
+    if os.getenv("KUBERNETES_SERVICE_HOST"):
+        config.load_incluster_config()
+    else:
+        config.load_kube_config()
+except config.config_exception.ConfigException as e:
+    logging.warning(f"Running without kube-config! e={e}")
+
+
+def find_service_url(label_selector):
+    """
+    Get the url of an in-cluster service with a specific label
+    """
+    # we do it this way because there is a weird issue with hikaru's ServiceList.listServiceForAllNamespaces()
+    try:
+        v1 = client.CoreV1Api()
+        svc_list: V1ServiceList = v1.list_service_for_all_namespaces(
+            label_selector=label_selector
+        )
+        if not svc_list.items:
+            return None
+        svc: V1Service = svc_list.items[0]
+        name = svc.metadata.name
+        namespace = svc.metadata.namespace
+        port = svc.spec.ports[0].port
+        url = f"http://{name}.{namespace}.svc.{CLUSTER_DOMAIN}:{port}"
+        logging.info(
+            f"discovered service with label-selector: `{label_selector}` at url: `{url}`"
+        )
+        return url
+    except Exception:
+        logging.warning("Error finding url")
+        return None
+
+
+class ServiceDiscovery:
+    @classmethod
+    def find_url(cls, selectors: List[str], error_msg: str) -> Optional[str]:
+        """
+        Try to autodiscover the url of an in-cluster service
+        """
+
+        for label_selector in selectors:
+            service_url = find_service_url(label_selector)
+            if service_url:
+                return service_url
+
+        logging.debug(error_msg)
+        return None
+
+
+class PrometheusDiscovery(ServiceDiscovery):
+    @classmethod
+    def find_prometheus_url(cls) -> Optional[str]:
+        return super().find_url(
+            selectors=[
+                "app=kube-prometheus-stack-prometheus",
+                "app=prometheus,component=server,release!=kubecost",
+                "app=prometheus-server",
+                "app=prometheus-operator-prometheus",
+                "app=rancher-monitoring-prometheus",
+                "app=prometheus-prometheus",
+                "app.kubernetes.io/component=query,app.kubernetes.io/name=thanos",
+                "app.kubernetes.io/name=thanos-query",
+                "app=thanos-query",
+                "app=thanos-querier",
+            ],
+            error_msg="Prometheus url could not be found. Add 'prometheus_url' under your prometheus tools config",
+        )
+
+    @classmethod
+    def find_vm_url(cls) -> Optional[str]:
+        return super().find_url(
+            selectors=[
+                "app.kubernetes.io/name=vmsingle",
+                "app.kubernetes.io/name=victoria-metrics-single",
+                "app.kubernetes.io/name=vmselect",
+                "app=vmselect",
+            ],
+            error_msg="Victoria Metrics url could not be found. Add 'prometheus_url' under your prometheus tools config",
+        )