Source code for smarter.apps.infrastructure.services.kubernetes

"""
The Kubernetes service: resources in the platform's Kubernetes cluster.

Kubernetes does not depend on a cloud: only the cluster's credentials do. So
:class:`KubernetesService` is implemented once, by :class:`KubectlKubernetesService`, with
`kubectl <https://kubernetes.io/docs/reference/kubectl/>`__, and the cloud provider contributes
only the kubeconfig, with
:meth:`~smarter.apps.infrastructure.providers.base.CloudProvider.update_kubeconfig`, e.g.
``aws eks update-kubeconfig`` for AWS EKS.

The platform uses it through :data:`smarter.apps.infrastructure.services.infrastructure`
``.kubernetes``. Tests replace it with :func:`configure_kubernetes`.

.. code-block:: python

    from smarter.apps.infrastructure.services import infrastructure

    infrastructure.kubernetes.apply_manifest(manifest)
"""

import os
import subprocess
import time
from abc import abstractmethod
from typing import TYPE_CHECKING, Callable, Optional

import yaml

from smarter.common.conf import smarter_settings
from smarter.common.utils import get_readonly_yaml_file
from smarter.lib import json, logging
from smarter.lib.django.waffle import SmarterWaffleSwitches
from smarter.lib.typing import SmarterDictType

from ..const import BILLABLE_KUBERNETES_KINDS, InfrastructureServiceNames
from ..exceptions import InfrastructureConfigurationError, KubernetesServiceError
from ..signals import resource_applied
from .base import InfrastructureService, refuse_in_unit_tests

if TYPE_CHECKING:
    from ..providers.base import CloudProvider

logger = logging.getSmarterLogger(__name__, any_switches=[SmarterWaffleSwitches.INFRASTRUCTURE_LOGGING])

RESOURCE_TYPE_PREFIX = "kubernetes"

SERVICE_ACCOUNT_DIRECTORY = "/var/run/secrets/kubernetes.io/serviceaccount"
"""Where Kubernetes mounts a pod's ServiceAccount token and the cluster's CA certificate."""


[docs] def in_cluster_server() -> Optional[str]: """ Return the Kubernetes api server's url, if Smarter runs in a pod that has a ServiceAccount token. Kubernetes sets ``KUBERNETES_SERVICE_HOST`` and ``KUBERNETES_SERVICE_PORT`` in every pod, and mounts the token in :data:`SERVICE_ACCOUNT_DIRECTORY`, unless ``automountServiceAccountToken`` is false. :returns: The url, or None outside a cluster, or without a token. """ host = os.environ.get("KUBERNETES_SERVICE_HOST") if not host: return None token_path = os.path.join(SERVICE_ACCOUNT_DIRECTORY, "token") if not os.path.isfile(token_path): return None port = os.environ.get("KUBERNETES_SERVICE_PORT", "443") if ":" in host: # an IPv6 address host = f"[{host}]" url = f"https://{host}:{port}" return url
[docs] def billable_resources(manifest: str) -> list[tuple[str, str]]: """ Return the resources of a manifest that provision billable cloud resources. - a PersistentVolumeClaim provisions a block storage volume. - a StatefulSet's volumeClaimTemplates provision one volume per replica. - a Service of type LoadBalancer provisions a cloud load balancer. :param manifest: A Kubernetes manifest, which may have several YAML documents. :returns: The (kind, name) of each, e.g. ``("persistentvolumeclaim", "data")``. """ retval: list[tuple[str, str]] = [] try: documents = [doc for doc in yaml.safe_load_all(manifest) if isinstance(doc, dict)] except yaml.YAMLError: return retval for doc in documents: kind = str(doc.get("kind", "")).lower() metadata = doc.get("metadata") or {} name = str(metadata.get("name", "")) spec = doc.get("spec") or {} if kind in BILLABLE_KUBERNETES_KINDS: retval.append((kind, name)) elif kind == "statefulset" and spec.get("volumeClaimTemplates"): retval.append((kind, name)) elif kind == "service" and spec.get("type") == "LoadBalancer": retval.append((kind, name)) return retval
[docs] def redacted_manifest(manifest: str) -> str: """ Return a manifest that can be logged: the values of its Secrets replaced with ``<redacted>``. A Secret's ``data`` and ``stringData`` hold credentials, e.g. an LLMHost's provider API key, in plain text or base64. :param manifest: A Kubernetes manifest, which may have several YAML documents. :returns: The manifest, with each Secret value redacted, or a placeholder if it isn't valid YAML. """ try: documents = list(yaml.safe_load_all(manifest)) except yaml.YAMLError: return "<a manifest that is not valid YAML, not logged>" for doc in documents: if isinstance(doc, dict) and str(doc.get("kind", "")).lower() == "secret": for field in ("data", "stringData"): if isinstance(doc.get(field), dict): doc[field] = {key: "<redacted>" for key in doc[field]} return yaml.safe_dump_all(documents, sort_keys=False)
[docs] def manifest_kinds(manifest: str) -> list[str]: """Return the kinds of a manifest's resources, e.g. ``["Ingress"]``.""" try: return [str(doc.get("kind")) for doc in yaml.safe_load_all(manifest) if isinstance(doc, dict)] except yaml.YAMLError: return []
[docs] class KubernetesService(InfrastructureService): """ The platform's Kubernetes cluster. Implementations provide the primitives: :meth:`apply_manifest`, :meth:`get_resource`, :meth:`list_resources`, :meth:`delete_resource`, :meth:`delete_resources` and :meth:`get_pod_logs`, and optionally :meth:`find_resources`, for the inventory. The ingress operations that LLMClient deployments use are built on them. """ service_name = InfrastructureServiceNames.KUBERNETES error_class = KubernetesServiceError certificate_wait_seconds: float = 60 """Seconds between the checks of a cert-manager certificate, in :meth:`verify_ingress_resources`.""" # -------------------------------------------------------------------------- # primitives # --------------------------------------------------------------------------
[docs] @abstractmethod def apply_manifest(self, manifest: str) -> None: """ Create or update the resources of a manifest. :raises KubernetesServiceError: If the cluster rejects them. """
[docs] @abstractmethod def get_resource(self, kind: str, name: str, namespace: str) -> Optional[SmarterDictType]: """ Return a resource, or None if it does not exist or the cluster is unavailable. :param kind: The kind, e.g. ``ingress``. :param name: The resource's name. :param namespace: The namespace. :returns: The resource, as ``kubectl get -o json`` returns it, or None. """
[docs] def get_resources(self, resources: list[tuple[str, str]], namespace: str) -> dict[tuple[str, str], SmarterDictType]: """ Return several resources at once. Implementations may override it to fetch them in one request. This one gets each in turn. :param resources: The (kind, name) of each resource, with a lowercase kind, e.g. ``("ingress", "app")``. :param namespace: The namespace. :returns: The resources that exist, keyed by their (kind, name). """ retval: dict[tuple[str, str], SmarterDictType] = {} for kind, name in resources: resource = self.get_resource(kind, name, namespace) if resource is not None: retval[(kind, name)] = resource return retval
[docs] @abstractmethod def list_resources(self, kind: str, namespace: str, selector: Optional[str] = None) -> list[SmarterDictType]: """ Return the resources of a kind, optionally those that match a label selector. :param kind: The kind, e.g. ``pods``. :param namespace: The namespace. :param selector: A label selector, e.g. ``app=qdrant``, or None for all. :returns: The resources, or an empty list. """
# pylint: disable=W0613
[docs] def find_resources(self, kind: str, namespace: Optional[str] = None) -> Optional[list[SmarterDictType]]: """ Return every resource of a kind, for the inventory. Unlike :meth:`list_resources`, it tells a kind that has no resources from one that could not be listed, so that the inventory never mistakes an unavailable cluster for an empty one. This implementation cannot list, and returns None. :param kind: The kind, e.g. ``node``. :param namespace: The namespace of a namespaced kind, or None for a cluster-scoped kind, e.g. a node. :returns: The resources, or None if they could not be listed, e.g. because the cluster is unavailable, or the kind's CustomResourceDefinition is not installed. """ return None
[docs] @abstractmethod def delete_resource(self, kind: str, name: str, namespace: str) -> bool: """Delete a resource by name. A resource that does not exist counts as deleted. """
[docs] @abstractmethod def delete_resources(self, kinds: list[str], namespace: str, selector: str) -> bool: """ Delete the resources of several kinds that match a label selector. Idempotent. A selector is required, so that a namespace is never emptied by mistake. """
[docs] @abstractmethod def get_pod_logs( self, namespace: str, selector: str, container: Optional[str] = None, tail: int = 200 ) -> Optional[str]: """ Return the most recent log lines of the pods that match a label selector. :param namespace: The namespace. :param selector: A label selector. :param container: The container, or None for the pod's default. :param tail: How many lines of each pod. :returns: The logs, or None if they could not be read. """
def _sleep(self, seconds: float) -> None: time.sleep(seconds) # -------------------------------------------------------------------------- # resources # --------------------------------------------------------------------------
[docs] def verify_ingress(self, name: str, namespace: str) -> bool: """ Whether an Ingress exists. :param name: The Ingress's name. :param namespace: The Ingress's namespace. :returns: True if it exists. """ return self.get_resource("ingress", name, namespace) is not None
[docs] def verify_secret(self, name: str, namespace: str) -> bool: """ Whether a Secret exists. :param name: The Secret's name. :param namespace: The Secret's namespace. :returns: True if it exists. """ return self.get_resource("secret", name, namespace) is not None
[docs] def verify_certificate(self, name: str, namespace: str) -> bool: """ Whether a cert-manager Certificate exists, and is Ready, i.e. issued. :param name: The Certificate's name. :param namespace: The Certificate's namespace. """ return self._certificate_ready(self.get_resource("certificate", name, namespace), name, namespace)
def _certificate_ready(self, certificate: Optional[SmarterDictType], name: str, namespace: str) -> bool: """Whether a cert-manager Certificate exists, and its Ready condition is True.""" if certificate is None: return False status = certificate.get("status") or {} conditions = status.get("conditions") or [] ready = None for condition in conditions: if condition.get("type") == "Ready": ready = condition.get("status") break if str(ready).lower() == "true": logger.info("certificate %s %s is issued", namespace, name) return True logger.warning( "certificate %s %s is not ready: %s", namespace, name, ready, ) return False
[docs] def verify_ingress_resources( self, hostname: str, namespace: str, max_attempts: int = 30 ) -> tuple[bool, bool, bool]: """ Verify that a host's Ingress, its cert-manager Certificate, and its TLS Secret exist. The Ingress is named after the host, and the Certificate and Secret ``<host>-tls``. :param hostname: The host, e.g. ``example.3141-5926-5359.api.example.com``. :param namespace: The namespace. :param max_attempts: How many times to check the Certificate, a minute apart. A Celery task passes 1, and checks again later, so that it does not block its worker. :returns: Whether the Ingress, the Certificate, and the Secret are verified. """ secret_name = f"{hostname}-tls" # one request for all three: each kubectl call also fetches a cluster token, which is slow. found = self.get_resources( [("ingress", hostname), ("certificate", secret_name), ("secret", secret_name)], namespace ) ingress_verified = ("ingress", hostname) in found secret_verified = ("secret", secret_name) in found certificate_verified = self._certificate_ready(found.get(("certificate", secret_name)), secret_name, namespace) for _ in range(1, max(1, max_attempts)): if certificate_verified: break self._sleep(self.certificate_wait_seconds) certificate_verified = self.verify_certificate(secret_name, namespace) return ingress_verified, certificate_verified, secret_verified
[docs] def delete_ingress(self, name: str, namespace: str) -> bool: """ Delete an Ingress. :param name: The Ingress's name. :param namespace: The Ingress's namespace. :returns: True if it was deleted, or did not exist. """ return self.delete_resource("ingress", name, namespace)
[docs] def delete_certificate(self, name: str, namespace: str) -> bool: """ Delete a cert-manager Certificate. :param name: The Certificate's name. :param namespace: The Certificate's namespace. :returns: True if it was deleted, or did not exist. """ return self.delete_resource("certificate", name, namespace)
[docs] def delete_secret(self, name: str, namespace: str) -> bool: """ Delete a Secret. :param name: The Secret's name. :param namespace: The Secret's namespace. :returns: True if it was deleted, or did not exist. """ return self.delete_resource("secret", name, namespace)
[docs] def delete_ingress_resources(self, hostname: str, namespace: str) -> tuple[bool, bool, bool]: """ Delete a host's Ingress, its cert-manager Certificate, and its TLS Secret. :returns: Whether the Ingress, the Certificate, and the Secret were deleted. """ secret_name = f"{hostname}-tls" return ( self.delete_ingress(hostname, namespace), self.delete_certificate(secret_name, namespace), self.delete_secret(secret_name, namespace), )
[docs] class KubectlKubernetesService(KubernetesService): """ The platform's Kubernetes cluster, through kubectl. The cluster is ready once kubectl is configured, with the provider's :meth:`~smarter.apps.infrastructure.providers.base.CloudProvider.update_kubeconfig`, and the environment's namespace, ``smarter_settings.environment_namespace``, exists. In a pod (see :func:`in_cluster_server`), kubectl authenticates as the pod's ServiceAccount instead, so Smarter has exactly the permissions of the Helm chart's Role and ClusterRole, and needs neither a kubeconfig nor the provider's token. :param provider: The cloud provider that writes the kubeconfig. None for a cluster whose kubeconfig is already in place. :param allow_in_tests: Allow kubectl in the unit tests, e.g. when subprocess is mocked. """
[docs] def __init__(self, provider: Optional["CloudProvider"] = None, allow_in_tests: bool = False, **kwargs): super().__init__(provider_name=provider.provider_name if provider else "kubectl", **kwargs) self.provider = provider self.allow_in_tests = allow_in_tests self._configured = False self._namespace_verified = False self._kubeconfig: Optional[SmarterDictType] = None self._token: Optional[str] = None self._token_expires: float = 0.0
# -------------------------------------------------------------------------- # readiness # -------------------------------------------------------------------------- @property def configured(self) -> bool: """Whether kubectl is configured for the cluster.""" if not self._configured: if self.provider is None: self._configured = True elif in_cluster_server(): logger.debug("using the pod's ServiceAccount") self._configured = True else: try: self._configured = self.provider.update_kubeconfig() except InfrastructureConfigurationError as e: logger.debug("%s", e) self._configured = False return self._configured @property def namespace_verified(self) -> bool: """Whether the environment's namespace exists.""" if not self._namespace_verified: self._namespace_verified = self.verify_namespace(smarter_settings.environment_namespace) return self._namespace_verified @property def ready(self) -> bool: """ Whether kubectl is configured for the cluster, and the environment's namespace exists. Records the connection state, which sends a signal when it changes. """ if not self.configured: return self.connection_state(False, "kubectl is not configured for the cluster") if not self.namespace_verified: return self.connection_state( False, f"the namespace {smarter_settings.environment_namespace} does not exist" ) return self.connection_state(True) @property def kubeconfig_path(self) -> str: """The path of the platform's kubeconfig file.""" return os.path.join(smarter_settings.data_directory, ".kube", "config") @property def kubeconfig(self) -> SmarterDictType: """The platform's kubeconfig file.""" if self._kubeconfig is None: self._kubeconfig = get_readonly_yaml_file(self.kubeconfig_path) return self._kubeconfig # -------------------------------------------------------------------------- # kubectl # -------------------------------------------------------------------------- def _bearer_token(self) -> Optional[str]: """ Return the provider's bearer token for the cluster, which is reused until it expires. Without one, kubectl authenticates with its kubeconfig, which for EKS runs ``aws eks get-token`` on every call: a Python process that takes seconds to start on a busy worker. In a pod, it is the pod's ServiceAccount token, read on every call, because the kubelet replaces the file before the token expires. """ if in_cluster_server(): token_path = os.path.join(SERVICE_ACCOUNT_DIRECTORY, "token") with open(token_path, encoding="utf-8") as token_file: token = token_file.read().strip() return token if self.provider is None: return None if self._token is None or time.time() >= self._token_expires: result = self.provider.get_kubernetes_token() self._token, self._token_expires = result if result else (None, 0.0) return self._token def _kubectl(self, *args: str, stdin: Optional[str] = None) -> subprocess.CompletedProcess: """ Run kubectl, with the provider's bearer token, if it has one. :raises InfrastructureConfigurationError: In the unit tests, unless allowed. """ refuse_in_unit_tests("the Kubernetes cluster", self.allow_in_tests) command = ["kubectl", *args] server = in_cluster_server() if server: # the in-cluster api server and CA replace the kubeconfig's cluster, if there is one. certificate_authority = os.path.join(SERVICE_ACCOUNT_DIRECTORY, "ca.crt") command += ["--server", server, "--certificate-authority", certificate_authority] token = self._bearer_token() if token: # a request that has a token does not run the kubeconfig's exec command. command += ["--token", token] return subprocess.run(command, input=stdin, capture_output=True, text=True, check=False) # nosec B603 B607 def _kubectl_json(self, *args: str) -> Optional[SmarterDictType]: """Run kubectl with ``-o json``, and return its output, or None if it fails or is empty.""" result = self._kubectl(*args, "-o", "json") if result.returncode != 0: logger.warning( "kubectl %s failed: %s", " ".join(args), result.stderr, ) return None if not result.stdout.strip(): return None try: return json.loads(result.stdout) except json.JSONDecodeError as e: logger.error( "kubectl %s returned invalid json: %s", " ".join(args), e, ) return None
[docs] def verify_namespace(self, namespace: str) -> bool: """Whether a namespace exists.""" if not self.configured: return False try: return self._kubectl_json("get", "namespace", namespace) is not None except InfrastructureConfigurationError as e: logger.debug("%s", e) return False
[docs] def apply_manifest(self, manifest: str) -> None: """ Create or update the resources of a manifest, with ``kubectl apply``. Resources that provision billable cloud resources, see :func:`billable_resources`, are announced with the billable resource signals. Nothing is applied if the cluster is not ready. :param manifest: The manifest, which may have several YAML documents. :raises KubernetesServiceError: If the cluster rejects it. """ logger.info( "applying a manifest to the cluster:\n%s", redacted_manifest(manifest), ) if not self.ready: logger.error( "the cluster is not ready. The manifest was not applied.", ) return None billable = [ self.creating_resource(f"{RESOURCE_TYPE_PREFIX}.{kind}", name, billable=True) for kind, name in billable_resources(manifest) ] with self.operation("apply_manifest"): result = self._kubectl("apply", "-f", "-", stdin=manifest) if result.returncode != 0: raise KubernetesServiceError(f"Failed to apply manifest: {result.stderr}") for resource in billable: self.created_resource(resource, resource_id=resource["resource_name"]) self.send(resource_applied, kinds=manifest_kinds(manifest)) return None
[docs] def get_resource(self, kind: str, name: str, namespace: str) -> Optional[SmarterDictType]: """ Return a resource with ``kubectl get``. :param kind: The kind, e.g. ``ingress``. :param name: The resource's name. :param namespace: The namespace. :returns: The resource, or None if it does not exist, kubectl fails, or the cluster is not ready. """ if not self.ready: return None return self._kubectl_json("get", kind, name, "-n", namespace, "--ignore-not-found")
[docs] def get_resources(self, resources: list[tuple[str, str]], namespace: str) -> dict[tuple[str, str], SmarterDictType]: """ Return several resources with one ``kubectl get``. Each kubectl call fetches a cluster token, e.g. with ``aws eks get-token``, which takes seconds, so one call rather than several frees a Celery worker sooner. If the call fails, e.g. because a kind's CustomResourceDefinition is not installed, each resource is fetched in turn. """ if not resources or not self.ready: return {} refs = [f"{kind}/{name}" for kind, name in resources] result = self._kubectl("get", *refs, "-n", namespace, "--ignore-not-found", "-o", "json") if result.returncode != 0: logger.warning( "kubectl get %s failed: %s", " ".join(refs), result.stderr, ) return super().get_resources(resources, namespace) if not result.stdout.strip(): return {} try: output = json.loads(result.stdout) except json.JSONDecodeError as e: logger.error( "kubectl get %s returned invalid json: %s", " ".join(refs), e, ) return {} # kubectl returns a List of several resources, or the resource itself when only one exists. if output.get("kind") == "List": items = output.get("items", []) else: items = [output] found: dict[tuple[str, str], SmarterDictType] = {} for item in items: item_kind = str(item.get("kind", "")).lower() metadata = item.get("metadata") or {} item_name = str(metadata.get("name", "")) found[(item_kind, item_name)] = item retval = {resource: found[resource] for resource in resources if resource in found} return retval
[docs] def list_resources(self, kind: str, namespace: str, selector: Optional[str] = None) -> list[SmarterDictType]: """ Return the resources of a kind with ``kubectl get``, optionally those that match a label selector. :param kind: The kind, e.g. ``pods``. :param namespace: The namespace. :param selector: A label selector, e.g. ``app=qdrant``, or None for all. :returns: The resources, or an empty list if there are none, kubectl fails, or the cluster is not ready. """ if not self.ready: return [] args = ["get", kind, "-n", namespace] if selector: args += ["-l", selector] output = self._kubectl_json(*args) return (output or {}).get("items", [])
[docs] def find_resources(self, kind: str, namespace: Optional[str] = None) -> Optional[list[SmarterDictType]]: """ Return every resource of a kind with ``kubectl get``, for the inventory. :param kind: The kind, e.g. ``node``. :param namespace: The namespace of a namespaced kind, or None for a cluster-scoped kind. :returns: The resources, or None if they could not be listed, e.g. because kubectl failed, its output is not a list, or the cluster is not ready. """ if not self.ready: return None args = ["get", kind] + (["-n", namespace] if namespace else []) result = self._kubectl(*args, "-o", "json") if result.returncode != 0: logger.warning( "kubectl %s failed: %s", " ".join(args), result.stderr, ) return None try: items = json.loads(result.stdout).get("items") except (json.JSONDecodeError, AttributeError) as e: logger.error( "kubectl %s returned invalid json: %s", " ".join(args), e, ) return None return items if isinstance(items, list) else None
[docs] def delete_resource(self, kind: str, name: str, namespace: str) -> bool: """ Delete a resource with ``kubectl delete``. A resource that does not exist counts as deleted. Sends the destroying and destroyed resource signals, which record it in the ledger, billable if its kind is in ``BILLABLE_KUBERNETES_KINDS``. :param kind: The kind, e.g. ``ingress``. :param name: The resource's name. :param namespace: The namespace. :returns: True if it was deleted, False if kubectl failed or the cluster is not ready. """ if not self.ready: return False billable = kind.lower() in BILLABLE_KUBERNETES_KINDS resource = self.destroying_resource(f"{RESOURCE_TYPE_PREFIX}.{kind.lower()}", name, name, billable=billable) # --ignore-not-found: a resource that is already gone counts as deleted. result = self._kubectl("delete", kind, name, "-n", namespace, "--ignore-not-found") if result.returncode != 0: logger.error( "failed to delete %s %s: %s", kind, name, result.stderr, ) return False self.destroyed_resource(resource) return True
[docs] def delete_resources(self, kinds: list[str], namespace: str, selector: str) -> bool: """ Delete the resources of several kinds that match a label selector, with one ``kubectl delete``. Sends the destroying and destroyed resource signals for each billable kind. :param kinds: The kinds, e.g. ``["statefulset", "service"]``. :param namespace: The namespace. :param selector: A label selector, which is required, so that a namespace is never emptied by mistake. :returns: True if they were deleted, False if kubectl failed or the cluster is not ready. :raises KubernetesServiceError: If ``kinds`` or ``selector`` is empty. """ if not kinds or not selector: raise KubernetesServiceError("delete_resources() requires kinds and a label selector.") if not self.ready: return False billable = [ self.destroying_resource(f"{RESOURCE_TYPE_PREFIX}.{kind.lower()}", selector, selector, billable=True) for kind in kinds if kind.lower() in BILLABLE_KUBERNETES_KINDS ] result = self._kubectl("delete", ",".join(kinds), "-n", namespace, "-l", selector, "--ignore-not-found") if result.returncode != 0: logger.error( "failed to delete %s %s: %s", kinds, selector, result.stderr, ) return False for resource in billable: self.destroyed_resource(resource) return True
[docs] def get_pod_logs( self, namespace: str, selector: str, container: Optional[str] = None, tail: int = 200 ) -> Optional[str]: """ Return the most recent log lines of the pods that match a label selector, with ``kubectl logs``. :param namespace: The namespace. :param selector: A label selector. :param container: The container, or None for the pod's default. :param tail: How many lines of each pod. :returns: The logs, each line prefixed with its pod. If kubectl fails, its output and error, which explain why, e.g. a pod that is still pulling its image. None if the cluster is not ready. """ if not self.ready: return None args = ["logs", "-n", namespace, "-l", selector, "--tail", str(tail), "--prefix"] if container: args += ["-c", container] result = self._kubectl(*args) if result.returncode == 0: return result.stdout # the error explains why there are no logs, e.g. a pod that is still pulling its image. logger.warning( "failed to get the logs of %s: %s", selector, result.stderr, ) return (result.stdout or "") + (result.stderr or "")
[docs] def matches_selector(item: SmarterDictType, selector: Optional[str]) -> bool: """ Decide whether a resource's labels match a label selector, as ``kubectl get -l`` does. Only equality requirements are supported, e.g. ``app.kubernetes.io/name=smarter,tier=web``, which are all that the platform uses. Every requirement must match. :param item: The resource, as ``kubectl get -o json`` returns it. :param selector: The selector, or None or an empty string, which every resource matches. :returns: True if every requirement of the selector matches one of the resource's labels. """ if not selector: return True metadata = item.get("metadata") or {} labels = metadata.get("labels") or {} for requirement in selector.split(","): key, _, value = requirement.strip().partition("=") if labels.get(key) != value: return False return True
[docs] class InMemoryKubernetesService(KubernetesService): """ A Kubernetes cluster in memory, for tests and local development. Its resources are :attr:`items`: lists of resources, as ``kubectl get -o json`` returns them, keyed by kind, e.g. ``{"pod": [...], "node": [...]}``. A kind that is not a key cannot be listed: :meth:`find_resources` returns None for it, as for a cluster that is unavailable, or a CustomResourceDefinition that is not installed. :param ready: Whether the cluster is ready. :param provider_name: The provider's name, in signals and logs. """
[docs] def __init__(self, ready: bool = True, provider_name: str = "memory", **kwargs): super().__init__(provider_name=provider_name, **kwargs) self._ready = ready self.items: dict[str, list[SmarterDictType]] = {} self.calls: list[tuple[str, Optional[str]]] = [] """The (kind, namespace) of every :meth:`find_resources` and :meth:`list_resources` call, in order.""" self.manifests: list[str] = [] """Every manifest that :meth:`apply_manifest` received, in order."""
@property def ready(self) -> bool: """Whether the cluster is ready, as set in the constructor.""" return self._ready def _in(self, kind: str, namespace: Optional[str]) -> list[SmarterDictType]: """ Return the resources of a kind in one namespace, or in every namespace. :param kind: The kind, e.g. ``pod``. :param namespace: The namespace, or None for every namespace, e.g. for a cluster-scoped kind such as ``node``. :returns: The resources, in the order of :attr:`items`. """ found = [] for item in self.items.get(kind, []): if namespace is None: found.append(item) continue metadata = item.get("metadata") or {} if metadata.get("namespace") == namespace: found.append(item) return found
[docs] def apply_manifest(self, manifest: str) -> None: """ Record a manifest in :attr:`manifests`, without creating its resources. :param manifest: The manifest. """ self.manifests.append(manifest)
[docs] def get_resource(self, kind: str, name: str, namespace: str) -> Optional[SmarterDictType]: """ Find one resource of :attr:`items` by its kind, name and namespace. :param kind: The kind, e.g. ``ingress``. :param name: The resource's name. :param namespace: The resource's namespace. :returns: The resource, or None if there is none. """ for item in self._in(kind, namespace): metadata = item.get("metadata") or {} if metadata.get("name") == name: return item return None
[docs] def list_resources(self, kind: str, namespace: str, selector: Optional[str] = None) -> list[SmarterDictType]: """ Return the resources of a kind in a namespace that match a label selector, and record the call. :param kind: The kind, e.g. ``pod``. :param namespace: The namespace. :param selector: An equality label selector, or None for every resource (see :func:`matches_selector`). :returns: The resources, or an empty list if the cluster is not ready, or has none. """ self.calls.append((kind, namespace)) if not self.ready: return [] found = [] for item in self._in(kind, namespace): if matches_selector(item, selector): found.append(item) return found
[docs] def find_resources(self, kind: str, namespace: Optional[str] = None) -> Optional[list[SmarterDictType]]: """ Return every resource of a kind, and record the call. :param kind: The kind, e.g. ``node``. :param namespace: The namespace of a namespaced kind, or None for every namespace. :returns: The resources, or None if the cluster is not ready, or the kind is not a key of :attr:`items`, as for a cluster that cannot be read. """ self.calls.append((kind, namespace)) if not self.ready: return None if kind not in self.items: return None return self._in(kind, namespace)
[docs] def delete_resource(self, kind: str, name: str, namespace: str) -> bool: """ Remove a resource from :attr:`items`. :param kind: The kind. :param name: The resource's name. :param namespace: The resource's namespace. :returns: True, also for a resource that does not exist, as Kubernetes' delete is idempotent. """ item = self.get_resource(kind, name, namespace) if item is not None: self.items[kind].remove(item) return True
[docs] def delete_resources(self, kinds: list[str], namespace: str, selector: str) -> bool: """ Remove the resources of several kinds that match a label selector from :attr:`items`. :param kinds: The kinds, e.g. ``["ingress", "service"]``. :param namespace: The namespace. :param selector: An equality label selector (see :func:`matches_selector`). :returns: True. """ for kind in kinds: matching = self.list_resources(kind, namespace, selector) for item in matching: self.items[kind].remove(item) return True
[docs] def get_pod_logs( self, namespace: str, selector: str, container: Optional[str] = None, tail: int = 200 ) -> Optional[str]: """ Return no log lines: the in-memory cluster runs no pods. :param namespace: Ignored. :param selector: Ignored. :param container: Ignored. :param tail: Ignored. :returns: An empty string. """ return ""
_kubernetes_factory: Optional[Callable[[], KubernetesService]] = None _kubernetes: Optional[KubernetesService] = None
[docs] def configure_kubernetes(factory: Optional[Callable[[], KubernetesService]]) -> None: """ Set the factory of the Kubernetes service that :func:`get_kubernetes` returns. :param factory: Returns the service, or None to restore the default, :class:`KubectlKubernetesService` with the configured cloud provider. """ global _kubernetes_factory, _kubernetes # pylint: disable=global-statement _kubernetes_factory = factory _kubernetes = None
[docs] def get_kubernetes() -> KubernetesService: """ Return the Kubernetes service. It is created once, and again if the cloud provider is reconfigured, so that it keeps its readiness, rather than configuring kubectl for each call. """ global _kubernetes # pylint: disable=global-statement if _kubernetes_factory is not None: if _kubernetes is None: _kubernetes = _kubernetes_factory() return _kubernetes # pylint: disable=import-outside-toplevel from ..providers import get_provider provider = get_provider() if not isinstance(_kubernetes, KubectlKubernetesService) or _kubernetes.provider is not provider: _kubernetes = KubectlKubernetesService(provider=provider) return _kubernetes
__all__ = [ "InMemoryKubernetesService", "KubectlKubernetesService", "KubernetesService", "billable_resources", "configure_kubernetes", "get_kubernetes", "in_cluster_server", "manifest_kinds", "matches_selector", "redacted_manifest", ]