diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index c577cd577e..01ce5e957c 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -254,8 +254,12 @@ SPIFFE JWT-SVIDs for dynamic provider token grants: 3. Redeploy: `mise run helm:skaffold:run` `ci/values-spire-stack.yaml` configures the local SPIRE trust domain as -`openshell.local` and adds a `ClusterSPIFFEID` that maps sandbox pod -annotations to `spiffe://openshell.local/openshell/sandbox/`. +`openshell.local` and adds a `ClusterSPIFFEID` that maps each sandbox pod to +`spiffe://openshell.local/openshell/sandbox/..`, +joining the pod's namespace, name, and `openshell.ai/sandbox-id` annotation into +one opaque final segment. Its `namespaceSelector` is left broad so SVIDs are +issued in all workspace modes; the overlay carries a commented-out stricter +selector for operators who label their managed namespaces. OpenShell mounts the SPIFFE CSI Workload API socket at `/spiffe-workload-api/spire-agent.sock` into sandbox pods for provider token grants. Supervisor-to-gateway authentication remains on the Kubernetes diff --git a/deploy/helm/openshell/ci/values-spire-stack.yaml b/deploy/helm/openshell/ci/values-spire-stack.yaml index 8a1e648829..181a88343f 100644 --- a/deploy/helm/openshell/ci/values-spire-stack.yaml +++ b/deploy/helm/openshell/ci/values-spire-stack.yaml @@ -15,10 +15,23 @@ spire-server: clusterSPIFFEIDs: openshell-sandboxes: enabled: true - spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/openshell/sandbox/{{ index .PodMeta.Annotations "openshell.ai/sandbox-id" }}' - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: openshell + # The final path segment joins namespace, pod-name, and sandbox-id with + # "." into a single opaque token. Treat the whole SPIFFE ID as an opaque + # principal: only the trust domain is authoritative, and the final + # sandbox segment (or any substring of it) must not drive authorization. + # The sandbox-id is appended conditionally, so pods without the + # openshell.ai/sandbox-id annotation render + # spiffe:///openshell/sandbox/. with no trailing dot. + spiffeIDTemplate: 'spiffe://{{ .TrustDomain }}/openshell/sandbox/{{ .PodMeta.Namespace }}.{{ .PodMeta.Name }}{{ with (index .PodMeta.Annotations "openshell.ai/sandbox-id") }}.{{ . }}{{ end }}' + # Match sandbox pods in shared, managed, and operator workspace modes. + # This CI/dev overlay intentionally leaves namespaceSelector broad so it + # works without requiring namespace labels. To restrict identity issuance + # to OpenShell-managed namespaces, label those namespaces and uncomment + # the namespaceSelector block below. + namespaceSelector: {} + # namespaceSelector: + # matchLabels: + # openshell.ai/managed-by: openshell podSelector: matchLabels: openshell.ai/managed-by: openshell diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index d6cb2140bd..1b48e06d19 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -25,7 +25,7 @@ Kubernetes sandbox supervisors authenticate back to the gateway as sandbox workl Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `server.providerTokenGrants.spiffe.enabled=true` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping the projected ServiceAccount token bootstrap and gateway-minted sandbox JWT path. -Provider token grants require a SPIFFE implementation such as SPIRE and identities for the gateway and sandbox pods. The repository's local SPIRE overlay assigns sandbox IDs from the pod's `openshell.ai/sandbox-id` annotation, but the gateway validation path only requires the supervisor SVID to be valid and in the same SPIFFE trust domain as the gateway SVID. Provider profiles with `token_grant` metadata cause the sandbox supervisor to request JWT-SVIDs and exchange them for upstream OAuth2 access tokens. Token-exchange profiles also require a gateway SPIFFE identity because the gateway brokers the intermediate token exchange with its own JWT-SVID. +Provider token grants require a SPIFFE implementation such as SPIRE and identities for the gateway and sandbox pods. The repository's local SPIRE overlay issues each sandbox a SPIFFE ID of the form `spiffe:///openshell/sandbox/..`, joining the sandbox pod's namespace, pod name, and `openshell.ai/sandbox-id` annotation into a single opaque final segment (the sandbox-id is appended only when the annotation is present). Treat the whole ID as an opaque principal: only the trust domain is authoritative, and the final segment must not be parsed or used to drive authorization. The overlay's `podSelector` matches the `openshell.ai/managed-by: openshell` label present on every sandbox pod, and its `namespaceSelector` is left broad so SVIDs are issued in all workspace modes (`Shared`, `Managed`, and `Operator`), whose sandbox pods land in different namespaces. To restrict issuance to OpenShell-managed namespaces, label those namespaces and enable the stricter `namespaceSelector` shown in the overlay; in `Operator` mode the platform team must label the workspace namespaces. The gateway validation path only requires the supervisor SVID to be valid and in the same SPIFFE trust domain as the gateway SVID. Provider profiles with `token_grant` metadata cause the sandbox supervisor to request JWT-SVIDs and exchange them for upstream OAuth2 access tokens. Token-exchange profiles also require a gateway SPIFFE identity because the gateway brokers the intermediate token exchange with its own JWT-SVID. The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so intermediate token exchange does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. diff --git a/examples/spiffe-token-exchange-demo/README.md b/examples/spiffe-token-exchange-demo/README.md index bd0ca44a89..ae309d5921 100644 --- a/examples/spiffe-token-exchange-demo/README.md +++ b/examples/spiffe-token-exchange-demo/README.md @@ -166,15 +166,15 @@ alpha called with path /: sub: demo-user aud: alpha, account scope: alpha profile email - azp: spiffe://openshell.local/openshell/sandbox/ - client_id: spiffe://openshell.local/openshell/sandbox/ + azp: spiffe://openshell.local/openshell/sandbox/.. + client_id: spiffe://openshell.local/openshell/sandbox/.. beta called with path /: sub: demo-user aud: beta, account scope: beta profile email - azp: spiffe://openshell.local/openshell/sandbox/ - client_id: spiffe://openshell.local/openshell/sandbox/ + azp: spiffe://openshell.local/openshell/sandbox/.. + client_id: spiffe://openshell.local/openshell/sandbox/.. ``` The token issuer logs both token exchange phases: @@ -186,9 +186,9 @@ KUBECONFIG="$DEMO_KUBECONFIG" kubectl -n default logs deployment/token-exchange- Example log lines: ```text -issued intermediate token for user=demo-user audience=spiffe://openshell.local/openshell/sandbox/ -issued final token for user=demo-user audience=alpha client=spiffe://openshell.local/openshell/sandbox/ -issued final token for user=demo-user audience=beta client=spiffe://openshell.local/openshell/sandbox/ +issued intermediate token for user=demo-user audience=spiffe://openshell.local/openshell/sandbox/.. +issued final token for user=demo-user audience=alpha client=spiffe://openshell.local/openshell/sandbox/.. +issued final token for user=demo-user audience=beta client=spiffe://openshell.local/openshell/sandbox/.. ``` ## Automated Demo diff --git a/examples/spiffe-token-grant-demo/README.md b/examples/spiffe-token-grant-demo/README.md index cb622ccbae..e9d81a8747 100644 --- a/examples/spiffe-token-grant-demo/README.md +++ b/examples/spiffe-token-grant-demo/README.md @@ -94,14 +94,20 @@ Expected output includes endpoint-specific token claims: alpha called with path /: aud: alpha, account scope: alpha profile email - azp: spiffe://openshell.local/openshell/sandbox/ + azp: spiffe://openshell.local/openshell/sandbox/.. beta called with path /: aud: beta, account scope: beta profile email - azp: spiffe://openshell.local/openshell/sandbox/ + azp: spiffe://openshell.local/openshell/sandbox/.. ``` +The final path segment joins the sandbox pod's namespace, pod name, and +`openshell.ai/sandbox-id` annotation with `.` into a single opaque identifier. +The `token-issuer` accepts any subject under the +`spiffe://openshell.local/openshell/sandbox/` prefix in the gateway trust +domain; the segment must be treated as opaque and not parsed. + The protected services also write proof-of-life logs when they accept a call: ```bash @@ -112,8 +118,8 @@ KUBECONFIG=kubeconfig kubectl -n default logs deployment/beta --tail=20 Example log lines: ```text -alpha accepted request path=/ aud="alpha, account" scope="alpha profile email" client_id=spiffe://openshell.local/openshell/sandbox/ -beta accepted request path=/ aud="beta, account" scope="beta profile email" client_id=spiffe://openshell.local/openshell/sandbox/ +alpha accepted request path=/ aud="alpha, account" scope="alpha profile email" client_id=spiffe://openshell.local/openshell/sandbox/.. +beta accepted request path=/ aud="beta, account" scope="beta profile email" client_id=spiffe://openshell.local/openshell/sandbox/.. ``` ## Automated Demo diff --git a/examples/spiffe-token-grant-demo/demo.sh b/examples/spiffe-token-grant-demo/demo.sh index f148d869fd..17c5727fdb 100755 --- a/examples/spiffe-token-grant-demo/demo.sh +++ b/examples/spiffe-token-grant-demo/demo.sh @@ -14,11 +14,20 @@ PROVIDER_NAME="${PROVIDER_NAME:-spiffe-token-demo}" PROFILE_ID="${PROFILE_ID:-spiffe-token-demo}" PORT_FORWARD_PORT="${PORT_FORWARD_PORT:-8097}" GATEWAY_ENDPOINT="${GATEWAY_ENDPOINT:-http://127.0.0.1:${PORT_FORWARD_PORT}}" +GATEWAY_NAME="${GATEWAY_NAME:-}" KEEP_SANDBOX="${KEEP_SANDBOX:-0}" ACCESS_TOKEN_SECRET="${ACCESS_TOKEN_SECRET:-$(openssl rand -hex 32)}" +# Optional `--gateway ` selector, shared by the OS array and cleanup. +GATEWAY_ARGS=() +if [[ -n "$GATEWAY_NAME" ]]; then + GATEWAY_ARGS=(--gateway "$GATEWAY_NAME") +fi + +# Only isolate config when no gateway is named; a named gateway must resolve +# from the caller's real config (registration and stored TLS bundle). TEMP_CONFIG_HOME="" -if [[ -z "${XDG_CONFIG_HOME:-}" ]]; then +if [[ -z "${XDG_CONFIG_HOME:-}" && -z "$GATEWAY_NAME" ]]; then TEMP_CONFIG_HOME="$(mktemp -d)" export XDG_CONFIG_HOME="$TEMP_CONFIG_HOME" fi @@ -27,7 +36,7 @@ PF_PID="" cleanup() { if [[ "$KEEP_SANDBOX" != "1" ]]; then - openshell --gateway-endpoint "$GATEWAY_ENDPOINT" sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + openshell "${GATEWAY_ARGS[@]}" --gateway-endpoint "$GATEWAY_ENDPOINT" sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true fi if [[ -n "$PF_PID" ]]; then kill "$PF_PID" >/dev/null 2>&1 || true @@ -89,7 +98,7 @@ sandbox_curl_until() { exit 1 } -OS=(openshell --gateway-endpoint "$GATEWAY_ENDPOINT") +OS=(openshell "${GATEWAY_ARGS[@]}" --gateway-endpoint "$GATEWAY_ENDPOINT") printf "\n$ kubectl -n default create secret generic openshell-spiffe-token-demo --from-literal=access-token-secret=*** --dry-run=client -o yaml | kubectl apply -f -\n" kubectl -n default create secret generic openshell-spiffe-token-demo \ diff --git a/examples/spiffe-token-grant-demo/k8s/kustomization.yaml b/examples/spiffe-token-grant-demo/k8s/kustomization.yaml index ccd3a94a3c..00948abbe5 100644 --- a/examples/spiffe-token-grant-demo/k8s/kustomization.yaml +++ b/examples/spiffe-token-grant-demo/k8s/kustomization.yaml @@ -12,6 +12,9 @@ configMapGenerator: files: - token-issuer.js - protected-service.js + - name: openshell-spiffe-token-demo-helper + files: + - spiffe-helper.conf resources: - workloads.yaml diff --git a/examples/spiffe-token-grant-demo/k8s/spiffe-helper.conf b/examples/spiffe-token-grant-demo/k8s/spiffe-helper.conf new file mode 100644 index 0000000000..93987d00c9 --- /dev/null +++ b/examples/spiffe-token-grant-demo/k8s/spiffe-helper.conf @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +agent_address = "/run/spire/sockets/spire-agent.sock" +cert_dir = "/target" +svid_file_name = "svid.pem" +svid_key_file_name = "key.pem" +svid_bundle_file_name = "bundle.pem" diff --git a/examples/spiffe-token-grant-demo/k8s/token-issuer.js b/examples/spiffe-token-grant-demo/k8s/token-issuer.js index 55e7ad48b6..55757f78e4 100644 --- a/examples/spiffe-token-grant-demo/k8s/token-issuer.js +++ b/examples/spiffe-token-grant-demo/k8s/token-issuer.js @@ -2,12 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 const http = require("http"); +const https = require("https"); const crypto = require("crypto"); +const fs = require("fs"); const PORT = Number(process.env.PORT || 8080); const JWKS_URI = process.env.SPIRE_JWKS_URI || "https://spire-spiffe-oidc-discovery-provider.spire.svc.cluster.local/keys"; +const SPIRE_JWKS_CA_FILE = process.env.SPIRE_JWKS_CA_FILE || ""; const SPIRE_ISSUER = process.env.SPIRE_ISSUER || "https://spire-spiffe-oidc-discovery-provider.spire.svc.cluster.local"; @@ -57,15 +60,42 @@ async function jwks() { if (cachedJwks && now - cachedJwksAt < 60000) { return cachedJwks; } - const response = await fetch(JWKS_URI); - if (!response.ok) { - throw new Error(`JWKS fetch failed with HTTP ${response.status}`); - } - cachedJwks = await response.json(); + cachedJwks = await fetchJson(JWKS_URI); cachedJwksAt = now; return cachedJwks; } +function fetchJson(url) { + return new Promise((resolve, reject) => { + const parsed = new URL(url); + const isHttps = parsed.protocol === "https:"; + const client = isHttps ? https : http; + const options = {}; + if (isHttps && SPIRE_JWKS_CA_FILE) { + options.ca = fs.readFileSync(SPIRE_JWKS_CA_FILE); + } + + const req = client.get(parsed, options, (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + if (res.statusCode < 200 || res.statusCode >= 300) { + reject(new Error(`JWKS fetch failed with HTTP ${res.statusCode}: ${body}`)); + return; + } + try { + resolve(JSON.parse(body)); + } catch (error) { + reject(error); + } + }); + }); + req.on("error", reject); + req.setTimeout(10000, () => req.destroy(new Error("JWKS fetch timed out"))); + }); +} + function hasAudience(payload, expected) { const aud = Array.isArray(payload.aud) ? payload.aud : [payload.aud]; return aud.includes(expected); diff --git a/examples/spiffe-token-grant-demo/k8s/workloads.yaml b/examples/spiffe-token-grant-demo/k8s/workloads.yaml index efba4e49f0..56c139b7a2 100644 --- a/examples/spiffe-token-grant-demo/k8s/workloads.yaml +++ b/examples/spiffe-token-grant-demo/k8s/workloads.yaml @@ -36,6 +36,8 @@ spec: value: http://token-issuer.default.svc.cluster.local - name: SPIRE_JWKS_URI value: https://spire-spiffe-oidc-discovery-provider.spire.svc.cluster.local/keys + - name: SPIRE_JWKS_CA_FILE + value: /etc/x509/spiffe-bundle/bundle.pem - name: SPIRE_ISSUER value: https://spire-spiffe-oidc-discovery-provider.spire.svc.cluster.local - name: JWT_SVID_AUDIENCE @@ -50,10 +52,35 @@ spec: - name: scripts mountPath: /demo readOnly: true + - name: spiffe-bundle + mountPath: /etc/x509/spiffe-bundle + readOnly: true + - name: spiffe-helper + image: ghcr.io/spiffe/spiffe-helper:0.11.0 + imagePullPolicy: IfNotPresent + args: ["-config", "/etc/spiffe-helper/spiffe-helper.conf"] + volumeMounts: + - name: spiffe-socket + mountPath: /run/spire/sockets + readOnly: true + - name: spiffe-bundle + mountPath: /target + - name: helper-config + mountPath: /etc/spiffe-helper + readOnly: true volumes: - name: scripts configMap: name: openshell-spiffe-token-demo-scripts + - name: helper-config + configMap: + name: openshell-spiffe-token-demo-helper + - name: spiffe-socket + csi: + driver: csi.spiffe.io + readOnly: true + - name: spiffe-bundle + emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/examples/spiffe-token-grant-demo/provider-profile.yaml b/examples/spiffe-token-grant-demo/provider-profile.yaml index caadfe2b3a..21db900b57 100644 --- a/examples/spiffe-token-grant-demo/provider-profile.yaml +++ b/examples/spiffe-token-grant-demo/provider-profile.yaml @@ -34,16 +34,12 @@ endpoints: tls: none access: read-write enforcement: enforce - allowed_ips: - - 10.43.0.0/16 - host: beta.default.svc.cluster.local port: 80 protocol: rest tls: none access: read-write enforcement: enforce - allowed_ips: - - 10.43.0.0/16 binaries: - /usr/bin/curl - /usr/local/bin/curl