Keep /api/issues fast on clusters with many Helm releases

docs/helm.md

@@ -54,6 +54,8 @@ Helm rewrites `Info.Deleted` at the start of every uninstall attempt, before the
 
 The issue suggests checking pre-delete hook Jobs and surviving resources with finalizers before retrying the uninstall (Helm re-runs an uninstall stuck in this state). When a hook is the blocker, `helm uninstall --no-hooks` completes the uninstall, but Helm does not remove the Jobs and Pods the hook created, and resources held by finalizers stay Terminating. Radar no longer ties either to the release afterwards, so the issue tells the operator to delete the hook's leftovers and check for Terminating resources. These are investigation steps, not verified blocker diagnoses.
 
+To find them, Radar first lists only the labels of the Helm release Secrets the caller can read. Helm labels each one with the release name, revision and status. Radar then reads in full only the releases whose newest revision is failed or pending, plus any release with the same name in the same read. If any release Secret lacks those labels, Radar reads them all in full, as before. Releases stored with Helm's ConfigMap or SQL drivers are not read for issues. `/api/issues` waits for this read, and the read stops as soon as the caller disconnects.
+
 ## Failed Upgrades And Rollback Inference
 
 Helm history does not persist whether `helm upgrade --atomic` was set. Radar infers an atomic-style rollback from the revision sequence:

internal/helm/client_test.go

@@ -407,11 +407,11 @@ func TestHelmReleaseRowsFromStorageSnapshot_AttachesLastOperation(t *testing.T)
 	}
 
 	client := fake.NewSimpleClientset(secretsToObjects(secrets)...)
-	snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+	rows := helmReleaseRowsForTest(t, snapshot)
 
 	if len(rows) != 1 {
 		t.Fatalf("len(rows) = %d, want 1", len(rows))
@@ -446,12 +446,12 @@ func TestHelmReleaseRowsFromStorageSnapshot_AttachesLastOperation(t *testing.T)
 func TestHelmReleaseRowsFromStorageSnapshot_KeepsHealthyRowsCompact(t *testing.T) {
 	rel := helmTestRelease("healthy", "demo", 1, release.StatusDeployed, "Install complete")
 	client := fake.NewSimpleClientset(helmReleaseSecret(t, "demo", rel, false))
-	snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+	rows := helmReleaseRowsForTest(t, snapshot)
 
 	if len(rows) != 1 {
 		t.Fatalf("len(rows) = %d, want 1", len(rows))
@@ -475,12 +475,12 @@ func TestHelmReleaseRowsFromStorageSnapshot_SkipsMalformedReleaseSecret(t *testi
 		},
 	}
 	client := fake.NewSimpleClientset(helmReleaseSecret(t, "demo", malformed, false))
-	snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+	rows := helmReleaseRowsForTest(t, snapshot)
 
 	if len(rows) != 0 {
 		t.Fatalf("len(rows) = %d, want 0 for malformed release secret", len(rows))
@@ -504,12 +504,12 @@ func TestHelmReleaseRowsFromStorageSnapshot_CapsOperations(t *testing.T) {
 		secrets = append(secrets, helmReleaseSecret(t, "demo", rel, false))
 	}
 	client := fake.NewSimpleClientset(secretsToObjects(secrets)...)
-	snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+	rows := helmReleaseRowsForTest(t, snapshot)
 
 	if len(rows) != 1 {
 		t.Fatalf("len(rows) = %d, want 1", len(rows))
@@ -543,12 +543,12 @@ func TestHelmReleaseRowsFromStorageSnapshot_UsesDetailHistoryWindow(t *testing.T
 		secrets = append(secrets, helmReleaseSecret(t, "demo", rel, false))
 	}
 	client := fake.NewSimpleClientset(secretsToObjects(secrets)...)
-	snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+	rows := helmReleaseRowsForTest(t, snapshot)
 
 	if len(rows) != 1 {
 		t.Fatalf("len(rows) = %d, want 1", len(rows))
@@ -1533,11 +1533,11 @@ func assertStorageNamespaceFromSecret(t *testing.T, gzipped bool) {
 	rel := helmTestRelease("podinfo", "demo-flux-helm", 1, release.StatusDeployed, "Install complete")
 	client := fake.NewSimpleClientset(helmReleaseSecret(t, "flux-system", rel, gzipped))
 
-	storageNamespaces, err := helmReleaseStorageNamespacesWithClient(client)
+	snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if got := storageNamespaces[releaseStorageKey(rel)]; got != "flux-system" {
+	if got := snapshot.storageNamespaces[releaseStorageKey(rel)]; got != "flux-system" {
 		t.Fatalf("storage namespace = %q, want flux-system", got)
 	}
 }
@@ -1656,7 +1656,7 @@ generated: "2026-05-05T00:00:00Z"
 }
 
 func TestListReleasesAcrossNamespacesEmptySerializesAsArray(t *testing.T) {
-	releases, err := (&Client{}).ListReleasesAcrossNamespaces([]string{}, "", nil)
+	releases, err := (&Client{}).ListReleasesAcrossNamespaces(context.Background(), []string{}, "", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2061,11 +2061,11 @@ func TestHelmReleaseUninstallTimestampFromStorage(t *testing.T) {
 			rel.Info.LastDeployed = helmtime.Time{Time: now.Add(-90 * 24 * time.Hour)}
 			rel.Info.Deleted = helmtime.Time{Time: tt.deleted}
 			client := fake.NewSimpleClientset(helmReleaseSecret(t, "helm-storage", rel, true))
-			snapshot, err := helmReleaseStorageSnapshotWithClient(client, "")
+			snapshot, err := helmReleaseStorageSnapshotWithClient(context.Background(), client, "")
 			if err != nil {
 				t.Fatal(err)
 			}
-			rows := helmReleaseRowsFromStorageSnapshot(snapshot, nil, "")
+			rows := helmReleaseRowsForTest(t, snapshot)
 			if len(rows) != 1 {
 				t.Fatalf("rows = %#v", rows)
 			}
@@ -2102,3 +2102,12 @@ func TestHelmReleaseDeploymentTimestampPreserved(t *testing.T) {
 		})
 	}
 }
+
+func helmReleaseRowsForTest(t *testing.T, snapshot *helmReleaseStorageSnapshot) []HelmRelease {
+	t.Helper()
+	rows, err := helmReleaseRowsFromStorageSnapshot(context.Background(), snapshot, nil, "", true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rows
+}

internal/helm/handlers.go

@@ -1,6 +1,7 @@
 package helm
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -182,8 +183,11 @@ func (h *Handlers) handleListReleases(w http.ResponseWriter, r *http.Request) {
 	}
 
 	username, groups := userCreds(r)
-	releases, err := client.ListReleasesAcrossNamespaces(namespaces, username, groups)
+	releases, err := client.ListReleasesAcrossNamespaces(r.Context(), namespaces, username, groups)
 	if err != nil {
+		if errors.Is(r.Context().Err(), context.Canceled) {
+			return // the caller left; this is not a Helm failure
+		}
 		if IsForbiddenError(err) {
 			writeError(w, http.StatusForbidden, "insufficient permissions to list Helm releases")
 			return
@@ -492,8 +496,11 @@ func (h *Handlers) handleBatchUpgradeCheck(w http.ResponseWriter, r *http.Reques
 	}
 
 	username, groups := userCreds(r)
-	info, err := client.BatchCheckUpgradesAcrossNamespaces(namespaces, username, groups)
+	info, err := client.BatchCheckUpgradesAcrossNamespaces(r.Context(), namespaces, username, groups)
 	if err != nil {
+		if errors.Is(r.Context().Err(), context.Canceled) {
+			return // the caller left; this is not a Helm failure
+		}
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
 	}

internal/helm/handlers_cancel_test.go

@@ -0,0 +1,86 @@
+package helm
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/skyhook-io/radar/internal/errorlog"
+	"github.com/skyhook-io/radar/internal/k8s"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// A caller that leaves while Helm storage is still being read is not a Helm
+// error. Recording it as one fills the diagnostics error log with failures
+// that never happened, and a 5xx Helm error is also what can mark the
+// cluster disconnected. A read that outlasts the server's own request
+// deadline is a real failure and is still recorded.
+func TestHelmListAndUpgradeCheckRecordOnlyRealFailures(t *testing.T) {
+	release := make(chan struct{})
+	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if !strings.HasSuffix(r.URL.Path, "/secrets") {
+			http.NotFound(w, r)
+			return
+		}
+		select {
+		case <-r.Context().Done():
+		case <-release:
+			_, _ = io.WriteString(w, `{"apiVersion":"v1","kind":"SecretList","items":[]}`)
+		}
+	}))
+	t.Cleanup(storage.Close)
+	t.Cleanup(func() { close(release) })
+	config := &rest.Config{Host: storage.URL}
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	previousClient := k8s.SetTestClient(client)
+	// A 5xx Helm error that reads as a connectivity failure can mark the
+	// cluster disconnected; keep that out of later tests.
+	previousStatus := k8s.GetConnectionStatus()
+	t.Cleanup(func() { k8s.SetConnectionStatus(previousStatus) })
+	ResetClient()
+	if err := InitializeWithRESTConfig(config); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		ResetClient()
+		k8s.SetTestClient(previousClient)
+	})
+
+	h := NewHandlers(func(*http.Request) ([]string, bool) { return []string{"shop"}, true })
+	for name, handle := range map[string]http.HandlerFunc{
+		"list":          h.handleListReleases,
+		"upgrade check": h.handleBatchUpgradeCheck,
+	} {
+		t.Run(name+", caller leaves", func(t *testing.T) {
+			errorlog.Reset()
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			time.AfterFunc(200*time.Millisecond, cancel)
+			request := httptest.NewRequest(http.MethodGet, "/api/helm/releases?namespace=shop", nil).WithContext(ctx)
+			handle(httptest.NewRecorder(), request)
+			if entries := errorlog.GetEntries(); len(entries) > 0 {
+				t.Errorf("a caller leaving was recorded as a Helm error: %+v", entries)
+			}
+		})
+		t.Run(name+", server deadline", func(t *testing.T) {
+			errorlog.Reset()
+			ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+			defer cancel()
+			request := httptest.NewRequest(http.MethodGet, "/api/helm/releases?namespace=shop", nil).WithContext(ctx)
+			recorder := httptest.NewRecorder()
+			handle(recorder, request)
+			if recorder.Code < 500 || len(errorlog.GetEntries()) == 0 {
+				t.Errorf("a read past the deadline answered %d and recorded %d errors; want a 5xx and a record", recorder.Code, len(errorlog.GetEntries()))
+			}
+		})
+	}
+}