Merge pull request #123642 from aramase/aramase/f/kep_3331_jwks_metrics

Add JWKS fetch metrics for jwt authenticator Kubernetes-commit: b4d4cc93840fa30a305b013acd1b1060ed3f8ee2

Author: k8s-publishing-bot

go.mod

@@ -48,7 +48,7 @@ require (
 gopkg.in/evanphx/json-patch.v4 v4.13.0
 gopkg.in/go-jose/go-jose.v2 v2.6.3
 gopkg.in/natefinch/lumberjack.v2 v2.2.1
-k8s.io/api v0.0.0-20251030032104-67e96f7d5671
+k8s.io/api v0.0.0-20251031002201-cc5ff10557fc
 k8s.io/apimachinery v0.0.0-20251029233601-5a85a549ac04
 k8s.io/client-go v0.0.0-20251030152624-63b5f5942509
 k8s.io/component-base v0.0.0-20251030233029-3a1cbfb75a20

go.sum

@@ -295,8 +295,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.0.0-20251030032104-67e96f7d5671 h1:144MaWZ/WJxhNAP2D+m1n80DpqGDVvC+i3n2a+NIubQ=
-k8s.io/api v0.0.0-20251030032104-67e96f7d5671/go.mod h1:t9ALtSJ/Lz32mvweSmEQ4A7Ixha3Bt7kzKKL/gNqQYI=
+k8s.io/api v0.0.0-20251031002201-cc5ff10557fc h1:c+B4s9hnmRzTTvUejC44aiqhPfuJCszorD8IN37j3W0=
+k8s.io/api v0.0.0-20251031002201-cc5ff10557fc/go.mod h1:t9ALtSJ/Lz32mvweSmEQ4A7Ixha3Bt7kzKKL/gNqQYI=
 k8s.io/apimachinery v0.0.0-20251029233601-5a85a549ac04 h1:cT+E5ifXtFO6HabSviCJKBWqZyRqjBesgYc95Bg1UuQ=
 k8s.io/apimachinery v0.0.0-20251029233601-5a85a549ac04/go.mod h1:khYq6ZZ3qxhyKXYGU64a438RVSfpfZZ4Xept0x/H3Qw=
 k8s.io/client-go v0.0.0-20251030152624-63b5f5942509 h1:K1IfKcFkYj6EP00JJJ6mW64i2iiJliaFBpQreDwY024=

pkg/features/kube_features.go

@@ -245,6 +245,12 @@ const (
 // Enables Egress Selector in Structured Authentication Configuration
 StructuredAuthenticationConfigurationEgressSelector featuregate.Feature = "StructuredAuthenticationConfigurationEgressSelector"
 
+// owner: @aramase, @enj, @nabokihms
+// kep: https://kep.k8s.io/3331
+//
+// Enables JWKs metrics for Structured Authentication Configuration
+StructuredAuthenticationConfigurationJWKSMetrics featuregate.Feature = "StructuredAuthenticationConfigurationJWKSMetrics"
+
 // owner: @palnabarun
 // kep: https://kep.k8s.io/3221
 //
@@ -472,6 +478,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta},
 },
 
+StructuredAuthenticationConfigurationJWKSMetrics: {
+{Version: version.MustParse("1.35"), Default: true, PreRelease: featuregate.Beta},
+},
+
 StructuredAuthorizationConfiguration: {
 {Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
 {Version: version.MustParse("1.30"), Default: true, PreRelease: featuregate.Beta},

plugin/pkg/authenticator/token/oidc/metrics.go

@@ -47,20 +47,160 @@ var (
 },
 []string{"result", "jwt_issuer_hash"},
 )
+
+jwksFetchLastTimestampSeconds = metrics.NewGaugeVec(
+&metrics.GaugeOpts{
+Namespace: namespace,
+Subsystem: subsystem,
+Name: "jwt_authenticator_jwks_fetch_last_timestamp_seconds",
+Help: "Timestamp of the last successful or failed JWKS fetch split by result, api server identity " +
+"and jwt issuer for the JWT authenticator.",
+StabilityLevel: metrics.ALPHA,
+},
+[]string{"result", "jwt_issuer_hash", "apiserver_id_hash"},
+)
+
+jwksFetchLastKeySetInfo = metrics.NewDesc(
+metrics.BuildFQName(namespace, subsystem, "jwt_authenticator_jwks_fetch_last_key_set_info"),
+"Information about the last JWKS fetched by the JWT authenticator with hash as label, split by api server identity and jwt issuer.",
+[]string{"jwt_issuer_hash", "apiserver_id_hash", "hash"},
+nil,
+metrics.ALPHA,
+"",
+)
 )
 
+// jwksHashKey uniquely identifies a JWKS by issuer and API server ID
+type jwksHashKey struct {
+jwtIssuerHash string
+apiServerIDHash string
+}
+
+// jwksHashProvider manages JWKS hashes for all authenticators
+type jwksHashProvider struct {
+hashes sync.Map // map[jwksHashKey]string
+}
+
+func newJWKSHashProvider() *jwksHashProvider {
+return &jwksHashProvider{}
+}
+
+func (p *jwksHashProvider) setHash(jwtIssuer, apiServerID, keySet string) {
+key := jwksHashKey{
+jwtIssuerHash: getHash(jwtIssuer),
+apiServerIDHash: getHash(apiServerID),
+}
+jwksHash := getHash(keySet)
+p.hashes.Store(key, jwksHash)
+}
+
+func (p *jwksHashProvider) getHashes() map[jwksHashKey]string {
+result := make(map[jwksHashKey]string)
+p.hashes.Range(func(k, v interface{}) bool {
+result[k.(jwksHashKey)] = v.(string)
+return true
+})
+return result
+}
+
+func (p *jwksHashProvider) reset() {
+p.hashes.Range(func(k, v interface{}) bool {
+p.hashes.Delete(k)
+return true
+})
+}
+
+func (p *jwksHashProvider) deleteHash(jwtIssuer, apiServerID string) {
+key := jwksHashKey{
+jwtIssuerHash: getHash(jwtIssuer),
+apiServerIDHash: getHash(apiServerID),
+}
+p.hashes.Delete(key)
+}
+
+// jwksHashCollector is a custom collector that emits JWKS hash metrics
+type jwksHashCollector struct {
+metrics.BaseStableCollector
+desc *metrics.Desc
+hashProvider *jwksHashProvider
+}
+
+func newJWKSHashCollector(desc *metrics.Desc, hashProvider *jwksHashProvider) metrics.StableCollector {
+return &jwksHashCollector{
+desc: desc,
+hashProvider: hashProvider,
+}
+}
+
+func (c *jwksHashCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
+ch <- c.desc
+}
+
+func (c *jwksHashCollector) CollectWithStability(ch chan<- metrics.Metric) {
+hashes := c.hashProvider.getHashes()
+for key, hash := range hashes {
+ch <- metrics.NewLazyConstMetric(
+c.desc,
+metrics.GaugeValue,
+1,
+key.jwtIssuerHash,
+key.apiServerIDHash,
+hash,
+)
+}
+}
+
 var registerMetrics sync.Once
+var hashProvider = newJWKSHashProvider()
 
 func RegisterMetrics() {
 registerMetrics.Do(func() {
 legacyregistry.MustRegister(jwtAuthenticatorLatencyMetric)
+legacyregistry.MustRegister(jwksFetchLastTimestampSeconds)
+legacyregistry.CustomMustRegister(newJWKSHashCollector(jwksFetchLastKeySetInfo, hashProvider))
 })
 }
 
 func recordAuthenticationLatency(result, jwtIssuerHash string, duration time.Duration) {
 jwtAuthenticatorLatencyMetric.WithLabelValues(result, jwtIssuerHash).Observe(duration.Seconds())
 }
 
+func recordJWKSFetchTimestamp(result, jwtIssuer, apiServerID string) {
+jwksFetchLastTimestampSeconds.WithLabelValues(result, getHash(jwtIssuer), getHash(apiServerID)).SetToCurrentTime()
+}
+
+func recordJWKSFetchKeySetSuccess(jwtIssuer, apiServerID, keySet string) {
+recordJWKSFetchKeySetHash(jwtIssuer, apiServerID, keySet)
+recordJWKSFetchTimestamp("success", jwtIssuer, apiServerID)
+}
+
+func recordJWKSFetchKeySetFailure(jwtIssuer, apiServerID string) {
+recordJWKSFetchTimestamp("failure", jwtIssuer, apiServerID)
+}
+
+func recordJWKSFetchKeySetHash(jwtIssuer, apiServerID, keySet string) {
+hashProvider.setHash(jwtIssuer, apiServerID, keySet)
+}
+
+// DeleteJWKSFetchMetrics deletes all JWKS-related metrics for a specific issuer and API server.
+// This includes the hash metric and timestamp metrics (both success and failure).
+// This should be called when an issuer is removed from the configuration to clean up stale metrics.
+func DeleteJWKSFetchMetrics(jwtIssuer, apiServerID string) {
+jwtIssuerHash := getHash(jwtIssuer)
+apiServerIDHash := getHash(apiServerID)
+
+hashProvider.deleteHash(jwtIssuer, apiServerID)
+
+jwksFetchLastTimestampSeconds.DeleteLabelValues("success", jwtIssuerHash, apiServerIDHash)
+jwksFetchLastTimestampSeconds.DeleteLabelValues("failure", jwtIssuerHash, apiServerIDHash)
+}
+
+func ResetMetrics() {
+jwtAuthenticatorLatencyMetric.Reset()
+jwksFetchLastTimestampSeconds.Reset()
+hashProvider.reset()
+}
+
 func getHash(data string) string {
 if len(data) > 0 {
 return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
@@ -73,7 +213,6 @@ func newInstrumentedAuthenticator(jwtIssuer string, delegate AuthenticatorTokenW
 }
 
 func newInstrumentedAuthenticatorWithClock(jwtIssuer string, delegate AuthenticatorTokenWithHealthCheck, clock clock.PassiveClock) *instrumentedAuthenticator {
-RegisterMetrics()
 return &instrumentedAuthenticator{
 jwtIssuerHash: getHash(jwtIssuer),
 delegate: delegate,

plugin/pkg/authenticator/token/oidc/metrics_test.go

@@ -29,7 +29,9 @@ import (
 )
 
 const (
-testIssuer = "testIssuer"
+testIssuer = "testIssuer"
+testAPIServerID = "testAPIServerID"
+testKeySet = `{"keys":[{"kty":"RSA","use":"sig","kid":"test"}]}`
 )
 
 func TestRecordAuthenticationLatency(t *testing.T) {
@@ -131,3 +133,125 @@ func (d dummyClock) Now() time.Time {
 func (d dummyClock) Since(t time.Time) time.Duration {
 return time.Duration(1)
 }
+
+func TestRecordJWKSFetchKeySetSuccess(t *testing.T) {
+expectedValue := `
+# HELP apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info [ALPHA] Information about the last JWKS fetched by the JWT authenticator with hash as label, split by api server identity and jwt issuer.
+# TYPE apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info gauge
+apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:d132d414ef2da3d863abd7bf0165c00403ef1d3510faf8fdf1d7cf335c888e53",jwt_issuer_hash="sha256:29b34beedc55b972f2428f21bc588f9d38e5e8f7a7af825486e7bb4fd9caa2ad"} 1
+`
+
+metrics := []string{
+namespace + "_" + subsystem + "_jwt_authenticator_jwks_fetch_last_key_set_info",
+}
+
+ResetMetrics()
+RegisterMetrics()
+
+recordJWKSFetchKeySetSuccess(testIssuer, testAPIServerID, testKeySet)
+
+if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+t.Fatal(err)
+}
+}
+
+func TestRecordJWKSFetchKeySetFailure(t *testing.T) {
+ResetMetrics()
+RegisterMetrics()
+
+recordJWKSFetchKeySetFailure(testIssuer, testAPIServerID)
+
+metrics, err := legacyregistry.DefaultGatherer.Gather()
+if err != nil {
+t.Fatal(err)
+}
+
+found := false
+for _, m := range metrics {
+if m.GetName() == namespace+"_"+subsystem+"_jwt_authenticator_jwks_fetch_last_timestamp_seconds" {
+found = true
+break
+}
+}
+if !found {
+t.Fatal("Expected jwt_authenticator_jwks_fetch_last_timestamp_seconds metric to be present")
+}
+}
+
+func TestJWKSHashCollector_MultipleAuthenticators(t *testing.T) {
+expectedValue := `
+# HELP apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info [ALPHA] Information about the last JWKS fetched by the JWT authenticator with hash as label, split by api server identity and jwt issuer.
+# TYPE apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info gauge
+apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:d132d414ef2da3d863abd7bf0165c00403ef1d3510faf8fdf1d7cf335c888e53",jwt_issuer_hash="sha256:29b34beedc55b972f2428f21bc588f9d38e5e8f7a7af825486e7bb4fd9caa2ad"} 1
+apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:1b5293c65ffc96e13f2d6fefae782190aec8cfb89957a3109419d4f47b80e3e8",jwt_issuer_hash="sha256:f10ab1bafaa1a8628d0fae41ee554948912b01957e4a2db1698fc1c3e4451682"} 1
+`
+
+metrics := []string{
+namespace + "_" + subsystem + "_jwt_authenticator_jwks_fetch_last_key_set_info",
+}
+
+ResetMetrics()
+RegisterMetrics()
+
+recordJWKSFetchKeySetSuccess(testIssuer, testAPIServerID, testKeySet)
+
+secondIssuer := "https://another-issuer.example.com"
+secondKeySet := `{"keys":[{"kty":"EC","use":"sig","kid":"test2"}]}`
+recordJWKSFetchKeySetSuccess(secondIssuer, testAPIServerID, secondKeySet)
+
+if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+t.Fatal(err)
+}
+}
+
+func TestJWKSHashCollector_UpdateExistingHash(t *testing.T) {
+expectedValue := `
+# HELP apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info [ALPHA] Information about the last JWKS fetched by the JWT authenticator with hash as label, split by api server identity and jwt issuer.
+# TYPE apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info gauge
+apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:1b5293c65ffc96e13f2d6fefae782190aec8cfb89957a3109419d4f47b80e3e8",jwt_issuer_hash="sha256:29b34beedc55b972f2428f21bc588f9d38e5e8f7a7af825486e7bb4fd9caa2ad"} 1
+`
+
+metrics := []string{
+namespace + "_" + subsystem + "_jwt_authenticator_jwks_fetch_last_key_set_info",
+}
+
+ResetMetrics()
+RegisterMetrics()
+
+recordJWKSFetchKeySetSuccess(testIssuer, testAPIServerID, testKeySet)
+
+// Update with new JWKS - should replace old hash with new one
+updatedKeySet := `{"keys":[{"kty":"EC","use":"sig","kid":"test2"}]}`
+recordJWKSFetchKeySetSuccess(testIssuer, testAPIServerID, updatedKeySet)
+
+if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+t.Fatal(err)
+}
+}
+
+func TestJWKSHashCollector_DeleteHash(t *testing.T) {
+ResetMetrics()
+RegisterMetrics()
+
+recordJWKSFetchKeySetSuccess(testIssuer, testAPIServerID, testKeySet)
+secondIssuer := "https://another-issuer.example.com"
+secondKeySet := `{"keys":[{"kty":"EC","use":"sig","kid":"test2"}]}`
+recordJWKSFetchKeySetSuccess(secondIssuer, testAPIServerID, secondKeySet)
+
+// Delete first authenticator's metrics and verify only second authenticator's hash remains
+DeleteJWKSFetchMetrics(testIssuer, testAPIServerID)
+
+expectedValue := `
+# HELP apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info [ALPHA] Information about the last JWKS fetched by the JWT authenticator with hash as label, split by api server identity and jwt issuer.
+# TYPE apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info gauge
+apiserver_authentication_jwt_authenticator_jwks_fetch_last_key_set_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:1b5293c65ffc96e13f2d6fefae782190aec8cfb89957a3109419d4f47b80e3e8",jwt_issuer_hash="sha256:f10ab1bafaa1a8628d0fae41ee554948912b01957e4a2db1698fc1c3e4451682"} 1
+`
+
+metrics := []string{
+namespace + "_" + subsystem + "_jwt_authenticator_jwks_fetch_last_key_set_info",
+}
+
+if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
+t.Fatal(err)
+}
+}