[kube-controller] create api service proxy

Author: theRealWardo

cmd/kube-controller/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+"flag"
+"os"
+
+corev1 "k8s.io/api/core/v1"
+discoveryv1 "k8s.io/api/discovery/v1"
+runtime "k8s.io/apimachinery/pkg/runtime"
+utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+"k8s.io/client-go/kubernetes"
+"k8s.io/client-go/rest"
+ctrl "sigs.k8s.io/controller-runtime"
+gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+corev1alpha "github.com/apoxy-dev/apoxy/api/core/v1alpha"
+"github.com/apoxy-dev/apoxy/pkg/kube-controller/apiregistration"
+"github.com/apoxy-dev/apoxy/pkg/kube-controller/apiserviceproxy"
+"github.com/apoxy-dev/apoxy/pkg/log"
+)
+
+var (
+projectID = flag.String("project_id", "", "Project ID.")
+
+// One of these must be set.
+bootstrapToken = flag.String("bootstrap_token", os.Getenv("APOXY_BOOTSTRAP_TOKEN"), "Token to bootstrap for Apoxy Cloud.")
+kubeconfigPath = flag.String("kubeconfig_path", "", "Path to kubeconfig file for a cluster to bootstrap against.")
+
+clusterName = flag.String("cluster_name", "", "Name of the cluster (can be used as Location in Proxy and other objects' specs).")
+namespace = flag.String("namespace", os.Getenv("POD_NAMESPACE"), "Namespace to watch for Proxy resources.")
+svcName = flag.String("service_name", "kube-controller", "Name of the service to register.")
+
+devMode = flag.Bool("dev", false, "Enable development mode.")
+logLevel = flag.String("log_level", "info", "Log level.")
+)
+
+var (
+scheme = runtime.NewScheme()
+)
+
+func init() {
+utilruntime.Must(corev1.AddToScheme(scheme))
+utilruntime.Must(discoveryv1.AddToScheme(scheme))
+utilruntime.Must(gwapiv1.AddToScheme(scheme))
+utilruntime.Must(corev1alpha.AddToScheme(scheme))
+}
+
+func main() {
+ctx := ctrl.SetupSignalHandler()
+
+flag.Parse()
+var lOpts []log.Option
+if *devMode {
+lOpts = append(lOpts, log.WithDevMode(), log.WithAlsoLogToStderr())
+} else if *logLevel != "" {
+lOpts = append(lOpts, log.WithLevelString(*logLevel))
+}
+log.Init(lOpts...)
+
+if *projectID == "" {
+log.Fatalf("--project_id must be set")
+}
+if *namespace == "" {
+log.Fatalf("--namespace must be set")
+}
+if *bootstrapToken == "" && *kubeconfigPath == "" {
+log.Fatalf("one of --bootstrap_token or --kubeconfig_path must be set")
+}
+
+log.Infof("Starting controllers...")
+
+kCluster, err := rest.InClusterConfig()
+if err != nil {
+log.Fatalf("failed to create in-cluster config: %v", err)
+}
+kc := kubernetes.NewForConfigOrDie(kCluster)
+
+apiReg, err := apiregistration.NewAPIRegistration(kCluster)
+if err != nil {
+log.Fatalf("failed to create api registration client: %v", err)
+}
+if err := apiReg.RegisterAPIServices(ctx, *svcName, *namespace, apiserviceproxy.DefaultPort); err != nil {
+log.Fatalf("failed to register api services: %v", err)
+}
+
+log.Infof("starting api service proxy")
+var proxyOpts []apiserviceproxy.Option
+if *projectID != "" {
+proxyOpts = append(proxyOpts, apiserviceproxy.WithProjectID(*projectID))
+}
+if *namespace != "" {
+proxyOpts = append(proxyOpts, apiserviceproxy.WithNamespace(*namespace))
+}
+if *clusterName != "" {
+proxyOpts = append(proxyOpts, apiserviceproxy.WithClusterName(*clusterName))
+}
+if *bootstrapToken != "" {
+proxyOpts = append(proxyOpts, apiserviceproxy.WithToken(*bootstrapToken))
+}
+if *kubeconfigPath != "" {
+proxyOpts = append(proxyOpts, apiserviceproxy.WithKubeconfigPath(*kubeconfigPath))
+}
+apiSvc, err := apiserviceproxy.NewAPIServiceProxy(ctx, kc, proxyOpts...)
+if err != nil {
+log.Fatalf("unable to create api service proxy: %v", err)
+}
+go func() {
+if err := apiSvc.Run(ctx); err != nil {
+log.Fatalf("unable to run api service proxy: %v", err)
+}
+}()
+
+<-ctx.Done()
+log.Infof("controllers shutting down")
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/apoxy-dev/apoxy
 
-go 1.24
+go 1.24.0
 
 toolchain go1.24.2
 
@@ -85,10 +85,10 @@ require (
 gopkg.in/yaml.v2 v2.4.0
 gopkg.in/yaml.v3 v3.0.1
 gvisor.dev/gvisor v0.0.0-20250314001526-eeca54973f8b
-k8s.io/api v0.31.1
-k8s.io/apimachinery v0.31.1
-k8s.io/apiserver v0.31.1
-k8s.io/client-go v0.31.1
+k8s.io/api v0.33.2
+k8s.io/apimachinery v0.33.2
+k8s.io/apiserver v0.33.2
+k8s.io/client-go v0.33.2
 k8s.io/klog/v2 v2.130.1
 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff
 k8s.io/kubernetes v1.31.1
@@ -191,7 +191,7 @@ require (
 github.com/gocql/gocql v1.6.0 // indirect
 github.com/godbus/dbus/v5 v5.1.0 // indirect
 github.com/gogo/protobuf v1.3.2 // indirect
-github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
+github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 github.com/golang/mock v1.7.0-rc.1 // indirect
 github.com/golang/snappy v1.0.0 // indirect
@@ -207,7 +207,7 @@ require (
 github.com/googleapis/gax-go/v2 v2.14.0 // indirect
 github.com/gophercloud/gophercloud v0.1.0 // indirect
 github.com/gorilla/securecookie v1.1.1 // indirect
-github.com/gorilla/websocket v1.5.3 // indirect
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect
@@ -392,9 +392,10 @@ require (
 gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 gopkg.in/validator.v2 v2.0.1 // indirect
 k8s.io/apiextensions-apiserver v0.31.1 // indirect
-k8s.io/component-base v0.31.1 // indirect
+k8s.io/component-base v0.33.2 // indirect
 k8s.io/component-helpers v0.31.1 // indirect
-k8s.io/kms v0.32.3 // indirect
+k8s.io/kms v0.33.2 // indirect
+k8s.io/kube-aggregator v0.33.2 // indirect
 lukechampine.com/uint128 v1.3.0 // indirect
 modernc.org/cc/v3 v3.41.0 // indirect
 modernc.org/ccgo/v3 v3.17.0 // indirect
@@ -405,7 +406,7 @@ require (
 modernc.org/sqlite v1.23.1 // indirect
 modernc.org/strutil v1.2.0 // indirect
 modernc.org/token v1.1.0 // indirect
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
 sigs.k8s.io/randfill v1.0.0 // indirect
 sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect

go.sum

@@ -472,6 +472,8 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
 github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4=
@@ -577,6 +579,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -1588,6 +1592,8 @@ k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78=
 k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4=
 k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU=
 k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI=
+k8s.io/api v0.33.2 h1:YgwIS5jKfA+BZg//OQhkJNIfie/kmRsO0BmNaVSimvY=
+k8s.io/api v0.33.2/go.mod h1:fhrbphQJSM2cXzCWgqU29xLDuks4mu7ti9vveEnpSXs=
 k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY=
 k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio=
 k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40=
@@ -1596,20 +1602,28 @@ k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftc
 k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
 k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
 k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY=
+k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
 k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw=
 k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8=
 k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c=
 k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM=
+k8s.io/apiserver v0.33.2 h1:KGTRbxn2wJagJowo29kKBp4TchpO1DRO3g+dB/KOJN4=
+k8s.io/apiserver v0.33.2/go.mod h1:9qday04wEAMLPWWo9AwqCZSiIn3OYSZacDyu/AcoM/M=
 k8s.io/client-go v0.18.2/go.mod h1:Xcm5wVGXX9HAA2JJ2sSBUn3tCJ+4SVlCbl2MNNv+CIU=
 k8s.io/client-go v0.18.4/go.mod h1:f5sXwL4yAZRkAtzOxRWUhA/N8XzGCb+nPZI8PfobZ9g=
 k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
 k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
+k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E=
+k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo=
 k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc=
 k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
 k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM=
 k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk=
 k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=
 k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w=
+k8s.io/component-base v0.33.2 h1:sCCsn9s/dG3ZrQTX/Us0/Sx2R0G5kwa0wbZFYoVp/+0=
+k8s.io/component-base v0.33.2/go.mod h1:/41uw9wKzuelhN+u+/C59ixxf4tYQKW7p32ddkYNe2k=
 k8s.io/component-helpers v0.31.1 h1:5hZUf3747atdgtR3gPntrG35rC2CkK7rYq2KUraz6Os=
 k8s.io/component-helpers v0.31.1/go.mod h1:ye0Gi8KzFNTfpIuzvVDtxJQMP/0Owkukf1vGf22Hl6U=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -1622,6 +1636,10 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kms v0.32.3 h1:HhHw5+pRCzEJp3oFFJ1q5W2N6gAI7YkUg4ay4Z0dgwM=
 k8s.io/kms v0.32.3/go.mod h1:Bk2evz/Yvk0oVrvm4MvZbgq8BD34Ksxs2SRHn4/UiOM=
+k8s.io/kms v0.33.2 h1:GFwNXX4CZGQCg9DPOaJi1/+iKidCtB9/OIAGdzRo8FI=
+k8s.io/kms v0.33.2/go.mod h1:C1I8mjFFBNzfUZXYt9FZVJ8MJl7ynFbGgZFbBzkBJ3E=
+k8s.io/kube-aggregator v0.33.2 h1:eMW63PNucP+3UxnwYcfn5Yt2w2Sj2jI+imA7UWkYHVc=
+k8s.io/kube-aggregator v0.33.2/go.mod h1:qQbliLwcdmx7/8mtvkc/9QV/ON2M6ZBMcffEUmrqKFw=
 k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
 k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
@@ -1676,6 +1694,8 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A=
 sigs.k8s.io/controller-runtime v0.19.7 h1:DLABZfMr20A+AwCZOHhcbcu+TqBXnJZaVBri9K3EO48=
 sigs.k8s.io/controller-runtime v0.19.7/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=

pkg/kube-controller/apiregistration/apiregistration.go

@@ -0,0 +1,54 @@
+// Package apiregistration handles Apoxy API service registration with Kubernetes API Aggregation
+package apiregistration
+
+import (
+"context"
+"fmt"
+
+apierrors "k8s.io/apimachinery/pkg/api/errors"
+metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+"k8s.io/client-go/rest"
+apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+"github.com/apoxy-dev/apoxy/pkg/log"
+)
+
+// APIRegistration handles registration of Apoxy APIs with Kubernetes API Aggregation
+type APIRegistration struct {
+apiRegC apiregistrationclient.Interface
+}
+
+// NewAPIRegistration creates a new APIRegistration with the given client-go config
+func NewAPIRegistration(config *rest.Config) (*APIRegistration, error) {
+apiRegC, err := apiregistrationclient.NewForConfig(config)
+if err != nil {
+return nil, fmt.Errorf("unable to create api registration client: %w", err)
+}
+
+return &APIRegistration{
+apiRegC: apiRegC,
+}, nil
+}
+
+// RegisterAPIServices registers all Apoxy API services with Kubernetes API Aggregation
+func (a *APIRegistration) RegisterAPIServices(ctx context.Context, serviceName, namespace string, port int) error {
+log.Infof("setting up API Aggregation")
+for _, apiSvcDef := range AllAPIServices {
+svc := apiSvcDef.ToAPIService(serviceName, namespace, port)
+if _, err := a.apiRegC.ApiregistrationV1().APIServices().Create(ctx, svc, metav1.CreateOptions{}); err != nil {
+if !apierrors.IsAlreadyExists(err) {
+return err
+}
+curSvc, err := a.apiRegC.ApiregistrationV1().APIServices().Get(ctx, svc.Name, metav1.GetOptions{})
+if err != nil {
+return fmt.Errorf("failed to get existing APIService %s: %w", svc.Name, err)
+}
+svc.ResourceVersion = curSvc.ResourceVersion
+if _, err := a.apiRegC.ApiregistrationV1().APIServices().Update(ctx, svc, metav1.UpdateOptions{}); err != nil {
+return fmt.Errorf("failed to update APIService %s: %w", svc.Name, err)
+}
+}
+log.Infof("Registered API service: %s", svc.Name)
+}
+return nil
+}