Upgrade the operator to use Operator SDK v1.33.0 (#182)

* Move controller package inside internal directory

Based on the go/v4 project structure, the following changed:
- Pakcage `controllers` is now named `controller`
- Package `controller` now lives inside new `internal` directory

* Move main.go in cmd directory

Based on the new go/v4 project structure, `main.go` now lives in the `cmd` directory.

* Change package import in main.go

* Update go mod dependencies

Update the dependencies based on the versions obtained by creating a new operator project using `kubebuilder init --domain onepassword.com --plugins=go/v4`.

This is based on the migration steps provided to go from go/v3 to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update vendor

* Adjust code for breaking changes from pkg update

sigs.k8s.io/controller-runtime package had breaking changes from v0.14.5 to v0.16.3. This commit brings the changes needed to achieve the same things using the new functionality avaialble.

* Adjust paths to connect yaml files

Since `main.go` is now in `cmd` directory, the paths to the files for deploying Connect have to be adjusted based on the new location `main.go` is executed from.

* Update files based on new structure and scaffolding

These changes are made based on the new project structure and scaffolding obtained when using the new go/v4 project structure.

These were done based on the migration steps mentioned when migrating to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4).

* Update config files

These updates are made based on the Kustomize v4 syntax.

This is part of the upgrate to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update dependencies and GO version

* Update vendor

* Update Kubernetes tools versions

* Update operator version in Makefile

Now the version in the Makefile matches the version of the operator

* Update Operator SDK version in version.go

* Adjust generated deepcopy

It seems that the +build tag is no longer needed based on the latest generated scaffolding, therefore it's removed.

* Update copyright year

* Bring back missing changes from migration

Some customization in Makefile was lost during the migration process. Specifically, the namespace customization for `make deploy` command.

Also, we push changes to kustomization.yaml for making the deploy process smoother.

* Add RBAC perms for coordination.k8s.io

It seems that with the latest changes to Kubernetes and Kustomize, we need to add additional RBAC to the service account used so that it can properly access the `leases` resource.

* Optimize Dockerfile

Dockerfile had a step for caching dependencies (go mod download). However, this is already done by the vendor directory, which we include. Therefore, this step can be removed to make the image build time faster.
This commit is contained in:
Eduard Filip
2024-01-25 14:21:31 +01:00
committed by GitHub
parent 8fc852a4dd
commit f72e5243b0
1356 changed files with 86780 additions and 43671 deletions

View File

@@ -19,7 +19,6 @@ package admission
import (
"fmt"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/json"
@@ -32,8 +31,11 @@ type Decoder struct {
}
// NewDecoder creates a Decoder given the runtime.Scheme.
func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) {
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil
func NewDecoder(scheme *runtime.Scheme) *Decoder {
if scheme == nil {
panic("scheme should never be nil")
}
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}
}
// Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object.
@@ -62,9 +64,14 @@ func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) er
if len(rawObj.Raw) == 0 {
return fmt.Errorf("there is no content to decode")
}
if unstructuredInto, isUnstructured := into.(*unstructured.Unstructured); isUnstructured {
if unstructuredInto, isUnstructured := into.(runtime.Unstructured); isUnstructured {
// unmarshal into unstructured's underlying object to avoid calling the decoder
return json.Unmarshal(rawObj.Raw, &unstructuredInto.Object)
var object map[string]interface{}
if err := json.Unmarshal(rawObj.Raw, &object); err != nil {
return err
}
unstructuredInto.SetUnstructuredContent(object)
return nil
}
deserializer := d.codecs.UniversalDeserializer()

View File

@@ -33,9 +33,9 @@ type Defaulter interface {
}
// DefaultingWebhookFor creates a new Webhook for Defaulting the provided type.
func DefaultingWebhookFor(defaulter Defaulter) *Webhook {
func DefaultingWebhookFor(scheme *runtime.Scheme, defaulter Defaulter) *Webhook {
return &Webhook{
Handler: &mutatingHandler{defaulter: defaulter},
Handler: &mutatingHandler{defaulter: defaulter, decoder: NewDecoder(scheme)},
}
}
@@ -44,16 +44,11 @@ type mutatingHandler struct {
decoder *Decoder
}
var _ DecoderInjector = &mutatingHandler{}
// InjectDecoder injects the decoder into a mutatingHandler.
func (h *mutatingHandler) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *mutatingHandler) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.defaulter == nil {
panic("defaulter should never be nil")
}

View File

@@ -34,9 +34,9 @@ type CustomDefaulter interface {
}
// WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface.
func WithCustomDefaulter(obj runtime.Object, defaulter CustomDefaulter) *Webhook {
func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter) *Webhook {
return &Webhook{
Handler: &defaulterForType{object: obj, defaulter: defaulter},
Handler: &defaulterForType{object: obj, defaulter: defaulter, decoder: NewDecoder(scheme)},
}
}
@@ -46,15 +46,11 @@ type defaulterForType struct {
decoder *Decoder
}
var _ DecoderInjector = &defaulterForType{}
func (h *defaulterForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *defaulterForType) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.defaulter == nil {
panic("defaulter should never be nil")
}

View File

@@ -20,9 +20,3 @@ Package admission provides implementation for admission webhook and methods to i
See examples/mutatingwebhook.go and examples/validatingwebhook.go for examples of admission webhooks.
*/
package admission
import (
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("admission")

View File

@@ -52,7 +52,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var reviewResponse Response
if r.Body == nil {
err = errors.New("request body is empty")
wh.log.Error(err, "bad request")
wh.getLogger(nil).Error(err, "bad request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -60,7 +60,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if body, err = io.ReadAll(r.Body); err != nil {
wh.log.Error(err, "unable to read the body from the incoming request")
wh.getLogger(nil).Error(err, "unable to read the body from the incoming request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -69,7 +69,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// verify the content type is accurate
if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
err = fmt.Errorf("contentType=%s, expected application/json", contentType)
wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType)
wh.getLogger(nil).Error(err, "unable to process a request with unknown content type")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -88,12 +88,12 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ar.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind("AdmissionReview"))
_, actualAdmRevGVK, err := admissionCodecs.UniversalDeserializer().Decode(body, nil, &ar)
if err != nil {
wh.log.Error(err, "unable to decode the request")
wh.getLogger(nil).Error(err, "unable to decode the request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
}
wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind, "resource", req.Resource)
wh.getLogger(&req).V(5).Info("received request")
reviewResponse = wh.Handle(ctx, req)
wh.writeResponseTyped(w, reviewResponse, actualAdmRevGVK)
@@ -124,7 +124,7 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, admRevGVK
// writeAdmissionResponse writes ar to w.
func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) {
if err := json.NewEncoder(w).Encode(ar); err != nil {
wh.log.Error(err, "unable to encode and write the response")
wh.getLogger(nil).Error(err, "unable to encode and write the response")
// Since the `ar v1.AdmissionReview` is a clear and legal object,
// it should not have problem to be marshalled into bytes.
// The error here is probably caused by the abnormal HTTP connection,
@@ -132,15 +132,15 @@ func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) {
// to avoid endless circular calling.
serverError := Errored(http.StatusInternalServerError, err)
if err = json.NewEncoder(w).Encode(v1.AdmissionReview{Response: &serverError.AdmissionResponse}); err != nil {
wh.log.Error(err, "still unable to encode and write the InternalServerError response")
wh.getLogger(nil).Error(err, "still unable to encode and write the InternalServerError response")
}
} else {
res := ar.Response
if log := wh.log; log.V(1).Enabled() {
if log := wh.getLogger(nil); log.V(5).Enabled() {
if res.Result != nil {
log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason)
log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason, "message", res.Result.Message)
}
log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed)
log.V(5).Info("wrote response", "requestID", res.UID, "allowed", res.Allowed)
}
}
}

View File

@@ -1,31 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
// DecoderInjector is used by the ControllerManager to inject decoder into webhook handlers.
type DecoderInjector interface {
InjectDecoder(*Decoder) error
}
// InjectDecoderInto will set decoder on i and return the result if it implements Decoder. Returns
// false if i does not implement Decoder.
func InjectDecoderInto(decoder *Decoder, i interface{}) (bool, error) {
if s, ok := i.(DecoderInjector); ok {
return true, s.InjectDecoder(decoder)
}
return false, nil
}

View File

@@ -25,8 +25,6 @@ import (
jsonpatch "gomodules.xyz/jsonpatch/v2"
admissionv1 "k8s.io/api/admission/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
type multiMutating []Handler
@@ -62,31 +60,6 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response {
}
}
// InjectFunc injects the field setter into the handlers.
func (hs multiMutating) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
for _, handler := range hs {
if err := f(handler); err != nil {
return err
}
}
return nil
}
// InjectDecoder injects the decoder into the handlers.
func (hs multiMutating) InjectDecoder(d *Decoder) error {
for _, handler := range hs {
if _, err := InjectDecoderInto(d, handler); err != nil {
return err
}
}
return nil
}
// MultiMutatingHandler combines multiple mutating webhook handlers into a single
// mutating webhook handler. Handlers are called in sequential order, and the first
// `allowed: false` response may short-circuit the rest. Users must take care to
@@ -120,28 +93,3 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response {
func MultiValidatingHandler(handlers ...Handler) Handler {
return multiValidating(handlers)
}
// InjectFunc injects the field setter into the handlers.
func (hs multiValidating) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
for _, handler := range hs {
if err := f(handler); err != nil {
return err
}
}
return nil
}
// InjectDecoder injects the decoder into the handlers.
func (hs multiValidating) InjectDecoder(d *Decoder) error {
for _, handler := range hs {
if _, err := InjectDecoderInto(d, handler); err != nil {
return err
}
}
return nil
}

View File

@@ -26,21 +26,21 @@ import (
// Allowed constructs a response indicating that the given operation
// is allowed (without any patches).
func Allowed(reason string) Response {
return ValidationResponse(true, reason)
func Allowed(message string) Response {
return ValidationResponse(true, message)
}
// Denied constructs a response indicating that the given operation
// is not allowed.
func Denied(reason string) Response {
return ValidationResponse(false, reason)
func Denied(message string) Response {
return ValidationResponse(false, message)
}
// Patched constructs a response indicating that the given operation is
// allowed, and that the target object should be modified by the given
// JSONPatch operations.
func Patched(reason string, patches ...jsonpatch.JsonPatchOperation) Response {
resp := Allowed(reason)
func Patched(message string, patches ...jsonpatch.JsonPatchOperation) Response {
resp := Allowed(message)
resp.Patches = patches
return resp
@@ -60,21 +60,24 @@ func Errored(code int32, err error) Response {
}
// ValidationResponse returns a response for admitting a request.
func ValidationResponse(allowed bool, reason string) Response {
func ValidationResponse(allowed bool, message string) Response {
code := http.StatusForbidden
reason := metav1.StatusReasonForbidden
if allowed {
code = http.StatusOK
reason = ""
}
resp := Response{
AdmissionResponse: admissionv1.AdmissionResponse{
Allowed: allowed,
Result: &metav1.Status{
Code: int32(code),
Code: int32(code),
Reason: reason,
},
},
}
if len(reason) > 0 {
resp.Result.Reason = metav1.StatusReason(reason)
if len(message) > 0 {
resp.Result.Message = message
}
return resp
}

View File

@@ -18,7 +18,8 @@ package admission
import (
"context"
goerrors "errors"
"errors"
"fmt"
"net/http"
v1 "k8s.io/api/admission/v1"
@@ -26,18 +27,35 @@ import (
"k8s.io/apimachinery/pkg/runtime"
)
// Warnings represents warning messages.
type Warnings []string
// Validator defines functions for validating an operation.
// The custom resource kind which implements this interface can validate itself.
// To validate the custom resource with another specific struct, use CustomValidator instead.
type Validator interface {
runtime.Object
ValidateCreate() error
ValidateUpdate(old runtime.Object) error
ValidateDelete() error
// ValidateCreate validates the object on creation.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateCreate() (warnings Warnings, err error)
// ValidateUpdate validates the object on update. The oldObj is the object before the update.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateUpdate(old runtime.Object) (warnings Warnings, err error)
// ValidateDelete validates the object on deletion.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateDelete() (warnings Warnings, err error)
}
// ValidatingWebhookFor creates a new Webhook for validating the provided type.
func ValidatingWebhookFor(validator Validator) *Webhook {
func ValidatingWebhookFor(scheme *runtime.Scheme, validator Validator) *Webhook {
return &Webhook{
Handler: &validatingHandler{validator: validator},
Handler: &validatingHandler{validator: validator, decoder: NewDecoder(scheme)},
}
}
@@ -46,42 +64,34 @@ type validatingHandler struct {
decoder *Decoder
}
var _ DecoderInjector = &validatingHandler{}
// InjectDecoder injects the decoder into a validatingHandler.
func (h *validatingHandler) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *validatingHandler) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.validator == nil {
panic("validator should never be nil")
}
// Get the object in the request
obj := h.validator.DeepCopyObject().(Validator)
if req.Operation == v1.Create {
err := h.decoder.Decode(req, obj)
if err != nil {
var err error
var warnings []string
switch req.Operation {
case v1.Connect:
// No validation for connect requests.
// TODO(vincepri): Should we validate CONNECT requests? In what cases?
case v1.Create:
if err = h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateCreate()
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
}
if req.Operation == v1.Update {
warnings, err = obj.ValidateCreate()
case v1.Update:
oldObj := obj.DeepCopyObject()
err := h.decoder.DecodeRaw(req.Object, obj)
err = h.decoder.DecodeRaw(req.Object, obj)
if err != nil {
return Errored(http.StatusBadRequest, err)
}
@@ -90,33 +100,26 @@ func (h *validatingHandler) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateUpdate(oldObj)
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
}
if req.Operation == v1.Delete {
warnings, err = obj.ValidateUpdate(oldObj)
case v1.Delete:
// In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346
// OldObject contains the object being deleted
err := h.decoder.DecodeRaw(req.OldObject, obj)
err = h.decoder.DecodeRaw(req.OldObject, obj)
if err != nil {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateDelete()
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
warnings, err = obj.ValidateDelete()
default:
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation))
}
return Allowed("")
if err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...)
}
return Denied(err.Error()).WithWarnings(warnings...)
}
return Allowed("").WithWarnings(warnings...)
}

View File

@@ -28,16 +28,29 @@ import (
)
// CustomValidator defines functions for validating an operation.
// The object to be validated is passed into methods as a parameter.
type CustomValidator interface {
ValidateCreate(ctx context.Context, obj runtime.Object) error
ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error
ValidateDelete(ctx context.Context, obj runtime.Object) error
// ValidateCreate validates the object on creation.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateCreate(ctx context.Context, obj runtime.Object) (warnings Warnings, err error)
// ValidateUpdate validates the object on update.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings Warnings, err error)
// ValidateDelete validates the object on deletion.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateDelete(ctx context.Context, obj runtime.Object) (warnings Warnings, err error)
}
// WithCustomValidator creates a new Webhook for validating the provided type.
func WithCustomValidator(obj runtime.Object, validator CustomValidator) *Webhook {
func WithCustomValidator(scheme *runtime.Scheme, obj runtime.Object, validator CustomValidator) *Webhook {
return &Webhook{
Handler: &validatorForType{object: obj, validator: validator},
Handler: &validatorForType{object: obj, validator: validator, decoder: NewDecoder(scheme)},
}
}
@@ -47,16 +60,11 @@ type validatorForType struct {
decoder *Decoder
}
var _ DecoderInjector = &validatorForType{}
// InjectDecoder injects the decoder into a validatingHandler.
func (h *validatorForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.validator == nil {
panic("validator should never be nil")
}
@@ -70,13 +78,18 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
obj := h.object.DeepCopyObject()
var err error
var warnings []string
switch req.Operation {
case v1.Connect:
// No validation for connect requests.
// TODO(vincepri): Should we validate CONNECT requests? In what cases?
case v1.Create:
if err := h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateCreate(ctx, obj)
warnings, err = h.validator.ValidateCreate(ctx, obj)
case v1.Update:
oldObj := obj.DeepCopyObject()
if err := h.decoder.DecodeRaw(req.Object, obj); err != nil {
@@ -86,7 +99,7 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateUpdate(ctx, oldObj, obj)
warnings, err = h.validator.ValidateUpdate(ctx, oldObj, obj)
case v1.Delete:
// In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346
// OldObject contains the object being deleted
@@ -94,20 +107,20 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateDelete(ctx, obj)
warnings, err = h.validator.ValidateDelete(ctx, obj)
default:
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation))
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation))
}
// Check the error message first.
if err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...)
}
return Denied(err.Error())
return Denied(err.Error()).WithWarnings(warnings...)
}
// Return allowed if everything succeeded.
return Allowed("")
return Allowed("").WithWarnings(warnings...)
}

View File

@@ -21,18 +21,17 @@ import (
"errors"
"fmt"
"net/http"
"sync"
"github.com/go-logr/logr"
jsonpatch "gomodules.xyz/jsonpatch/v2"
"gomodules.xyz/jsonpatch/v2"
admissionv1 "k8s.io/api/admission/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog/v2"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
@@ -131,16 +130,14 @@ type Webhook struct {
// headers thus allowing you to read them from within the handler
WithContextFunc func(context.Context, *http.Request) context.Context
// decoder is constructed on receiving a scheme and passed down to then handler
decoder *Decoder
// LogConstructor is used to construct a logger for logging messages during webhook calls
// based on the given base logger (which might carry more values like the webhook's path).
// Note: LogConstructor has to be able to handle nil requests as we are also using it
// outside the context of requests.
LogConstructor func(base logr.Logger, req *Request) logr.Logger
log logr.Logger
}
// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook.
func (wh *Webhook) InjectLogger(l logr.Logger) error {
wh.log = l
return nil
setupLogOnce sync.Once
log logr.Logger
}
// WithRecoverPanic takes a bool flag which indicates whether the panic caused by webhook should be recovered.
@@ -166,79 +163,47 @@ func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response)
}()
}
reqLog := wh.getLogger(&req)
ctx = logf.IntoContext(ctx, reqLog)
resp := wh.Handler.Handle(ctx, req)
if err := resp.Complete(req); err != nil {
wh.log.Error(err, "unable to encode response")
reqLog.Error(err, "unable to encode response")
return Errored(http.StatusInternalServerError, errUnableToEncodeResponse)
}
return resp
}
// InjectScheme injects a scheme into the webhook, in order to construct a Decoder.
func (wh *Webhook) InjectScheme(s *runtime.Scheme) error {
// TODO(directxman12): we should have a better way to pass this down
var err error
wh.decoder, err = NewDecoder(s)
if err != nil {
return err
}
// inject the decoder here too, just in case the order of calling this is not
// scheme first, then inject func
if wh.Handler != nil {
if _, err := InjectDecoderInto(wh.GetDecoder(), wh.Handler); err != nil {
return err
// getLogger constructs a logger from the injected log and LogConstructor.
func (wh *Webhook) getLogger(req *Request) logr.Logger {
wh.setupLogOnce.Do(func() {
if wh.log.GetSink() == nil {
wh.log = logf.Log.WithName("admission")
}
}
})
return nil
logConstructor := wh.LogConstructor
if logConstructor == nil {
logConstructor = DefaultLogConstructor
}
return logConstructor(wh.log, req)
}
// GetDecoder returns a decoder to decode the objects embedded in admission requests.
// It may be nil if we haven't received a scheme to use to determine object types yet.
func (wh *Webhook) GetDecoder() *Decoder {
return wh.decoder
}
// InjectFunc injects the field setter into the webhook.
func (wh *Webhook) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
// also inject a decoder, and wrap this so that we get a setFields
// that injects a decoder (hopefully things don't ignore the duplicate
// InjectorInto call).
var setFields inject.Func
setFields = func(target interface{}) error {
if err := f(target); err != nil {
return err
}
if _, err := inject.InjectorInto(setFields, target); err != nil {
return err
}
if _, err := InjectDecoderInto(wh.GetDecoder(), target); err != nil {
return err
}
return nil
// DefaultLogConstructor adds some commonly interesting fields to the given logger.
func DefaultLogConstructor(base logr.Logger, req *Request) logr.Logger {
if req != nil {
return base.WithValues("object", klog.KRef(req.Namespace, req.Name),
"namespace", req.Namespace, "name", req.Name,
"resource", req.Resource, "user", req.UserInfo.Username,
"requestID", req.UID,
)
}
return setFields(wh.Handler)
return base
}
// StandaloneOptions let you configure a StandaloneWebhook.
type StandaloneOptions struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// idea to pass your own scheme in. See the documentation in pkg/scheme for more information.
Scheme *runtime.Scheme
// Logger to be used by the webhook.
// If none is set, it defaults to log.Log global logger.
Logger logr.Logger
@@ -258,19 +223,9 @@ type StandaloneOptions struct {
// in your own server/mux. In order to be accessed by a kubernetes cluster,
// all webhook servers require TLS.
func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) {
if opts.Scheme == nil {
opts.Scheme = scheme.Scheme
if opts.Logger.GetSink() != nil {
hook.log = opts.Logger
}
if err := hook.InjectScheme(opts.Scheme); err != nil {
return nil, err
}
if opts.Logger.GetSink() == nil {
opts.Logger = logf.RuntimeLog.WithName("webhook")
}
hook.log = opts.Logger
if opts.MetricsPath == "" {
return hook, nil
}

View File

@@ -39,28 +39,20 @@ var (
log = logf.Log.WithName("conversion-webhook")
)
// Webhook implements a CRD conversion webhook HTTP handler.
type Webhook struct {
func NewWebhookHandler(scheme *runtime.Scheme) http.Handler {
return &webhook{scheme: scheme, decoder: NewDecoder(scheme)}
}
// webhook implements a CRD conversion webhook HTTP handler.
type webhook struct {
scheme *runtime.Scheme
decoder *Decoder
}
// InjectScheme injects a scheme into the webhook, in order to construct a Decoder.
func (wh *Webhook) InjectScheme(s *runtime.Scheme) error {
var err error
wh.scheme = s
wh.decoder, err = NewDecoder(s)
if err != nil {
return err
}
return nil
}
// ensure Webhook implements http.Handler
var _ http.Handler = &Webhook{}
var _ http.Handler = &webhook{}
func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
convertReview := &apix.ConversionReview{}
err := json.NewDecoder(r.Body).Decode(convertReview)
if err != nil {
@@ -95,7 +87,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// handles a version conversion request.
func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) {
func (wh *webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) {
if req == nil {
return nil, fmt.Errorf("conversion request is nil")
}
@@ -128,7 +120,7 @@ func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.Conv
// convertObject will convert given a src object to dst object.
// Note(droot): couldn't find a way to reduce the cyclomatic complexity under 10
// without compromising readability, so disabling gocyclo linter
func (wh *Webhook) convertObject(src, dst runtime.Object) error {
func (wh *webhook) convertObject(src, dst runtime.Object) error {
srcGVK := src.GetObjectKind().GroupVersionKind()
dstGVK := dst.GetObjectKind().GroupVersionKind()
@@ -155,7 +147,7 @@ func (wh *Webhook) convertObject(src, dst runtime.Object) error {
}
}
func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error {
func (wh *webhook) convertViaHub(src, dst conversion.Convertible) error {
hub, err := wh.getHub(src)
if err != nil {
return err
@@ -179,7 +171,7 @@ func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error {
}
// getHub returns an instance of the Hub for passed-in object's group/kind.
func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
func (wh *webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
gvks, err := objectGVKs(wh.scheme, obj)
if err != nil {
return nil, err
@@ -207,7 +199,7 @@ func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
}
// allocateDstObject returns an instance for a given GVK.
func (wh *Webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) {
func (wh *webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) {
gvk := schema.FromAPIVersionAndKind(apiVersion, kind)
obj, err := wh.scheme.New(gvk)

View File

@@ -30,8 +30,11 @@ type Decoder struct {
}
// NewDecoder creates a Decoder given the runtime.Scheme
func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) {
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil
func NewDecoder(scheme *runtime.Scheme) *Decoder {
if scheme == nil {
panic("scheme should never be nil")
}
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}
}
// Decode decodes the inlined object.

View File

@@ -29,12 +29,9 @@ import (
"sync"
"time"
"k8s.io/apimachinery/pkg/runtime"
kscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
@@ -49,7 +46,29 @@ var DefaultPort = 9443
// at the default locations (tls.crt and tls.key). If you do not
// want to configure TLS (i.e for testing purposes) run an
// admission.StandaloneWebhook in your own server.
type Server struct {
type Server interface {
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the webhook server doesn't need leader election.
NeedLeaderElection() bool
// Register marks the given webhook as being served at the given path.
// It panics if two hooks are registered on the same path.
Register(path string, hook http.Handler)
// Start runs the server.
// It will install the webhook related resources depend on the server configuration.
Start(ctx context.Context) error
// StartedChecker returns an healthz.Checker which is healthy after the
// server has been started.
StartedChecker() healthz.Checker
// WebhookMux returns the servers WebhookMux
WebhookMux() *http.ServeMux
}
// Options are all the available options for a webhook.Server
type Options struct {
// Host is the address that the server will listen on.
// Defaults to "" - all addresses.
Host string
@@ -58,38 +77,46 @@ type Server struct {
// It will be defaulted to 9443 if unspecified.
Port int
// CertDir is the directory that contains the server key and certificate. The
// server key and certificate.
// CertDir is the directory that contains the server key and certificate. Defaults to
// <temp-dir>/k8s-webhook-server/serving-certs.
CertDir string
// CertName is the server certificate name. Defaults to tls.crt.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
CertName string
// KeyName is the server key name. Defaults to tls.key.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
KeyName string
// ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate.
// Defaults to "", which means server does not verify client's certificate.
ClientCAName string
// TLSVersion is the minimum version of TLS supported. Accepts
// "", "1.0", "1.1", "1.2" and "1.3" only ("" is equivalent to "1.0" for backwards compatibility)
// Deprecated: Use TLSOpts instead.
TLSMinVersion string
// TLSOpts is used to allow configuring the TLS config used for the server
// TLSOpts is used to allow configuring the TLS config used for the server.
// This also allows providing a certificate via GetCertificate.
TLSOpts []func(*tls.Config)
// WebhookMux is the multiplexer that handles different webhooks.
WebhookMux *http.ServeMux
}
// webhooks keep track of all registered webhooks for dependency injection,
// and to provide better panic messages on duplicate webhook registration.
// NewServer constructs a new webhook.Server from the provided options.
func NewServer(o Options) Server {
return &DefaultServer{
Options: o,
}
}
// DefaultServer is the default implementation used for Server.
type DefaultServer struct {
Options Options
// webhooks keep track of all registered webhooks
webhooks map[string]http.Handler
// setFields allows injecting dependencies from an external source
setFields inject.Func
// defaultingOnce ensures that the default fields are only ever set once.
defaultingOnce sync.Once
@@ -99,41 +126,49 @@ type Server struct {
// mu protects access to the webhook map & setFields for Start, Register, etc
mu sync.Mutex
webhookMux *http.ServeMux
}
// setDefaults does defaulting for the Server.
func (s *Server) setDefaults() {
func (o *Options) setDefaults() {
if o.WebhookMux == nil {
o.WebhookMux = http.NewServeMux()
}
if o.Port <= 0 {
o.Port = DefaultPort
}
if len(o.CertDir) == 0 {
o.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
}
if len(o.CertName) == 0 {
o.CertName = "tls.crt"
}
if len(o.KeyName) == 0 {
o.KeyName = "tls.key"
}
}
func (s *DefaultServer) setDefaults() {
s.webhooks = map[string]http.Handler{}
if s.WebhookMux == nil {
s.WebhookMux = http.NewServeMux()
}
s.Options.setDefaults()
if s.Port <= 0 {
s.Port = DefaultPort
}
if len(s.CertDir) == 0 {
s.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
}
if len(s.CertName) == 0 {
s.CertName = "tls.crt"
}
if len(s.KeyName) == 0 {
s.KeyName = "tls.key"
}
s.webhookMux = s.Options.WebhookMux
}
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the webhook server doesn't need leader election.
func (*Server) NeedLeaderElection() bool {
func (*DefaultServer) NeedLeaderElection() bool {
return false
}
// Register marks the given webhook as being served at the given path.
// It panics if two hooks are registered on the same path.
func (s *Server) Register(path string, hook http.Handler) {
func (s *DefaultServer) Register(path string, hook http.Handler) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -141,110 +176,51 @@ func (s *Server) Register(path string, hook http.Handler) {
if _, found := s.webhooks[path]; found {
panic(fmt.Errorf("can't register duplicate path: %v", path))
}
// TODO(directxman12): call setfields if we've already started the server
s.webhooks[path] = hook
s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook))
s.webhookMux.Handle(path, metrics.InstrumentedHook(path, hook))
regLog := log.WithValues("path", path)
regLog.Info("Registering webhook")
// we've already been "started", inject dependencies here.
// Otherwise, InjectFunc will do this for us later.
if s.setFields != nil {
if err := s.setFields(hook); err != nil {
// TODO(directxman12): swallowing this error isn't great, but we'd have to
// change the signature to fix that
regLog.Error(err, "unable to inject fields into webhook during registration")
}
baseHookLog := log.WithName("webhooks")
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", path), hook); err != nil {
regLog.Error(err, "unable to logger into webhook during registration")
}
}
}
// StartStandalone runs a webhook server without
// a controller manager.
func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error {
// Use the Kubernetes client-go scheme if none is specified
if scheme == nil {
scheme = kscheme.Scheme
}
if err := s.InjectFunc(func(i interface{}) error {
if _, err := inject.SchemeInto(scheme, i); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return s.Start(ctx)
}
// tlsVersion converts from human-readable TLS version (for example "1.1")
// to the values accepted by tls.Config (for example 0x301).
func tlsVersion(version string) (uint16, error) {
switch version {
// default is previous behaviour
case "":
return tls.VersionTLS10, nil
case "1.0":
return tls.VersionTLS10, nil
case "1.1":
return tls.VersionTLS11, nil
case "1.2":
return tls.VersionTLS12, nil
case "1.3":
return tls.VersionTLS13, nil
default:
return 0, fmt.Errorf("invalid TLSMinVersion %v: expects 1.0, 1.1, 1.2, 1.3 or empty", version)
}
}
// Start runs the server.
// It will install the webhook related resources depend on the server configuration.
func (s *Server) Start(ctx context.Context) error {
func (s *DefaultServer) Start(ctx context.Context) error {
s.defaultingOnce.Do(s.setDefaults)
baseHookLog := log.WithName("webhooks")
baseHookLog.Info("Starting webhook server")
certPath := filepath.Join(s.CertDir, s.CertName)
keyPath := filepath.Join(s.CertDir, s.KeyName)
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
return err
}
go func() {
if err := certWatcher.Start(ctx); err != nil {
log.Error(err, "certificate watcher error")
}
}()
tlsMinVersion, err := tlsVersion(s.TLSMinVersion)
if err != nil {
return err
}
log.Info("Starting webhook server")
cfg := &tls.Config{ //nolint:gosec
NextProtos: []string{"h2"},
GetCertificate: certWatcher.GetCertificate,
MinVersion: tlsMinVersion,
NextProtos: []string{"h2"},
}
// fallback TLS config ready, will now mutate if passer wants full control over it
for _, op := range s.Options.TLSOpts {
op(cfg)
}
// load CA to verify client certificate
if s.ClientCAName != "" {
if cfg.GetCertificate == nil {
certPath := filepath.Join(s.Options.CertDir, s.Options.CertName)
keyPath := filepath.Join(s.Options.CertDir, s.Options.KeyName)
// Create the certificate watcher and
// set the config's GetCertificate on the TLSConfig
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
return err
}
cfg.GetCertificate = certWatcher.GetCertificate
go func() {
if err := certWatcher.Start(ctx); err != nil {
log.Error(err, "certificate watcher error")
}
}()
}
// Load CA to verify client certificate, if configured.
if s.Options.ClientCAName != "" {
certPool := x509.NewCertPool()
clientCABytes, err := os.ReadFile(filepath.Join(s.CertDir, s.ClientCAName))
clientCABytes, err := os.ReadFile(filepath.Join(s.Options.CertDir, s.Options.ClientCAName))
if err != nil {
return fmt.Errorf("failed to read client CA cert: %w", err)
}
@@ -258,27 +234,23 @@ func (s *Server) Start(ctx context.Context) error {
cfg.ClientAuth = tls.RequireAndVerifyClientCert
}
// fallback TLS config ready, will now mutate if passer wants full control over it
for _, op := range s.TLSOpts {
op(cfg)
}
listener, err := tls.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), cfg)
listener, err := tls.Listen("tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), cfg)
if err != nil {
return err
}
log.Info("Serving webhook server", "host", s.Host, "port", s.Port)
log.Info("Serving webhook server", "host", s.Options.Host, "port", s.Options.Port)
srv := httpserver.New(s.WebhookMux)
srv := httpserver.New(s.webhookMux)
idleConnsClosed := make(chan struct{})
go func() {
<-ctx.Done()
log.Info("shutting down webhook server")
log.Info("Shutting down webhook server with timeout of 1 minute")
// TODO: use a context with reasonable timeout
if err := srv.Shutdown(context.Background()); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
// Error from closing listeners, or context timeout
log.Error(err, "error shutting down the HTTP server")
}
@@ -298,7 +270,7 @@ func (s *Server) Start(ctx context.Context) error {
// StartedChecker returns an healthz.Checker which is healthy after the
// server has been started.
func (s *Server) StartedChecker() healthz.Checker {
func (s *DefaultServer) StartedChecker() healthz.Checker {
config := &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // config is used to connect to our own webhook port.
}
@@ -311,7 +283,7 @@ func (s *Server) StartedChecker() healthz.Checker {
}
d := &net.Dialer{Timeout: 10 * time.Second}
conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), config)
conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), config)
if err != nil {
return fmt.Errorf("webhook server is not reachable: %w", err)
}
@@ -324,23 +296,7 @@ func (s *Server) StartedChecker() healthz.Checker {
}
}
// InjectFunc injects the field setter into the server.
func (s *Server) InjectFunc(f inject.Func) error {
s.setFields = f
// inject fields here that weren't injected in Register because we didn't have setFields yet.
baseHookLog := log.WithName("webhooks")
for hookPath, webhook := range s.webhooks {
if err := s.setFields(webhook); err != nil {
return err
}
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil {
return err
}
}
return nil
// WebhookMux returns the servers WebhookMux
func (s *DefaultServer) WebhookMux() *http.ServeMux {
return s.webhookMux
}