Upgrade the operator to use Operator SDK v1.33.0 (#182)

* Move controller package inside internal directory

Based on the go/v4 project structure, the following changed:
- Pakcage `controllers` is now named `controller`
- Package `controller` now lives inside new `internal` directory

* Move main.go in cmd directory

Based on the new go/v4 project structure, `main.go` now lives in the `cmd` directory.

* Change package import in main.go

* Update go mod dependencies

Update the dependencies based on the versions obtained by creating a new operator project using `kubebuilder init --domain onepassword.com --plugins=go/v4`.

This is based on the migration steps provided to go from go/v3 to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update vendor

* Adjust code for breaking changes from pkg update

sigs.k8s.io/controller-runtime package had breaking changes from v0.14.5 to v0.16.3. This commit brings the changes needed to achieve the same things using the new functionality avaialble.

* Adjust paths to connect yaml files

Since `main.go` is now in `cmd` directory, the paths to the files for deploying Connect have to be adjusted based on the new location `main.go` is executed from.

* Update files based on new structure and scaffolding

These changes are made based on the new project structure and scaffolding obtained when using the new go/v4 project structure.

These were done based on the migration steps mentioned when migrating to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4).

* Update config files

These updates are made based on the Kustomize v4 syntax.

This is part of the upgrate to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update dependencies and GO version

* Update vendor

* Update Kubernetes tools versions

* Update operator version in Makefile

Now the version in the Makefile matches the version of the operator

* Update Operator SDK version in version.go

* Adjust generated deepcopy

It seems that the +build tag is no longer needed based on the latest generated scaffolding, therefore it's removed.

* Update copyright year

* Bring back missing changes from migration

Some customization in Makefile was lost during the migration process. Specifically, the namespace customization for `make deploy` command.

Also, we push changes to kustomization.yaml for making the deploy process smoother.

* Add RBAC perms for coordination.k8s.io

It seems that with the latest changes to Kubernetes and Kustomize, we need to add additional RBAC to the service account used so that it can properly access the `leases` resource.

* Optimize Dockerfile

Dockerfile had a step for caching dependencies (go mod download). However, this is already done by the vendor directory, which we include. Therefore, this step can be removed to make the image build time faster.
This commit is contained in:
Eduard Filip
2024-01-25 14:21:31 +01:00
committed by GitHub
parent 8fc852a4dd
commit f72e5243b0
1356 changed files with 86780 additions and 43671 deletions

View File

@@ -1,38 +1,44 @@
linters:
disable-all: true
enable:
- asciicheck
- bodyclose
- depguard
- dogsled
- errcheck
- errorlint
- exportloopref
- goconst
- gocritic
- gocyclo
- gofmt
- goimports
- goprintffuncname
- gosec
- gosimple
- govet
- importas
- ineffassign
- misspell
- nakedret
- nilerr
- nolintlint
- prealloc
- revive
- rowserrcheck
- staticcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- whitespace
- asasalint
- asciicheck
- bidichk
- bodyclose
- dogsled
- dupl
- errcheck
- errchkjson
- errorlint
- exhaustive
- exportloopref
- ginkgolinter
- goconst
- gocritic
- gocyclo
- gofmt
- goimports
- goprintffuncname
- gosec
- gosimple
- govet
- importas
- ineffassign
- makezero
- misspell
- nakedret
- nilerr
- nolintlint
- prealloc
- revive
- staticcheck
- stylecheck
- tagliatelle
- typecheck
- unconvert
- unparam
- unused
- whitespace
linters-settings:
importas:
@@ -53,13 +59,38 @@ linters-settings:
- pkg: sigs.k8s.io/controller-runtime
alias: ctrl
staticcheck:
go: "1.19"
go: "1.20"
stylecheck:
go: "1.19"
depguard:
include-go-root: true
packages:
- io/ioutil # https://go.dev/doc/go1.16#ioutil
go: "1.20"
revive:
rules:
# The following rules are recommended https://github.com/mgechev/revive#recommended-configuration
- name: blank-imports
- name: context-as-argument
- name: context-keys-type
- name: dot-imports
- name: error-return
- name: error-strings
- name: error-naming
- name: exported
- name: if-return
- name: increment-decrement
- name: var-naming
- name: var-declaration
- name: range
- name: receiver-naming
- name: time-naming
- name: unexported-return
- name: indent-error-flow
- name: errorf
- name: superfluous-else
- name: unreachable-code
- name: redefines-builtin-id
#
# Rules in addition to the recommended configuration above.
#
- name: bool-literal-in-expr
- name: constant-logical-expr
issues:
max-same-issues: 0
@@ -69,68 +100,71 @@ issues:
exclude-use-default: false
# List of regexps of issue texts to exclude, empty list by default.
exclude:
# The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time.
# If it is decided they will not be addressed they should be moved above this comment.
- Subprocess launch(ed with variable|ing should be audited)
- (G204|G104|G307)
- "ST1000: at least one file in a package should have a package comment"
# The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time.
# If it is decided they will not be addressed they should be moved above this comment.
- Subprocess launch(ed with variable|ing should be audited)
- (G204|G104|G307)
- "ST1000: at least one file in a package should have a package comment"
exclude-rules:
- linters:
- gosec
text: "G108: Profiling endpoint is automatically exposed on /debug/pprof"
- linters:
- revive
text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported"
- linters:
- errcheck
text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
# With Go 1.16, the new embed directive can be used with an un-named import,
# revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us.
# This directive allows the embed package to be imported with an underscore everywhere.
- linters:
- revive
source: _ "embed"
# Exclude some packages or code to require comments, for example test code, or fake clients.
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
source: (func|type).*Fake.*
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
path: fake_\.go
# Disable unparam "always receives" which might not be really
# useful when building libraries.
- linters:
- unparam
text: always receives
# Dot imports for gomega or ginkgo are allowed
# within test files.
- path: _test\.go
text: should not use dot imports
- path: _test\.go
text: cyclomatic complexity
- path: _test\.go
text: "G107: Potential HTTP request made with variable url"
# Append should be able to assign to a different var/slice.
- linters:
- gocritic
text: "appendAssign: append result not assigned to the same slice"
- linters:
- gocritic
text: "singleCaseSwitch: should rewrite switch statement to if statement"
# It considers all file access to a filename that comes from a variable problematic,
# which is naiv at best.
- linters:
- gosec
text: "G304: Potential file inclusion via variable"
- linters:
- revive
text: "package-comments: should have a package comment"
- linters:
- gosec
text: "G108: Profiling endpoint is automatically exposed on /debug/pprof"
- linters:
- revive
text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported"
- linters:
- errcheck
text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
- linters:
- staticcheck
text: "SA1019: .*The component config package has been deprecated and will be removed in a future release."
# With Go 1.16, the new embed directive can be used with an un-named import,
# revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us.
# This directive allows the embed package to be imported with an underscore everywhere.
- linters:
- revive
source: _ "embed"
# Exclude some packages or code to require comments, for example test code, or fake clients.
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
source: (func|type).*Fake.*
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
path: fake_\.go
# Disable unparam "always receives" which might not be really
# useful when building libraries.
- linters:
- unparam
text: always receives
# Dot imports for gomega and ginkgo are allowed
# within test files.
- path: _test\.go
text: should not use dot imports
- path: _test\.go
text: cyclomatic complexity
- path: _test\.go
text: "G107: Potential HTTP request made with variable url"
# Append should be able to assign to a different var/slice.
- linters:
- gocritic
text: "appendAssign: append result not assigned to the same slice"
- linters:
- gocritic
text: "singleCaseSwitch: should rewrite switch statement to if statement"
# It considers all file access to a filename that comes from a variable problematic,
# which is naiv at best.
- linters:
- gosec
text: "G304: Potential file inclusion via variable"
- linters:
- dupl
path: _test\.go
run:
timeout: 10m
skip-files:
- "zz_generated.*\\.go$"
- ".*conversion.*\\.go$"
- "zz_generated.*\\.go$"
- ".*conversion.*\\.go$"
allow-parallel-runners: true

View File

@@ -41,6 +41,7 @@ GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint)
GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff
CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen
ENVTEST_DIR := $(abspath tools/setup-envtest)
SCRATCH_ENV_DIR := $(abspath examples/scratch-env)
# The help will print out all targets with their descriptions organized bellow their categories. The categories are represented by `##@` and the target descriptions by `##`.
# The awk commands is responsible to read the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help. Then, if there's a line with ##@ something, that gets pretty-printed as a category.
@@ -75,7 +76,7 @@ $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder.
$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golanci-lint using hack script into tools folder.
hack/ensure-golangci-lint.sh \
-b $(TOOLS_BIN_DIR) \
$(shell cat .github/workflows/golangci-lint.yml | grep version | sed 's/.*version: //')
$(shell cat .github/workflows/golangci-lint.yml | grep "version: v" | sed 's/.*version: //')
## --------------------------------------
## Linting
@@ -99,6 +100,7 @@ modules: ## Runs go mod to ensure modules are up to date.
go mod tidy
cd $(TOOLS_DIR); go mod tidy
cd $(ENVTEST_DIR); go mod tidy
cd $(SCRATCH_ENV_DIR); go mod tidy
.PHONY: generate
generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config file
@@ -110,6 +112,7 @@ generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config
.PHONY: clean
clean: ## Cleanup.
$(GOLANGCI_LINT) cache clean
$(MAKE) clean-bin
.PHONY: clean-bin
@@ -117,7 +120,15 @@ clean-bin: ## Remove all generated binaries.
rm -rf hack/tools/bin
.PHONY: verify-modules
verify-modules: modules
@if !(git diff --quiet HEAD -- go.sum go.mod); then \
verify-modules: modules ## Verify go modules are up to date
@if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum $(SCRATCH_ENV_DIR)/go.sum); then \
git diff; \
echo "go module files are out of date, please run 'make modules'"; exit 1; \
fi
.PHONY: verify-generate
verify-generate: generate ## Verify generated files are up to date
@if !(git diff --quiet HEAD); then \
git diff; \
echo "generated files are out of date, run make generate"; exit 1; \
fi

View File

@@ -9,23 +9,21 @@ aliases:
# non-admin folks who have write-access and can approve any PRs in the repo
controller-runtime-maintainers:
- vincepri
- alvaroaleman
- joelanford
- sbueringer
- vincepri
# non-admin folks who can approve any PRs in the repo
controller-runtime-approvers:
- alvaroaleman
- fillzpp
- sbueringer
# folks who can review and LGTM any PRs in the repo (doesn't
# include approvers & admins -- those count too via the OWNERS
# file)
controller-runtime-reviewers:
- vincepri
- varshaprasad96
- fillzpp
- sbueringer
- inteon
# folks to can approve things in the directly-ported
# testing_frameworks portions of the codebase

View File

@@ -16,8 +16,8 @@ Documentation:
- [Basic controller using builder](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/builder#example-Builder)
- [Creating a manager](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#example-New)
- [Creating a controller](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#example-New)
- [Examples](https://github.com/kubernetes-sigs/controller-runtime/blob/master/examples)
- [Designs](https://github.com/kubernetes-sigs/controller-runtime/blob/master/designs)
- [Examples](https://github.com/kubernetes-sigs/controller-runtime/blob/main/examples)
- [Designs](https://github.com/kubernetes-sigs/controller-runtime/blob/main/designs)
# Versioning, Maintenance, and Compatibility
@@ -27,7 +27,7 @@ Users:
- We follow [Semantic Versioning (semver)](https://semver.org)
- Use releases with your dependency management to ensure that you get compatible code
- The master branch contains all the latest code, some of which may break compatibility (so "normal" `go get` is not recommended)
- The main branch contains all the latest code, some of which may break compatibility (so "normal" `go get` is not recommended)
Contributors:

View File

@@ -10,7 +10,7 @@ to create a new branch you will just need to ensure that all big fixes are cherr
### Create the new branch and the release tag
1. Create a new branch `git checkout -b release-<MAJOR.MINOR>` from master
1. Create a new branch `git checkout -b release-<MAJOR.MINOR>` from main
2. Push the new branch to the remote repository
### Now, let's generate the changelog

View File

@@ -10,5 +10,6 @@
# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
# INSTRUCTIONS AT https://kubernetes.io/security/
pwittrock
droot
alvaroaleman
sbueringer
vincepri

View File

@@ -9,7 +9,7 @@ exactly.
[guidelines]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md
## Compatiblity and Release Support
## Compatibility and Release Support
For release branches, we generally tend to support backporting one (1)
major release (`release-{X-1}` or `release-0.{Y-1}`), but may go back
@@ -19,12 +19,12 @@ further if the need arises and is very pressing (e.g. security updates).
Note the [guidelines on dependency versions][dep-versions]. Particularly:
- We **DO** guarantee Kubernetes REST API compability -- if a given
- We **DO** guarantee Kubernetes REST API compatibility -- if a given
version of controller-runtime stops working with what should be
a supported version of Kubernetes, this is almost certainly a bug.
- We **DO NOT** guarantee any particular compability matrix between
- We **DO NOT** guarantee any particular compatibility matrix between
kubernetes library dependencies (client-go, apimachinery, etc); Such
compability is infeasible due to the way those libraries are versioned.
compatibility is infeasible due to the way those libraries are versioned.
[dep-versions]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md#kubernetes-version-compatibility

View File

@@ -99,6 +99,8 @@ var (
// ConfigFile returns the cfg.File function for deferred config file loading,
// this is passed into Options{}.From() to populate the Options fields for
// the manager.
//
// Deprecated: This is deprecated in favor of using Options directly.
ConfigFile = cfg.File
// NewControllerManagedBy returns a new controller builder that will be started by the provided Manager.
@@ -108,6 +110,9 @@ var (
NewWebhookManagedBy = builder.WebhookManagedBy
// NewManager returns a new Manager for creating Controllers.
// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf"
// will be used for all built-in resources of Kubernetes, and "application/json" is for other types
// including all CRD resources.
NewManager = manager.New
// CreateOrUpdate creates or updates the given object obj in the Kubernetes
@@ -139,7 +144,7 @@ var (
// The logger, when used with controllers, can be expected to contain basic information about the object
// that's being reconciled like:
// - `reconciler group` and `reconciler kind` coming from the For(...) object passed in when building a controller.
// - `name` and `namespace` injected from the reconciliation request.
// - `name` and `namespace` from the reconciliation request.
//
// This is meant to be used with the context supplied in a struct that satisfies the Reconciler interface.
LoggerFrom = log.FromContext

View File

@@ -46,13 +46,13 @@ limitations under the License.
//
// Frequently asked questions about using controller-runtime and designing
// controllers can be found at
// https://github.com/kubernetes-sigs/controller-runtime/blob/master/FAQ.md.
// https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md.
//
// # Managers
//
// Every controller and webhook is ultimately run by a Manager (pkg/manager). A
// manager is responsible for running controllers and webhooks, and setting up
// common dependencies (pkg/runtime/inject), like shared caches and clients, as
// common dependencies, like shared caches and clients, as
// well as managing leader election (pkg/leaderelection). Managers are
// generally configured to gracefully shut down controllers on pod termination
// by wiring up a signal handler (pkg/manager/signals).

View File

@@ -30,6 +30,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/handler"
internalsource "sigs.k8s.io/controller-runtime/pkg/internal/source"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -40,14 +41,14 @@ import (
var newController = controller.New
var getGvk = apiutil.GVKForObject
// project represents other forms that the we can use to
// project represents other forms that we can use to
// send/receive a given resource (metadata-only, unstructured, etc).
type objectProjection int
const (
// projectAsNormal doesn't change the object from the form given.
projectAsNormal objectProjection = iota
// projectAsMetadata turns this into an metadata-only watch.
// projectAsMetadata turns this into a metadata-only watch.
projectAsMetadata
)
@@ -68,7 +69,7 @@ func ControllerManagedBy(m manager.Manager) *Builder {
return &Builder{mgr: m}
}
// ForInput represents the information set by For method.
// ForInput represents the information set by the For method.
type ForInput struct {
object client.Object
predicates []predicate.Predicate
@@ -96,14 +97,20 @@ func (blder *Builder) For(object client.Object, opts ...ForOption) *Builder {
// OwnsInput represents the information set by Owns method.
type OwnsInput struct {
matchEveryOwner bool
object client.Object
predicates []predicate.Predicate
objectProjection objectProjection
}
// Owns defines types of Objects being *generated* by the ControllerManagedBy, and configures the ControllerManagedBy to respond to
// create / delete / update events by *reconciling the owner object*. This is the equivalent of calling
// Watches(&source.Kind{Type: <ForType-forInput>}, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true}).
// create / delete / update events by *reconciling the owner object*.
//
// The default behavior reconciles only the first controller-type OwnerReference of the given type.
// Use Owns(object, builder.MatchEveryOwner) to reconcile all owners.
//
// By default, this is the equivalent of calling
// Watches(object, handler.EnqueueRequestForOwner([...], ownerType, OnlyControllerOwner())).
func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder {
input := OwnsInput{object: object}
for _, opt := range opts {
@@ -117,16 +124,60 @@ func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder {
// WatchesInput represents the information set by Watches method.
type WatchesInput struct {
src source.Source
eventhandler handler.EventHandler
eventHandler handler.EventHandler
predicates []predicate.Predicate
objectProjection objectProjection
}
// Watches exposes the lower-level ControllerManagedBy Watches functions through the builder. Consider using
// Owns or For instead of Watches directly.
// Watches defines the type of Object to watch, and configures the ControllerManagedBy to respond to create / delete /
// update events by *reconciling the object* with the given EventHandler.
//
// This is the equivalent of calling
// WatchesRawSource(source.Kind(cache, object), eventHandler, opts...).
func (blder *Builder) Watches(object client.Object, eventHandler handler.EventHandler, opts ...WatchesOption) *Builder {
src := source.Kind(blder.mgr.GetCache(), object)
return blder.WatchesRawSource(src, eventHandler, opts...)
}
// WatchesMetadata is the same as Watches, but forces the internal cache to only watch PartialObjectMetadata.
//
// This is useful when watching lots of objects, really big objects, or objects for which you only know
// the GVK, but not the structure. You'll need to pass metav1.PartialObjectMetadata to the client
// when fetching objects in your reconciler, otherwise you'll end up with a duplicate structured or unstructured cache.
//
// When watching a resource with metadata only, for example the v1.Pod, you should not Get and List using the v1.Pod type.
// Instead, you should use the special metav1.PartialObjectMetadata type.
//
// ❌ Incorrect:
//
// pod := &v1.Pod{}
// mgr.GetClient().Get(ctx, nsAndName, pod)
//
// ✅ Correct:
//
// pod := &metav1.PartialObjectMetadata{}
// pod.SetGroupVersionKind(schema.GroupVersionKind{
// Group: "",
// Version: "v1",
// Kind: "Pod",
// })
// mgr.GetClient().Get(ctx, nsAndName, pod)
//
// In the first case, controller-runtime will create another cache for the
// concrete type on top of the metadata cache; this increases memory
// consumption and leads to race conditions as caches are not in sync.
func (blder *Builder) WatchesMetadata(object client.Object, eventHandler handler.EventHandler, opts ...WatchesOption) *Builder {
opts = append(opts, OnlyMetadata)
return blder.Watches(object, eventHandler, opts...)
}
// WatchesRawSource exposes the lower-level ControllerManagedBy Watches functions through the builder.
// Specified predicates are registered only for given source.
func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder {
input := WatchesInput{src: src, eventhandler: eventhandler}
//
// STOP! Consider using For(...), Owns(...), Watches(...), WatchesMetadata(...) instead.
// This method is only exposed for more advanced use cases, most users should use one of the higher level functions.
func (blder *Builder) WatchesRawSource(src source.Source, eventHandler handler.EventHandler, opts ...WatchesOption) *Builder {
input := WatchesInput{src: src, eventHandler: eventHandler}
for _, opt := range opts {
opt.ApplyToWatches(&input)
}
@@ -136,7 +187,7 @@ func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandl
}
// WithEventFilter sets the event filters, to filter which create/update/delete/generic events eventually
// trigger reconciliations. For example, filtering on whether the resource version has changed.
// trigger reconciliations. For example, filtering on whether the resource version has changed.
// Given predicate is added for all watched objects.
// Defaults to the empty list.
func (blder *Builder) WithEventFilter(p predicate.Predicate) *Builder {
@@ -144,7 +195,7 @@ func (blder *Builder) WithEventFilter(p predicate.Predicate) *Builder {
return blder
}
// WithOptions overrides the controller options use in doController. Defaults to empty.
// WithOptions overrides the controller options used in doController. Defaults to empty.
func (blder *Builder) WithOptions(options controller.Options) *Builder {
blder.ctrlOptions = options
return blder
@@ -156,7 +207,7 @@ func (blder *Builder) WithLogConstructor(logConstructor func(*reconcile.Request)
return blder
}
// Named sets the name of the controller to the given name. The name shows up
// Named sets the name of the controller to the given name. The name shows up
// in metrics, among other things, and thus should be a prometheus compatible name
// (underscores and alphanumeric characters only).
//
@@ -217,13 +268,14 @@ func (blder *Builder) project(obj client.Object, proj objectProjection) (client.
func (blder *Builder) doWatch() error {
// Reconcile type
if blder.forInput.object != nil {
typeForSrc, err := blder.project(blder.forInput.object, blder.forInput.objectProjection)
obj, err := blder.project(blder.forInput.object, blder.forInput.objectProjection)
if err != nil {
return err
}
src := &source.Kind{Type: typeForSrc}
src := source.Kind(blder.mgr.GetCache(), obj)
hdler := &handler.EnqueueRequestForObject{}
allPredicates := append(blder.globalPredicates, blder.forInput.predicates...)
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, blder.forInput.predicates...)
if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil {
return err
}
@@ -234,15 +286,20 @@ func (blder *Builder) doWatch() error {
return errors.New("Owns() can only be used together with For()")
}
for _, own := range blder.ownsInput {
typeForSrc, err := blder.project(own.object, own.objectProjection)
obj, err := blder.project(own.object, own.objectProjection)
if err != nil {
return err
}
src := &source.Kind{Type: typeForSrc}
hdler := &handler.EnqueueRequestForOwner{
OwnerType: blder.forInput.object,
IsController: true,
src := source.Kind(blder.mgr.GetCache(), obj)
opts := []handler.OwnerOption{}
if !own.matchEveryOwner {
opts = append(opts, handler.OnlyControllerOwner())
}
hdler := handler.EnqueueRequestForOwner(
blder.mgr.GetScheme(), blder.mgr.GetRESTMapper(),
blder.forInput.object,
opts...,
)
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, own.predicates...)
if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil {
@@ -255,19 +312,17 @@ func (blder *Builder) doWatch() error {
return errors.New("there are no watches configured, controller will never get triggered. Use For(), Owns() or Watches() to set them up")
}
for _, w := range blder.watchesInput {
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, w.predicates...)
// If the source of this watch is of type *source.Kind, project it.
if srckind, ok := w.src.(*source.Kind); ok {
typeForSrc, err := blder.project(srckind.Type, w.objectProjection)
// If the source of this watch is of type Kind, project it.
if srcKind, ok := w.src.(*internalsource.Kind); ok {
typeForSrc, err := blder.project(srcKind.Type, w.objectProjection)
if err != nil {
return err
}
srckind.Type = typeForSrc
srcKind.Type = typeForSrc
}
if err := blder.ctrl.Watch(w.src, w.eventhandler, allPredicates...); err != nil {
allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...)
allPredicates = append(allPredicates, w.predicates...)
if err := blder.ctrl.Watch(w.src, w.eventHandler, allPredicates...); err != nil {
return err
}
}
@@ -288,12 +343,15 @@ func (blder *Builder) doController(r reconcile.Reconciler) error {
globalOpts := blder.mgr.GetControllerOptions()
ctrlOptions := blder.ctrlOptions
if ctrlOptions.Reconciler != nil && r != nil {
return errors.New("reconciler was set via WithOptions() and via Build() or Complete()")
}
if ctrlOptions.Reconciler == nil {
ctrlOptions.Reconciler = r
}
// Retrieve the GVK from the object we're reconciling
// to prepopulate logger information, and to optionally generate a default name.
// to pre-populate logger information, and to optionally generate a default name.
var gvk schema.GroupVersionKind
hasGVK := blder.forInput.object != nil
if hasGVK {
@@ -314,8 +372,8 @@ func (blder *Builder) doController(r reconcile.Reconciler) error {
}
// Setup cache sync timeout.
if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout != nil {
ctrlOptions.CacheSyncTimeout = *globalOpts.CacheSyncTimeout
if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout > 0 {
ctrlOptions.CacheSyncTimeout = globalOpts.CacheSyncTimeout
}
controllerName, err := blder.getControllerName(gvk, hasGVK)

View File

@@ -28,7 +28,7 @@ type ForOption interface {
ApplyToFor(*ForInput)
}
// OwnsOption is some configuration that modifies options for a owns request.
// OwnsOption is some configuration that modifies options for an owns request.
type OwnsOption interface {
// ApplyToOwns applies this configuration to the given owns input.
ApplyToOwns(*OwnsInput)
@@ -79,8 +79,8 @@ var _ WatchesOption = &Predicates{}
// {{{ For & Owns Dual-Type options
// asProjection configures the projection (currently only metadata) on the input.
// Currently only metadata is supported. We might want to expand
// projectAs configures the projection on the input.
// Currently only OnlyMetadata is supported. We might want to expand
// this to arbitrary non-special local projections in the future.
type projectAs objectProjection
@@ -101,9 +101,9 @@ func (p projectAs) ApplyToWatches(opts *WatchesInput) {
var (
// OnlyMetadata tells the controller to *only* cache metadata, and to watch
// the API server in metadata-only form. This is useful when watching
// the API server in metadata-only form. This is useful when watching
// lots of objects, really big objects, or objects for which you only know
// the GVK, but not the structure. You'll need to pass
// the GVK, but not the structure. You'll need to pass
// metav1.PartialObjectMetadata to the client when fetching objects in your
// reconciler, otherwise you'll end up with a duplicate structured or
// unstructured cache.
@@ -138,3 +138,19 @@ var (
)
// }}}
// MatchEveryOwner determines whether the watch should be filtered based on
// controller ownership. As in, when the OwnerReference.Controller field is set.
//
// If passed as an option,
// the handler receives notification for every owner of the object with the given type.
// If unset (default), the handler receives notification only for the first
// OwnerReference with `Controller: true`.
var MatchEveryOwner = &matchEveryOwner{}
type matchEveryOwner struct{}
// ApplyToOwns applies this configuration to the given OwnsInput options.
func (o matchEveryOwner) ApplyToOwns(opts *OwnsInput) {
opts.matchEveryOwner = true
}

View File

@@ -22,9 +22,12 @@ import (
"net/url"
"strings"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -33,16 +36,17 @@ import (
// WebhookBuilder builds a Webhook.
type WebhookBuilder struct {
apiType runtime.Object
withDefaulter admission.CustomDefaulter
withValidator admission.CustomValidator
gvk schema.GroupVersionKind
mgr manager.Manager
config *rest.Config
recoverPanic bool
apiType runtime.Object
customDefaulter admission.CustomDefaulter
customValidator admission.CustomValidator
gvk schema.GroupVersionKind
mgr manager.Manager
config *rest.Config
recoverPanic bool
logConstructor func(base logr.Logger, req *admission.Request) logr.Logger
}
// WebhookManagedBy allows inform its manager.Manager.
// WebhookManagedBy returns a new webhook builder.
func WebhookManagedBy(m manager.Manager) *WebhookBuilder {
return &WebhookBuilder{mgr: m}
}
@@ -57,19 +61,25 @@ func (blder *WebhookBuilder) For(apiType runtime.Object) *WebhookBuilder {
return blder
}
// WithDefaulter takes a admission.WithDefaulter interface, a MutatingWebhook will be wired for this type.
// WithDefaulter takes an admission.CustomDefaulter interface, a MutatingWebhook will be wired for this type.
func (blder *WebhookBuilder) WithDefaulter(defaulter admission.CustomDefaulter) *WebhookBuilder {
blder.withDefaulter = defaulter
blder.customDefaulter = defaulter
return blder
}
// WithValidator takes a admission.WithValidator interface, a ValidatingWebhook will be wired for this type.
// WithValidator takes a admission.CustomValidator interface, a ValidatingWebhook will be wired for this type.
func (blder *WebhookBuilder) WithValidator(validator admission.CustomValidator) *WebhookBuilder {
blder.withValidator = validator
blder.customValidator = validator
return blder
}
// RecoverPanic indicates whether the panic caused by webhook should be recovered.
// WithLogConstructor overrides the webhook's LogConstructor.
func (blder *WebhookBuilder) WithLogConstructor(logConstructor func(base logr.Logger, req *admission.Request) logr.Logger) *WebhookBuilder {
blder.logConstructor = logConstructor
return blder
}
// RecoverPanic indicates whether panics caused by the webhook should be recovered.
func (blder *WebhookBuilder) RecoverPanic() *WebhookBuilder {
blder.recoverPanic = true
return blder
@@ -80,6 +90,9 @@ func (blder *WebhookBuilder) Complete() error {
// Set the Config
blder.loadRestConfig()
// Configure the default LogConstructor
blder.setLogConstructor()
// Set the Webhook if needed
return blder.registerWebhooks()
}
@@ -90,18 +103,38 @@ func (blder *WebhookBuilder) loadRestConfig() {
}
}
func (blder *WebhookBuilder) setLogConstructor() {
if blder.logConstructor == nil {
blder.logConstructor = func(base logr.Logger, req *admission.Request) logr.Logger {
log := base.WithValues(
"webhookGroup", blder.gvk.Group,
"webhookKind", blder.gvk.Kind,
)
if req != nil {
return log.WithValues(
blder.gvk.Kind, klog.KRef(req.Namespace, req.Name),
"namespace", req.Namespace, "name", req.Name,
"resource", req.Resource, "user", req.UserInfo.Username,
"requestID", req.UID,
)
}
return log
}
}
}
func (blder *WebhookBuilder) registerWebhooks() error {
typ, err := blder.getType()
if err != nil {
return err
}
// Create webhook(s) for each type
blder.gvk, err = apiutil.GVKForObject(typ, blder.mgr.GetScheme())
if err != nil {
return err
}
// Register webhook(s) for type
blder.registerDefaultingWebhook()
blder.registerValidatingWebhook()
@@ -112,10 +145,11 @@ func (blder *WebhookBuilder) registerWebhooks() error {
return nil
}
// registerDefaultingWebhook registers a defaulting webhook if th.
// registerDefaultingWebhook registers a defaulting webhook if necessary.
func (blder *WebhookBuilder) registerDefaultingWebhook() {
mwh := blder.getDefaultingWebhook()
if mwh != nil {
mwh.LogConstructor = blder.logConstructor
path := generateMutatePath(blder.gvk)
// Checking if the path is already registered.
@@ -130,11 +164,11 @@ func (blder *WebhookBuilder) registerDefaultingWebhook() {
}
func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook {
if defaulter := blder.withDefaulter; defaulter != nil {
return admission.WithCustomDefaulter(blder.apiType, defaulter).WithRecoverPanic(blder.recoverPanic)
if defaulter := blder.customDefaulter; defaulter != nil {
return admission.WithCustomDefaulter(blder.mgr.GetScheme(), blder.apiType, defaulter).WithRecoverPanic(blder.recoverPanic)
}
if defaulter, ok := blder.apiType.(admission.Defaulter); ok {
return admission.DefaultingWebhookFor(defaulter).WithRecoverPanic(blder.recoverPanic)
return admission.DefaultingWebhookFor(blder.mgr.GetScheme(), defaulter).WithRecoverPanic(blder.recoverPanic)
}
log.Info(
"skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called",
@@ -142,9 +176,11 @@ func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook {
return nil
}
// registerValidatingWebhook registers a validating webhook if necessary.
func (blder *WebhookBuilder) registerValidatingWebhook() {
vwh := blder.getValidatingWebhook()
if vwh != nil {
vwh.LogConstructor = blder.logConstructor
path := generateValidatePath(blder.gvk)
// Checking if the path is already registered.
@@ -159,11 +195,11 @@ func (blder *WebhookBuilder) registerValidatingWebhook() {
}
func (blder *WebhookBuilder) getValidatingWebhook() *admission.Webhook {
if validator := blder.withValidator; validator != nil {
return admission.WithCustomValidator(blder.apiType, validator).WithRecoverPanic(blder.recoverPanic)
if validator := blder.customValidator; validator != nil {
return admission.WithCustomValidator(blder.mgr.GetScheme(), blder.apiType, validator).WithRecoverPanic(blder.recoverPanic)
}
if validator, ok := blder.apiType.(admission.Validator); ok {
return admission.ValidatingWebhookFor(validator).WithRecoverPanic(blder.recoverPanic)
return admission.ValidatingWebhookFor(blder.mgr.GetScheme(), validator).WithRecoverPanic(blder.recoverPanic)
}
log.Info(
"skip registering a validating webhook, object does not implement admission.Validator or WithValidator wasn't called",
@@ -179,7 +215,7 @@ func (blder *WebhookBuilder) registerConversionWebhook() error {
}
if ok {
if !blder.isAlreadyHandled("/convert") {
blder.mgr.GetWebhookServer().Register("/convert", &conversion.Webhook{})
blder.mgr.GetWebhookServer().Register("/convert", conversion.NewWebhookHandler(blder.mgr.GetScheme()))
}
log.Info("Conversion webhook enabled", "GVK", blder.gvk)
}
@@ -195,10 +231,10 @@ func (blder *WebhookBuilder) getType() (runtime.Object, error) {
}
func (blder *WebhookBuilder) isAlreadyHandled(path string) bool {
if blder.mgr.GetWebhookServer().WebhookMux == nil {
if blder.mgr.GetWebhookServer().WebhookMux() == nil {
return false
}
h, p := blder.mgr.GetWebhookServer().WebhookMux.Handler(&http.Request{URL: &url.URL{Path: path}})
h, p := blder.mgr.GetWebhookServer().WebhookMux().Handler(&http.Request{URL: &url.URL{Path: path}})
if p == path && h != nil {
return true
}

View File

@@ -19,10 +19,13 @@ package cache
import (
"context"
"fmt"
"reflect"
"net/http"
"time"
"golang.org/x/exp/maps"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
@@ -30,6 +33,7 @@ import (
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
toolscache "k8s.io/client-go/tools/cache"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/cache/internal"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -37,16 +41,33 @@ import (
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("object-cache")
var (
log = logf.RuntimeLog.WithName("object-cache")
defaultSyncPeriod = 10 * time.Hour
)
// InformerGetOptions defines the behavior of how informers are retrieved.
type InformerGetOptions internal.GetOptions
// InformerGetOption defines an option that alters the behavior of how informers are retrieved.
type InformerGetOption func(*InformerGetOptions)
// BlockUntilSynced determines whether a get request for an informer should block
// until the informer's cache has synced.
func BlockUntilSynced(shouldBlock bool) InformerGetOption {
return func(opts *InformerGetOptions) {
opts.BlockUntilSynced = &shouldBlock
}
}
// Cache knows how to load Kubernetes objects, fetch informers to request
// to receive events for Kubernetes objects (at a low-level),
// and add indices to fields on the objects stored in the cache.
type Cache interface {
// Cache acts as a client to objects stored in the cache.
// Reader acts as a client to objects stored in the cache.
client.Reader
// Cache loads informers and adds field indices.
// Informers loads informers and adds field indices.
Informers
}
@@ -56,352 +77,321 @@ type Cache interface {
type Informers interface {
// GetInformer fetches or constructs an informer for the given object that corresponds to a single
// API kind and resource.
GetInformer(ctx context.Context, obj client.Object) (Informer, error)
GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error)
// GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead
// of the underlying object.
GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error)
GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error)
// Start runs all the informers known to this cache until the context is closed.
// It blocks.
Start(ctx context.Context) error
// WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache.
// WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache.
WaitForCacheSync(ctx context.Context) bool
// Informers knows how to add indices to the caches (informers) that it manages.
// FieldIndexer adds indices to the managed informers.
client.FieldIndexer
}
// Informer - informer allows you interact with the underlying informer.
// Informer allows you to interact with the underlying informer.
type Informer interface {
// AddEventHandler adds an event handler to the shared informer using the shared informer's resync
// period. Events to a single handler are delivered sequentially, but there is no coordination
// period. Events to a single handler are delivered sequentially, but there is no coordination
// between different handlers.
// It returns a registration handle for the handler that can be used to remove
// the handler again.
// the handler again and an error if the handler cannot be added.
AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error)
// AddEventHandlerWithResyncPeriod adds an event handler to the shared informer using the
// specified resync period. Events to a single handler are delivered sequentially, but there is
// specified resync period. Events to a single handler are delivered sequentially, but there is
// no coordination between different handlers.
// It returns a registration handle for the handler that can be used to remove
// the handler again and an error if the handler cannot be added.
AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error)
// RemoveEventHandler removes a formerly added event handler given by
// RemoveEventHandler removes a previously added event handler given by
// its registration handle.
// This function is guaranteed to be idempotent, and thread-safe.
// This function is guaranteed to be idempotent and thread-safe.
RemoveEventHandler(handle toolscache.ResourceEventHandlerRegistration) error
// AddIndexers adds more indexers to this store. If you call this after you already have data
// AddIndexers adds indexers to this store. If this is called after there is already data
// in the store, the results are undefined.
AddIndexers(indexers toolscache.Indexers) error
// HasSynced return true if the informers underlying store has synced.
HasSynced() bool
}
// ObjectSelector is an alias name of internal.Selector.
type ObjectSelector internal.Selector
// AllNamespaces should be used as the map key to deliminate namespace settings
// that apply to all namespaces that themselves do not have explicit settings.
const AllNamespaces = metav1.NamespaceAll
// SelectorsByObject associate a client.Object's GVK to a field/label selector.
// There is also `DefaultSelector` to set a global default (which will be overridden by
// a more specific setting here, if any).
type SelectorsByObject map[client.Object]ObjectSelector
// Options are the optional arguments for creating a new InformersMap object.
// Options are the optional arguments for creating a new Cache object.
type Options struct {
// HTTPClient is the http client to use for the REST client
HTTPClient *http.Client
// Scheme is the scheme to use for mapping objects to GroupVersionKinds
Scheme *runtime.Scheme
// Mapper is the RESTMapper to use for mapping GroupVersionKinds to Resources
Mapper meta.RESTMapper
// Resync is the base frequency the informers are resynced.
// Defaults to defaultResyncTime.
// A 10 percent jitter will be added to the Resync period between informers
// So that all informers will not send list requests simultaneously.
Resync *time.Duration
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
//
// This applies to all controllers.
//
// A period sync happens for two reasons:
// 1. To insure against a bug in the controller that causes an object to not
// be requeued, when it otherwise should be requeued.
// 2. To insure against an unknown bug in controller-runtime, or its dependencies,
// that causes an object to not be requeued, when it otherwise should be
// requeued, or to be removed from the queue, when it otherwise should not
// be removed.
//
// If you want
// 1. to insure against missed watch events, or
// 2. to poll services that cannot be watched,
// then we recommend that, instead of changing the default period, the
// controller requeue, with a constant duration `t`, whenever the controller
// is "done" with an object, and would otherwise not requeue it, i.e., we
// recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`,
// instead of `reconcile.Result{}`.
SyncPeriod *time.Duration
// Namespace restricts the cache's ListWatch to the desired namespace
// Default watches all namespaces
Namespace string
// ReaderFailOnMissingInformer configures the cache to return a ErrResourceNotCached error when a user
// requests, using Get() and List(), a resource the cache does not already have an informer for.
//
// This error is distinct from an errors.NotFound.
//
// Defaults to false, which means that the cache will start a new informer
// for every new requested resource.
ReaderFailOnMissingInformer bool
// SelectorsByObject restricts the cache's ListWatch to the desired
// fields per GVK at the specified object, the map's value must implement
// Selector [1] using for example a Set [2]
// [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Selector
// [2] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Set
SelectorsByObject SelectorsByObject
// DefaultNamespaces maps namespace names to cache configs. If set, only
// the namespaces in here will be watched and it will by used to default
// ByObject.Namespaces for all objects if that is nil.
//
// It is possible to have specific Config for just some namespaces
// but cache all namespaces by using the AllNamespaces const as the map key.
// This will then include all namespaces that do not have a more specific
// setting.
//
// The options in the Config that are nil will be defaulted from
// the respective Default* settings.
DefaultNamespaces map[string]Config
// DefaultSelector will be used as selectors for all object types
// that do not have a selector in SelectorsByObject defined.
DefaultSelector ObjectSelector
// DefaultLabelSelector will be used as a label selector for all objects
// unless there is already one set in ByObject or DefaultNamespaces.
DefaultLabelSelector labels.Selector
// UnsafeDisableDeepCopyByObject indicates not to deep copy objects during get or
// DefaultFieldSelector will be used as a field selector for all object types
// unless there is already one set in ByObject or DefaultNamespaces.
DefaultFieldSelector fields.Selector
// DefaultTransform will be used as transform for all object types
// unless there is already one set in ByObject or DefaultNamespaces.
DefaultTransform toolscache.TransformFunc
// DefaultUnsafeDisableDeepCopy is the default for UnsafeDisableDeepCopy
// for everything that doesn't specify this.
//
// Be very careful with this, when enabled you must DeepCopy any object before mutating it,
// otherwise you will mutate the object in the cache.
//
// This will be used for all object types, unless it is set in ByObject or
// DefaultNamespaces.
DefaultUnsafeDisableDeepCopy *bool
// ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object.
// object, this will fall through to Default* settings.
ByObject map[client.Object]ByObject
// newInformer allows overriding of NewSharedIndexInformer for testing.
newInformer *func(toolscache.ListerWatcher, runtime.Object, time.Duration, toolscache.Indexers) toolscache.SharedIndexInformer
}
// ByObject offers more fine-grained control over the cache's ListWatch by object.
type ByObject struct {
// Namespaces maps a namespace name to cache configs. If set, only the
// namespaces in this map will be cached.
//
// Settings in the map value that are unset will be defaulted.
// Use an empty value for the specific setting to prevent that.
//
// It is possible to have specific Config for just some namespaces
// but cache all namespaces by using the AllNamespaces const as the map key.
// This will then include all namespaces that do not have a more specific
// setting.
//
// A nil map allows to default this to the cache's DefaultNamespaces setting.
// An empty map prevents this and means that all namespaces will be cached.
//
// The defaulting follows the following precedence order:
// 1. ByObject
// 2. DefaultNamespaces[namespace]
// 3. Default*
//
// This must be unset for cluster-scoped objects.
Namespaces map[string]Config
// Label represents a label selector for the object.
Label labels.Selector
// Field represents a field selector for the object.
Field fields.Selector
// Transform is a transformer function for the object which gets applied
// when objects of the transformation are about to be committed to the cache.
//
// This function is called both for new objects to enter the cache,
// and for updated objects.
Transform toolscache.TransformFunc
// UnsafeDisableDeepCopy indicates not to deep copy objects during get or
// list objects per GVK at the specified object.
// Be very careful with this, when enabled you must DeepCopy any object before mutating it,
// otherwise you will mutate the object in the cache.
UnsafeDisableDeepCopyByObject DisableDeepCopyByObject
// TransformByObject is a map from GVKs to transformer functions which
// get applied when objects of the transformation are about to be committed
// to cache.
//
// This function is called both for new objects to enter the cache,
// and for updated objects.
TransformByObject TransformByObject
// DefaultTransform is the transform used for all GVKs which do
// not have an explicit transform func set in TransformByObject
DefaultTransform toolscache.TransformFunc
UnsafeDisableDeepCopy *bool
}
var defaultResyncTime = 10 * time.Hour
// Config describes all potential options for a given watch.
type Config struct {
// LabelSelector specifies a label selector. A nil value allows to
// default this.
//
// Set to labels.Everything() if you don't want this defaulted.
LabelSelector labels.Selector
// FieldSelector specifics a field selector. A nil value allows to
// default this.
//
// Set to fields.Everything() if you don't want this defaulted.
FieldSelector fields.Selector
// Transform specifies a transform func. A nil value allows to default
// this.
//
// Set to an empty func to prevent this:
// func(in interface{}) (interface{}, error) { return in, nil }
Transform toolscache.TransformFunc
// UnsafeDisableDeepCopy specifies if List and Get requests against the
// cache should not DeepCopy. A nil value allows to default this.
UnsafeDisableDeepCopy *bool
}
// NewCacheFunc - Function for creating a new cache from the options and a rest config.
type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
// New initializes and returns a new Cache.
func New(config *rest.Config, opts Options) (Cache, error) {
opts, err := defaultOpts(config, opts)
func New(cfg *rest.Config, opts Options) (Cache, error) {
opts, err := defaultOpts(cfg, opts)
if err != nil {
return nil, err
}
selectorsByGVK, err := convertToByGVK(opts.SelectorsByObject, opts.DefaultSelector, opts.Scheme)
if err != nil {
return nil, err
}
disableDeepCopyByGVK, err := convertToDisableDeepCopyByGVK(opts.UnsafeDisableDeepCopyByObject, opts.Scheme)
if err != nil {
return nil, err
}
transformByGVK, err := convertToByGVK(opts.TransformByObject, opts.DefaultTransform, opts.Scheme)
if err != nil {
return nil, err
}
transformByObj := internal.TransformFuncByObjectFromMap(transformByGVK)
internalSelectorsByGVK := internal.SelectorsByGVK{}
for gvk, selector := range selectorsByGVK {
internalSelectorsByGVK[gvk] = internal.Selector(selector)
}
im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, internalSelectorsByGVK, disableDeepCopyByGVK, transformByObj)
return &informerCache{InformersMap: im}, nil
}
newCacheFunc := newCache(cfg, opts)
// BuilderWithOptions returns a Cache constructor that will build a cache
// honoring the options argument, this is useful to specify options like
// SelectorsByObject
// WARNING: If SelectorsByObject is specified, filtered out resources are not
// returned.
// WARNING: If UnsafeDisableDeepCopy is enabled, you must DeepCopy any object
// returned from cache get/list before mutating it.
func BuilderWithOptions(options Options) NewCacheFunc {
return func(config *rest.Config, inherited Options) (Cache, error) {
var err error
inherited, err = defaultOpts(config, inherited)
var defaultCache Cache
if len(opts.DefaultNamespaces) > 0 {
defaultConfig := optionDefaultsToConfig(&opts)
defaultCache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, opts.DefaultNamespaces, &defaultConfig)
} else {
defaultCache = newCacheFunc(optionDefaultsToConfig(&opts), corev1.NamespaceAll)
}
if len(opts.ByObject) == 0 {
return defaultCache, nil
}
delegating := &delegatingByGVKCache{
scheme: opts.Scheme,
caches: make(map[schema.GroupVersionKind]Cache, len(opts.ByObject)),
defaultCache: defaultCache,
}
for obj, config := range opts.ByObject {
gvk, err := apiutil.GVKForObject(obj, opts.Scheme)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to get GVK for type %T: %w", obj, err)
}
options, err = defaultOpts(config, options)
if err != nil {
return nil, err
var cache Cache
if len(config.Namespaces) > 0 {
cache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, config.Namespaces, nil)
} else {
cache = newCacheFunc(byObjectToConfig(config), corev1.NamespaceAll)
}
combined, err := options.inheritFrom(inherited)
if err != nil {
return nil, err
delegating.caches[gvk] = cache
}
return delegating, nil
}
func optionDefaultsToConfig(opts *Options) Config {
return Config{
LabelSelector: opts.DefaultLabelSelector,
FieldSelector: opts.DefaultFieldSelector,
Transform: opts.DefaultTransform,
UnsafeDisableDeepCopy: opts.DefaultUnsafeDisableDeepCopy,
}
}
func byObjectToConfig(byObject ByObject) Config {
return Config{
LabelSelector: byObject.Label,
FieldSelector: byObject.Field,
Transform: byObject.Transform,
UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy,
}
}
type newCacheFunc func(config Config, namespace string) Cache
func newCache(restConfig *rest.Config, opts Options) newCacheFunc {
return func(config Config, namespace string) Cache {
return &informerCache{
scheme: opts.Scheme,
Informers: internal.NewInformers(restConfig, &internal.InformersOpts{
HTTPClient: opts.HTTPClient,
Scheme: opts.Scheme,
Mapper: opts.Mapper,
ResyncPeriod: *opts.SyncPeriod,
Namespace: namespace,
Selector: internal.Selector{
Label: config.LabelSelector,
Field: config.FieldSelector,
},
Transform: config.Transform,
UnsafeDisableDeepCopy: pointer.BoolDeref(config.UnsafeDisableDeepCopy, false),
NewInformer: opts.newInformer,
}),
readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer,
}
return New(config, *combined)
}
}
func (options Options) inheritFrom(inherited Options) (*Options, error) {
var (
combined Options
err error
)
combined.Scheme = combineScheme(inherited.Scheme, options.Scheme)
combined.Mapper = selectMapper(inherited.Mapper, options.Mapper)
combined.Resync = selectResync(inherited.Resync, options.Resync)
combined.Namespace = selectNamespace(inherited.Namespace, options.Namespace)
combined.SelectorsByObject, combined.DefaultSelector, err = combineSelectors(inherited, options, combined.Scheme)
if err != nil {
return nil, err
}
combined.UnsafeDisableDeepCopyByObject, err = combineUnsafeDeepCopy(inherited, options, combined.Scheme)
if err != nil {
return nil, err
}
combined.TransformByObject, combined.DefaultTransform, err = combineTransforms(inherited, options, combined.Scheme)
if err != nil {
return nil, err
}
return &combined, nil
}
func combineScheme(schemes ...*runtime.Scheme) *runtime.Scheme {
var out *runtime.Scheme
for _, sch := range schemes {
if sch == nil {
continue
}
for gvk, t := range sch.AllKnownTypes() {
if out == nil {
out = runtime.NewScheme()
}
out.AddKnownTypeWithName(gvk, reflect.New(t).Interface().(runtime.Object))
}
}
return out
}
func selectMapper(def, override meta.RESTMapper) meta.RESTMapper {
if override != nil {
return override
}
return def
}
func selectResync(def, override *time.Duration) *time.Duration {
if override != nil {
return override
}
return def
}
func selectNamespace(def, override string) string {
if override != "" {
return override
}
return def
}
func combineSelectors(inherited, options Options, scheme *runtime.Scheme) (SelectorsByObject, ObjectSelector, error) {
// Selectors are combined via logical AND.
// - Combined label selector is a union of the selectors requirements from both sets of options.
// - Combined field selector uses fields.AndSelectors with the combined list of non-nil field selectors
// defined in both sets of options.
//
// There is a bunch of complexity here because we need to convert to SelectorsByGVK
// to be able to match keys between options and inherited and then convert back to SelectorsByObject
optionsSelectorsByGVK, err := convertToByGVK(options.SelectorsByObject, options.DefaultSelector, scheme)
if err != nil {
return nil, ObjectSelector{}, err
}
inheritedSelectorsByGVK, err := convertToByGVK(inherited.SelectorsByObject, inherited.DefaultSelector, inherited.Scheme)
if err != nil {
return nil, ObjectSelector{}, err
}
for gvk, inheritedSelector := range inheritedSelectorsByGVK {
optionsSelectorsByGVK[gvk] = combineSelector(inheritedSelector, optionsSelectorsByGVK[gvk])
}
return convertToByObject(optionsSelectorsByGVK, scheme)
}
func combineSelector(selectors ...ObjectSelector) ObjectSelector {
ls := make([]labels.Selector, 0, len(selectors))
fs := make([]fields.Selector, 0, len(selectors))
for _, s := range selectors {
ls = append(ls, s.Label)
fs = append(fs, s.Field)
}
return ObjectSelector{
Label: combineLabelSelectors(ls...),
Field: combineFieldSelectors(fs...),
}
}
func combineLabelSelectors(ls ...labels.Selector) labels.Selector {
var combined labels.Selector
for _, l := range ls {
if l == nil {
continue
}
if combined == nil {
combined = labels.NewSelector()
}
reqs, _ := l.Requirements()
combined = combined.Add(reqs...)
}
return combined
}
func combineFieldSelectors(fs ...fields.Selector) fields.Selector {
nonNil := fs[:0]
for _, f := range fs {
if f == nil {
continue
}
nonNil = append(nonNil, f)
}
if len(nonNil) == 0 {
return nil
}
if len(nonNil) == 1 {
return nonNil[0]
}
return fields.AndSelectors(nonNil...)
}
func combineUnsafeDeepCopy(inherited, options Options, scheme *runtime.Scheme) (DisableDeepCopyByObject, error) {
// UnsafeDisableDeepCopyByObject is combined via precedence. Only if a value for a particular GVK is unset
// in options will a value from inherited be used.
optionsDisableDeepCopyByGVK, err := convertToDisableDeepCopyByGVK(options.UnsafeDisableDeepCopyByObject, options.Scheme)
if err != nil {
return nil, err
}
inheritedDisableDeepCopyByGVK, err := convertToDisableDeepCopyByGVK(inherited.UnsafeDisableDeepCopyByObject, inherited.Scheme)
if err != nil {
return nil, err
}
for gvk, inheritedDeepCopy := range inheritedDisableDeepCopyByGVK {
if _, ok := optionsDisableDeepCopyByGVK[gvk]; !ok {
if optionsDisableDeepCopyByGVK == nil {
optionsDisableDeepCopyByGVK = map[schema.GroupVersionKind]bool{}
}
optionsDisableDeepCopyByGVK[gvk] = inheritedDeepCopy
}
}
return convertToDisableDeepCopyByObject(optionsDisableDeepCopyByGVK, scheme)
}
func combineTransforms(inherited, options Options, scheme *runtime.Scheme) (TransformByObject, toolscache.TransformFunc, error) {
// Transform functions are combined via chaining. If both inherited and options define a transform
// function, the transform function from inherited will be called first, and the transform function from
// options will be called second.
optionsTransformByGVK, err := convertToByGVK(options.TransformByObject, options.DefaultTransform, options.Scheme)
if err != nil {
return nil, nil, err
}
inheritedTransformByGVK, err := convertToByGVK(inherited.TransformByObject, inherited.DefaultTransform, inherited.Scheme)
if err != nil {
return nil, nil, err
}
for gvk, inheritedTransform := range inheritedTransformByGVK {
if optionsTransformByGVK == nil {
optionsTransformByGVK = map[schema.GroupVersionKind]toolscache.TransformFunc{}
}
optionsTransformByGVK[gvk] = combineTransform(inheritedTransform, optionsTransformByGVK[gvk])
}
return convertToByObject(optionsTransformByGVK, scheme)
}
func combineTransform(inherited, current toolscache.TransformFunc) toolscache.TransformFunc {
if inherited == nil {
return current
}
if current == nil {
return inherited
}
return func(in interface{}) (interface{}, error) {
mid, err := inherited(in)
if err != nil {
return nil, err
}
return current(mid)
}
}
func defaultOpts(config *rest.Config, opts Options) (Options, error) {
config = rest.CopyConfig(config)
if config.UserAgent == "" {
config.UserAgent = rest.DefaultKubernetesUserAgent()
}
// Use the rest HTTP client for the provided config if unset
if opts.HTTPClient == nil {
var err error
opts.HTTPClient, err = rest.HTTPClientFor(config)
if err != nil {
return Options{}, fmt.Errorf("could not create HTTP client from config: %w", err)
}
}
// Use the default Kubernetes Scheme if unset
if opts.Scheme == nil {
opts.Scheme = scheme.Scheme
@@ -410,107 +400,106 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
// Construct a new Mapper if unset
if opts.Mapper == nil {
var err error
opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config)
opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config, opts.HTTPClient)
if err != nil {
log.WithName("setup").Error(err, "Failed to get API Group-Resources")
return opts, fmt.Errorf("could not create RESTMapper from config")
return Options{}, fmt.Errorf("could not create RESTMapper from config: %w", err)
}
}
for namespace, cfg := range opts.DefaultNamespaces {
cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts))
if namespace == metav1.NamespaceAll {
cfg.FieldSelector = fields.AndSelectors(appendIfNotNil(namespaceAllSelector(maps.Keys(opts.DefaultNamespaces)), cfg.FieldSelector)...)
}
opts.DefaultNamespaces[namespace] = cfg
}
for obj, byObject := range opts.ByObject {
isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper)
if err != nil {
return opts, fmt.Errorf("failed to determine if %T is namespaced: %w", obj, err)
}
if !isNamespaced && byObject.Namespaces != nil {
return opts, fmt.Errorf("type %T is not namespaced, but its ByObject.Namespaces setting is not nil", obj)
}
// Default the namespace-level configs first, because they need to use the undefaulted type-level config.
for namespace, config := range byObject.Namespaces {
// 1. Default from the undefaulted type-level config
config = defaultConfig(config, byObjectToConfig(byObject))
// 2. Default from the namespace-level config. This was defaulted from the global default config earlier, but
// might not have an entry for the current namespace.
if defaultNamespaceSettings, hasDefaultNamespace := opts.DefaultNamespaces[namespace]; hasDefaultNamespace {
config = defaultConfig(config, defaultNamespaceSettings)
}
// 3. Default from the global defaults
config = defaultConfig(config, optionDefaultsToConfig(&opts))
if namespace == metav1.NamespaceAll {
config.FieldSelector = fields.AndSelectors(
appendIfNotNil(
namespaceAllSelector(maps.Keys(byObject.Namespaces)),
config.FieldSelector,
)...,
)
}
byObject.Namespaces[namespace] = config
}
defaultedConfig := defaultConfig(byObjectToConfig(byObject), optionDefaultsToConfig(&opts))
byObject.Label = defaultedConfig.LabelSelector
byObject.Field = defaultedConfig.FieldSelector
byObject.Transform = defaultedConfig.Transform
byObject.UnsafeDisableDeepCopy = defaultedConfig.UnsafeDisableDeepCopy
if isNamespaced && byObject.Namespaces == nil {
byObject.Namespaces = opts.DefaultNamespaces
}
opts.ByObject[obj] = byObject
}
// Default the resync period to 10 hours if unset
if opts.Resync == nil {
opts.Resync = &defaultResyncTime
if opts.SyncPeriod == nil {
opts.SyncPeriod = &defaultSyncPeriod
}
return opts, nil
}
func convertToByGVK[T any](byObject map[client.Object]T, def T, scheme *runtime.Scheme) (map[schema.GroupVersionKind]T, error) {
byGVK := map[schema.GroupVersionKind]T{}
for object, value := range byObject {
gvk, err := apiutil.GVKForObject(object, scheme)
if err != nil {
return nil, err
}
byGVK[gvk] = value
func defaultConfig(toDefault, defaultFrom Config) Config {
if toDefault.LabelSelector == nil {
toDefault.LabelSelector = defaultFrom.LabelSelector
}
byGVK[schema.GroupVersionKind{}] = def
return byGVK, nil
}
func convertToByObject[T any](byGVK map[schema.GroupVersionKind]T, scheme *runtime.Scheme) (map[client.Object]T, T, error) {
var byObject map[client.Object]T
def := byGVK[schema.GroupVersionKind{}]
for gvk, value := range byGVK {
if gvk == (schema.GroupVersionKind{}) {
continue
}
obj, err := scheme.New(gvk)
if err != nil {
return nil, def, err
}
cObj, ok := obj.(client.Object)
if !ok {
return nil, def, fmt.Errorf("object %T for GVK %q does not implement client.Object", obj, gvk)
}
if byObject == nil {
byObject = map[client.Object]T{}
}
byObject[cObj] = value
if toDefault.FieldSelector == nil {
toDefault.FieldSelector = defaultFrom.FieldSelector
}
return byObject, def, nil
if toDefault.Transform == nil {
toDefault.Transform = defaultFrom.Transform
}
if toDefault.UnsafeDisableDeepCopy == nil {
toDefault.UnsafeDisableDeepCopy = defaultFrom.UnsafeDisableDeepCopy
}
return toDefault
}
// DisableDeepCopyByObject associate a client.Object's GVK to disable DeepCopy during get or list from cache.
type DisableDeepCopyByObject map[client.Object]bool
var _ client.Object = &ObjectAll{}
// ObjectAll is the argument to represent all objects' types.
type ObjectAll struct {
client.Object
}
func convertToDisableDeepCopyByGVK(disableDeepCopyByObject DisableDeepCopyByObject, scheme *runtime.Scheme) (internal.DisableDeepCopyByGVK, error) {
disableDeepCopyByGVK := internal.DisableDeepCopyByGVK{}
for obj, disable := range disableDeepCopyByObject {
switch obj.(type) {
case ObjectAll, *ObjectAll:
disableDeepCopyByGVK[internal.GroupVersionKindAll] = disable
default:
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return nil, err
}
disableDeepCopyByGVK[gvk] = disable
func namespaceAllSelector(namespaces []string) fields.Selector {
selectors := make([]fields.Selector, 0, len(namespaces)-1)
for _, namespace := range namespaces {
if namespace != metav1.NamespaceAll {
selectors = append(selectors, fields.OneTermNotEqualSelector("metadata.namespace", namespace))
}
}
return disableDeepCopyByGVK, nil
return fields.AndSelectors(selectors...)
}
func convertToDisableDeepCopyByObject(byGVK internal.DisableDeepCopyByGVK, scheme *runtime.Scheme) (DisableDeepCopyByObject, error) {
var byObject DisableDeepCopyByObject
for gvk, value := range byGVK {
if byObject == nil {
byObject = DisableDeepCopyByObject{}
}
if gvk == (schema.GroupVersionKind{}) {
byObject[ObjectAll{}] = value
continue
}
obj, err := scheme.New(gvk)
if err != nil {
return nil, err
}
cObj, ok := obj.(client.Object)
if !ok {
return nil, fmt.Errorf("object %T for GVK %q does not implement client.Object", obj, gvk)
}
byObject[cObj] = value
func appendIfNotNil[T comparable](a, b T) []T {
if b != *new(T) {
return []T{a, b}
}
return byObject, nil
return []T{a}
}
// TransformByObject associate a client.Object's GVK to a transformer function
// to be applied when storing the object into the cache.
type TransformByObject map[client.Object]toolscache.TransformFunc

View File

@@ -0,0 +1,127 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cache
import (
"context"
"strings"
"sync"
"golang.org/x/exp/maps"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// delegatingByGVKCache delegates to a type-specific cache if present
// and uses the defaultCache otherwise.
type delegatingByGVKCache struct {
scheme *runtime.Scheme
caches map[schema.GroupVersionKind]Cache
defaultCache Cache
}
func (dbt *delegatingByGVKCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
cache, err := dbt.cacheForObject(obj)
if err != nil {
return err
}
return cache.Get(ctx, key, obj, opts...)
}
func (dbt *delegatingByGVKCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
cache, err := dbt.cacheForObject(list)
if err != nil {
return err
}
return cache.List(ctx, list, opts...)
}
func (dbt *delegatingByGVKCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) {
cache, err := dbt.cacheForObject(obj)
if err != nil {
return nil, err
}
return cache.GetInformer(ctx, obj, opts...)
}
func (dbt *delegatingByGVKCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) {
return dbt.cacheForGVK(gvk).GetInformerForKind(ctx, gvk, opts...)
}
func (dbt *delegatingByGVKCache) Start(ctx context.Context) error {
allCaches := maps.Values(dbt.caches)
allCaches = append(allCaches, dbt.defaultCache)
wg := &sync.WaitGroup{}
errs := make(chan error)
for idx := range allCaches {
cache := allCaches[idx]
wg.Add(1)
go func() {
defer wg.Done()
if err := cache.Start(ctx); err != nil {
errs <- err
}
}()
}
select {
case err := <-errs:
return err
case <-ctx.Done():
wg.Wait()
return nil
}
}
func (dbt *delegatingByGVKCache) WaitForCacheSync(ctx context.Context) bool {
synced := true
for _, cache := range append(maps.Values(dbt.caches), dbt.defaultCache) {
if !cache.WaitForCacheSync(ctx) {
synced = false
}
}
return synced
}
func (dbt *delegatingByGVKCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
cache, err := dbt.cacheForObject(obj)
if err != nil {
return err
}
return cache.IndexField(ctx, obj, field, extractValue)
}
func (dbt *delegatingByGVKCache) cacheForObject(o runtime.Object) (Cache, error) {
gvk, err := apiutil.GVKForObject(o, dbt.scheme)
if err != nil {
return nil, err
}
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
return dbt.cacheForGVK(gvk), nil
}
func (dbt *delegatingByGVKCache) cacheForGVK(gvk schema.GroupVersionKind) Cache {
if specific, hasSpecific := dbt.caches[gvk]; hasSpecific {
return specific
}
return dbt.defaultCache
}

View File

@@ -19,14 +19,15 @@ package cache
import (
"context"
"fmt"
"reflect"
"strings"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/cache/internal"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
@@ -45,19 +46,38 @@ func (*ErrCacheNotStarted) Error() string {
return "the cache is not started, can not read objects"
}
// informerCache is a Kubernetes Object cache populated from InformersMap. informerCache wraps an InformersMap.
var _ error = (*ErrCacheNotStarted)(nil)
// ErrResourceNotCached indicates that the resource type
// the client asked the cache for is not cached, i.e. the
// corresponding informer does not exist yet.
type ErrResourceNotCached struct {
GVK schema.GroupVersionKind
}
// Error returns the error
func (r ErrResourceNotCached) Error() string {
return fmt.Sprintf("%s is not cached", r.GVK.String())
}
var _ error = (*ErrResourceNotCached)(nil)
// informerCache is a Kubernetes Object cache populated from internal.Informers.
// informerCache wraps internal.Informers.
type informerCache struct {
*internal.InformersMap
scheme *runtime.Scheme
*internal.Informers
readerFailOnMissingInformer bool
}
// Get implements Reader.
func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error {
gvk, err := apiutil.GVKForObject(out, ip.Scheme)
func (ic *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error {
gvk, err := apiutil.GVKForObject(out, ic.scheme)
if err != nil {
return err
}
started, cache, err := ip.InformersMap.Get(ctx, gvk, out)
started, cache, err := ic.getInformerForKind(ctx, gvk, out)
if err != nil {
return err
}
@@ -65,17 +85,17 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out clie
if !started {
return &ErrCacheNotStarted{}
}
return cache.Reader.Get(ctx, key, out)
return cache.Reader.Get(ctx, key, out, opts...)
}
// List implements Reader.
func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error {
gvk, cacheTypeObj, err := ip.objectTypeForListObject(out)
func (ic *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error {
gvk, cacheTypeObj, err := ic.objectTypeForListObject(out)
if err != nil {
return err
}
started, cache, err := ip.InformersMap.Get(ctx, *gvk, cacheTypeObj)
started, cache, err := ic.getInformerForKind(ctx, *gvk, cacheTypeObj)
if err != nil {
return err
}
@@ -90,94 +110,106 @@ func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts .
// objectTypeForListObject tries to find the runtime.Object and associated GVK
// for a single object corresponding to the passed-in list type. We need them
// because they are used as cache map key.
func (ip *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) {
gvk, err := apiutil.GVKForObject(list, ip.Scheme)
func (ic *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) {
gvk, err := apiutil.GVKForObject(list, ic.scheme)
if err != nil {
return nil, nil, err
}
// we need the non-list GVK, so chop off the "List" from the end of the kind
if strings.HasSuffix(gvk.Kind, "List") && apimeta.IsListType(list) {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
// We need the non-list GVK, so chop off the "List" from the end of the kind.
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
_, isUnstructured := list.(*unstructured.UnstructuredList)
var cacheTypeObj runtime.Object
if isUnstructured {
// Handle unstructured.UnstructuredList.
if _, isUnstructured := list.(runtime.Unstructured); isUnstructured {
u := &unstructured.Unstructured{}
u.SetGroupVersionKind(gvk)
cacheTypeObj = u
} else {
itemsPtr, err := apimeta.GetItemsPtr(list)
if err != nil {
return nil, nil, err
}
// http://knowyourmeme.com/memes/this-is-fine
elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem()
if elemType.Kind() != reflect.Ptr {
elemType = reflect.PtrTo(elemType)
}
cacheTypeValue := reflect.Zero(elemType)
var ok bool
cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object)
if !ok {
return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface())
}
return &gvk, u, nil
}
// Handle metav1.PartialObjectMetadataList.
if _, isPartialObjectMetadata := list.(*metav1.PartialObjectMetadataList); isPartialObjectMetadata {
pom := &metav1.PartialObjectMetadata{}
pom.SetGroupVersionKind(gvk)
return &gvk, pom, nil
}
// Any other list type should have a corresponding non-list type registered
// in the scheme. Use that to create a new instance of the non-list type.
cacheTypeObj, err := ic.scheme.New(gvk)
if err != nil {
return nil, nil, err
}
return &gvk, cacheTypeObj, nil
}
// GetInformerForKind returns the informer for the GroupVersionKind.
func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
// Map the gvk to an object
obj, err := ip.Scheme.New(gvk)
if err != nil {
return nil, err
func applyGetOptions(opts ...InformerGetOption) *internal.GetOptions {
cfg := &InformerGetOptions{}
for _, opt := range opts {
opt(cfg)
}
_, i, err := ip.InformersMap.Get(ctx, gvk, obj)
if err != nil {
return nil, err
}
return i.Informer, err
return (*internal.GetOptions)(cfg)
}
// GetInformer returns the informer for the obj.
func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) {
gvk, err := apiutil.GVKForObject(obj, ip.Scheme)
// GetInformerForKind returns the informer for the GroupVersionKind. If no informer exists, one will be started.
func (ic *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) {
// Map the gvk to an object
obj, err := ic.scheme.New(gvk)
if err != nil {
return nil, err
}
_, i, err := ip.InformersMap.Get(ctx, gvk, obj)
_, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...))
if err != nil {
return nil, err
}
return i.Informer, err
return i.Informer, nil
}
// GetInformer returns the informer for the obj. If no informer exists, one will be started.
func (ic *informerCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) {
gvk, err := apiutil.GVKForObject(obj, ic.scheme)
if err != nil {
return nil, err
}
_, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...))
if err != nil {
return nil, err
}
return i.Informer, nil
}
func (ic *informerCache) getInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *internal.Cache, error) {
if ic.readerFailOnMissingInformer {
cache, started, ok := ic.Informers.Peek(gvk, obj)
if !ok {
return false, nil, &ErrResourceNotCached{GVK: gvk}
}
return started, cache, nil
}
return ic.Informers.Get(ctx, gvk, obj, &internal.GetOptions{})
}
// NeedLeaderElection implements the LeaderElectionRunnable interface
// to indicate that this can be started without requiring the leader lock.
func (ip *informerCache) NeedLeaderElection() bool {
func (ic *informerCache) NeedLeaderElection() bool {
return false
}
// IndexField adds an indexer to the underlying cache, using extraction function to get
// value(s) from the given field. This index can then be used by passing a field selector
// IndexField adds an indexer to the underlying informer, using extractValue function to get
// value(s) from the given field. This index can then be used by passing a field selector
// to List. For one-to-one compatibility with "normal" field selectors, only return one value.
// The values may be anything. They will automatically be prefixed with the namespace of the
// given object, if present. The objects passed are guaranteed to be objects of the correct type.
func (ip *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
informer, err := ip.GetInformer(ctx, obj)
// The values may be anything. They will automatically be prefixed with the namespace of the
// given object, if present. The objects passed are guaranteed to be objects of the correct type.
func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
informer, err := ic.GetInformer(ctx, obj)
if err != nil {
return err
}
return indexByField(informer, field, extractValue)
}
func indexByField(indexer Informer, field string, extractor client.IndexerFunc) error {
func indexByField(informer Informer, field string, extractValue client.IndexerFunc) error {
indexFunc := func(objRaw interface{}) ([]string, error) {
// TODO(directxman12): check if this is the correct type?
obj, isObj := objRaw.(client.Object)
@@ -190,7 +222,7 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc)
}
ns := meta.GetNamespace()
rawVals := extractor(obj)
rawVals := extractValue(obj)
var vals []string
if ns == "" {
// if we're not doubling the keys for the namespaced case, just create a new slice with same length
@@ -213,5 +245,5 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc)
return vals, nil
}
return indexer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc})
return informer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc})
}

View File

@@ -27,9 +27,9 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/internal/field/selector"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/field/selector"
)
// CacheReader is a client.Reader.
@@ -53,7 +53,7 @@ type CacheReader struct {
}
// Get checks the indexer for the object and writes a copy of it if found.
func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error {
func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, _ ...client.GetOption) error {
if c.scopeName == apimeta.RESTScopeNameRoot {
key.Namespace = ""
}
@@ -67,9 +67,9 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob
// Not found, return an error
if !exists {
// Resource gets transformed into Kind in the error anyway, so this is fine
return apierrors.NewNotFound(schema.GroupResource{
Group: c.groupVersionKind.Group,
Group: c.groupVersionKind.Group,
// Resource gets set as Kind in the error so this is fine
Resource: c.groupVersionKind.Kind,
}, key.Name)
}
@@ -111,6 +111,10 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
if listOpts.Continue != "" {
return fmt.Errorf("continue list option is not supported by the cache")
}
switch {
case listOpts.FieldSelector != nil:
// TODO(directxman12): support more complicated field selectors by
@@ -119,8 +123,8 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
if !requiresExact {
return fmt.Errorf("non-exact field matches are not supported by the cache")
}
// list all objects by the field selector. If this is namespaced and we have one, ask for the
// namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces"
// list all objects by the field selector. If this is namespaced and we have one, ask for the
// namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces"
// namespace.
objs, err = c.indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val))
case listOpts.Namespace != "":
@@ -147,7 +151,7 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
}
obj, isObj := item.(runtime.Object)
if !isObj {
return fmt.Errorf("cache contained %T, which is not an Object", obj)
return fmt.Errorf("cache contained %T, which is not an Object", item)
}
meta, err := apimeta.Accessor(obj)
if err != nil {
@@ -175,7 +179,7 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli
}
// objectKeyToStorageKey converts an object key to store key.
// It's akin to MetaNamespaceKeyFunc. It's separate from
// It's akin to MetaNamespaceKeyFunc. It's separate from
// String to allow keeping the key format easily in sync with
// MetaNamespaceKeyFunc.
func objectKeyToStoreKey(k client.ObjectKey) string {
@@ -191,7 +195,7 @@ func FieldIndexName(field string) string {
return "field:" + field
}
// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces.
// allNamespacesNamespace is used as the "namespace" when we want to list across all namespaces.
const allNamespacesNamespace = "__all_namespaces"
// KeyToNamespacedKey prefixes the given index key with a namespace

View File

@@ -1,126 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"context"
"time"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
)
// InformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs.
// It uses a standard parameter codec constructed based on the given generated Scheme.
type InformersMap struct {
// we abstract over the details of structured/unstructured/metadata with the specificInformerMaps
// TODO(directxman12): genericize this over different projections now that we have 3 different maps
structured *specificInformersMap
unstructured *specificInformersMap
metadata *specificInformersMap
// Scheme maps runtime.Objects to GroupVersionKinds
Scheme *runtime.Scheme
}
// NewInformersMap creates a new InformersMap that can create informers for
// both structured and unstructured objects.
func NewInformersMap(config *rest.Config,
scheme *runtime.Scheme,
mapper meta.RESTMapper,
resync time.Duration,
namespace string,
selectors SelectorsByGVK,
disableDeepCopy DisableDeepCopyByGVK,
transformers TransformFuncByObject,
) *InformersMap {
return &InformersMap{
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers),
Scheme: scheme,
}
}
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
func (m *InformersMap) Start(ctx context.Context) error {
go m.structured.Start(ctx)
go m.unstructured.Start(ctx)
go m.metadata.Start(ctx)
<-ctx.Done()
return nil
}
// WaitForCacheSync waits until all the caches have been started and synced.
func (m *InformersMap) WaitForCacheSync(ctx context.Context) bool {
syncedFuncs := append([]cache.InformerSynced(nil), m.structured.HasSyncedFuncs()...)
syncedFuncs = append(syncedFuncs, m.unstructured.HasSyncedFuncs()...)
syncedFuncs = append(syncedFuncs, m.metadata.HasSyncedFuncs()...)
if !m.structured.waitForStarted(ctx) {
return false
}
if !m.unstructured.waitForStarted(ctx) {
return false
}
if !m.metadata.waitForStarted(ctx) {
return false
}
return cache.WaitForCacheSync(ctx.Done(), syncedFuncs...)
}
// Get will create a new Informer and add it to the map of InformersMap if none exists. Returns
// the Informer from the map.
func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
switch obj.(type) {
case *unstructured.Unstructured:
return m.unstructured.Get(ctx, gvk, obj)
case *unstructured.UnstructuredList:
return m.unstructured.Get(ctx, gvk, obj)
case *metav1.PartialObjectMetadata:
return m.metadata.Get(ctx, gvk, obj)
case *metav1.PartialObjectMetadataList:
return m.metadata.Get(ctx, gvk, obj)
default:
return m.structured.Get(ctx, gvk, obj)
}
}
// newStructuredInformersMap creates a new InformersMap for structured objects.
func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createStructuredListWatch)
}
// newUnstructuredInformersMap creates a new InformersMap for unstructured objects.
func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createUnstructuredListWatch)
}
// newMetadataInformersMap creates a new InformersMap for metadata-only objects.
func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK, transformers TransformFuncByObject) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, transformers, createMetadataListWatch)
}

View File

@@ -1,35 +0,0 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import "k8s.io/apimachinery/pkg/runtime/schema"
// GroupVersionKindAll is the argument to represent all GroupVersionKind types.
var GroupVersionKindAll = schema.GroupVersionKind{}
// DisableDeepCopyByGVK associate a GroupVersionKind to disable DeepCopy during get or list from cache.
type DisableDeepCopyByGVK map[schema.GroupVersionKind]bool
// IsDisabled returns whether a GroupVersionKind is disabled DeepCopy.
func (disableByGVK DisableDeepCopyByGVK) IsDisabled(gvk schema.GroupVersionKind) bool {
if d, ok := disableByGVK[gvk]; ok {
return d
} else if d, ok = disableByGVK[GroupVersionKindAll]; ok {
return d
}
return false
}

View File

@@ -0,0 +1,542 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"context"
"fmt"
"math/rand"
"net/http"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// InformersOpts configures an InformerMap.
type InformersOpts struct {
HTTPClient *http.Client
Scheme *runtime.Scheme
Mapper meta.RESTMapper
ResyncPeriod time.Duration
Namespace string
NewInformer *func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer
Selector Selector
Transform cache.TransformFunc
UnsafeDisableDeepCopy bool
}
// NewInformers creates a new InformersMap that can create informers under the hood.
func NewInformers(config *rest.Config, options *InformersOpts) *Informers {
newInformer := cache.NewSharedIndexInformer
if options.NewInformer != nil {
newInformer = *options.NewInformer
}
return &Informers{
config: config,
httpClient: options.HTTPClient,
scheme: options.Scheme,
mapper: options.Mapper,
tracker: tracker{
Structured: make(map[schema.GroupVersionKind]*Cache),
Unstructured: make(map[schema.GroupVersionKind]*Cache),
Metadata: make(map[schema.GroupVersionKind]*Cache),
},
codecs: serializer.NewCodecFactory(options.Scheme),
paramCodec: runtime.NewParameterCodec(options.Scheme),
resync: options.ResyncPeriod,
startWait: make(chan struct{}),
namespace: options.Namespace,
selector: options.Selector,
transform: options.Transform,
unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy,
newInformer: newInformer,
}
}
// Cache contains the cached data for an Cache.
type Cache struct {
// Informer is the cached informer
Informer cache.SharedIndexInformer
// CacheReader wraps Informer and implements the CacheReader interface for a single type
Reader CacheReader
}
type tracker struct {
Structured map[schema.GroupVersionKind]*Cache
Unstructured map[schema.GroupVersionKind]*Cache
Metadata map[schema.GroupVersionKind]*Cache
}
// GetOptions provides configuration to customize the behavior when
// getting an informer.
type GetOptions struct {
// BlockUntilSynced controls if the informer retrieval will block until the informer is synced. Defaults to `true`.
BlockUntilSynced *bool
}
// Informers create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs.
// It uses a standard parameter codec constructed based on the given generated Scheme.
type Informers struct {
// httpClient is used to create a new REST client
httpClient *http.Client
// scheme maps runtime.Objects to GroupVersionKinds
scheme *runtime.Scheme
// config is used to talk to the apiserver
config *rest.Config
// mapper maps GroupVersionKinds to Resources
mapper meta.RESTMapper
// tracker tracks informers keyed by their type and groupVersionKind
tracker tracker
// codecs is used to create a new REST client
codecs serializer.CodecFactory
// paramCodec is used by list and watch
paramCodec runtime.ParameterCodec
// resync is the base frequency the informers are resynced
// a 10 percent jitter will be added to the resync period between informers
// so that all informers will not send list requests simultaneously.
resync time.Duration
// mu guards access to the map
mu sync.RWMutex
// started is true if the informers have been started
started bool
// startWait is a channel that is closed after the
// informer has been started.
startWait chan struct{}
// waitGroup is the wait group that is used to wait for all informers to stop
waitGroup sync.WaitGroup
// stopped is true if the informers have been stopped
stopped bool
// ctx is the context to stop informers
ctx context.Context
// namespace is the namespace that all ListWatches are restricted to
// default or empty string means all namespaces
namespace string
selector Selector
transform cache.TransformFunc
unsafeDisableDeepCopy bool
// NewInformer allows overriding of the shared index informer constructor for testing.
newInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer
}
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
// It doesn't return start because it can't return an error, and it's not a runnable directly.
func (ip *Informers) Start(ctx context.Context) error {
func() {
ip.mu.Lock()
defer ip.mu.Unlock()
// Set the context so it can be passed to informers that are added later
ip.ctx = ctx
// Start each informer
for _, i := range ip.tracker.Structured {
ip.startInformerLocked(i.Informer)
}
for _, i := range ip.tracker.Unstructured {
ip.startInformerLocked(i.Informer)
}
for _, i := range ip.tracker.Metadata {
ip.startInformerLocked(i.Informer)
}
// Set started to true so we immediately start any informers added later.
ip.started = true
close(ip.startWait)
}()
<-ctx.Done() // Block until the context is done
ip.mu.Lock()
ip.stopped = true // Set stopped to true so we don't start any new informers
ip.mu.Unlock()
ip.waitGroup.Wait() // Block until all informers have stopped
return nil
}
func (ip *Informers) startInformerLocked(informer cache.SharedIndexInformer) {
// Don't start the informer in case we are already waiting for the items in
// the waitGroup to finish, since waitGroups don't support waiting and adding
// at the same time.
if ip.stopped {
return
}
ip.waitGroup.Add(1)
go func() {
defer ip.waitGroup.Done()
informer.Run(ip.ctx.Done())
}()
}
func (ip *Informers) waitForStarted(ctx context.Context) bool {
select {
case <-ip.startWait:
return true
case <-ctx.Done():
return false
}
}
// getHasSyncedFuncs returns all the HasSynced functions for the informers in this map.
func (ip *Informers) getHasSyncedFuncs() []cache.InformerSynced {
ip.mu.RLock()
defer ip.mu.RUnlock()
res := make([]cache.InformerSynced, 0,
len(ip.tracker.Structured)+len(ip.tracker.Unstructured)+len(ip.tracker.Metadata),
)
for _, i := range ip.tracker.Structured {
res = append(res, i.Informer.HasSynced)
}
for _, i := range ip.tracker.Unstructured {
res = append(res, i.Informer.HasSynced)
}
for _, i := range ip.tracker.Metadata {
res = append(res, i.Informer.HasSynced)
}
return res
}
// WaitForCacheSync waits until all the caches have been started and synced.
func (ip *Informers) WaitForCacheSync(ctx context.Context) bool {
if !ip.waitForStarted(ctx) {
return false
}
return cache.WaitForCacheSync(ctx.Done(), ip.getHasSyncedFuncs()...)
}
// Peek attempts to get the informer for the GVK, but does not start one if one does not exist.
func (ip *Informers) Peek(gvk schema.GroupVersionKind, obj runtime.Object) (res *Cache, started bool, ok bool) {
ip.mu.RLock()
defer ip.mu.RUnlock()
i, ok := ip.informersByType(obj)[gvk]
return i, ip.started, ok
}
// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns
// the Informer from the map.
func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object, opts *GetOptions) (bool, *Cache, error) {
// Return the informer if it is found
i, started, ok := ip.Peek(gvk, obj)
if !ok {
var err error
if i, started, err = ip.addInformerToMap(gvk, obj); err != nil {
return started, nil, err
}
}
shouldBlock := true
if opts.BlockUntilSynced != nil {
shouldBlock = *opts.BlockUntilSynced
}
if shouldBlock && started && !i.Informer.HasSynced() {
// Wait for it to sync before returning the Informer so that folks don't read from a stale cache.
if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) {
return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0)
}
}
return started, i, nil
}
func (ip *Informers) informersByType(obj runtime.Object) map[schema.GroupVersionKind]*Cache {
switch obj.(type) {
case runtime.Unstructured:
return ip.tracker.Unstructured
case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList:
return ip.tracker.Metadata
default:
return ip.tracker.Structured
}
}
// addInformerToMap either returns an existing informer or creates a new informer, adds it to the map and returns it.
func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*Cache, bool, error) {
ip.mu.Lock()
defer ip.mu.Unlock()
// Check the cache to see if we already have an Informer. If we do, return the Informer.
// This is for the case where 2 routines tried to get the informer when it wasn't in the map
// so neither returned early, but the first one created it.
if i, ok := ip.informersByType(obj)[gvk]; ok {
return i, ip.started, nil
}
// Create a NewSharedIndexInformer and add it to the map.
listWatcher, err := ip.makeListWatcher(gvk, obj)
if err != nil {
return nil, false, err
}
sharedIndexInformer := ip.newInformer(&cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selector.ApplyToList(&opts)
return listWatcher.ListFunc(opts)
},
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selector.ApplyToList(&opts)
opts.Watch = true // Watch needs to be set to true separately
return listWatcher.WatchFunc(opts)
},
}, obj, calculateResyncPeriod(ip.resync), cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
})
// Check to see if there is a transformer for this gvk
if err := sharedIndexInformer.SetTransform(ip.transform); err != nil {
return nil, false, err
}
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, false, err
}
// Create the new entry and set it in the map.
i := &Cache{
Informer: sharedIndexInformer,
Reader: CacheReader{
indexer: sharedIndexInformer.GetIndexer(),
groupVersionKind: gvk,
scopeName: mapping.Scope.Name(),
disableDeepCopy: ip.unsafeDisableDeepCopy,
},
}
ip.informersByType(obj)[gvk] = i
// Start the informer in case the InformersMap has started, otherwise it will be
// started when the InformersMap starts.
if ip.started {
ip.startInformerLocked(i.Informer)
}
return i, ip.started, nil
}
func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Object) (*cache.ListWatch, error) {
// Kubernetes APIs work against Resources, not GroupVersionKinds. Map the
// groupVersionKind to the Resource API we will use.
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
// Figure out if the GVK we're dealing with is global, or namespace scoped.
var namespace string
if mapping.Scope.Name() == meta.RESTScopeNameNamespace {
namespace = restrictNamespaceBySelector(ip.namespace, ip.selector)
}
switch obj.(type) {
//
// Unstructured
//
case runtime.Unstructured:
// If the rest configuration has a negotiated serializer passed in,
// we should remove it and use the one that the dynamic client sets for us.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
dynamicClient, err := dynamic.NewForConfigAndClient(cfg, ip.httpClient)
if err != nil {
return nil, err
}
resources := dynamicClient.Resource(mapping.Resource)
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
if namespace != "" {
return resources.Namespace(namespace).List(ip.ctx, opts)
}
return resources.List(ip.ctx, opts)
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
if namespace != "" {
return resources.Namespace(namespace).Watch(ip.ctx, opts)
}
return resources.Watch(ip.ctx, opts)
},
}, nil
//
// Metadata
//
case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList:
// Always clear the negotiated serializer and use the one
// set from the metadata client.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
// Grab the metadata metadataClient.
metadataClient, err := metadata.NewForConfigAndClient(cfg, ip.httpClient)
if err != nil {
return nil, err
}
resources := metadataClient.Resource(mapping.Resource)
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
var (
list *metav1.PartialObjectMetadataList
err error
)
if namespace != "" {
list, err = resources.Namespace(namespace).List(ip.ctx, opts)
} else {
list, err = resources.List(ip.ctx, opts)
}
if list != nil {
for i := range list.Items {
list.Items[i].SetGroupVersionKind(gvk)
}
}
return list, err
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watcher watch.Interface, err error) {
if namespace != "" {
watcher, err = resources.Namespace(namespace).Watch(ip.ctx, opts)
} else {
watcher, err = resources.Watch(ip.ctx, opts)
}
if err != nil {
return nil, err
}
return newGVKFixupWatcher(gvk, watcher), nil
},
}, nil
//
// Structured.
//
default:
client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs, ip.httpClient)
if err != nil {
return nil, err
}
listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List")
listObj, err := ip.scheme.New(listGVK)
if err != nil {
return nil, err
}
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
// Build the request.
req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec)
if namespace != "" {
req.Namespace(namespace)
}
// Create the resulting object, and execute the request.
res := listObj.DeepCopyObject()
if err := req.Do(ip.ctx).Into(res); err != nil {
return nil, err
}
return res, nil
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
// Build the request.
req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec)
if namespace != "" {
req.Namespace(namespace)
}
// Call the watch.
return req.Watch(ip.ctx)
},
}, nil
}
}
// newGVKFixupWatcher adds a wrapper that preserves the GVK information when
// events come in.
//
// This works around a bug where GVK information is not passed into mapping
// functions when using the OnlyMetadata option in the builder.
// This issue is most likely caused by kubernetes/kubernetes#80609.
// See kubernetes-sigs/controller-runtime#1484.
//
// This was originally implemented as a cache.ResourceEventHandler wrapper but
// that contained a data race which was resolved by setting the GVK in a watch
// wrapper, before the objects are written to the cache.
// See kubernetes-sigs/controller-runtime#1650.
//
// The original watch wrapper was found to be incompatible with
// k8s.io/client-go/tools/cache.Reflector so it has been re-implemented as a
// watch.Filter which is compatible.
// See kubernetes-sigs/controller-runtime#1789.
func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface {
return watch.Filter(
watcher,
func(in watch.Event) (watch.Event, bool) {
in.Object.GetObjectKind().SetGroupVersionKind(gvk)
return in, true
},
)
}
// calculateResyncPeriod returns a duration based on the desired input
// this is so that multiple controllers don't get into lock-step and all
// hammer the apiserver with list requests simultaneously.
func calculateResyncPeriod(resync time.Duration) time.Duration {
// the factor will fall into [0.9, 1.1)
factor := rand.Float64()/5.0 + 0.9 //nolint:gosec
return time.Duration(float64(resync.Nanoseconds()) * factor)
}
// restrictNamespaceBySelector returns either a global restriction for all ListWatches
// if not default/empty, or the namespace that a ListWatch for the specific resource
// is restricted to, based on a specified field selector for metadata.namespace field.
func restrictNamespaceBySelector(namespaceOpt string, s Selector) string {
if namespaceOpt != "" {
// namespace is already restricted
return namespaceOpt
}
fieldSelector := s.Field
if fieldSelector == nil || fieldSelector.Empty() {
return ""
}
// check whether a selector includes the namespace field
value, found := fieldSelector.RequiresExactMatch("metadata.namespace")
if found {
return value
}
return ""
}

View File

@@ -1,480 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"context"
"fmt"
"math/rand"
"sync"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
func init() {
rand.Seed(time.Now().UnixNano())
}
// clientListWatcherFunc knows how to create a ListWatcher.
type createListWatcherFunc func(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error)
// newSpecificInformersMap returns a new specificInformersMap (like
// the generical InformersMap, except that it doesn't implement WaitForCacheSync).
func newSpecificInformersMap(config *rest.Config,
scheme *runtime.Scheme,
mapper meta.RESTMapper,
resync time.Duration,
namespace string,
selectors SelectorsByGVK,
disableDeepCopy DisableDeepCopyByGVK,
transformers TransformFuncByObject,
createListWatcher createListWatcherFunc,
) *specificInformersMap {
ip := &specificInformersMap{
config: config,
Scheme: scheme,
mapper: mapper,
informersByGVK: make(map[schema.GroupVersionKind]*MapEntry),
codecs: serializer.NewCodecFactory(scheme),
paramCodec: runtime.NewParameterCodec(scheme),
resync: resync,
startWait: make(chan struct{}),
createListWatcher: createListWatcher,
namespace: namespace,
selectors: selectors.forGVK,
disableDeepCopy: disableDeepCopy,
transformers: transformers,
}
return ip
}
// MapEntry contains the cached data for an Informer.
type MapEntry struct {
// Informer is the cached informer
Informer cache.SharedIndexInformer
// CacheReader wraps Informer and implements the CacheReader interface for a single type
Reader CacheReader
}
// specificInformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs.
// It uses a standard parameter codec constructed based on the given generated Scheme.
type specificInformersMap struct {
// Scheme maps runtime.Objects to GroupVersionKinds
Scheme *runtime.Scheme
// config is used to talk to the apiserver
config *rest.Config
// mapper maps GroupVersionKinds to Resources
mapper meta.RESTMapper
// informersByGVK is the cache of informers keyed by groupVersionKind
informersByGVK map[schema.GroupVersionKind]*MapEntry
// codecs is used to create a new REST client
codecs serializer.CodecFactory
// paramCodec is used by list and watch
paramCodec runtime.ParameterCodec
// stop is the stop channel to stop informers
stop <-chan struct{}
// resync is the base frequency the informers are resynced
// a 10 percent jitter will be added to the resync period between informers
// so that all informers will not send list requests simultaneously.
resync time.Duration
// mu guards access to the map
mu sync.RWMutex
// start is true if the informers have been started
started bool
// startWait is a channel that is closed after the
// informer has been started.
startWait chan struct{}
// createClient knows how to create a client and a list object,
// and allows for abstracting over the particulars of structured vs
// unstructured objects.
createListWatcher createListWatcherFunc
// namespace is the namespace that all ListWatches are restricted to
// default or empty string means all namespaces
namespace string
// selectors are the label or field selectors that will be added to the
// ListWatch ListOptions.
selectors func(gvk schema.GroupVersionKind) Selector
// disableDeepCopy indicates not to deep copy objects during get or list objects.
disableDeepCopy DisableDeepCopyByGVK
// transform funcs are applied to objects before they are committed to the cache
transformers TransformFuncByObject
}
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
// It doesn't return start because it can't return an error, and it's not a runnable directly.
func (ip *specificInformersMap) Start(ctx context.Context) {
func() {
ip.mu.Lock()
defer ip.mu.Unlock()
// Set the stop channel so it can be passed to informers that are added later
ip.stop = ctx.Done()
// Start each informer
for _, informer := range ip.informersByGVK {
go informer.Informer.Run(ctx.Done())
}
// Set started to true so we immediately start any informers added later.
ip.started = true
close(ip.startWait)
}()
<-ctx.Done()
}
func (ip *specificInformersMap) waitForStarted(ctx context.Context) bool {
select {
case <-ip.startWait:
return true
case <-ctx.Done():
return false
}
}
// HasSyncedFuncs returns all the HasSynced functions for the informers in this map.
func (ip *specificInformersMap) HasSyncedFuncs() []cache.InformerSynced {
ip.mu.RLock()
defer ip.mu.RUnlock()
syncedFuncs := make([]cache.InformerSynced, 0, len(ip.informersByGVK))
for _, informer := range ip.informersByGVK {
syncedFuncs = append(syncedFuncs, informer.Informer.HasSynced)
}
return syncedFuncs
}
// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns
// the Informer from the map.
func (ip *specificInformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) {
// Return the informer if it is found
i, started, ok := func() (*MapEntry, bool, bool) {
ip.mu.RLock()
defer ip.mu.RUnlock()
i, ok := ip.informersByGVK[gvk]
return i, ip.started, ok
}()
if !ok {
var err error
if i, started, err = ip.addInformerToMap(gvk, obj); err != nil {
return started, nil, err
}
}
if started && !i.Informer.HasSynced() {
// Wait for it to sync before returning the Informer so that folks don't read from a stale cache.
if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) {
return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0)
}
}
return started, i, nil
}
func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*MapEntry, bool, error) {
ip.mu.Lock()
defer ip.mu.Unlock()
// Check the cache to see if we already have an Informer. If we do, return the Informer.
// This is for the case where 2 routines tried to get the informer when it wasn't in the map
// so neither returned early, but the first one created it.
if i, ok := ip.informersByGVK[gvk]; ok {
return i, ip.started, nil
}
// Create a NewSharedIndexInformer and add it to the map.
var lw *cache.ListWatch
lw, err := ip.createListWatcher(gvk, ip)
if err != nil {
return nil, false, err
}
ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
})
// Check to see if there is a transformer for this gvk
if err := ni.SetTransform(ip.transformers.Get(gvk)); err != nil {
return nil, false, err
}
rm, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, false, err
}
i := &MapEntry{
Informer: ni,
Reader: CacheReader{
indexer: ni.GetIndexer(),
groupVersionKind: gvk,
scopeName: rm.Scope.Name(),
disableDeepCopy: ip.disableDeepCopy.IsDisabled(gvk),
},
}
ip.informersByGVK[gvk] = i
// Start the Informer if need by
// TODO(seans): write thorough tests and document what happens here - can you add indexers?
// can you add eventhandlers?
if ip.started {
go i.Informer.Run(ip.stop)
}
return i, ip.started, nil
}
// newListWatch returns a new ListWatch object that can be used to create a SharedIndexInformer.
func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) {
// Kubernetes APIs work against Resources, not GroupVersionKinds. Map the
// groupVersionKind to the Resource API we will use.
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs)
if err != nil {
return nil, err
}
listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List")
listObj, err := ip.Scheme.New(listGVK)
if err != nil {
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors(gvk).ApplyToList(&opts)
res := listObj.DeepCopyObject()
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
err := client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res)
return res, err
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
return client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch(ctx)
},
}, nil
}
func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) {
// Kubernetes APIs work against Resources, not GroupVersionKinds. Map the
// groupVersionKind to the Resource API we will use.
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
// If the rest configuration has a negotiated serializer passed in,
// we should remove it and use the one that the dynamic client sets for us.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
dynamicClient, err := dynamic.NewForConfig(cfg)
if err != nil {
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors(gvk).ApplyToList(&opts)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts)
}
return dynamicClient.Resource(mapping.Resource).List(ctx, opts)
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts)
}
return dynamicClient.Resource(mapping.Resource).Watch(ctx, opts)
},
}, nil
}
func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) {
// Kubernetes APIs work against Resources, not GroupVersionKinds. Map the
// groupVersionKind to the Resource API we will use.
mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, err
}
// Always clear the negotiated serializer and use the one
// set from the metadata client.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
// grab the metadata client
client, err := metadata.NewForConfig(cfg)
if err != nil {
return nil, err
}
// TODO: the functions that make use of this ListWatch should be adapted to
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// create the relevant listwatch
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors(gvk).ApplyToList(&opts)
var (
list *metav1.PartialObjectMetadataList
err error
)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
list, err = client.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts)
} else {
list, err = client.Resource(mapping.Resource).List(ctx, opts)
}
if list != nil {
for i := range list.Items {
list.Items[i].SetGroupVersionKind(gvk)
}
}
return list, err
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors(gvk).ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
var (
watcher watch.Interface
err error
)
namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors(gvk))
if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
watcher, err = client.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts)
} else {
watcher, err = client.Resource(mapping.Resource).Watch(ctx, opts)
}
if watcher != nil {
watcher = newGVKFixupWatcher(gvk, watcher)
}
return watcher, err
},
}, nil
}
// newGVKFixupWatcher adds a wrapper that preserves the GVK information when
// events come in.
//
// This works around a bug where GVK information is not passed into mapping
// functions when using the OnlyMetadata option in the builder.
// This issue is most likely caused by kubernetes/kubernetes#80609.
// See kubernetes-sigs/controller-runtime#1484.
//
// This was originally implemented as a cache.ResourceEventHandler wrapper but
// that contained a data race which was resolved by setting the GVK in a watch
// wrapper, before the objects are written to the cache.
// See kubernetes-sigs/controller-runtime#1650.
//
// The original watch wrapper was found to be incompatible with
// k8s.io/client-go/tools/cache.Reflector so it has been re-implemented as a
// watch.Filter which is compatible.
// See kubernetes-sigs/controller-runtime#1789.
func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface {
return watch.Filter(
watcher,
func(in watch.Event) (watch.Event, bool) {
in.Object.GetObjectKind().SetGroupVersionKind(gvk)
return in, true
},
)
}
// resyncPeriod returns a function which generates a duration each time it is
// invoked; this is so that multiple controllers don't get into lock-step and all
// hammer the apiserver with list requests simultaneously.
func resyncPeriod(resync time.Duration) func() time.Duration {
return func() time.Duration {
// the factor will fall into [0.9, 1.1)
factor := rand.Float64()/5.0 + 0.9 //nolint:gosec
return time.Duration(float64(resync.Nanoseconds()) * factor)
}
}
// restrictNamespaceBySelector returns either a global restriction for all ListWatches
// if not default/empty, or the namespace that a ListWatch for the specific resource
// is restricted to, based on a specified field selector for metadata.namespace field.
func restrictNamespaceBySelector(namespaceOpt string, s Selector) string {
if namespaceOpt != "" {
// namespace is already restricted
return namespaceOpt
}
fieldSelector := s.Field
if fieldSelector == nil || fieldSelector.Empty() {
return ""
}
// check whether a selector includes the namespace field
value, found := fieldSelector.RequiresExactMatch("metadata.namespace")
if found {
return value
}
return ""
}

View File

@@ -20,23 +20,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// SelectorsByGVK associate a GroupVersionKind to a field/label selector.
type SelectorsByGVK map[schema.GroupVersionKind]Selector
func (s SelectorsByGVK) forGVK(gvk schema.GroupVersionKind) Selector {
if specific, found := s[gvk]; found {
return specific
}
if defaultSelector, found := s[schema.GroupVersionKind{}]; found {
return defaultSelector
}
return Selector{}
}
// Selector specify the label/field selector to fill in ListOptions.
type Selector struct {
Label labels.Selector

View File

@@ -1,55 +0,0 @@
package internal
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// TransformFuncByObject provides access to the correct transform function for
// any given GVK.
type TransformFuncByObject interface {
Set(runtime.Object, *runtime.Scheme, cache.TransformFunc) error
Get(schema.GroupVersionKind) cache.TransformFunc
SetDefault(transformer cache.TransformFunc)
}
type transformFuncByGVK struct {
defaultTransform cache.TransformFunc
transformers map[schema.GroupVersionKind]cache.TransformFunc
}
// TransformFuncByObjectFromMap creates a TransformFuncByObject from a map that
// maps GVKs to TransformFuncs.
func TransformFuncByObjectFromMap(in map[schema.GroupVersionKind]cache.TransformFunc) TransformFuncByObject {
byGVK := &transformFuncByGVK{}
if defaultFunc, hasDefault := in[schema.GroupVersionKind{}]; hasDefault {
byGVK.defaultTransform = defaultFunc
}
delete(in, schema.GroupVersionKind{})
byGVK.transformers = in
return byGVK
}
func (t *transformFuncByGVK) SetDefault(transformer cache.TransformFunc) {
t.defaultTransform = transformer
}
func (t *transformFuncByGVK) Set(obj runtime.Object, scheme *runtime.Scheme, transformer cache.TransformFunc) error {
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return err
}
t.transformers[gvk] = transformer
return nil
}
func (t transformFuncByGVK) Get(gvk schema.GroupVersionKind) cache.TransformFunc {
if val, ok := t.transformers[gvk]; ok {
return val
}
return t.defaultTransform
}

View File

@@ -23,50 +23,42 @@ import (
corev1 "k8s.io/api/core/v1"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
toolscache "k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
)
// NewCacheFunc - Function for creating a new cache from the options and a rest config.
type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// a new global namespaced cache to handle cluster scoped resources.
const globalCache = "_cluster-scope"
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
// This will scope the cache to a list of namespaces. Listing for all namespaces
// will list for all the namespaces that this knows about. By default this will create
// a global cache for cluster scoped resource. Note that this is not intended
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
// you may face performance issues when using this with a high number of namespaces.
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
return func(config *rest.Config, opts Options) (Cache, error) {
opts, err := defaultOpts(config, opts)
if err != nil {
return nil, err
}
func newMultiNamespaceCache(
newCache newCacheFunc,
scheme *runtime.Scheme,
restMapper apimeta.RESTMapper,
namespaces map[string]Config,
globalConfig *Config, // may be nil in which case no cache for cluster-scoped objects will be created
) Cache {
// Create every namespace cache.
caches := map[string]Cache{}
for namespace, config := range namespaces {
caches[namespace] = newCache(config, namespace)
}
caches := map[string]Cache{}
// Create a cache for cluster scoped resources if requested
var clusterCache Cache
if globalConfig != nil {
clusterCache = newCache(*globalConfig, corev1.NamespaceAll)
}
// create a cache for cluster scoped resources
gCache, err := New(config, opts)
if err != nil {
return nil, fmt.Errorf("error creating global cache: %w", err)
}
for _, ns := range namespaces {
opts.Namespace = ns
c, err := New(config, opts)
if err != nil {
return nil, err
}
caches[ns] = c
}
return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil
return &multiNamespaceCache{
namespaceToCache: caches,
Scheme: scheme,
RESTMapper: restMapper,
clusterCache: clusterCache,
}
}
@@ -75,90 +67,96 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
// operator to a list of namespaces instead of watching every namespace
// in the cluster.
type multiNamespaceCache struct {
namespaceToCache map[string]Cache
Scheme *runtime.Scheme
RESTMapper apimeta.RESTMapper
namespaceToCache map[string]Cache
clusterCache Cache
}
var _ Cache = &multiNamespaceCache{}
// Methods for multiNamespaceCache to conform to the Informers interface.
func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) {
informers := map[string]Informer{}
// If the object is clusterscoped, get the informer from clusterCache,
func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) {
// If the object is cluster scoped, get the informer from clusterCache,
// if not use the namespaced caches.
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return nil, err
}
if !isNamespaced {
clusterCacheInf, err := c.clusterCache.GetInformer(ctx, obj)
clusterCacheInformer, err := c.clusterCache.GetInformer(ctx, obj, opts...)
if err != nil {
return nil, err
}
informers[globalCache] = clusterCacheInf
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
return &multiNamespaceInformer{
namespaceToInformer: map[string]Informer{
globalCache: clusterCacheInformer,
},
}, nil
}
namespaceToInformer := map[string]Informer{}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformer(ctx, obj)
informer, err := cache.GetInformer(ctx, obj, opts...)
if err != nil {
return nil, err
}
informers[ns] = informer
namespaceToInformer[ns] = informer
}
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil
}
func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
informers := map[string]Informer{}
// If the object is clusterscoped, get the informer from clusterCache,
func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) {
// If the object is cluster scoped, get the informer from clusterCache,
// if not use the namespaced caches.
isNamespaced, err := objectutil.IsAPINamespacedWithGVK(gvk, c.Scheme, c.RESTMapper)
isNamespaced, err := apiutil.IsGVKNamespaced(gvk, c.RESTMapper)
if err != nil {
return nil, err
}
if !isNamespaced {
clusterCacheInf, err := c.clusterCache.GetInformerForKind(ctx, gvk)
clusterCacheInformer, err := c.clusterCache.GetInformerForKind(ctx, gvk, opts...)
if err != nil {
return nil, err
}
informers[globalCache] = clusterCacheInf
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
return &multiNamespaceInformer{
namespaceToInformer: map[string]Informer{
globalCache: clusterCacheInformer,
},
}, nil
}
namespaceToInformer := map[string]Informer{}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformerForKind(ctx, gvk)
informer, err := cache.GetInformerForKind(ctx, gvk, opts...)
if err != nil {
return nil, err
}
informers[ns] = informer
namespaceToInformer[ns] = informer
}
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil
}
func (c *multiNamespaceCache) Start(ctx context.Context) error {
// start global cache
go func() {
err := c.clusterCache.Start(ctx)
if err != nil {
log.Error(err, "cluster scoped cache failed to start")
}
}()
if c.clusterCache != nil {
go func() {
err := c.clusterCache.Start(ctx)
if err != nil {
log.Error(err, "cluster scoped cache failed to start")
}
}()
}
// start namespaced caches
for ns, cache := range c.namespaceToCache {
go func(ns string, cache Cache) {
err := cache.Start(ctx)
if err != nil {
log.Error(err, "multinamespace cache failed to start namespaced informer", "namespace", ns)
if err := cache.Start(ctx); err != nil {
log.Error(err, "multi-namespace cache failed to start namespaced informer", "namespace", ns)
}
}(ns, cache)
}
@@ -170,22 +168,22 @@ func (c *multiNamespaceCache) Start(ctx context.Context) error {
func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool {
synced := true
for _, cache := range c.namespaceToCache {
if s := cache.WaitForCacheSync(ctx); !s {
synced = s
if !cache.WaitForCacheSync(ctx) {
synced = false
}
}
// check if cluster scoped cache has synced
if !c.clusterCache.WaitForCacheSync(ctx) {
if c.clusterCache != nil && !c.clusterCache.WaitForCacheSync(ctx) {
synced = false
}
return synced
}
func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return nil //nolint:nilerr
return err
}
if !isNamespaced {
@@ -201,7 +199,7 @@ func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object,
}
func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return err
}
@@ -213,9 +211,12 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj
cache, ok := c.namespaceToCache[key.Namespace]
if !ok {
if global, hasGlobal := c.namespaceToCache[metav1.NamespaceAll]; hasGlobal {
return global.Get(ctx, key, obj, opts...)
}
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key)
}
return cache.Get(ctx, key, obj)
return cache.Get(ctx, key, obj, opts...)
}
// List multi namespace cache will get all the objects in the namespaces that the cache is watching if asked for all namespaces.
@@ -223,7 +224,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
isNamespaced, err := objectutil.IsAPINamespaced(list, c.Scheme, c.RESTMapper)
isNamespaced, err := apiutil.IsObjectNamespaced(list, c.Scheme, c.RESTMapper)
if err != nil {
return err
}
@@ -236,7 +237,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
if listOpts.Namespace != corev1.NamespaceAll {
cache, ok := c.namespaceToCache[listOpts.Namespace]
if !ok {
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace)
return fmt.Errorf("unable to list: %v because of unknown namespace for the cache", listOpts.Namespace)
}
return cache.List(ctx, list, opts...)
}
@@ -269,12 +270,14 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
return fmt.Errorf("object: %T must be a list type", list)
}
allItems = append(allItems, items...)
// The last list call should have the most correct resource version.
resourceVersion = accessor.GetResourceVersion()
if limitSet {
// decrement Limit by the number of items
// fetched from the current namespace.
listOpts.Limit -= int64(len(items))
// if a Limit was set and the number of
// items read has reached this set limit,
// then stop reading.
@@ -293,42 +296,71 @@ type multiNamespaceInformer struct {
namespaceToInformer map[string]Informer
}
type handlerRegistration struct {
handles map[string]toolscache.ResourceEventHandlerRegistration
}
type syncer interface {
HasSynced() bool
}
// HasSynced asserts that the handler has been called for the full initial state of the informer.
// This uses syncer to be compatible between client-go 1.27+ and older versions when the interface changed.
func (h handlerRegistration) HasSynced() bool {
for _, reg := range h.handles {
if s, ok := reg.(syncer); ok {
if !s.HasSynced() {
return false
}
}
}
return true
}
var _ Informer = &multiNamespaceInformer{}
// AddEventHandler adds the handler to each namespaced informer.
// AddEventHandler adds the handler to each informer.
func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) {
handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))
handles := handlerRegistration{
handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)),
}
for ns, informer := range i.namespaceToInformer {
registration, err := informer.AddEventHandler(handler)
if err != nil {
return nil, err
}
handles[ns] = registration
handles.handles[ns] = registration
}
return handles, nil
}
// AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer.
func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) {
handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))
handles := handlerRegistration{
handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)),
}
for ns, informer := range i.namespaceToInformer {
registration, err := informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod)
if err != nil {
return nil, err
}
handles[ns] = registration
handles.handles[ns] = registration
}
return handles, nil
}
// RemoveEventHandler removes a formerly added event handler given by its registration handle.
// RemoveEventHandler removes a previously added event handler given by its registration handle.
func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHandlerRegistration) error {
handles, ok := h.(map[string]toolscache.ResourceEventHandlerRegistration)
handles, ok := h.(handlerRegistration)
if !ok {
return fmt.Errorf("it is not the registration returned by multiNamespaceInformer")
return fmt.Errorf("registration is not a registration returned by multiNamespaceInformer")
}
for ns, informer := range i.namespaceToInformer {
registration, ok := handles[ns]
registration, ok := handles.handles[ns]
if !ok {
continue
}
@@ -339,7 +371,7 @@ func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHa
return nil
}
// AddIndexers adds the indexer for each namespaced informer.
// AddIndexers adds the indexers to each informer.
func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error {
for _, informer := range i.namespaceToInformer {
err := informer.AddIndexers(indexers)
@@ -350,11 +382,11 @@ func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error
return nil
}
// HasSynced checks if each namespaced informer has synced.
// HasSynced checks if each informer has synced.
func (i *multiNamespaceInformer) HasSynced() bool {
for _, informer := range i.namespaceToInformer {
if ok := informer.HasSynced(); !ok {
return ok
if !informer.HasSynced() {
return false
}
}
return true

View File

@@ -19,9 +19,14 @@ package certwatcher
import (
"context"
"crypto/tls"
"fmt"
"sync"
"time"
"github.com/fsnotify/fsnotify"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -39,6 +44,9 @@ type CertWatcher struct {
certPath string
keyPath string
// callback is a function to be invoked when the certificate changes.
callback func(tls.Certificate)
}
// New returns a new CertWatcher watching the given certificate and key.
@@ -63,6 +71,17 @@ func New(certPath, keyPath string) (*CertWatcher, error) {
return cw, nil
}
// RegisterCallback registers a callback to be invoked when the certificate changes.
func (cw *CertWatcher) RegisterCallback(callback func(tls.Certificate)) {
cw.Lock()
defer cw.Unlock()
// If the current certificate is not nil, invoke the callback immediately.
if cw.currentCert != nil {
callback(*cw.currentCert)
}
cw.callback = callback
}
// GetCertificate fetches the currently loaded certificate, which may be nil.
func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
cw.RLock()
@@ -72,11 +91,22 @@ func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate,
// Start starts the watch on the certificate and key files.
func (cw *CertWatcher) Start(ctx context.Context) error {
files := []string{cw.certPath, cw.keyPath}
files := sets.New(cw.certPath, cw.keyPath)
for _, f := range files {
if err := cw.watcher.Add(f); err != nil {
return err
{
var watchErr error
if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, true, func(ctx context.Context) (done bool, err error) {
for _, f := range files.UnsortedList() {
if err := cw.watcher.Add(f); err != nil {
watchErr = err
return false, nil //nolint:nilerr // We want to keep trying.
}
// We've added the watch, remove it from the set.
files.Delete(f)
}
return true, nil
}); err != nil {
return fmt.Errorf("failed to add watches: %w", kerrors.NewAggregate([]error{err, watchErr}))
}
}
@@ -130,6 +160,14 @@ func (cw *CertWatcher) ReadCertificate() error {
log.Info("Updated current TLS certificate")
// If a callback is registered, invoke it with the new certificate.
cw.RLock()
defer cw.RUnlock()
if cw.callback != nil {
go func() {
cw.callback(cert)
}()
}
return nil
}
@@ -154,13 +192,13 @@ func (cw *CertWatcher) handleEvent(event fsnotify.Event) {
}
func isWrite(event fsnotify.Event) bool {
return event.Op&fsnotify.Write == fsnotify.Write
return event.Op.Has(fsnotify.Write)
}
func isCreate(event fsnotify.Event) bool {
return event.Op&fsnotify.Create == fsnotify.Create
return event.Op.Has(fsnotify.Create)
}
func isRemove(event fsnotify.Event) bool {
return event.Op&fsnotify.Remove == fsnotify.Remove
return event.Op.Has(fsnotify.Remove)
}

View File

@@ -20,7 +20,9 @@ limitations under the License.
package apiutil
import (
"errors"
"fmt"
"net/http"
"reflect"
"sync"
@@ -30,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
@@ -59,9 +62,13 @@ func AddToProtobufScheme(addToScheme func(*runtime.Scheme) error) error {
// NewDiscoveryRESTMapper constructs a new RESTMapper based on discovery
// information fetched by a new client with the given config.
func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) {
func NewDiscoveryRESTMapper(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) {
if httpClient == nil {
return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client")
}
// Get a mapper
dc, err := discovery.NewDiscoveryClientForConfig(c)
dc, err := discovery.NewDiscoveryClientForConfigAndClient(c, httpClient)
if err != nil {
return nil, err
}
@@ -72,6 +79,36 @@ func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) {
return restmapper.NewDiscoveryRESTMapper(gr), nil
}
// IsObjectNamespaced returns true if the object is namespace scoped.
// For unstructured objects the gvk is found from the object itself.
func IsObjectNamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper meta.RESTMapper) (bool, error) {
gvk, err := GVKForObject(obj, scheme)
if err != nil {
return false, err
}
return IsGVKNamespaced(gvk, restmapper)
}
// IsGVKNamespaced returns true if the object having the provided
// GVK is namespace scoped.
func IsGVKNamespaced(gvk schema.GroupVersionKind, restmapper meta.RESTMapper) (bool, error) {
restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind})
if err != nil {
return false, fmt.Errorf("failed to get restmapping: %w", err)
}
scope := restmapping.Scope.Name()
if scope == "" {
return false, errors.New("scope cannot be identified, empty scope returned")
}
if scope != meta.RESTScopeNameRoot {
return true, nil
}
return false, nil
}
// GVKForObject finds the GroupVersionKind associated with the given object, if there is only a single such GVK.
func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionKind, error) {
// TODO(directxman12): do we want to generalize this to arbitrary container types?
@@ -142,21 +179,11 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi
// RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated
// with the given GroupVersionKind. The REST client will be configured to use the negotiated serializer from
// baseConfig, if set, otherwise a default serializer will be set.
func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) (rest.Interface, error) {
return rest.RESTClientFor(createRestConfig(gvk, isUnstructured, baseConfig, codecs))
}
// serializerWithDecodedGVK is a CodecFactory that overrides the DecoderToVersion of a WithoutConversionCodecFactory
// in order to avoid clearing the GVK from the decoded object.
//
// See https://github.com/kubernetes/kubernetes/issues/80609.
type serializerWithDecodedGVK struct {
serializer.WithoutConversionCodecFactory
}
// DecoderToVersion returns an decoder that does not do conversion.
func (f serializerWithDecodedGVK) DecoderToVersion(serializer runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder {
return serializer
func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory, httpClient *http.Client) (rest.Interface, error) {
if httpClient == nil {
return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client")
}
return rest.RESTClientForConfigAndClient(createRestConfig(gvk, isUnstructured, baseConfig, codecs), httpClient)
}
// createRestConfig copies the base config and updates needed fields for a new rest config.
@@ -183,9 +210,8 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf
}
if isUnstructured {
// If the object is unstructured, we need to preserve the GVK information.
// Use our own custom serializer.
cfg.NegotiatedSerializer = serializerWithDecodedGVK{serializer.WithoutConversionCodecFactory{CodecFactory: codecs}}
// If the object is unstructured, we use the client-go dynamic serializer.
cfg = dynamic.ConfigFor(cfg)
} else {
cfg.NegotiatedSerializer = serializerWithTargetZeroingDecode{NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: codecs}}
}

View File

@@ -1,301 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apiutil
import (
"sync"
"sync/atomic"
"golang.org/x/time/rate"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
)
// dynamicRESTMapper is a RESTMapper that dynamically discovers resource
// types at runtime.
type dynamicRESTMapper struct {
mu sync.RWMutex // protects the following fields
staticMapper meta.RESTMapper
limiter *rate.Limiter
newMapper func() (meta.RESTMapper, error)
lazy bool
// Used for lazy init.
inited uint32
initMtx sync.Mutex
useLazyRestmapper bool
}
// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper.
type DynamicRESTMapperOption func(*dynamicRESTMapper) error
// WithLimiter sets the RESTMapper's underlying limiter to lim.
func WithLimiter(lim *rate.Limiter) DynamicRESTMapperOption {
return func(drm *dynamicRESTMapper) error {
drm.limiter = lim
return nil
}
}
// WithLazyDiscovery prevents the RESTMapper from discovering REST mappings
// until an API call is made.
var WithLazyDiscovery DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error {
drm.lazy = true
return nil
}
// WithExperimentalLazyMapper enables experimental more advanced Lazy Restmapping mechanism.
var WithExperimentalLazyMapper DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error {
drm.useLazyRestmapper = true
return nil
}
// WithCustomMapper supports setting a custom RESTMapper refresher instead of
// the default method, which uses a discovery client.
//
// This exists mainly for testing, but can be useful if you need tighter control
// over how discovery is performed, which discovery endpoints are queried, etc.
func WithCustomMapper(newMapper func() (meta.RESTMapper, error)) DynamicRESTMapperOption {
return func(drm *dynamicRESTMapper) error {
drm.newMapper = newMapper
return nil
}
}
// NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic
// RESTMapper dynamically discovers resource types at runtime. opts
// configure the RESTMapper.
func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (meta.RESTMapper, error) {
client, err := discovery.NewDiscoveryClientForConfig(cfg)
if err != nil {
return nil, err
}
drm := &dynamicRESTMapper{
limiter: rate.NewLimiter(rate.Limit(defaultRefillRate), defaultLimitSize),
newMapper: func() (meta.RESTMapper, error) {
groupResources, err := restmapper.GetAPIGroupResources(client)
if err != nil {
return nil, err
}
return restmapper.NewDiscoveryRESTMapper(groupResources), nil
},
}
for _, opt := range opts {
if err = opt(drm); err != nil {
return nil, err
}
}
if drm.useLazyRestmapper {
return newLazyRESTMapperWithClient(client)
}
if !drm.lazy {
if err := drm.setStaticMapper(); err != nil {
return nil, err
}
}
return drm, nil
}
var (
// defaultRefilRate is the default rate at which potential calls are
// added back to the "bucket" of allowed calls.
defaultRefillRate = 5
// defaultLimitSize is the default starting/max number of potential calls
// per second. Once a call is used, it's added back to the bucket at a rate
// of defaultRefillRate per second.
defaultLimitSize = 5
)
// setStaticMapper sets drm's staticMapper by querying its client, regardless
// of reload backoff.
func (drm *dynamicRESTMapper) setStaticMapper() error {
newMapper, err := drm.newMapper()
if err != nil {
return err
}
drm.staticMapper = newMapper
return nil
}
// init initializes drm only once if drm is lazy.
func (drm *dynamicRESTMapper) init() (err error) {
// skip init if drm is not lazy or has initialized
if !drm.lazy || atomic.LoadUint32(&drm.inited) != 0 {
return nil
}
drm.initMtx.Lock()
defer drm.initMtx.Unlock()
if drm.inited == 0 {
if err = drm.setStaticMapper(); err == nil {
atomic.StoreUint32(&drm.inited, 1)
}
}
return err
}
// checkAndReload attempts to call the given callback, which is assumed to be dependent
// on the data in the restmapper.
//
// If the callback returns an error matching meta.IsNoMatchErr, it will attempt to reload
// the RESTMapper's data and re-call the callback once that's occurred.
// If the callback returns any other error, the function will return immediately regardless.
//
// It will take care of ensuring that reloads are rate-limited and that extraneous calls
// aren't made. If a reload would exceed the limiters rate, it returns the error return by
// the callback.
// It's thread-safe, and worries about thread-safety for the callback (so the callback does
// not need to attempt to lock the restmapper).
func (drm *dynamicRESTMapper) checkAndReload(checkNeedsReload func() error) error {
// first, check the common path -- data is fresh enough
// (use an IIFE for the lock's defer)
err := func() error {
drm.mu.RLock()
defer drm.mu.RUnlock()
return checkNeedsReload()
}()
needsReload := meta.IsNoMatchError(err)
if !needsReload {
return err
}
// if the data wasn't fresh, we'll need to try and update it, so grab the lock...
drm.mu.Lock()
defer drm.mu.Unlock()
// ... and double-check that we didn't reload in the meantime
err = checkNeedsReload()
needsReload = meta.IsNoMatchError(err)
if !needsReload {
return err
}
// we're still stale, so grab a rate-limit token if we can...
if !drm.limiter.Allow() {
// return error from static mapper here, we have refreshed often enough (exceeding rate of provided limiter)
// so that client's can handle this the same way as a "normal" NoResourceMatchError / NoKindMatchError
return err
}
// ...reload...
if err := drm.setStaticMapper(); err != nil {
return err
}
// ...and return the results of the closure regardless
return checkNeedsReload()
}
// TODO: wrap reload errors on NoKindMatchError with go 1.13 errors.
func (drm *dynamicRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) {
if err := drm.init(); err != nil {
return schema.GroupVersionKind{}, err
}
var gvk schema.GroupVersionKind
err := drm.checkAndReload(func() error {
var err error
gvk, err = drm.staticMapper.KindFor(resource)
return err
})
return gvk, err
}
func (drm *dynamicRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) {
if err := drm.init(); err != nil {
return nil, err
}
var gvks []schema.GroupVersionKind
err := drm.checkAndReload(func() error {
var err error
gvks, err = drm.staticMapper.KindsFor(resource)
return err
})
return gvks, err
}
func (drm *dynamicRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) {
if err := drm.init(); err != nil {
return schema.GroupVersionResource{}, err
}
var gvr schema.GroupVersionResource
err := drm.checkAndReload(func() error {
var err error
gvr, err = drm.staticMapper.ResourceFor(input)
return err
})
return gvr, err
}
func (drm *dynamicRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) {
if err := drm.init(); err != nil {
return nil, err
}
var gvrs []schema.GroupVersionResource
err := drm.checkAndReload(func() error {
var err error
gvrs, err = drm.staticMapper.ResourcesFor(input)
return err
})
return gvrs, err
}
func (drm *dynamicRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) {
if err := drm.init(); err != nil {
return nil, err
}
var mapping *meta.RESTMapping
err := drm.checkAndReload(func() error {
var err error
mapping, err = drm.staticMapper.RESTMapping(gk, versions...)
return err
})
return mapping, err
}
func (drm *dynamicRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) {
if err := drm.init(); err != nil {
return nil, err
}
var mappings []*meta.RESTMapping
err := drm.checkAndReload(func() error {
var err error
mappings, err = drm.staticMapper.RESTMappings(gk, versions...)
return err
})
return mappings, err
}
func (drm *dynamicRESTMapper) ResourceSingularizer(resource string) (string, error) {
if err := drm.init(); err != nil {
return "", err
}
var singular string
err := drm.checkAndReload(func() error {
var err error
singular, err = drm.staticMapper.ResourceSingularizer(resource)
return err
})
return singular, err
}

View File

@@ -0,0 +1,54 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apiutil
import (
"fmt"
"sort"
"strings"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// ErrResourceDiscoveryFailed is returned if the RESTMapper cannot discover supported resources for some GroupVersions.
// It wraps the errors encountered, except "NotFound" errors are replaced with meta.NoResourceMatchError, for
// backwards compatibility with code that uses meta.IsNoMatchError() to check for unsupported APIs.
type ErrResourceDiscoveryFailed map[schema.GroupVersion]error
// Error implements the error interface.
func (e *ErrResourceDiscoveryFailed) Error() string {
subErrors := []string{}
for k, v := range *e {
subErrors = append(subErrors, fmt.Sprintf("%s: %v", k, v))
}
sort.Strings(subErrors)
return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(subErrors, ", "))
}
func (e *ErrResourceDiscoveryFailed) Unwrap() []error {
subErrors := []error{}
for gv, err := range *e {
if apierrors.IsNotFound(err) {
err = &meta.NoResourceMatchError{PartialResource: gv.WithResource("")}
}
subErrors = append(subErrors, err)
}
return subErrors
}

View File

@@ -18,137 +18,151 @@ package apiutil
import (
"fmt"
"net/http"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
)
// lazyRESTMapper is a RESTMapper that will lazily query the provided
// client for discovery information to do REST mappings.
type lazyRESTMapper struct {
mapper meta.RESTMapper
client *discovery.DiscoveryClient
knownGroups map[string]*restmapper.APIGroupResources
apiGroups []metav1.APIGroup
// NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic
// RESTMapper dynamically discovers resource types at runtime.
func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) {
if httpClient == nil {
return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client")
}
// mutex to provide thread-safe mapper reloading.
mu sync.Mutex
}
// newLazyRESTMapperWithClient initializes a LazyRESTMapper with a custom discovery client.
func newLazyRESTMapperWithClient(discoveryClient *discovery.DiscoveryClient) (meta.RESTMapper, error) {
return &lazyRESTMapper{
client, err := discovery.NewDiscoveryClientForConfigAndClient(cfg, httpClient)
if err != nil {
return nil, err
}
return &mapper{
mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}),
client: discoveryClient,
client: client,
knownGroups: map[string]*restmapper.APIGroupResources{},
apiGroups: []metav1.APIGroup{},
apiGroups: map[string]*metav1.APIGroup{},
}, nil
}
// KindFor implements Mapper.KindFor.
func (m *lazyRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) {
res, err := m.mapper.KindFor(resource)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil {
return res, err
}
// mapper is a RESTMapper that will lazily query the provided
// client for discovery information to do REST mappings.
type mapper struct {
mapper meta.RESTMapper
client *discovery.DiscoveryClient
knownGroups map[string]*restmapper.APIGroupResources
apiGroups map[string]*metav1.APIGroup
res, err = m.mapper.KindFor(resource)
// mutex to provide thread-safe mapper reloading.
mu sync.RWMutex
}
// KindFor implements Mapper.KindFor.
func (m *mapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) {
res, err := m.getMapper().KindFor(resource)
if meta.IsNoMatchError(err) {
if err := m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil {
return schema.GroupVersionKind{}, err
}
res, err = m.getMapper().KindFor(resource)
}
return res, err
}
// KindsFor implements Mapper.KindsFor.
func (m *lazyRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) {
res, err := m.mapper.KindsFor(resource)
func (m *mapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) {
res, err := m.getMapper().KindsFor(resource)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil {
return res, err
if err := m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil {
return nil, err
}
res, err = m.mapper.KindsFor(resource)
res, err = m.getMapper().KindsFor(resource)
}
return res, err
}
// ResourceFor implements Mapper.ResourceFor.
func (m *lazyRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) {
res, err := m.mapper.ResourceFor(input)
func (m *mapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) {
res, err := m.getMapper().ResourceFor(input)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil {
return res, err
if err := m.addKnownGroupAndReload(input.Group, input.Version); err != nil {
return schema.GroupVersionResource{}, err
}
res, err = m.mapper.ResourceFor(input)
res, err = m.getMapper().ResourceFor(input)
}
return res, err
}
// ResourcesFor implements Mapper.ResourcesFor.
func (m *lazyRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) {
res, err := m.mapper.ResourcesFor(input)
func (m *mapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) {
res, err := m.getMapper().ResourcesFor(input)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil {
return res, err
if err := m.addKnownGroupAndReload(input.Group, input.Version); err != nil {
return nil, err
}
res, err = m.mapper.ResourcesFor(input)
res, err = m.getMapper().ResourcesFor(input)
}
return res, err
}
// RESTMapping implements Mapper.RESTMapping.
func (m *lazyRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) {
res, err := m.mapper.RESTMapping(gk, versions...)
func (m *mapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) {
res, err := m.getMapper().RESTMapping(gk, versions...)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil {
return res, err
if err := m.addKnownGroupAndReload(gk.Group, versions...); err != nil {
return nil, err
}
res, err = m.mapper.RESTMapping(gk, versions...)
res, err = m.getMapper().RESTMapping(gk, versions...)
}
return res, err
}
// RESTMappings implements Mapper.RESTMappings.
func (m *lazyRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) {
res, err := m.mapper.RESTMappings(gk, versions...)
func (m *mapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) {
res, err := m.getMapper().RESTMappings(gk, versions...)
if meta.IsNoMatchError(err) {
if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil {
return res, err
if err := m.addKnownGroupAndReload(gk.Group, versions...); err != nil {
return nil, err
}
res, err = m.mapper.RESTMappings(gk, versions...)
res, err = m.getMapper().RESTMappings(gk, versions...)
}
return res, err
}
// ResourceSingularizer implements Mapper.ResourceSingularizer.
func (m *lazyRESTMapper) ResourceSingularizer(resource string) (string, error) {
return m.mapper.ResourceSingularizer(resource)
func (m *mapper) ResourceSingularizer(resource string) (string, error) {
return m.getMapper().ResourceSingularizer(resource)
}
func (m *mapper) getMapper() meta.RESTMapper {
m.mu.RLock()
defer m.mu.RUnlock()
return m.mapper
}
// addKnownGroupAndReload reloads the mapper with updated information about missing API group.
// versions can be specified for partial updates, for instance for v1beta1 version only.
func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...string) error {
m.mu.Lock()
defer m.mu.Unlock()
func (m *mapper) addKnownGroupAndReload(groupName string, versions ...string) error {
// versions will here be [""] if the forwarded Version value of
// GroupVersionResource (in calling method) was not specified.
if len(versions) == 1 && versions[0] == "" {
versions = nil
}
// If no specific versions are set by user, we will scan all available ones for the API group.
// This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls
// this data will be taken from cache.
if len(versions) == 0 {
apiGroup, err := m.findAPIGroupByNameLocked(groupName)
apiGroup, err := m.findAPIGroupByName(groupName)
if err != nil {
return err
}
@@ -157,6 +171,9 @@ func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...st
}
}
m.mu.Lock()
defer m.mu.Unlock()
// Create or fetch group resources from cache.
groupResources := &restmapper.APIGroupResources{
Group: metav1.APIGroup{Name: groupName},
@@ -205,43 +222,53 @@ func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...st
}
m.mapper = restmapper.NewDiscoveryRESTMapper(updatedGroupResources)
return nil
}
// findAPIGroupByNameLocked returns API group by its name.
func (m *lazyRESTMapper) findAPIGroupByNameLocked(groupName string) (metav1.APIGroup, error) {
func (m *mapper) findAPIGroupByName(groupName string) (*metav1.APIGroup, error) {
// Looking in the cache first.
for _, apiGroup := range m.apiGroups {
if groupName == apiGroup.Name {
return apiGroup, nil
{
m.mu.RLock()
group, ok := m.apiGroups[groupName]
m.mu.RUnlock()
if ok {
return group, nil
}
}
// Update the cache if nothing was found.
apiGroups, err := m.client.ServerGroups()
if err != nil {
return metav1.APIGroup{}, fmt.Errorf("failed to get server groups: %w", err)
return nil, fmt.Errorf("failed to get server groups: %w", err)
}
if len(apiGroups.Groups) == 0 {
return metav1.APIGroup{}, fmt.Errorf("received an empty API groups list")
return nil, fmt.Errorf("received an empty API groups list")
}
m.apiGroups = apiGroups.Groups
m.mu.Lock()
for i := range apiGroups.Groups {
group := &apiGroups.Groups[i]
m.apiGroups[group.Name] = group
}
m.mu.Unlock()
// Looking in the cache again.
for _, apiGroup := range m.apiGroups {
if groupName == apiGroup.Name {
return apiGroup, nil
{
m.mu.RLock()
group, ok := m.apiGroups[groupName]
m.mu.RUnlock()
if ok {
return group, nil
}
}
// If there is still nothing, return an error.
return metav1.APIGroup{}, fmt.Errorf("failed to find API group %s", groupName)
return nil, fmt.Errorf("failed to find API group %q", groupName)
}
// fetchGroupVersionResources fetches the resources for the specified group and its versions.
func (m *lazyRESTMapper) fetchGroupVersionResources(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) {
func (m *mapper) fetchGroupVersionResources(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) {
groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList)
failedGroups := make(map[schema.GroupVersion]error)
@@ -259,7 +286,8 @@ func (m *lazyRESTMapper) fetchGroupVersionResources(groupName string, versions .
}
if len(failedGroups) > 0 {
return nil, &discovery.ErrGroupDiscoveryFailed{Groups: failedGroups}
err := ErrResourceDiscoveryFailed(failedGroups)
return nil, &err
}
return groupVersionResources, nil

View File

@@ -20,11 +20,11 @@ import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -36,6 +36,28 @@ import (
"sigs.k8s.io/controller-runtime/pkg/log"
)
// Options are creation options for a Client.
type Options struct {
// HTTPClient is the HTTP client to use for requests.
HTTPClient *http.Client
// Scheme, if provided, will be used to map go structs to GroupVersionKinds
Scheme *runtime.Scheme
// Mapper, if provided, will be used to map GroupVersionKinds to Resources
Mapper meta.RESTMapper
// Cache, if provided, is used to read objects from the cache.
Cache *CacheOptions
// WarningHandler is used to configure the warning handler responsible for
// surfacing and handling warnings messages sent by the API server.
WarningHandler WarningHandlerOptions
// DryRun instructs the client to only perform dry run requests.
DryRun *bool
}
// WarningHandlerOptions are options for configuring a
// warning handler for the client which is responsible
// for surfacing API Server warnings.
@@ -50,19 +72,23 @@ type WarningHandlerOptions struct {
AllowDuplicateLogs bool
}
// Options are creation options for a Client.
type Options struct {
// Scheme, if provided, will be used to map go structs to GroupVersionKinds
Scheme *runtime.Scheme
// Mapper, if provided, will be used to map GroupVersionKinds to Resources
Mapper meta.RESTMapper
// Opts is used to configure the warning handler responsible for
// surfacing and handling warnings messages sent by the API server.
Opts WarningHandlerOptions
// CacheOptions are options for creating a cache-backed client.
type CacheOptions struct {
// Reader is a cache-backed reader that will be used to read objects from the cache.
// +required
Reader Reader
// DisableFor is a list of objects that should never be read from the cache.
// Objects configured here always result in a live lookup.
DisableFor []Object
// Unstructured is a flag that indicates whether the cache-backed client should
// read unstructured objects or lists from the cache.
// If false, unstructured objects will always result in a live lookup.
Unstructured bool
}
// NewClientFunc allows a user to define how to create a client.
type NewClientFunc func(config *rest.Config, options Options) (Client, error)
// New returns a new Client using the provided config and Options.
// The returned client reads *and* writes directly from the server
// (it doesn't use object caches). It understands how to work with
@@ -73,8 +99,12 @@ type Options struct {
// corresponding group, version, and kind for the given type. In the
// case of unstructured types, the group, version, and kind will be extracted
// from the corresponding fields on the object.
func New(config *rest.Config, options Options) (Client, error) {
return newClient(config, options)
func New(config *rest.Config, options Options) (c Client, err error) {
c, err = newClient(config, options)
if err == nil && options.DryRun != nil && *options.DryRun {
c = NewDryRunClient(c)
}
return c, err
}
func newClient(config *rest.Config, options Options) (*client, error) {
@@ -82,22 +112,35 @@ func newClient(config *rest.Config, options Options) (*client, error) {
return nil, fmt.Errorf("must provide non-nil rest.Config to client.New")
}
if !options.Opts.SuppressWarnings {
config = rest.CopyConfig(config)
if config.UserAgent == "" {
config.UserAgent = rest.DefaultKubernetesUserAgent()
}
if !options.WarningHandler.SuppressWarnings {
// surface warnings
logger := log.Log.WithName("KubeAPIWarningLogger")
// Set a WarningHandler, the default WarningHandler
// is log.KubeAPIWarningLogger with deduplication enabled.
// See log.KubeAPIWarningLoggerOptions for considerations
// regarding deduplication.
config = rest.CopyConfig(config)
config.WarningHandler = log.NewKubeAPIWarningLogger(
logger,
log.KubeAPIWarningLoggerOptions{
Deduplicate: !options.Opts.AllowDuplicateLogs,
Deduplicate: !options.WarningHandler.AllowDuplicateLogs,
},
)
}
// Use the rest HTTP client for the provided config if unset
if options.HTTPClient == nil {
var err error
options.HTTPClient, err = rest.HTTPClientFor(config)
if err != nil {
return nil, err
}
}
// Init a scheme if none provided
if options.Scheme == nil {
options.Scheme = scheme.Scheme
@@ -106,34 +149,35 @@ func newClient(config *rest.Config, options Options) (*client, error) {
// Init a Mapper if none provided
if options.Mapper == nil {
var err error
options.Mapper, err = apiutil.NewDynamicRESTMapper(config)
options.Mapper, err = apiutil.NewDynamicRESTMapper(config, options.HTTPClient)
if err != nil {
return nil, err
}
}
clientcache := &clientCache{
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
resources := &clientRestResources{
httpClient: options.HTTPClient,
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
}
rawMetaClient, err := metadata.NewForConfig(config)
rawMetaClient, err := metadata.NewForConfigAndClient(metadata.ConfigFor(config), options.HTTPClient)
if err != nil {
return nil, fmt.Errorf("unable to construct metadata-only client for use as part of client: %w", err)
}
c := &client{
typedClient: typedClient{
cache: clientcache,
resources: resources,
paramCodec: runtime.NewParameterCodec(options.Scheme),
},
unstructuredClient: unstructuredClient{
cache: clientcache,
resources: resources,
paramCodec: noConversionParamCodec{},
},
metadataClient: metadataClient{
@@ -143,20 +187,65 @@ func newClient(config *rest.Config, options Options) (*client, error) {
scheme: options.Scheme,
mapper: options.Mapper,
}
if options.Cache == nil || options.Cache.Reader == nil {
return c, nil
}
// We want a cache if we're here.
// Set the cache.
c.cache = options.Cache.Reader
// Load uncached GVKs.
c.cacheUnstructured = options.Cache.Unstructured
c.uncachedGVKs = map[schema.GroupVersionKind]struct{}{}
for _, obj := range options.Cache.DisableFor {
gvk, err := c.GroupVersionKindFor(obj)
if err != nil {
return nil, err
}
c.uncachedGVKs[gvk] = struct{}{}
}
return c, nil
}
var _ Client = &client{}
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
// client is a client.Client that reads and writes directly from/to an API server.
// It lazily initializes new clients at the time they are used.
type client struct {
typedClient typedClient
unstructuredClient unstructuredClient
metadataClient metadataClient
scheme *runtime.Scheme
mapper meta.RESTMapper
cache Reader
uncachedGVKs map[schema.GroupVersionKind]struct{}
cacheUnstructured bool
}
func (c *client) shouldBypassCache(obj runtime.Object) (bool, error) {
if c.cache == nil {
return true, nil
}
gvk, err := c.GroupVersionKindFor(obj)
if err != nil {
return false, err
}
// TODO: this is producing unsafe guesses that don't actually work,
// but it matches ~99% of the cases out there.
if meta.IsListType(obj) {
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
}
if _, isUncached := c.uncachedGVKs[gvk]; isUncached {
return true, nil
}
if !c.cacheUnstructured {
_, isUnstructured := obj.(runtime.Unstructured)
return isUnstructured, nil
}
return false, nil
}
// resetGroupVersionKind is a helper function to restore and preserve GroupVersionKind on an object.
@@ -168,6 +257,16 @@ func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersi
}
}
// GroupVersionKindFor returns the GroupVersionKind for the given object.
func (c *client) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
return apiutil.GVKForObject(obj, c.scheme)
}
// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced.
func (c *client) IsObjectNamespaced(obj runtime.Object) (bool, error) {
return apiutil.IsObjectNamespaced(obj, c.scheme, c.mapper)
}
// Scheme returns the scheme this client is using.
func (c *client) Scheme() *runtime.Scheme {
return c.scheme
@@ -181,7 +280,7 @@ func (c *client) RESTMapper() meta.RESTMapper {
// Create implements client.Client.
func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.Create(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot create using only metadata")
@@ -194,7 +293,7 @@ func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) e
func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.Update(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot update using only metadata -- did you mean to patch?")
@@ -206,7 +305,7 @@ func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) e
// Delete implements client.Client.
func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.Delete(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.Delete(ctx, obj, opts...)
@@ -218,7 +317,7 @@ func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) e
// DeleteAllOf implements client.Client.
func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.DeleteAllOf(ctx, obj, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.DeleteAllOf(ctx, obj, opts...)
@@ -231,7 +330,7 @@ func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllO
func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.Patch(ctx, obj, patch, opts...)
case *metav1.PartialObjectMetadata:
return c.metadataClient.Patch(ctx, obj, patch, opts...)
@@ -242,8 +341,16 @@ func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...Pat
// Get implements client.Client.
func (c *client) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error {
if isUncached, err := c.shouldBypassCache(obj); err != nil {
return err
} else if !isUncached {
// Attempt to get from the cache.
return c.cache.Get(ctx, key, obj, opts...)
}
// Perform a live lookup.
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return c.unstructuredClient.Get(ctx, key, obj, opts...)
case *metav1.PartialObjectMetadata:
// Metadata only object should always preserve the GVK coming in from the caller.
@@ -256,8 +363,16 @@ func (c *client) Get(ctx context.Context, key ObjectKey, obj Object, opts ...Get
// List implements client.Client.
func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
if isUncached, err := c.shouldBypassCache(obj); err != nil {
return err
} else if !isUncached {
// Attempt to get from the cache.
return c.cache.List(ctx, obj, opts...)
}
// Perform a live lookup.
switch x := obj.(type) {
case *unstructured.UnstructuredList:
case runtime.Unstructured:
return c.unstructuredClient.List(ctx, obj, opts...)
case *metav1.PartialObjectMetadataList:
// Metadata only object should always preserve the GVK.
@@ -431,7 +546,7 @@ func (po *SubResourcePatchOptions) ApplyToSubResourcePatch(o *SubResourcePatchOp
func (sc *subResourceClient) Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return sc.client.unstructuredClient.GetSubResource(ctx, obj, subResource, sc.subResource, opts...)
case *metav1.PartialObjectMetadata:
return errors.New("can not get subresource using only metadata")
@@ -446,7 +561,7 @@ func (sc *subResourceClient) Create(ctx context.Context, obj Object, subResource
defer sc.client.resetGroupVersionKind(subResource, subResource.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return sc.client.unstructuredClient.CreateSubResource(ctx, obj, subResource, sc.subResource, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?")
@@ -459,7 +574,7 @@ func (sc *subResourceClient) Create(ctx context.Context, obj Object, subResource
func (sc *subResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error {
defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return sc.client.unstructuredClient.UpdateSubResource(ctx, obj, sc.subResource, opts...)
case *metav1.PartialObjectMetadata:
return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?")
@@ -472,7 +587,7 @@ func (sc *subResourceClient) Update(ctx context.Context, obj Object, opts ...Sub
func (sc *subResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error {
defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
case runtime.Unstructured:
return sc.client.unstructuredClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...)
case *metav1.PartialObjectMetadata:
return sc.client.metadataClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...)

View File

@@ -17,12 +17,12 @@ limitations under the License.
package client
import (
"net/http"
"strings"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -30,8 +30,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// clientCache creates and caches rest clients and metadata for Kubernetes types.
type clientCache struct {
// clientRestResources creates and stores rest clients and metadata for Kubernetes types.
type clientRestResources struct {
// httpClient is the http client to use for requests
httpClient *http.Client
// config is the rest.Config to talk to an apiserver
config *rest.Config
@@ -44,22 +47,22 @@ type clientCache struct {
// codecs are used to create a REST client for a gvk
codecs serializer.CodecFactory
// structuredResourceByType caches structured type metadata
// structuredResourceByType stores structured type metadata
structuredResourceByType map[schema.GroupVersionKind]*resourceMeta
// unstructuredResourceByType caches unstructured type metadata
// unstructuredResourceByType stores unstructured type metadata
unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta
mu sync.RWMutex
}
// newResource maps obj to a Kubernetes Resource and constructs a client for that Resource.
// If the object is a list, the resource represents the item's type instead.
func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) {
func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) {
if strings.HasSuffix(gvk.Kind, "List") && isList {
// if this was a list, treat it as a request for the item's resource
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs)
client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs, c.httpClient)
if err != nil {
return nil, err
}
@@ -72,15 +75,13 @@ func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstruc
// getResource returns the resource meta information for the given type of object.
// If the object is a list, the resource represents the item's type instead.
func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, error) {
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return nil, err
}
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
isUnstructured = isUnstructured || isUnstructuredList
_, isUnstructured := obj.(runtime.Unstructured)
// It's better to do creation work twice than to not let multiple
// people make requests at once
@@ -108,7 +109,7 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
}
// getObjMeta returns objMeta containing both type and object metadata and state.
func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) {
func (c *clientRestResources) getObjMeta(obj runtime.Object) (*objMeta, error) {
r, err := c.getResource(obj)
if err != nil {
return nil, err
@@ -120,7 +121,7 @@ func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) {
return &objMeta{resourceMeta: r, Object: m}, err
}
// resourceMeta caches state for a Kubernetes type.
// resourceMeta stores state for a Kubernetes type.
type resourceMeta struct {
// client is the rest client used to talk to the apiserver
rest.Interface

View File

@@ -98,12 +98,12 @@ func GetConfigWithContext(context string) (*rest.Config, error) {
if err != nil {
return nil, err
}
if cfg.QPS == 0.0 {
cfg.QPS = 20.0
cfg.Burst = 30.0
}
if cfg.Burst == 0 {
cfg.Burst = 30
}
return cfg, nil
}

View File

@@ -26,8 +26,7 @@ limitations under the License.
// to the API server.
//
// It is a common pattern in Kubernetes to read from a cache and write to the API
// server. This pattern is covered by the DelegatingClient type, which can
// be used to have a client whose Reader is different from the Writer.
// server. This pattern is covered by the creating the Client with a Cache.
//
// # Options
//

View File

@@ -21,6 +21,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// NewDryRunClient wraps an existing client and enforces DryRun mode
@@ -46,6 +47,16 @@ func (c *dryRunClient) RESTMapper() meta.RESTMapper {
return c.client.RESTMapper()
}
// GroupVersionKindFor returns the GroupVersionKind for the given object.
func (c *dryRunClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
return c.client.GroupVersionKindFor(obj)
}
// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced.
func (c *dryRunClient) IsObjectNamespaced(obj runtime.Object) (bool, error) {
return c.client.IsObjectNamespaced(obj)
}
// Create implements client.Client.
func (c *dryRunClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
return c.client.Create(ctx, obj, append(opts, DryRunAll)...)

View File

@@ -17,15 +17,23 @@ limitations under the License.
package fake
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
// Using v4 to match upstream
jsonpatch "github.com/evanphx/json-patch"
corev1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1"
policyv1beta1 "k8s.io/api/policy/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,27 +42,33 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/testing"
"sigs.k8s.io/controller-runtime/pkg/internal/field/selector"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"sigs.k8s.io/controller-runtime/pkg/internal/field/selector"
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
)
type versionedTracker struct {
testing.ObjectTracker
scheme *runtime.Scheme
scheme *runtime.Scheme
withStatusSubresource sets.Set[schema.GroupVersionKind]
}
type fakeClient struct {
tracker versionedTracker
scheme *runtime.Scheme
restMapper meta.RESTMapper
tracker versionedTracker
scheme *runtime.Scheme
restMapper meta.RESTMapper
withStatusSubresource sets.Set[schema.GroupVersionKind]
// indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK.
// The inner map maps from index name to IndexerFunc.
@@ -73,21 +87,10 @@ const (
// NewFakeClient creates a new fake client for testing.
// You can choose to initialize it with a slice of runtime.Object.
//
// Deprecated: Please use NewClientBuilder instead.
func NewFakeClient(initObjs ...runtime.Object) client.WithWatch {
return NewClientBuilder().WithRuntimeObjects(initObjs...).Build()
}
// NewFakeClientWithScheme creates a new fake client with the given scheme
// for testing.
// You can choose to initialize it with a slice of runtime.Object.
//
// Deprecated: Please use NewClientBuilder instead.
func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch {
return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build()
}
// NewClientBuilder returns a new builder to create a fake client.
func NewClientBuilder() *ClientBuilder {
return &ClientBuilder{}
@@ -95,12 +98,14 @@ func NewClientBuilder() *ClientBuilder {
// ClientBuilder builds a fake client.
type ClientBuilder struct {
scheme *runtime.Scheme
restMapper meta.RESTMapper
initObject []client.Object
initLists []client.ObjectList
initRuntimeObjects []runtime.Object
objectTracker testing.ObjectTracker
scheme *runtime.Scheme
restMapper meta.RESTMapper
initObject []client.Object
initLists []client.ObjectList
initRuntimeObjects []runtime.Object
withStatusSubresource []client.Object
objectTracker testing.ObjectTracker
interceptorFuncs *interceptor.Funcs
// indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK.
// The inner map maps from index name to IndexerFunc.
@@ -185,6 +190,19 @@ func (f *ClientBuilder) WithIndex(obj runtime.Object, field string, extractValue
return f
}
// WithStatusSubresource configures the passed object with a status subresource, which means
// calls to Update and Patch will not alter its status.
func (f *ClientBuilder) WithStatusSubresource(o ...client.Object) *ClientBuilder {
f.withStatusSubresource = append(f.withStatusSubresource, o...)
return f
}
// WithInterceptorFuncs configures the client methods to be intercepted using the provided interceptor.Funcs.
func (f *ClientBuilder) WithInterceptorFuncs(interceptorFuncs interceptor.Funcs) *ClientBuilder {
f.interceptorFuncs = &interceptorFuncs
return f
}
// Build builds and returns a new fake client.
func (f *ClientBuilder) Build() client.WithWatch {
if f.scheme == nil {
@@ -196,10 +214,19 @@ func (f *ClientBuilder) Build() client.WithWatch {
var tracker versionedTracker
withStatusSubResource := sets.New(inTreeResourcesWithStatus()...)
for _, o := range f.withStatusSubresource {
gvk, err := apiutil.GVKForObject(o, f.scheme)
if err != nil {
panic(fmt.Errorf("failed to get gvk for object %T: %w", withStatusSubResource, err))
}
withStatusSubResource.Insert(gvk)
}
if f.objectTracker == nil {
tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme}
tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme, withStatusSubresource: withStatusSubResource}
} else {
tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme}
tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme, withStatusSubresource: withStatusSubResource}
}
for _, obj := range f.initObject {
@@ -217,12 +244,20 @@ func (f *ClientBuilder) Build() client.WithWatch {
panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err))
}
}
return &fakeClient{
tracker: tracker,
scheme: f.scheme,
restMapper: f.restMapper,
indexes: f.indexes,
var result client.WithWatch = &fakeClient{
tracker: tracker,
scheme: f.scheme,
restMapper: f.restMapper,
indexes: f.indexes,
withStatusSubresource: withStatusSubResource,
}
if f.interceptorFuncs != nil {
result = interceptor.NewClient(result, *f.interceptorFuncs)
}
return result
}
const trackerAddResourceVersion = "999"
@@ -243,6 +278,9 @@ func (t versionedTracker) Add(obj runtime.Object) error {
if err != nil {
return fmt.Errorf("failed to get accessor for object: %w", err)
}
if accessor.GetDeletionTimestamp() != nil && len(accessor.GetFinalizers()) == 0 {
return fmt.Errorf("refusing to create obj %s with metadata.deletionTimestamp but no finalizers", accessor.GetName())
}
if accessor.GetResourceVersion() == "" {
// We use a "magic" value of 999 here because this field
// is parsed as uint and and 0 is already used in Update.
@@ -290,20 +328,22 @@ func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Ob
return nil
}
// convertFromUnstructuredIfNecessary will convert *unstructured.Unstructured for a GVK that is recocnized
// convertFromUnstructuredIfNecessary will convert runtime.Unstructured for a GVK that is recognized
// by the schema into the whatever the schema produces with New() for said GVK.
// This is required because the tracker unconditionally saves on manipulations, but its List() implementation
// tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure
// we save as the very same type, otherwise subsequent List requests will fail.
func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) {
u, isUnstructured := o.(*unstructured.Unstructured)
if !isUnstructured || !s.Recognizes(u.GroupVersionKind()) {
gvk := o.GetObjectKind().GroupVersionKind()
u, isUnstructured := o.(runtime.Unstructured)
if !isUnstructured || !s.Recognizes(gvk) {
return o, nil
}
typed, err := s.New(u.GroupVersionKind())
typed, err := s.New(gvk)
if err != nil {
return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", u.GroupVersionKind().String(), err)
return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", gvk, err)
}
unstructuredSerialized, err := json.Marshal(u)
@@ -318,6 +358,16 @@ func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (ru
}
func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
isStatus := false
// We apply patches using a client-go reaction that ends up calling the trackers Update. As we can't change
// that reaction, we use the callstack to figure out if this originated from the status client.
if bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) {
isStatus = true
}
return t.update(gvr, obj, ns, isStatus, false)
}
func (t versionedTracker) update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus bool, deleting bool) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return fmt.Errorf("failed to get accessor for object: %w", err)
@@ -348,6 +398,25 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob
return err
}
if t.withStatusSubresource.Has(gvk) {
if isStatus { // copy everything but status and metadata.ResourceVersion from original object
if err := copyStatusFrom(obj, oldObject); err != nil {
return fmt.Errorf("failed to copy non-status field for object with status subresouce: %w", err)
}
passedRV := accessor.GetResourceVersion()
if err := copyFrom(oldObject, obj); err != nil {
return fmt.Errorf("failed to restore non-status fields: %w", err)
}
accessor.SetResourceVersion(passedRV)
} else { // copy status from original object
if err := copyStatusFrom(oldObject, obj); err != nil {
return fmt.Errorf("failed to copy the status for object with status subresource: %w", err)
}
}
} else if isStatus {
return apierrors.NewNotFound(gvr.GroupResource(), accessor.GetName())
}
oldAccessor, err := meta.Accessor(oldObject)
if err != nil {
return err
@@ -370,6 +439,11 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob
}
intResourceVersion++
accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10))
if !deleting && !deletionTimestampEqual(accessor, oldAccessor) {
return fmt.Errorf("error: Unable to edit %s: metadata.deletionTimestamp field is immutable", accessor.GetName())
}
if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 {
return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
}
@@ -436,7 +510,7 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) {
if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(gvk) {
// We need to register the ListKind with UnstructuredList:
// https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51
c.schemeWriteLock.Lock()
@@ -563,6 +637,16 @@ func (c *fakeClient) RESTMapper() meta.RESTMapper {
return c.restMapper
}
// GroupVersionKindFor returns the GroupVersionKind for the given object.
func (c *fakeClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
return apiutil.GVKForObject(obj, c.scheme)
}
// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced.
func (c *fakeClient) IsObjectNamespaced(obj runtime.Object) (bool, error) {
return apiutil.IsObjectNamespaced(obj, c.scheme, c.restMapper)
}
func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
createOptions := &client.CreateOptions{}
createOptions.ApplyOptions(opts)
@@ -589,6 +673,10 @@ func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...clie
}
accessor.SetName(fmt.Sprintf("%s%s", base, utilrand.String(randomLength)))
}
// Ignore attempts to set deletion timestamp
if !accessor.GetDeletionTimestamp().IsZero() {
accessor.SetDeletionTimestamp(nil)
}
return c.tracker.Create(gvr, obj, accessor.GetNamespace())
}
@@ -679,6 +767,10 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ..
}
func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
return c.update(obj, false, opts...)
}
func (c *fakeClient) update(obj client.Object, isStatus bool, opts ...client.UpdateOption) error {
updateOptions := &client.UpdateOptions{}
updateOptions.ApplyOptions(opts)
@@ -696,10 +788,14 @@ func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...clie
if err != nil {
return err
}
return c.tracker.Update(gvr, obj, accessor.GetNamespace())
return c.tracker.update(gvr, obj, accessor.GetNamespace(), isStatus, false)
}
func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
return c.patch(obj, patch, opts...)
}
func (c *fakeClient) patch(obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
patchOptions := &client.PatchOptions{}
patchOptions.ApplyOptions(opts)
@@ -722,19 +818,50 @@ func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.
return err
}
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return err
}
oldObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName())
if err != nil {
return err
}
oldAccessor, err := meta.Accessor(oldObj)
if err != nil {
return err
}
// Apply patch without updating object.
// To remain in accordance with the behavior of k8s api behavior,
// a patch must not allow for changes to the deletionTimestamp of an object.
// The reaction() function applies the patch to the object and calls Update(),
// whereas dryPatch() replicates this behavior but skips the call to Update().
// This ensures that the patch may be rejected if a deletionTimestamp is modified, prior
// to updating the object.
action := testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data)
o, err := dryPatch(action, c.tracker)
if err != nil {
return err
}
newObj, err := meta.Accessor(o)
if err != nil {
return err
}
// Validate that deletionTimestamp has not been changed
if !deletionTimestampEqual(newObj, oldAccessor) {
return fmt.Errorf("rejected patch, metadata.deletionTimestamp immutable")
}
reaction := testing.ObjectReaction(c.tracker)
handled, o, err := reaction(testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data))
handled, o, err := reaction(action)
if err != nil {
return err
}
if !handled {
panic("tracker could not handle patch method")
}
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return err
}
ta, err := meta.TypeAccessor(o)
if err != nil {
return err
@@ -752,12 +879,153 @@ func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.
return err
}
// Applying a patch results in a deletionTimestamp that is truncated to the nearest second.
// Check that the diff between a new and old deletion timestamp is within a reasonable threshold
// to be considered unchanged.
func deletionTimestampEqual(newObj metav1.Object, obj metav1.Object) bool {
newTime := newObj.GetDeletionTimestamp()
oldTime := obj.GetDeletionTimestamp()
if newTime == nil || oldTime == nil {
return newTime == oldTime
}
return newTime.Time.Sub(oldTime.Time).Abs() < time.Second
}
// The behavior of applying the patch is pulled out into dryPatch(),
// which applies the patch and returns an object, but does not Update() the object.
// This function returns a patched runtime object that may then be validated before a call to Update() is executed.
// This results in some code duplication, but was found to be a cleaner alternative than unmarshalling and introspecting the patch data
// and easier than refactoring the k8s client-go method upstream.
// Duplicate of upstream: https://github.com/kubernetes/client-go/blob/783d0d33626e59d55d52bfd7696b775851f92107/testing/fixture.go#L146-L194
func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (runtime.Object, error) {
ns := action.GetNamespace()
gvr := action.GetResource()
obj, err := tracker.Get(gvr, ns, action.GetName())
if err != nil {
return nil, err
}
old, err := json.Marshal(obj)
if err != nil {
return nil, err
}
// reset the object in preparation to unmarshal, since unmarshal does not guarantee that fields
// in obj that are removed by patch are cleared
value := reflect.ValueOf(obj)
value.Elem().Set(reflect.New(value.Type().Elem()).Elem())
switch action.GetPatchType() {
case types.JSONPatchType:
patch, err := jsonpatch.DecodePatch(action.GetPatch())
if err != nil {
return nil, err
}
modified, err := patch.Apply(old)
if err != nil {
return nil, err
}
if err = json.Unmarshal(modified, obj); err != nil {
return nil, err
}
case types.MergePatchType:
modified, err := jsonpatch.MergePatch(old, action.GetPatch())
if err != nil {
return nil, err
}
if err := json.Unmarshal(modified, obj); err != nil {
return nil, err
}
case types.StrategicMergePatchType, types.ApplyPatchType:
mergedByte, err := strategicpatch.StrategicMergePatch(old, action.GetPatch(), obj)
if err != nil {
return nil, err
}
if err = json.Unmarshal(mergedByte, obj); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("PatchType is not supported")
}
return obj, nil
}
// copyStatusFrom copies the status from old into new
func copyStatusFrom(old, new runtime.Object) error {
oldMapStringAny, err := toMapStringAny(old)
if err != nil {
return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err)
}
newMapStringAny, err := toMapStringAny(new)
if err != nil {
return fmt.Errorf("failed to convert new to *unststructured.Unstructured: %w", err)
}
newMapStringAny["status"] = oldMapStringAny["status"]
if err := fromMapStringAny(newMapStringAny, new); err != nil {
return fmt.Errorf("failed to convert back from map[string]any: %w", err)
}
return nil
}
// copyFrom copies from old into new
func copyFrom(old, new runtime.Object) error {
oldMapStringAny, err := toMapStringAny(old)
if err != nil {
return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err)
}
if err := fromMapStringAny(oldMapStringAny, new); err != nil {
return fmt.Errorf("failed to convert back from map[string]any: %w", err)
}
return nil
}
func toMapStringAny(obj runtime.Object) (map[string]any, error) {
if unstructured, isUnstructured := obj.(*unstructured.Unstructured); isUnstructured {
return unstructured.Object, nil
}
serialized, err := json.Marshal(obj)
if err != nil {
return nil, err
}
u := map[string]any{}
return u, json.Unmarshal(serialized, &u)
}
func fromMapStringAny(u map[string]any, target runtime.Object) error {
if targetUnstructured, isUnstructured := target.(*unstructured.Unstructured); isUnstructured {
targetUnstructured.Object = u
return nil
}
serialized, err := json.Marshal(u)
if err != nil {
return fmt.Errorf("failed to serialize: %w", err)
}
zero(target)
if err := json.Unmarshal(serialized, &target); err != nil {
return fmt.Errorf("failed to deserialize: %w", err)
}
return nil
}
func (c *fakeClient) Status() client.SubResourceWriter {
return c.SubResource("status")
}
func (c *fakeClient) SubResource(subResource string) client.SubResourceClient {
return &fakeSubResourceClient{client: c}
return &fakeSubResourceClient{client: c, subResource: subResource}
}
func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error {
@@ -768,7 +1036,9 @@ func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor meta
if len(oldAccessor.GetFinalizers()) > 0 {
now := metav1.Now()
oldAccessor.SetDeletionTimestamp(&now)
return c.tracker.Update(gvr, old, accessor.GetNamespace())
// Call update directly with mutability parameter set to true to allow
// changes to deletionTimestamp
return c.tracker.update(gvr, old, accessor.GetNamespace(), false, true)
}
}
}
@@ -787,7 +1057,8 @@ func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupV
}
type fakeSubResourceClient struct {
client *fakeClient
client *fakeClient
subResource string
}
func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceGetOption) error {
@@ -795,12 +1066,26 @@ func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource clien
}
func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error {
panic("fakeSubResourceWriter does not support create")
switch sw.subResource {
case "eviction":
_, isEviction := subResource.(*policyv1beta1.Eviction)
if !isEviction {
_, isEviction = subResource.(*policyv1.Eviction)
}
if !isEviction {
return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %t, expected Eviction", subResource))
}
if _, isPod := obj.(*corev1.Pod); !isPod {
return apierrors.NewNotFound(schema.GroupResource{}, "")
}
return sw.client.Delete(ctx, obj)
default:
return fmt.Errorf("fakeSubResourceWriter does not support create for %s", sw.subResource)
}
}
func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
// TODO(droot): This results in full update of the obj (spec + subresources). Need
// a way to update subresource only.
updateOptions := client.SubResourceUpdateOptions{}
updateOptions.ApplyOptions(opts)
@@ -808,13 +1093,10 @@ func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object,
if updateOptions.SubResourceBody != nil {
body = updateOptions.SubResourceBody
}
return sw.client.Update(ctx, body, &updateOptions.UpdateOptions)
return sw.client.update(body, true, &updateOptions.UpdateOptions)
}
func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
// TODO(droot): This results in full update of the obj (spec + subresources). Need
// a way to update subresource only.
patchOptions := client.SubResourcePatchOptions{}
patchOptions.ApplyOptions(opts)
@@ -823,7 +1105,16 @@ func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, p
body = patchOptions.SubResourceBody
}
return sw.client.Patch(ctx, body, patch, &patchOptions.PatchOptions)
// this is necessary to identify that last call was made for status patch, through stack trace.
if sw.subResource == "status" {
return sw.statusPatch(body, patch, patchOptions)
}
return sw.client.patch(body, patch, &patchOptions.PatchOptions)
}
func (sw *fakeSubResourceClient) statusPatch(body client.Object, patch client.Patch, patchOptions client.SubResourcePatchOptions) error {
return sw.client.patch(body, patch, &patchOptions.PatchOptions)
}
func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool {
@@ -863,7 +1154,7 @@ func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool {
case "PodSecurityPolicy":
return true
}
case "rbac":
case "rbac.authorization.k8s.io":
switch gvk.Kind {
case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding":
return true
@@ -923,6 +1214,42 @@ func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool {
return false
}
func inTreeResourcesWithStatus() []schema.GroupVersionKind {
return []schema.GroupVersionKind{
{Version: "v1", Kind: "Namespace"},
{Version: "v1", Kind: "Node"},
{Version: "v1", Kind: "PersistentVolumeClaim"},
{Version: "v1", Kind: "PersistentVolume"},
{Version: "v1", Kind: "Pod"},
{Version: "v1", Kind: "ReplicationController"},
{Version: "v1", Kind: "Service"},
{Group: "apps", Version: "v1", Kind: "Deployment"},
{Group: "apps", Version: "v1", Kind: "DaemonSet"},
{Group: "apps", Version: "v1", Kind: "ReplicaSet"},
{Group: "apps", Version: "v1", Kind: "StatefulSet"},
{Group: "autoscaling", Version: "v1", Kind: "HorizontalPodAutoscaler"},
{Group: "batch", Version: "v1", Kind: "CronJob"},
{Group: "batch", Version: "v1", Kind: "Job"},
{Group: "certificates.k8s.io", Version: "v1", Kind: "CertificateSigningRequest"},
{Group: "networking.k8s.io", Version: "v1", Kind: "Ingress"},
{Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"},
{Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"},
{Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"},
{Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema"},
{Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"},
}
}
// zero zeros the value of a pointer.
func zero(x interface{}) {
if x == nil {

View File

@@ -20,7 +20,7 @@ Package fake provides a fake client for testing.
A fake client is backed by its simple object store indexed by GroupVersionResource.
You can create a fake client with optional objects.
client := NewFakeClientWithScheme(scheme, initObjs...) // initObjs is a slice of runtime.Object
client := NewClientBuilder().WithScheme(scheme).WithObj(initObjs...).Build()
You can invoke the methods defined in the Client interface.

View File

@@ -0,0 +1,166 @@
package interceptor
import (
"context"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// Funcs contains functions that are called instead of the underlying client's methods.
type Funcs struct {
Get func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error
List func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error
Create func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error
Delete func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error
DeleteAllOf func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error
Update func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error
Patch func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error
Watch func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error)
SubResource func(client client.WithWatch, subResource string) client.SubResourceClient
SubResourceGet func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error
SubResourceCreate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error
SubResourceUpdate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error
SubResourcePatch func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error
}
// NewClient returns a new interceptor client that calls the functions in funcs instead of the underlying client's methods, if they are not nil.
func NewClient(interceptedClient client.WithWatch, funcs Funcs) client.WithWatch {
return interceptor{
client: interceptedClient,
funcs: funcs,
}
}
type interceptor struct {
client client.WithWatch
funcs Funcs
}
var _ client.WithWatch = &interceptor{}
func (c interceptor) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
return c.client.GroupVersionKindFor(obj)
}
func (c interceptor) IsObjectNamespaced(obj runtime.Object) (bool, error) {
return c.client.IsObjectNamespaced(obj)
}
func (c interceptor) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if c.funcs.Get != nil {
return c.funcs.Get(ctx, c.client, key, obj, opts...)
}
return c.client.Get(ctx, key, obj, opts...)
}
func (c interceptor) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
if c.funcs.List != nil {
return c.funcs.List(ctx, c.client, list, opts...)
}
return c.client.List(ctx, list, opts...)
}
func (c interceptor) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
if c.funcs.Create != nil {
return c.funcs.Create(ctx, c.client, obj, opts...)
}
return c.client.Create(ctx, obj, opts...)
}
func (c interceptor) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
if c.funcs.Delete != nil {
return c.funcs.Delete(ctx, c.client, obj, opts...)
}
return c.client.Delete(ctx, obj, opts...)
}
func (c interceptor) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
if c.funcs.Update != nil {
return c.funcs.Update(ctx, c.client, obj, opts...)
}
return c.client.Update(ctx, obj, opts...)
}
func (c interceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
if c.funcs.Patch != nil {
return c.funcs.Patch(ctx, c.client, obj, patch, opts...)
}
return c.client.Patch(ctx, obj, patch, opts...)
}
func (c interceptor) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
if c.funcs.DeleteAllOf != nil {
return c.funcs.DeleteAllOf(ctx, c.client, obj, opts...)
}
return c.client.DeleteAllOf(ctx, obj, opts...)
}
func (c interceptor) Status() client.SubResourceWriter {
return c.SubResource("status")
}
func (c interceptor) SubResource(subResource string) client.SubResourceClient {
if c.funcs.SubResource != nil {
return c.funcs.SubResource(c.client, subResource)
}
return subResourceInterceptor{
subResourceName: subResource,
client: c.client,
funcs: c.funcs,
}
}
func (c interceptor) Scheme() *runtime.Scheme {
return c.client.Scheme()
}
func (c interceptor) RESTMapper() meta.RESTMapper {
return c.client.RESTMapper()
}
func (c interceptor) Watch(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) {
if c.funcs.Watch != nil {
return c.funcs.Watch(ctx, c.client, obj, opts...)
}
return c.client.Watch(ctx, obj, opts...)
}
type subResourceInterceptor struct {
subResourceName string
client client.Client
funcs Funcs
}
var _ client.SubResourceClient = &subResourceInterceptor{}
func (s subResourceInterceptor) Get(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error {
if s.funcs.SubResourceGet != nil {
return s.funcs.SubResourceGet(ctx, s.client, s.subResourceName, obj, subResource, opts...)
}
return s.client.SubResource(s.subResourceName).Get(ctx, obj, subResource, opts...)
}
func (s subResourceInterceptor) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error {
if s.funcs.SubResourceCreate != nil {
return s.funcs.SubResourceCreate(ctx, s.client, s.subResourceName, obj, subResource, opts...)
}
return s.client.SubResource(s.subResourceName).Create(ctx, obj, subResource, opts...)
}
func (s subResourceInterceptor) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
if s.funcs.SubResourceUpdate != nil {
return s.funcs.SubResourceUpdate(ctx, s.client, s.subResourceName, obj, opts...)
}
return s.client.SubResource(s.subResourceName).Update(ctx, obj, opts...)
}
func (s subResourceInterceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
if s.funcs.SubResourcePatch != nil {
return s.funcs.SubResourcePatch(ctx, s.client, s.subResourceName, obj, patch, opts...)
}
return s.client.SubResource(s.subResourceName).Patch(ctx, obj, patch, opts...)
}

View File

@@ -20,6 +20,7 @@ import (
"context"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
@@ -141,6 +142,7 @@ type SubResourceWriter interface {
// Create saves the subResource object in the Kubernetes cluster. obj must be a
// struct pointer so that obj can be updated with the content returned by the Server.
Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error
// Update updates the fields corresponding to the status subresource for the
// given obj. obj must be a struct pointer so that obj can be updated
// with the content returned by the Server.
@@ -169,6 +171,10 @@ type Client interface {
Scheme() *runtime.Scheme
// RESTMapper returns the rest this client is using.
RESTMapper() meta.RESTMapper
// GroupVersionKindFor returns the GroupVersionKind for the given object.
GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error)
// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced.
IsObjectNamespaced(obj runtime.Object) (bool, error)
}
// WithWatch supports Watch on top of the CRUD operations supported by

View File

@@ -22,7 +22,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// NewNamespacedClient wraps an existing client enforcing the namespace value.
@@ -52,9 +52,19 @@ func (n *namespacedClient) RESTMapper() meta.RESTMapper {
return n.client.RESTMapper()
}
// GroupVersionKindFor returns the GroupVersionKind for the given object.
func (n *namespacedClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
return n.client.GroupVersionKindFor(obj)
}
// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced.
func (n *namespacedClient) IsObjectNamespaced(obj runtime.Object) (bool, error) {
return n.client.IsObjectNamespaced(obj)
}
// Create implements client.Client.
func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -72,7 +82,7 @@ func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...Creat
// Update implements client.Client.
func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -90,7 +100,7 @@ func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...Updat
// Delete implements client.Client.
func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -108,7 +118,7 @@ func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...Delet
// DeleteAllOf implements client.Client.
func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -121,7 +131,7 @@ func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...
// Patch implements client.Client.
func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -139,7 +149,7 @@ func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, o
// Get implements client.Client.
func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper())
isNamespaceScoped, err := n.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -180,7 +190,7 @@ type namespacedClientSubResourceClient struct {
}
func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -198,7 +208,7 @@ func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subR
}
func (nsw *namespacedClientSubResourceClient) Create(ctx context.Context, obj, subResource Object, opts ...SubResourceCreateOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -217,7 +227,7 @@ func (nsw *namespacedClientSubResourceClient) Create(ctx context.Context, obj, s
// Update implements client.SubResourceWriter.
func (nsw *namespacedClientSubResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}
@@ -235,8 +245,7 @@ func (nsw *namespacedClientSubResourceClient) Update(ctx context.Context, obj Ob
// Patch implements client.SubResourceWriter.
func (nsw *namespacedClientSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error {
isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper())
isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %w", err)
}

View File

@@ -513,8 +513,15 @@ type MatchingLabels map[string]string
// ApplyToList applies this configuration to the given list options.
func (m MatchingLabels) ApplyToList(opts *ListOptions) {
// TODO(directxman12): can we avoid reserializing this over and over?
sel := labels.SelectorFromValidatedSet(map[string]string(m))
opts.LabelSelector = sel
if opts.LabelSelector == nil {
opts.LabelSelector = labels.NewSelector()
}
// If there's already a selector, we need to AND the two together.
noValidSel := labels.SelectorFromValidatedSet(map[string]string(m))
reqs, _ := noValidSel.Requirements()
for _, req := range reqs {
opts.LabelSelector = opts.LabelSelector.Add(req)
}
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
@@ -528,14 +535,17 @@ type HasLabels []string
// ApplyToList applies this configuration to the given list options.
func (m HasLabels) ApplyToList(opts *ListOptions) {
sel := labels.NewSelector()
if opts.LabelSelector == nil {
opts.LabelSelector = labels.NewSelector()
}
// TODO: ignore invalid labels will result in an empty selector.
// This is inconsistent to the that of MatchingLabels.
for _, label := range m {
r, err := labels.NewRequirement(label, selection.Exists, nil)
if err == nil {
sel = sel.Add(*r)
opts.LabelSelector = opts.LabelSelector.Add(*r)
}
}
opts.LabelSelector = sel
}
// ApplyToDeleteAllOf applies this configuration to the given an List options.
@@ -606,6 +616,11 @@ func (n InNamespace) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
n.ApplyToList(&opts.ListOptions)
}
// AsSelector returns a selector that matches objects in the given namespace.
func (n InNamespace) AsSelector() fields.Selector {
return fields.SelectorFromSet(fields.Set{"metadata.namespace": string(n)})
}
// Limit specifies the maximum number of results to return from the server.
// Limit does not implement DeleteAllOfOption interface because the server
// does not support setting it for deletecollection operations.
@@ -788,6 +803,11 @@ func (forceOwnership) ApplyToPatch(opts *PatchOptions) {
opts.Force = &definitelyTrue
}
func (forceOwnership) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) {
definitelyTrue := true
opts.Force = &definitelyTrue
}
// }}}
// {{{ DeleteAllOf Options

View File

@@ -1,143 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// NewDelegatingClientInput encapsulates the input parameters to create a new delegating client.
type NewDelegatingClientInput struct {
CacheReader Reader
Client Client
UncachedObjects []Object
CacheUnstructured bool
}
// NewDelegatingClient creates a new delegating client.
//
// A delegating client forms a Client by composing separate reader, writer and
// statusclient interfaces. This way, you can have an Client that reads from a
// cache and writes to the API server.
func NewDelegatingClient(in NewDelegatingClientInput) (Client, error) {
uncachedGVKs := map[schema.GroupVersionKind]struct{}{}
for _, obj := range in.UncachedObjects {
gvk, err := apiutil.GVKForObject(obj, in.Client.Scheme())
if err != nil {
return nil, err
}
uncachedGVKs[gvk] = struct{}{}
}
return &delegatingClient{
scheme: in.Client.Scheme(),
mapper: in.Client.RESTMapper(),
Reader: &delegatingReader{
CacheReader: in.CacheReader,
ClientReader: in.Client,
scheme: in.Client.Scheme(),
uncachedGVKs: uncachedGVKs,
cacheUnstructured: in.CacheUnstructured,
},
Writer: in.Client,
StatusClient: in.Client,
SubResourceClientConstructor: in.Client,
}, nil
}
type delegatingClient struct {
Reader
Writer
StatusClient
SubResourceClientConstructor
scheme *runtime.Scheme
mapper meta.RESTMapper
}
// Scheme returns the scheme this client is using.
func (d *delegatingClient) Scheme() *runtime.Scheme {
return d.scheme
}
// RESTMapper returns the rest mapper this client is using.
func (d *delegatingClient) RESTMapper() meta.RESTMapper {
return d.mapper
}
// delegatingReader forms a Reader that will cause Get and List requests for
// unstructured types to use the ClientReader while requests for any other type
// of object with use the CacheReader. This avoids accidentally caching the
// entire cluster in the common case of loading arbitrary unstructured objects
// (e.g. from OwnerReferences).
type delegatingReader struct {
CacheReader Reader
ClientReader Reader
uncachedGVKs map[schema.GroupVersionKind]struct{}
scheme *runtime.Scheme
cacheUnstructured bool
}
func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) {
gvk, err := apiutil.GVKForObject(obj, d.scheme)
if err != nil {
return false, err
}
// TODO: this is producing unsafe guesses that don't actually work,
// but it matches ~99% of the cases out there.
if meta.IsListType(obj) {
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
}
if _, isUncached := d.uncachedGVKs[gvk]; isUncached {
return true, nil
}
if !d.cacheUnstructured {
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
return isUnstructured || isUnstructuredList, nil
}
return false, nil
}
// Get retrieves an obj for a given object key from the Kubernetes Cluster.
func (d *delegatingReader) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error {
if isUncached, err := d.shouldBypassCache(obj); err != nil {
return err
} else if isUncached {
return d.ClientReader.Get(ctx, key, obj, opts...)
}
return d.CacheReader.Get(ctx, key, obj, opts...)
}
// List retrieves list of objects for a given namespace and list options.
func (d *delegatingReader) List(ctx context.Context, list ObjectList, opts ...ListOption) error {
if isUncached, err := d.shouldBypassCache(list); err != nil {
return err
} else if isUncached {
return d.ClientReader.List(ctx, list, opts...)
}
return d.CacheReader.List(ctx, list, opts...)
}

View File

@@ -25,16 +25,14 @@ import (
var _ Reader = &typedClient{}
var _ Writer = &typedClient{}
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type typedClient struct {
cache *clientCache
resources *clientRestResources
paramCodec runtime.ParameterCodec
}
// Create implements client.Client.
func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -53,7 +51,7 @@ func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOpti
// Update implements client.Client.
func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -73,7 +71,7 @@ func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOpti
// Delete implements client.Client.
func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -92,7 +90,7 @@ func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOpti
// DeleteAllOf implements client.Client.
func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -111,7 +109,7 @@ func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...Delet
// Patch implements client.Client.
func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -136,7 +134,7 @@ func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts .
// Get implements client.Client.
func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error {
r, err := c.cache.getResource(obj)
r, err := c.resources.getResource(obj)
if err != nil {
return err
}
@@ -151,7 +149,7 @@ func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts .
// List implements client.Client.
func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
r, err := c.cache.getResource(obj)
r, err := c.resources.getResource(obj)
if err != nil {
return err
}
@@ -168,7 +166,7 @@ func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOpti
}
func (c *typedClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -191,7 +189,7 @@ func (c *typedClient) GetSubResource(ctx context.Context, obj, subResourceObj Ob
}
func (c *typedClient) CreateSubResource(ctx context.Context, obj Object, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -216,7 +214,7 @@ func (c *typedClient) CreateSubResource(ctx context.Context, obj Object, subReso
// UpdateSubResource used by SubResourceWriter to write status.
func (c *typedClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -251,7 +249,7 @@ func (c *typedClient) UpdateSubResource(ctx context.Context, obj Object, subReso
// PatchSubResource used by SubResourceWriter to write subresource.
func (c *typedClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error {
o, err := c.cache.getObjMeta(obj)
o, err := c.resources.getObjMeta(obj)
if err != nil {
return err
}

View File

@@ -21,30 +21,27 @@ import (
"fmt"
"strings"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
)
var _ Reader = &unstructuredClient{}
var _ Writer = &unstructuredClient{}
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type unstructuredClient struct {
cache *clientCache
resources *clientRestResources
paramCodec runtime.ParameterCodec
}
// Create implements client.Client.
func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
u, ok := obj.(*unstructured.Unstructured)
u, ok := obj.(runtime.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
gvk := u.GetObjectKind().GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -60,20 +57,20 @@ func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...Cr
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
u.GetObjectKind().SetGroupVersionKind(gvk)
return result
}
// Update implements client.Client.
func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
u, ok := obj.(*unstructured.Unstructured)
u, ok := obj.(runtime.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
gvk := u.GetObjectKind().GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -90,17 +87,17 @@ func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...Up
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
u.GetObjectKind().SetGroupVersionKind(gvk)
return result
}
// Delete implements client.Client.
func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -119,11 +116,11 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...De
// DeleteAllOf implements client.Client.
func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -142,11 +139,11 @@ func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts
// Patch implements client.Client.
func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -171,17 +168,17 @@ func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch
// Get implements client.Client.
func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error {
u, ok := obj.(*unstructured.Unstructured)
u, ok := obj.(runtime.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
gvk := u.GetObjectKind().GroupVersionKind()
getOpts := GetOptions{}
getOpts.ApplyOptions(opts)
r, err := uc.cache.getResource(obj)
r, err := uc.resources.getResource(obj)
if err != nil {
return err
}
@@ -194,22 +191,22 @@ func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object
Do(ctx).
Into(obj)
u.SetGroupVersionKind(gvk)
u.GetObjectKind().SetGroupVersionKind(gvk)
return result
}
// List implements client.Client.
func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
u, ok := obj.(*unstructured.UnstructuredList)
u, ok := obj.(runtime.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
gvk := u.GetObjectKind().GroupVersionKind()
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
r, err := uc.cache.getResource(obj)
r, err := uc.resources.getResource(obj)
if err != nil {
return err
}
@@ -226,19 +223,19 @@ func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...
}
func (uc *unstructuredClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", subResource)
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
if _, ok := subResourceObj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
if _, ok := subResourceObj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj)
}
if subResourceObj.GetName() == "" {
subResourceObj.SetName(obj.GetName())
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -257,19 +254,19 @@ func (uc *unstructuredClient) GetSubResource(ctx context.Context, obj, subResour
}
func (uc *unstructuredClient) CreateSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj)
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
if _, ok := subResourceObj.(*unstructured.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
if _, ok := subResourceObj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj)
}
if subResourceObj.GetName() == "" {
subResourceObj.SetName(obj.GetName())
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -289,11 +286,11 @@ func (uc *unstructuredClient) CreateSubResource(ctx context.Context, obj, subRes
}
func (uc *unstructuredClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error {
if _, ok := obj.(*unstructured.Unstructured); !ok {
if _, ok := obj.(runtime.Unstructured); !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -324,14 +321,14 @@ func (uc *unstructuredClient) UpdateSubResource(ctx context.Context, obj Object,
}
func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error {
u, ok := obj.(*unstructured.Unstructured)
u, ok := obj.(runtime.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
}
gvk := u.GroupVersionKind()
gvk := u.GetObjectKind().GroupVersionKind()
o, err := uc.cache.getObjMeta(obj)
o, err := uc.resources.getObjMeta(obj)
if err != nil {
return err
}
@@ -359,6 +356,6 @@ func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object,
Do(ctx).
Into(body)
u.SetGroupVersionKind(gvk)
u.GetObjectKind().SetGroupVersionKind(gvk)
return result
}

View File

@@ -21,9 +21,8 @@ import (
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
)
@@ -33,21 +32,16 @@ func NewWithWatch(config *rest.Config, options Options) (WithWatch, error) {
if err != nil {
return nil, err
}
dynamicClient, err := dynamic.NewForConfig(config)
if err != nil {
return nil, err
}
return &watchingClient{client: client, dynamic: dynamicClient}, nil
return &watchingClient{client: client}, nil
}
type watchingClient struct {
*client
dynamic dynamic.Interface
}
func (w *watchingClient) Watch(ctx context.Context, list ObjectList, opts ...ListOption) (watch.Interface, error) {
switch l := list.(type) {
case *unstructured.UnstructuredList:
case runtime.Unstructured:
return w.unstructuredWatch(ctx, l, opts...)
case *metav1.PartialObjectMetadataList:
return w.metadataWatch(ctx, l, opts...)
@@ -81,25 +75,23 @@ func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialO
return resInt.Watch(ctx, *listOpts.AsListOptions())
}
func (w *watchingClient) unstructuredWatch(ctx context.Context, obj *unstructured.UnstructuredList, opts ...ListOption) (watch.Interface, error) {
gvk := obj.GroupVersionKind()
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
r, err := w.client.unstructuredClient.cache.getResource(obj)
func (w *watchingClient) unstructuredWatch(ctx context.Context, obj runtime.Unstructured, opts ...ListOption) (watch.Interface, error) {
r, err := w.client.unstructuredClient.resources.getResource(obj)
if err != nil {
return nil, err
}
listOpts := w.listOpts(opts...)
if listOpts.Namespace != "" && r.isNamespaced() {
return w.dynamic.Resource(r.mapping.Resource).Namespace(listOpts.Namespace).Watch(ctx, *listOpts.AsListOptions())
}
return w.dynamic.Resource(r.mapping.Resource).Watch(ctx, *listOpts.AsListOptions())
return r.Get().
NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()).
Resource(r.resource()).
VersionedParams(listOpts.AsListOptions(), w.client.unstructuredClient.paramCodec).
Watch(ctx)
}
func (w *watchingClient) typedWatch(ctx context.Context, obj ObjectList, opts ...ListOption) (watch.Interface, error) {
r, err := w.client.typedClient.cache.getResource(obj)
r, err := w.client.typedClient.resources.getResource(obj)
if err != nil {
return nil, err
}

View File

@@ -19,6 +19,7 @@ package cluster
import (
"context"
"errors"
"net/http"
"time"
"github.com/go-logr/logr"
@@ -27,24 +28,25 @@ import (
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
)
// Cluster provides various methods to interact with a cluster.
type Cluster interface {
// SetFields will set any dependencies on an object for which the object has implemented the inject
// interface - e.g. inject.Client.
// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10.
SetFields(interface{}) error
// GetHTTPClient returns an HTTP client that can be used to talk to the apiserver
GetHTTPClient() *http.Client
// GetConfig returns an initialized Config
GetConfig() *rest.Config
// GetCache returns a cache.Cache
GetCache() cache.Cache
// GetScheme returns an initialized Scheme
GetScheme() *runtime.Scheme
@@ -57,9 +59,6 @@ type Cluster interface {
// GetFieldIndexer returns a client.FieldIndexer configured with the client
GetFieldIndexer() client.FieldIndexer
// GetCache returns a cache.Cache
GetCache() cache.Cache
// GetEventRecorderFor returns a new EventRecorder for the provided name
GetEventRecorderFor(name string) record.EventRecorder
@@ -83,7 +82,7 @@ type Options struct {
Scheme *runtime.Scheme
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
MapperProvider func(c *rest.Config) (meta.RESTMapper, error)
MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error)
// Logger is the logger that should be used by this Cluster.
// If none is set, it defaults to log.Log global logger.
@@ -95,33 +94,43 @@ type Options struct {
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
//
// Deprecated: Use Cache.SyncPeriod instead.
SyncPeriod *time.Duration
// Namespace if specified restricts the manager's cache to watch objects in
// the desired namespace Defaults to all namespaces
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources the cache
// will only hold objects from the desired namespace.
Namespace string
// HTTPClient is the http client that will be used to create the default
// Cache and Client. If not set the rest.HTTPClientFor function will be used
// to create the http client.
HTTPClient *http.Client
// Cache is the cache.Options that will be used to create the default Cache.
// By default, the cache will watch and list requested objects in all namespaces.
Cache cache.Options
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
//
// When using a custom NewCache, the Cache options will be passed to the
// NewCache function.
//
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewCache if you know what you are doing.
NewCache cache.NewCacheFunc
// Client is the client.Options that will be used to create the default Client.
// By default, the client will use the cache for reads and direct calls for writes.
Client client.Options
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create the default DelegatingClient that will
// use the cache for reads and the client for writes.
// NOTE: The default client will not cache Unstructured.
NewClient NewClientFunc
// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
// for the given objects.
ClientDisableCacheFor []client.Object
// DryRunClient specifies whether the client should be configured to enforce
// dryRun mode.
DryRunClient bool
// If not set this will create a Client backed by a Cache for read operations
// and a direct Client for write operations.
//
// When using a custom NewClient, the Client options will be passed to the
// NewClient function.
//
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewClient if you know what you are doing.
NewClient client.NewClientFunc
// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
// Use this to customize the event correlator and spam filter
@@ -137,7 +146,7 @@ type Options struct {
makeBroadcaster intrec.EventBroadcasterProducer
// Dependency injection for testing
newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
}
// Option can be used to manipulate Options.
@@ -149,56 +158,103 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) {
return nil, errors.New("must specify Config")
}
originalConfig := config
config = rest.CopyConfig(config)
if config.UserAgent == "" {
config.UserAgent = rest.DefaultKubernetesUserAgent()
}
options := Options{}
for _, opt := range opts {
opt(&options)
}
options = setOptionsDefaults(options)
options, err := setOptionsDefaults(options, config)
if err != nil {
options.Logger.Error(err, "Failed to set defaults")
return nil, err
}
// Create the mapper provider
mapper, err := options.MapperProvider(config)
mapper, err := options.MapperProvider(config, options.HTTPClient)
if err != nil {
options.Logger.Error(err, "Failed to get API Group-Resources")
return nil, err
}
// Create the cache for the cached read client and registering informers
cache, err := options.NewCache(config, cache.Options{Scheme: options.Scheme, Mapper: mapper, Resync: options.SyncPeriod, Namespace: options.Namespace})
cacheOpts := options.Cache
{
if cacheOpts.Scheme == nil {
cacheOpts.Scheme = options.Scheme
}
if cacheOpts.Mapper == nil {
cacheOpts.Mapper = mapper
}
if cacheOpts.HTTPClient == nil {
cacheOpts.HTTPClient = options.HTTPClient
}
if cacheOpts.SyncPeriod == nil {
cacheOpts.SyncPeriod = options.SyncPeriod
}
}
cache, err := options.NewCache(config, cacheOpts)
if err != nil {
return nil, err
}
clientOptions := client.Options{Scheme: options.Scheme, Mapper: mapper}
apiReader, err := client.New(config, clientOptions)
// Create the client, and default its options.
clientOpts := options.Client
{
if clientOpts.Scheme == nil {
clientOpts.Scheme = options.Scheme
}
if clientOpts.Mapper == nil {
clientOpts.Mapper = mapper
}
if clientOpts.HTTPClient == nil {
clientOpts.HTTPClient = options.HTTPClient
}
if clientOpts.Cache == nil {
clientOpts.Cache = &client.CacheOptions{
Unstructured: false,
}
}
if clientOpts.Cache.Reader == nil {
clientOpts.Cache.Reader = cache
}
}
clientWriter, err := options.NewClient(config, clientOpts)
if err != nil {
return nil, err
}
writeObj, err := options.NewClient(cache, config, clientOptions, options.ClientDisableCacheFor...)
// Create the API Reader, a client with no cache.
clientReader, err := client.New(config, client.Options{
HTTPClient: options.HTTPClient,
Scheme: options.Scheme,
Mapper: mapper,
})
if err != nil {
return nil, err
}
if options.DryRunClient {
writeObj = client.NewDryRunClient(writeObj)
}
// Create the recorder provider to inject event recorders for the components.
// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
// to the particular controller that it's being injected into, rather than a generic one like is here.
recorderProvider, err := options.newRecorderProvider(config, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster)
recorderProvider, err := options.newRecorderProvider(config, options.HTTPClient, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
return &cluster{
config: config,
config: originalConfig,
httpClient: options.HTTPClient,
scheme: options.Scheme,
cache: cache,
fieldIndexes: cache,
client: writeObj,
apiReader: apiReader,
client: clientWriter,
apiReader: clientReader,
recorderProvider: recorderProvider,
mapper: mapper,
logger: options.Logger,
@@ -206,21 +262,27 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) {
}
// setOptionsDefaults set default values for Options fields.
func setOptionsDefaults(options Options) Options {
func setOptionsDefaults(options Options, config *rest.Config) (Options, error) {
if options.HTTPClient == nil {
var err error
options.HTTPClient, err = rest.HTTPClientFor(config)
if err != nil {
return options, err
}
}
// Use the Kubernetes client-go scheme if none is specified
if options.Scheme == nil {
options.Scheme = scheme.Scheme
}
if options.MapperProvider == nil {
options.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) {
return apiutil.NewDynamicRESTMapper(c)
}
options.MapperProvider = apiutil.NewDynamicRESTMapper
}
// Allow users to define how to create a new client
if options.NewClient == nil {
options.NewClient = DefaultNewClient
options.NewClient = client.New
}
// Allow newCache to be mocked
@@ -250,39 +312,5 @@ func setOptionsDefaults(options Options) Options {
options.Logger = logf.RuntimeLog.WithName("cluster")
}
return options
}
// NewClientFunc allows a user to define how to create a client.
type NewClientFunc func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error)
// ClientOptions are the optional arguments for tuning the caching client.
type ClientOptions struct {
UncachedObjects []client.Object
CacheUnstructured bool
}
// DefaultNewClient creates the default caching client, that will never cache Unstructured.
func DefaultNewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) {
return ClientBuilderWithOptions(ClientOptions{})(cache, config, options, uncachedObjects...)
}
// ClientBuilderWithOptions returns a Client constructor that will build a client
// honoring the options argument
func ClientBuilderWithOptions(options ClientOptions) NewClientFunc {
return func(cache cache.Cache, config *rest.Config, clientOpts client.Options, uncachedObjects ...client.Object) (client.Client, error) {
options.UncachedObjects = append(options.UncachedObjects, uncachedObjects...)
c, err := client.New(config, clientOpts)
if err != nil {
return nil, err
}
return client.NewDelegatingClient(client.NewDelegatingClientInput{
CacheReader: cache,
Client: c,
UncachedObjects: options.UncachedObjects,
CacheUnstructured: options.CacheUnstructured,
})
}
return options, nil
}

View File

@@ -18,6 +18,7 @@ package cluster
import (
"context"
"net/http"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/meta"
@@ -28,22 +29,16 @@ import (
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
type cluster struct {
// config is the rest.config used to talk to the apiserver. Required.
config *rest.Config
// scheme is the scheme injected into Controllers, EventHandlers, Sources and Predicates. Defaults
// to scheme.scheme.
scheme *runtime.Scheme
cache cache.Cache
// TODO(directxman12): Provide an escape hatch to get individual indexers
// client is the client injected into Controllers (and EventHandlers, Sources and Predicates).
client client.Client
httpClient *http.Client
scheme *runtime.Scheme
cache cache.Cache
client client.Client
// apiReader is the reader that will make requests to the api server and not the cache.
apiReader client.Reader
@@ -64,32 +59,14 @@ type cluster struct {
logger logr.Logger
}
func (c *cluster) SetFields(i interface{}) error {
if _, err := inject.ConfigInto(c.config, i); err != nil {
return err
}
if _, err := inject.ClientInto(c.client, i); err != nil {
return err
}
if _, err := inject.APIReaderInto(c.apiReader, i); err != nil {
return err
}
if _, err := inject.SchemeInto(c.scheme, i); err != nil {
return err
}
if _, err := inject.CacheInto(c.cache, i); err != nil {
return err
}
if _, err := inject.MapperInto(c.mapper, i); err != nil {
return err
}
return nil
}
func (c *cluster) GetConfig() *rest.Config {
return c.config
}
func (c *cluster) GetHTTPClient() *http.Client {
return c.httpClient
}
func (c *cluster) GetClient() client.Client {
return c.client
}

View File

@@ -29,6 +29,8 @@ import (
// ControllerManagerConfiguration defines the functions necessary to parse a config file
// and to configure the Options struct for the ctrl.Manager.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerManagerConfiguration interface {
runtime.Object
@@ -38,6 +40,8 @@ type ControllerManagerConfiguration interface {
// DeferredFileLoader is used to configure the decoder for loading controller
// runtime component config types.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type DeferredFileLoader struct {
ControllerManagerConfiguration
path string
@@ -52,6 +56,8 @@ type DeferredFileLoader struct {
// Defaults:
// * Path: "./config.yaml"
// * Kind: GenericControllerManagerConfiguration
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
func File() *DeferredFileLoader {
scheme := runtime.NewScheme()
utilruntime.Must(v1alpha1.AddToScheme(scheme))
@@ -83,12 +89,6 @@ func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *Deferre
return d
}
// InjectScheme will configure the scheme to be used for decoding the file.
func (d *DeferredFileLoader) InjectScheme(scheme *runtime.Scheme) error {
d.scheme = scheme
return nil
}
// loadFile is used from the mutex.Once to load the file.
func (d *DeferredFileLoader) loadFile() {
if d.scheme == nil {

View File

@@ -0,0 +1,49 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import "time"
// Controller contains configuration options for a controller.
type Controller struct {
// GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation
// allowed for that controller.
//
// When a controller is registered within this manager using the builder utilities,
// users have to specify the type the controller reconciles in the For(...) call.
// If the object's kind passed matches one of the keys in this map, the concurrency
// for that controller is set to the number specified.
//
// The key is expected to be consistent in form with GroupKind.String(),
// e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`.
GroupKindConcurrency map[string]int
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
MaxConcurrentReconciles int
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
// Defaults to the Controller.RecoverPanic setting from the Manager if unset.
RecoverPanic *bool
// NeedLeaderElection indicates whether the controller needs to use leader election.
// Defaults to true, which means the controller will use leader election.
NeedLeaderElection *bool
}

View File

@@ -14,12 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package config contains functionality for interacting with ComponentConfig
// files
//
// # DeferredFileLoader
//
// This uses a deferred file decoding allowing you to chain your configuration
// setup. You can pass this into manager.Options#File and it will load your
// config.
// Package config contains functionality for interacting with
// configuration for controller-runtime components.
package config

View File

@@ -17,4 +17,6 @@ limitations under the License.
// Package v1alpha1 provides the ControllerManagerConfiguration used for
// configuring ctrl.Manager
// +kubebuilder:object:generate=true
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
package v1alpha1

View File

@@ -23,12 +23,18 @@ import (
var (
// GroupVersion is group version used to register these objects.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"}
// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
AddToScheme = SchemeBuilder.AddToScheme
)

View File

@@ -25,6 +25,8 @@ import (
)
// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerManagerConfigurationSpec struct {
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
@@ -60,7 +62,7 @@ type ControllerManagerConfigurationSpec struct {
// +optional
Controller *ControllerConfigurationSpec `json:"controller,omitempty"`
// Metrics contains thw controller metrics configuration
// Metrics contains the controller metrics configuration
// +optional
Metrics ControllerMetrics `json:"metrics,omitempty"`
@@ -75,6 +77,11 @@ type ControllerManagerConfigurationSpec struct {
// ControllerConfigurationSpec defines the global configuration for
// controllers registered with the manager.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
//
// Deprecated: Controller global configuration can now be set at the manager level,
// using the manager.Options.Controller field.
type ControllerConfigurationSpec struct {
// GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation
// allowed for that controller.
@@ -101,6 +108,8 @@ type ControllerConfigurationSpec struct {
}
// ControllerMetrics defines the metrics configs.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerMetrics struct {
// BindAddress is the TCP address that the controller should bind to
// for serving prometheus metrics.
@@ -110,6 +119,8 @@ type ControllerMetrics struct {
}
// ControllerHealth defines the health configs.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerHealth struct {
// HealthProbeBindAddress is the TCP address that the controller should bind to
// for serving health probes
@@ -127,6 +138,8 @@ type ControllerHealth struct {
}
// ControllerWebhook defines the webhook server for the controller.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerWebhook struct {
// Port is the port that the webhook server serves at.
// It is used to set webhook.Server.Port.
@@ -149,6 +162,8 @@ type ControllerWebhook struct {
// +kubebuilder:object:root=true
// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
type ControllerManagerConfiguration struct {
metav1.TypeMeta `json:",inline"`
@@ -157,6 +172,8 @@ type ControllerManagerConfiguration struct {
}
// Complete returns the configuration for controller-runtime.
//
// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895.
func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) {
return *c, nil
}

View File

@@ -1,5 +1,4 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
// Code generated by controller-gen. DO NOT EDIT.

View File

@@ -39,6 +39,18 @@ type Options struct {
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
MaxConcurrentReconciles int
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
// Defaults to the Controller.RecoverPanic setting from the Manager if unset.
RecoverPanic *bool
// NeedLeaderElection indicates whether the controller needs to use leader election.
// Defaults to true, which means the controller will use leader election.
NeedLeaderElection *bool
// Reconciler reconciles an object
Reconciler reconcile.Reconciler
@@ -50,14 +62,6 @@ type Options struct {
// LogConstructor is used to construct a logger used for this controller and passed
// to each reconciliation via the context field.
LogConstructor func(request *reconcile.Request) logr.Logger
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
// Defaults to the Controller.RecoverPanic setting from the Manager if unset.
RecoverPanic *bool
}
// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests
@@ -124,38 +128,47 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
}
if options.MaxConcurrentReconciles <= 0 {
options.MaxConcurrentReconciles = 1
if mgr.GetControllerOptions().MaxConcurrentReconciles > 0 {
options.MaxConcurrentReconciles = mgr.GetControllerOptions().MaxConcurrentReconciles
} else {
options.MaxConcurrentReconciles = 1
}
}
if options.CacheSyncTimeout == 0 {
options.CacheSyncTimeout = 2 * time.Minute
if mgr.GetControllerOptions().CacheSyncTimeout != 0 {
options.CacheSyncTimeout = mgr.GetControllerOptions().CacheSyncTimeout
} else {
options.CacheSyncTimeout = 2 * time.Minute
}
}
if options.RateLimiter == nil {
options.RateLimiter = workqueue.DefaultControllerRateLimiter()
}
// Inject dependencies into Reconciler
if err := mgr.SetFields(options.Reconciler); err != nil {
return nil, err
}
if options.RecoverPanic == nil {
options.RecoverPanic = mgr.GetControllerOptions().RecoverPanic
}
if options.NeedLeaderElection == nil {
options.NeedLeaderElection = mgr.GetControllerOptions().NeedLeaderElection
}
// Create controller with dependencies set
return &controller.Controller{
Do: options.Reconciler,
MakeQueue: func() workqueue.RateLimitingInterface {
return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name)
return workqueue.NewRateLimitingQueueWithConfig(options.RateLimiter, workqueue.RateLimitingQueueConfig{
Name: name,
})
},
MaxConcurrentReconciles: options.MaxConcurrentReconciles,
CacheSyncTimeout: options.CacheSyncTimeout,
SetFields: mgr.SetFields,
Name: name,
LogConstructor: options.LogConstructor,
RecoverPanic: options.RecoverPanic,
LeaderElected: options.NeedLeaderElection,
}, nil
}

View File

@@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
@@ -365,15 +366,18 @@ func AddFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) {
// It returns an indication of whether it updated the object's list of finalizers.
func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) {
f := o.GetFinalizers()
for i := 0; i < len(f); i++ {
length := len(f)
index := 0
for i := 0; i < length; i++ {
if f[i] == finalizer {
f = append(f[:i], f[i+1:]...)
i--
finalizersUpdated = true
continue
}
f[index] = f[i]
index++
}
o.SetFinalizers(f)
return
o.SetFinalizers(f[:index])
return length != index
}
// ContainsFinalizer checks an Object that the provided finalizer is present.
@@ -386,9 +390,3 @@ func ContainsFinalizer(o client.Object, finalizer string) bool {
}
return false
}
// Object allows functions to work indistinctly with any resource that
// implements both Object interfaces.
//
// Deprecated: Use client.Object instead.
type Object = client.Object

View File

@@ -166,7 +166,7 @@ func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefi
// Poll until all resources are found in discovery
p := &poller{config: config, waitingFor: waitingFor}
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
return wait.PollUntilContextTimeout(context.TODO(), options.PollInterval, options.MaxTime, true, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not.
@@ -179,7 +179,7 @@ type poller struct {
}
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *poller) poll() (done bool, err error) {
func (p *poller) poll(ctx context.Context) (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
cs, err := clientset.NewForConfig(p.config)
if err != nil {

View File

@@ -161,11 +161,6 @@ type Environment struct {
// environment variable or 20 seconds if unspecified
ControlPlaneStopTimeout time.Duration
// KubeAPIServerFlags is the set of flags passed while starting the api server.
//
// Deprecated: use ControlPlane.GetAPIServer().Configure() instead.
KubeAPIServerFlags []string
// AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr.
// Enable this to get more visibility of the testing control plane.
// It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable.
@@ -210,19 +205,7 @@ func (te *Environment) Start() (*rest.Config, error) {
}
} else {
apiServer := te.ControlPlane.GetAPIServer()
if len(apiServer.Args) == 0 { //nolint:staticcheck
// pass these through separately from above in case something like
// AddUser defaults APIServer.
//
// TODO(directxman12): if/when we feel like making a bigger
// breaking change here, just make APIServer and Etcd non-pointers
// in ControlPlane.
// NB(directxman12): we still pass these in so that things work if the
// user manually specifies them, but in most cases we expect them to
// be nil so that we use the new .Configure() logic.
apiServer.Args = te.KubeAPIServerFlags //nolint:staticcheck
}
if te.ControlPlane.Etcd == nil {
te.ControlPlane.Etcd = &controlplane.Etcd{}
}
@@ -230,17 +213,19 @@ func (te *Environment) Start() (*rest.Config, error) {
if os.Getenv(envAttachOutput) == "true" {
te.AttachControlPlaneOutput = true
}
if apiServer.Out == nil && te.AttachControlPlaneOutput {
apiServer.Out = os.Stdout
}
if apiServer.Err == nil && te.AttachControlPlaneOutput {
apiServer.Err = os.Stderr
}
if te.ControlPlane.Etcd.Out == nil && te.AttachControlPlaneOutput {
te.ControlPlane.Etcd.Out = os.Stdout
}
if te.ControlPlane.Etcd.Err == nil && te.AttachControlPlaneOutput {
te.ControlPlane.Etcd.Err = os.Stderr
if te.AttachControlPlaneOutput {
if apiServer.Out == nil {
apiServer.Out = os.Stdout
}
if apiServer.Err == nil {
apiServer.Err = os.Stderr
}
if te.ControlPlane.Etcd.Out == nil {
te.ControlPlane.Etcd.Out = os.Stdout
}
if te.ControlPlane.Etcd.Err == nil {
te.ControlPlane.Etcd.Err = os.Stderr
}
}
apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
@@ -287,6 +272,9 @@ func (te *Environment) Start() (*rest.Config, error) {
}
log.V(1).Info("installing CRDs")
if te.CRDInstallOptions.Scheme == nil {
te.CRDInstallOptions.Scheme = te.Scheme
}
te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs)
te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths)
te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing

View File

@@ -147,6 +147,8 @@ func (o *WebhookInstallOptions) PrepWithoutInstalling() error {
// Install installs specified webhooks to the API server.
func (o *WebhookInstallOptions) Install(config *rest.Config) error {
defaultWebhookOptions(o)
if len(o.LocalServingCAData) == 0 {
if err := o.PrepWithoutInstalling(); err != nil {
return err
@@ -168,6 +170,16 @@ func (o *WebhookInstallOptions) Cleanup() error {
return nil
}
// defaultWebhookOptions sets the default values for Webhooks.
func defaultWebhookOptions(o *WebhookInstallOptions) {
if o.MaxTime == 0 {
o.MaxTime = defaultMaxWait
}
if o.PollInterval == 0 {
o.PollInterval = defaultPollInterval
}
}
// WaitForWebhooks waits for the Webhooks to be available through API server.
func WaitForWebhooks(config *rest.Config,
mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration,
@@ -203,7 +215,7 @@ func WaitForWebhooks(config *rest.Config,
// Poll until all resources are found in discovery
p := &webhookPoller{config: config, waitingFor: waitingFor}
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
return wait.PollUntilContextTimeout(context.TODO(), options.PollInterval, options.MaxTime, true, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not.
@@ -216,7 +228,7 @@ type webhookPoller struct {
}
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *webhookPoller) poll() (done bool, err error) {
func (p *webhookPoller) poll(ctx context.Context) (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
c, err := client.New(p.config, client.Options{})
if err != nil {

View File

@@ -17,6 +17,8 @@ limitations under the License.
package handler
import (
"context"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/event"
@@ -36,7 +38,7 @@ var _ EventHandler = &EnqueueRequestForObject{}
type EnqueueRequestForObject struct{}
// Create implements EventHandler.
func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
func (e *EnqueueRequestForObject) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) {
if evt.Object == nil {
enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt)
return
@@ -48,7 +50,7 @@ func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.Rate
}
// Update implements EventHandler.
func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
func (e *EnqueueRequestForObject) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
switch {
case evt.ObjectNew != nil:
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
@@ -66,7 +68,7 @@ func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.Rate
}
// Delete implements EventHandler.
func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
func (e *EnqueueRequestForObject) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
if evt.Object == nil {
enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt)
return
@@ -78,7 +80,7 @@ func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.Rate
}
// Generic implements EventHandler.
func (e *EnqueueRequestForObject) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
func (e *EnqueueRequestForObject) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
if evt.Object == nil {
enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt)
return

View File

@@ -17,16 +17,17 @@ limitations under the License.
package handler
import (
"context"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
// MapFunc is the signature required for enqueueing requests from a generic function.
// This type is usually used with EnqueueRequestsFromMapFunc when registering an event handler.
type MapFunc func(client.Object) []reconcile.Request
type MapFunc func(context.Context, client.Object) []reconcile.Request
// EnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection
// of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects
@@ -52,32 +53,32 @@ type enqueueRequestsFromMapFunc struct {
}
// Create implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestsFromMapFunc) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
e.mapAndEnqueue(ctx, q, evt.Object, reqs)
}
// Update implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestsFromMapFunc) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.ObjectOld, reqs)
e.mapAndEnqueue(q, evt.ObjectNew, reqs)
e.mapAndEnqueue(ctx, q, evt.ObjectOld, reqs)
e.mapAndEnqueue(ctx, q, evt.ObjectNew, reqs)
}
// Delete implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestsFromMapFunc) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
e.mapAndEnqueue(ctx, q, evt.Object, reqs)
}
// Generic implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestsFromMapFunc) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
e.mapAndEnqueue(ctx, q, evt.Object, reqs)
}
func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) {
for _, req := range e.toRequests(object) {
func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(ctx context.Context, q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) {
for _, req := range e.toRequests(ctx, object) {
_, ok := reqs[req]
if !ok {
q.Add(req)
@@ -85,13 +86,3 @@ func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInter
}
}
}
// EnqueueRequestsFromMapFunc can inject fields into the mapper.
// InjectFunc implements inject.Injector.
func (e *enqueueRequestsFromMapFunc) InjectFunc(f inject.Func) error {
if f == nil {
return nil
}
return f(e.toRequests)
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package handler
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
@@ -25,15 +26,18 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
var _ EventHandler = &EnqueueRequestForOwner{}
var _ EventHandler = &enqueueRequestForOwner{}
var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOwner")
var log = logf.RuntimeLog.WithName("eventhandler").WithName("enqueueRequestForOwner")
// OwnerOption modifies an EnqueueRequestForOwner EventHandler.
type OwnerOption func(e *enqueueRequestForOwner)
// EnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created
// the object that was the source of the Event.
@@ -42,13 +46,34 @@ var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOw
//
// - a source.Kind Source with Type of Pod.
//
// - a handler.EnqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and IsController set to true.
type EnqueueRequestForOwner struct {
// OwnerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared.
OwnerType runtime.Object
// - a handler.enqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true.
func EnqueueRequestForOwner(scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) EventHandler {
e := &enqueueRequestForOwner{
ownerType: ownerType,
mapper: mapper,
}
if err := e.parseOwnerTypeGroupKind(scheme); err != nil {
panic(err)
}
for _, opt := range opts {
opt(e)
}
return e
}
// IsController if set will only look at the first OwnerReference with Controller: true.
IsController bool
// OnlyControllerOwner if provided will only look at the first OwnerReference with Controller: true.
func OnlyControllerOwner() OwnerOption {
return func(e *enqueueRequestForOwner) {
e.isController = true
}
}
type enqueueRequestForOwner struct {
// ownerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared.
ownerType runtime.Object
// isController if set will only look at the first OwnerReference with Controller: true.
isController bool
// groupKind is the cached Group and Kind from OwnerType
groupKind schema.GroupKind
@@ -58,7 +83,7 @@ type EnqueueRequestForOwner struct {
}
// Create implements EventHandler.
func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestForOwner) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
@@ -67,7 +92,7 @@ func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateL
}
// Update implements EventHandler.
func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestForOwner) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.ObjectOld, reqs)
e.getOwnerReconcileRequest(evt.ObjectNew, reqs)
@@ -77,7 +102,7 @@ func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateL
}
// Delete implements EventHandler.
func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestForOwner) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
@@ -86,7 +111,7 @@ func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateL
}
// Generic implements EventHandler.
func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
func (e *enqueueRequestForOwner) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
@@ -96,17 +121,17 @@ func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.Rat
// parseOwnerTypeGroupKind parses the OwnerType into a Group and Kind and caches the result. Returns false
// if the OwnerType could not be parsed using the scheme.
func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error {
func (e *enqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error {
// Get the kinds of the type
kinds, _, err := scheme.ObjectKinds(e.OwnerType)
kinds, _, err := scheme.ObjectKinds(e.ownerType)
if err != nil {
log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType))
log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType))
return err
}
// Expect only 1 kind. If there is more than one kind this is probably an edge case such as ListOptions.
if len(kinds) != 1 {
err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.OwnerType, kinds)
log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType), "kinds", kinds)
err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.ownerType, kinds)
log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType), "kinds", kinds)
return err
}
// Cache the Group and Kind for the OwnerType
@@ -116,7 +141,7 @@ func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme)
// getOwnerReconcileRequest looks at object and builds a map of reconcile.Request to reconcile
// owners of object that match e.OwnerType.
func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) {
func (e *enqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) {
// Iterate through the OwnerReferences looking for a match on Group and Kind against what was requested
// by the user
for _, ref := range e.getOwnersReferences(object) {
@@ -138,7 +163,7 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object,
Name: ref.Name,
}}
// if owner is not namespaced then we should set the namespace to the empty
// if owner is not namespaced then we should not set the namespace
mapping, err := e.mapper.RESTMapping(e.groupKind, refGV.Version)
if err != nil {
log.Error(err, "Could not retrieve rest mapping", "kind", e.groupKind)
@@ -153,16 +178,16 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object,
}
}
// getOwnersReferences returns the OwnerReferences for an object as specified by the EnqueueRequestForOwner
// getOwnersReferences returns the OwnerReferences for an object as specified by the enqueueRequestForOwner
// - if IsController is true: only take the Controller OwnerReference (if found)
// - if IsController is false: take all OwnerReferences.
func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference {
func (e *enqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference {
if object == nil {
return nil
}
// If not filtered as Controller only, then use all the OwnerReferences
if !e.IsController {
if !e.isController {
return object.GetOwnerReferences()
}
// If filtered to a Controller, only take the Controller OwnerReference
@@ -172,18 +197,3 @@ func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []met
// No Controller OwnerReference found
return nil
}
var _ inject.Scheme = &EnqueueRequestForOwner{}
// InjectScheme is called by the Controller to provide a singleton scheme to the EnqueueRequestForOwner.
func (e *EnqueueRequestForOwner) InjectScheme(s *runtime.Scheme) error {
return e.parseOwnerTypeGroupKind(s)
}
var _ inject.Mapper = &EnqueueRequestForOwner{}
// InjectMapper is called by the Controller to provide the rest mapper used by the manager.
func (e *EnqueueRequestForOwner) InjectMapper(m meta.RESTMapper) error {
e.mapper = m
return nil
}

View File

@@ -17,6 +17,8 @@ limitations under the License.
package handler
import (
"context"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/event"
)
@@ -41,17 +43,17 @@ import (
// Most users shouldn't need to implement their own EventHandler.
type EventHandler interface {
// Create is called in response to an create event - e.g. Pod Creation.
Create(event.CreateEvent, workqueue.RateLimitingInterface)
Create(context.Context, event.CreateEvent, workqueue.RateLimitingInterface)
// Update is called in response to an update event - e.g. Pod Updated.
Update(event.UpdateEvent, workqueue.RateLimitingInterface)
Update(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface)
// Delete is called in response to a delete event - e.g. Pod Deleted.
Delete(event.DeleteEvent, workqueue.RateLimitingInterface)
Delete(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface)
// Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or
// external trigger request - e.g. reconcile Autoscaling, or a Webhook.
Generic(event.GenericEvent, workqueue.RateLimitingInterface)
Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface)
}
var _ EventHandler = Funcs{}
@@ -60,45 +62,45 @@ var _ EventHandler = Funcs{}
type Funcs struct {
// Create is called in response to an add event. Defaults to no-op.
// RateLimitingInterface is used to enqueue reconcile.Requests.
CreateFunc func(event.CreateEvent, workqueue.RateLimitingInterface)
CreateFunc func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface)
// Update is called in response to an update event. Defaults to no-op.
// RateLimitingInterface is used to enqueue reconcile.Requests.
UpdateFunc func(event.UpdateEvent, workqueue.RateLimitingInterface)
UpdateFunc func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface)
// Delete is called in response to a delete event. Defaults to no-op.
// RateLimitingInterface is used to enqueue reconcile.Requests.
DeleteFunc func(event.DeleteEvent, workqueue.RateLimitingInterface)
DeleteFunc func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface)
// GenericFunc is called in response to a generic event. Defaults to no-op.
// RateLimitingInterface is used to enqueue reconcile.Requests.
GenericFunc func(event.GenericEvent, workqueue.RateLimitingInterface)
GenericFunc func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface)
}
// Create implements EventHandler.
func (h Funcs) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) {
func (h Funcs) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) {
if h.CreateFunc != nil {
h.CreateFunc(e, q)
h.CreateFunc(ctx, e, q)
}
}
// Delete implements EventHandler.
func (h Funcs) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) {
func (h Funcs) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) {
if h.DeleteFunc != nil {
h.DeleteFunc(e, q)
h.DeleteFunc(ctx, e, q)
}
}
// Update implements EventHandler.
func (h Funcs) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) {
func (h Funcs) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) {
if h.UpdateFunc != nil {
h.UpdateFunc(e, q)
h.UpdateFunc(ctx, e, q)
}
}
// Generic implements EventHandler.
func (h Funcs) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) {
func (h Funcs) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) {
if h.GenericFunc != nil {
h.GenericFunc(e, q)
h.GenericFunc(ctx, e, q)
}
}

View File

@@ -28,17 +28,15 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/handler"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/source"
)
var _ inject.Injector = &Controller{}
// Controller implements controller.Controller.
type Controller struct {
// Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required.
@@ -61,10 +59,6 @@ type Controller struct {
// the Queue for processing
Queue workqueue.RateLimitingInterface
// SetFields is used to inject dependencies into other objects such as Sources, EventHandlers and Predicates
// Deprecated: the caller should handle injected fields itself.
SetFields func(i interface{}) error
// mu is used to synchronize Controller setup
mu sync.Mutex
@@ -93,6 +87,9 @@ type Controller struct {
// RecoverPanic indicates whether the panic caused by reconcile should be recovered.
RecoverPanic *bool
// LeaderElected indicates whether the controller is leader elected or always running.
LeaderElected *bool
}
// watchDescription contains all the information necessary to start a watch.
@@ -127,19 +124,6 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
c.mu.Lock()
defer c.mu.Unlock()
// Inject Cache into arguments
if err := c.SetFields(src); err != nil {
return err
}
if err := c.SetFields(evthdler); err != nil {
return err
}
for _, pr := range prct {
if err := c.SetFields(pr); err != nil {
return err
}
}
// Controller hasn't started yet, store the watches locally and return.
//
// These watches are going to be held on the controller struct until the manager or user calls Start(...).
@@ -152,6 +136,14 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
return src.Start(c.ctx, evthdler, c.Queue, prct...)
}
// NeedLeaderElection implements the manager.LeaderElectionRunnable interface.
func (c *Controller) NeedLeaderElection() bool {
if c.LeaderElected == nil {
return true
}
return *c.LeaderElected
}
// Start implements controller.Controller.
func (c *Controller) Start(ctx context.Context) error {
// use an IIFE to get proper lock handling
@@ -320,14 +312,23 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) {
// RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the
// resource to be synced.
log.V(5).Info("Reconciling")
result, err := c.Reconcile(ctx, req)
switch {
case err != nil:
c.Queue.AddRateLimited(req)
if errors.Is(err, reconcile.TerminalError(nil)) {
ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Inc()
} else {
c.Queue.AddRateLimited(req)
}
ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Inc()
if !result.IsZero() {
log.Info("Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes reqeueuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler")
}
log.Error(err, "Reconciler error")
case result.RequeueAfter > 0:
log.V(5).Info(fmt.Sprintf("Reconcile done, requeueing after %s", result.RequeueAfter))
// The result.RequeueAfter request will be lost, if it is returned
// along with a non-nil error. But this is intended as
// We need to drive to stable reconcile loops before queuing due
@@ -336,9 +337,11 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) {
c.Queue.AddAfter(req, result.RequeueAfter)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Inc()
case result.Requeue:
log.V(5).Info("Reconcile done, requeueing")
c.Queue.AddRateLimited(req)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Inc()
default:
log.V(5).Info("Reconcile successful")
// Finally, if no error occurs we Forget this item so it does not
// get queued again until another change happens.
c.Queue.Forget(obj)
@@ -351,12 +354,6 @@ func (c *Controller) GetLogger() logr.Logger {
return c.LogConstructor(nil)
}
// InjectFunc implement SetFields.Injector.
func (c *Controller) InjectFunc(f inject.Func) error {
c.SetFields = f
return nil
}
// updateMetrics updates prometheus metrics within the controller.
func (c *Controller) updateMetrics(reconcileTime time.Duration) {
ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())

View File

@@ -39,6 +39,13 @@ var (
Help: "Total number of reconciliation errors per controller",
}, []string{"controller"})
// TerminalReconcileErrors is a prometheus counter metrics which holds the total
// number of terminal errors from the Reconciler.
TerminalReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "controller_runtime_terminal_reconcile_errors_total",
Help: "Total number of terminal reconciliation errors per controller",
}, []string{"controller"})
// ReconcileTime is a prometheus metric which keeps track of the duration
// of reconciliations.
ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
@@ -67,6 +74,7 @@ func init() {
metrics.Registry.MustRegister(
ReconcileTotal,
ReconcileErrors,
TerminalReconcileErrors,
ReconcileTime,
WorkerCount,
ActiveWorkers,

View File

@@ -17,14 +17,9 @@ limitations under the License.
package objectutil
import (
"errors"
"fmt"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// FilterWithLabels returns a copy of the items in objs matching labelSel.
@@ -45,34 +40,3 @@ func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtim
}
return outItems, nil
}
// IsAPINamespaced returns true if the object is namespace scoped.
// For unstructured objects the gvk is found from the object itself.
func IsAPINamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) {
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return false, err
}
return IsAPINamespacedWithGVK(gvk, scheme, restmapper)
}
// IsAPINamespacedWithGVK returns true if the object having the provided
// GVK is namespace scoped.
func IsAPINamespacedWithGVK(gk schema.GroupVersionKind, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) {
restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gk.Group, Kind: gk.Kind})
if err != nil {
return false, fmt.Errorf("failed to get restmapping: %w", err)
}
scope := restmapping.Scope.Name()
if scope == "" {
return false, errors.New("scope cannot be identified, empty scope returned")
}
if scope != apimeta.RESTScopeNameRoot {
return true, nil
}
return false, nil
}

View File

@@ -19,6 +19,7 @@ package recorder
import (
"context"
"fmt"
"net/http"
"sync"
"github.com/go-logr/logr"
@@ -110,8 +111,12 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster {
}
// NewProvider create a new Provider instance.
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) {
corev1Client, err := corev1client.NewForConfig(config)
func NewProvider(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) {
if httpClient == nil {
panic("httpClient must not be nil")
}
corev1Client, err := corev1client.NewForConfigAndClient(config, httpClient)
if err != nil {
return nil, fmt.Errorf("failed to init client: %w", err)
}

View File

@@ -17,6 +17,7 @@ limitations under the License.
package internal
import (
"context"
"fmt"
"k8s.io/client-go/tools/cache"
@@ -31,17 +32,39 @@ import (
var log = logf.RuntimeLog.WithName("source").WithName("EventHandler")
var _ cache.ResourceEventHandler = EventHandler{}
// NewEventHandler creates a new EventHandler.
func NewEventHandler(ctx context.Context, queue workqueue.RateLimitingInterface, handler handler.EventHandler, predicates []predicate.Predicate) *EventHandler {
return &EventHandler{
ctx: ctx,
handler: handler,
queue: queue,
predicates: predicates,
}
}
// EventHandler adapts a handler.EventHandler interface to a cache.ResourceEventHandler interface.
type EventHandler struct {
EventHandler handler.EventHandler
Queue workqueue.RateLimitingInterface
Predicates []predicate.Predicate
// ctx stores the context that created the event handler
// that is used to propagate cancellation signals to each handler function.
ctx context.Context
handler handler.EventHandler
queue workqueue.RateLimitingInterface
predicates []predicate.Predicate
}
// HandlerFuncs converts EventHandler to a ResourceEventHandlerFuncs
// TODO: switch to ResourceEventHandlerDetailedFuncs with client-go 1.27
func (e *EventHandler) HandlerFuncs() cache.ResourceEventHandlerFuncs {
return cache.ResourceEventHandlerFuncs{
AddFunc: e.OnAdd,
UpdateFunc: e.OnUpdate,
DeleteFunc: e.OnDelete,
}
}
// OnAdd creates CreateEvent and calls Create on EventHandler.
func (e EventHandler) OnAdd(obj interface{}) {
func (e *EventHandler) OnAdd(obj interface{}) {
c := event.CreateEvent{}
// Pull Object out of the object
@@ -53,18 +76,20 @@ func (e EventHandler) OnAdd(obj interface{}) {
return
}
for _, p := range e.Predicates {
for _, p := range e.predicates {
if !p.Create(c) {
return
}
}
// Invoke create handler
e.EventHandler.Create(c, e.Queue)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
e.handler.Create(ctx, c, e.queue)
}
// OnUpdate creates UpdateEvent and calls Update on EventHandler.
func (e EventHandler) OnUpdate(oldObj, newObj interface{}) {
func (e *EventHandler) OnUpdate(oldObj, newObj interface{}) {
u := event.UpdateEvent{}
if o, ok := oldObj.(client.Object); ok {
@@ -84,18 +109,20 @@ func (e EventHandler) OnUpdate(oldObj, newObj interface{}) {
return
}
for _, p := range e.Predicates {
for _, p := range e.predicates {
if !p.Update(u) {
return
}
}
// Invoke update handler
e.EventHandler.Update(u, e.Queue)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
e.handler.Update(ctx, u, e.queue)
}
// OnDelete creates DeleteEvent and calls Delete on EventHandler.
func (e EventHandler) OnDelete(obj interface{}) {
func (e *EventHandler) OnDelete(obj interface{}) {
d := event.DeleteEvent{}
// Deal with tombstone events by pulling the object out. Tombstone events wrap the object in a
@@ -114,6 +141,9 @@ func (e EventHandler) OnDelete(obj interface{}) {
return
}
// Set DeleteStateUnknown to true
d.DeleteStateUnknown = true
// Set obj to the tombstone obj
obj = tombstone.Obj
}
@@ -127,12 +157,14 @@ func (e EventHandler) OnDelete(obj interface{}) {
return
}
for _, p := range e.Predicates {
for _, p := range e.predicates {
if !p.Delete(d) {
return
}
}
// Invoke delete handler
e.EventHandler.Delete(d, e.Queue)
ctx, cancel := context.WithCancel(e.ctx)
defer cancel()
e.handler.Delete(ctx, d, e.queue)
}

View File

@@ -0,0 +1,117 @@
package internal
import (
"context"
"errors"
"fmt"
"time"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create).
type Kind struct {
// Type is the type of object to watch. e.g. &v1.Pod{}
Type client.Object
// Cache used to watch APIs
Cache cache.Cache
// started may contain an error if one was encountered during startup. If its closed and does not
// contain an error, startup and syncing finished.
started chan error
startCancel func()
}
// Start is internal and should be called only by the Controller to register an EventHandler with the Informer
// to enqueue reconcile.Requests.
func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface,
prct ...predicate.Predicate) error {
if ks.Type == nil {
return fmt.Errorf("must create Kind with a non-nil object")
}
if ks.Cache == nil {
return fmt.Errorf("must create Kind with a non-nil cache")
}
// cache.GetInformer will block until its context is cancelled if the cache was already started and it can not
// sync that informer (most commonly due to RBAC issues).
ctx, ks.startCancel = context.WithCancel(ctx)
ks.started = make(chan error)
go func() {
var (
i cache.Informer
lastErr error
)
// Tries to get an informer until it returns true,
// an error or the specified context is cancelled or expired.
if err := wait.PollUntilContextCancel(ctx, 10*time.Second, true, func(ctx context.Context) (bool, error) {
// Lookup the Informer from the Cache and add an EventHandler which populates the Queue
i, lastErr = ks.Cache.GetInformer(ctx, ks.Type)
if lastErr != nil {
kindMatchErr := &meta.NoKindMatchError{}
switch {
case errors.As(lastErr, &kindMatchErr):
log.Error(lastErr, "if kind is a CRD, it should be installed before calling Start",
"kind", kindMatchErr.GroupKind)
case runtime.IsNotRegisteredError(lastErr):
log.Error(lastErr, "kind must be registered to the Scheme")
default:
log.Error(lastErr, "failed to get informer from cache")
}
return false, nil // Retry.
}
return true, nil
}); err != nil {
if lastErr != nil {
ks.started <- fmt.Errorf("failed to get informer from cache: %w", lastErr)
return
}
ks.started <- err
return
}
_, err := i.AddEventHandler(NewEventHandler(ctx, queue, handler, prct).HandlerFuncs())
if err != nil {
ks.started <- err
return
}
if !ks.Cache.WaitForCacheSync(ctx) {
// Would be great to return something more informative here
ks.started <- errors.New("cache did not sync")
}
close(ks.started)
}()
return nil
}
func (ks *Kind) String() string {
if ks.Type != nil {
return fmt.Sprintf("kind source: %T", ks.Type)
}
return "kind source: unknown type"
}
// WaitForSync implements SyncingSource to allow controllers to wait with starting
// workers until the cache is synced.
func (ks *Kind) WaitForSync(ctx context.Context) error {
select {
case err := <-ks.started:
return err
case <-ctx.Done():
ks.startCancel()
if errors.Is(ctx.Err(), context.Canceled) {
return nil
}
return fmt.Errorf("timed out waiting for cache to be synced for Kind %T", ks.Type)
}
}

View File

@@ -25,7 +25,7 @@ import (
// loggerPromise knows how to populate a concrete logr.Logger
// with options, given an actual base logger later on down the line.
type loggerPromise struct {
logger *DelegatingLogSink
logger *delegatingLogSink
childPromises []*loggerPromise
promisesLock sync.Mutex
@@ -33,7 +33,7 @@ type loggerPromise struct {
tags []interface{}
}
func (p *loggerPromise) WithName(l *DelegatingLogSink, name string) *loggerPromise {
func (p *loggerPromise) WithName(l *delegatingLogSink, name string) *loggerPromise {
res := &loggerPromise{
logger: l,
name: &name,
@@ -47,7 +47,7 @@ func (p *loggerPromise) WithName(l *DelegatingLogSink, name string) *loggerPromi
}
// WithValues provides a new Logger with the tags appended.
func (p *loggerPromise) WithValues(l *DelegatingLogSink, tags ...interface{}) *loggerPromise {
func (p *loggerPromise) WithValues(l *delegatingLogSink, tags ...interface{}) *loggerPromise {
res := &loggerPromise{
logger: l,
tags: tags,
@@ -84,12 +84,12 @@ func (p *loggerPromise) Fulfill(parentLogSink logr.LogSink) {
}
}
// DelegatingLogSink is a logsink that delegates to another logr.LogSink.
// delegatingLogSink is a logsink that delegates to another logr.LogSink.
// If the underlying promise is not nil, it registers calls to sub-loggers with
// the logging factory to be populated later, and returns a new delegating
// logger. It expects to have *some* logr.Logger set at all times (generally
// a no-op logger before the promises are fulfilled).
type DelegatingLogSink struct {
type delegatingLogSink struct {
lock sync.RWMutex
logger logr.LogSink
promise *loggerPromise
@@ -97,7 +97,8 @@ type DelegatingLogSink struct {
}
// Init implements logr.LogSink.
func (l *DelegatingLogSink) Init(info logr.RuntimeInfo) {
func (l *delegatingLogSink) Init(info logr.RuntimeInfo) {
eventuallyFulfillRoot()
l.lock.Lock()
defer l.lock.Unlock()
l.info = info
@@ -106,7 +107,8 @@ func (l *DelegatingLogSink) Init(info logr.RuntimeInfo) {
// Enabled tests whether this Logger is enabled. For example, commandline
// flags might be used to set the logging verbosity and disable some info
// logs.
func (l *DelegatingLogSink) Enabled(level int) bool {
func (l *delegatingLogSink) Enabled(level int) bool {
eventuallyFulfillRoot()
l.lock.RLock()
defer l.lock.RUnlock()
return l.logger.Enabled(level)
@@ -118,7 +120,8 @@ func (l *DelegatingLogSink) Enabled(level int) bool {
// the log line. The key/value pairs can then be used to add additional
// variable information. The key/value pairs should alternate string
// keys and arbitrary values.
func (l *DelegatingLogSink) Info(level int, msg string, keysAndValues ...interface{}) {
func (l *delegatingLogSink) Info(level int, msg string, keysAndValues ...interface{}) {
eventuallyFulfillRoot()
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Info(level, msg, keysAndValues...)
@@ -132,14 +135,16 @@ func (l *DelegatingLogSink) Info(level int, msg string, keysAndValues ...interfa
// The msg field should be used to add context to any underlying error,
// while the err field should be used to attach the actual error that
// triggered this log line, if present.
func (l *DelegatingLogSink) Error(err error, msg string, keysAndValues ...interface{}) {
func (l *delegatingLogSink) Error(err error, msg string, keysAndValues ...interface{}) {
eventuallyFulfillRoot()
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Error(err, msg, keysAndValues...)
}
// WithName provides a new Logger with the name appended.
func (l *DelegatingLogSink) WithName(name string) logr.LogSink {
func (l *delegatingLogSink) WithName(name string) logr.LogSink {
eventuallyFulfillRoot()
l.lock.RLock()
defer l.lock.RUnlock()
@@ -151,7 +156,7 @@ func (l *DelegatingLogSink) WithName(name string) logr.LogSink {
return sink
}
res := &DelegatingLogSink{logger: l.logger}
res := &delegatingLogSink{logger: l.logger}
promise := l.promise.WithName(res, name)
res.promise = promise
@@ -159,7 +164,8 @@ func (l *DelegatingLogSink) WithName(name string) logr.LogSink {
}
// WithValues provides a new Logger with the tags appended.
func (l *DelegatingLogSink) WithValues(tags ...interface{}) logr.LogSink {
func (l *delegatingLogSink) WithValues(tags ...interface{}) logr.LogSink {
eventuallyFulfillRoot()
l.lock.RLock()
defer l.lock.RUnlock()
@@ -171,7 +177,7 @@ func (l *DelegatingLogSink) WithValues(tags ...interface{}) logr.LogSink {
return sink
}
res := &DelegatingLogSink{logger: l.logger}
res := &delegatingLogSink{logger: l.logger}
promise := l.promise.WithValues(res, tags...)
res.promise = promise
@@ -181,16 +187,19 @@ func (l *DelegatingLogSink) WithValues(tags ...interface{}) logr.LogSink {
// Fulfill switches the logger over to use the actual logger
// provided, instead of the temporary initial one, if this method
// has not been previously called.
func (l *DelegatingLogSink) Fulfill(actual logr.LogSink) {
func (l *delegatingLogSink) Fulfill(actual logr.LogSink) {
if actual == nil {
actual = NullLogSink{}
}
if l.promise != nil {
l.promise.Fulfill(actual)
}
}
// NewDelegatingLogSink constructs a new DelegatingLogSink which uses
// newDelegatingLogSink constructs a new DelegatingLogSink which uses
// the given logger before its promise is fulfilled.
func NewDelegatingLogSink(initial logr.LogSink) *DelegatingLogSink {
l := &DelegatingLogSink{
func newDelegatingLogSink(initial logr.LogSink) *delegatingLogSink {
l := &delegatingLogSink{
logger: initial,
promise: &loggerPromise{promisesLock: sync.Mutex{}},
}

View File

@@ -34,8 +34,12 @@ limitations under the License.
package log
import (
"bytes"
"context"
"sync"
"fmt"
"os"
"runtime/debug"
"sync/atomic"
"time"
"github.com/go-logr/logr"
@@ -43,35 +47,32 @@ import (
// SetLogger sets a concrete logging implementation for all deferred Loggers.
func SetLogger(l logr.Logger) {
loggerWasSetLock.Lock()
defer loggerWasSetLock.Unlock()
loggerWasSet = true
dlog.Fulfill(l.GetSink())
logFullfilled.Store(true)
rootLog.Fulfill(l.GetSink())
}
// It is safe to assume that if this wasn't set within the first 30 seconds of a binaries
// lifetime, it will never get set. The DelegatingLogSink causes a high number of memory
// allocations when not given an actual Logger, so we set a NullLogSink to avoid that.
//
// We need to keep the DelegatingLogSink because we have various inits() that get a logger from
// here. They will always get executed before any code that imports controller-runtime
// has a chance to run and hence to set an actual logger.
func init() {
// Init is blocking, so start a new goroutine
go func() {
time.Sleep(30 * time.Second)
loggerWasSetLock.Lock()
defer loggerWasSetLock.Unlock()
if !loggerWasSet {
dlog.Fulfill(NullLogSink{})
func eventuallyFulfillRoot() {
if logFullfilled.Load() {
return
}
if time.Since(rootLogCreated).Seconds() >= 30 {
if logFullfilled.CompareAndSwap(false, true) {
stack := debug.Stack()
stackLines := bytes.Count(stack, []byte{'\n'})
sep := []byte{'\n', '\t', '>', ' ', ' '}
fmt.Fprintf(os.Stderr,
"[controller-runtime] log.SetLogger(...) was never called; logs will not be displayed.\nDetected at:%s%s", sep,
// prefix every line, so it's clear this is a stack trace related to the above message
bytes.Replace(stack, []byte{'\n'}, sep, stackLines-1),
)
SetLogger(logr.New(NullLogSink{}))
}
}()
}
}
var (
loggerWasSetLock sync.Mutex
loggerWasSet bool
logFullfilled atomic.Bool
)
// Log is the base logger used by kubebuilder. It delegates
@@ -80,8 +81,10 @@ var (
// the first 30 seconds of a binaries lifetime, it will get
// set to a NullLogSink.
var (
dlog = NewDelegatingLogSink(NullLogSink{})
Log = logr.New(dlog)
rootLog, rootLogCreated = func() (*delegatingLogSink, time.Time) {
return newDelegatingLogSink(NullLogSink{}), time.Now()
}()
Log = logr.New(rootLog)
)
// FromContext returns a logger with predefined values from a context.Context.

View File

@@ -24,7 +24,6 @@ import (
"go.uber.org/zap/zapcore"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
)
// KubeAwareEncoder is a Kubernetes-aware Zap Encoder.
@@ -41,21 +40,6 @@ type KubeAwareEncoder struct {
Verbose bool
}
// namespacedNameWrapper is a zapcore.ObjectMarshaler for Kubernetes NamespacedName.
type namespacedNameWrapper struct {
types.NamespacedName
}
func (w namespacedNameWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error {
if w.Namespace != "" {
enc.AddString("namespace", w.Namespace)
}
enc.AddString("name", w.Name)
return nil
}
// kubeObjectWrapper is a zapcore.ObjectMarshaler for Kubernetes objects.
type kubeObjectWrapper struct {
obj runtime.Object
@@ -119,12 +103,6 @@ func (k *KubeAwareEncoder) EncodeEntry(entry zapcore.Entry, fields []zapcore.Fie
Key: field.Key,
Interface: kubeObjectWrapper{obj: val},
}
case types.NamespacedName:
fields[i] = zapcore.Field{
Type: zapcore.ObjectMarshalerType,
Key: field.Key,
Interface: namespacedNameWrapper{NamespacedName: val},
}
}
}
}

View File

@@ -148,11 +148,6 @@ type Options struct {
// DestWriter controls the destination of the log output. Defaults to
// os.Stderr.
DestWriter io.Writer
// DestWritter controls the destination of the log output. Defaults to
// os.Stderr.
//
// Deprecated: Use DestWriter instead
DestWritter io.Writer
// Level configures the verbosity of the logging.
// Defaults to Debug when Development is true and Info otherwise.
// A zap log level should be multiplied by -1 to get the logr verbosity.
@@ -174,11 +169,8 @@ type Options struct {
// addDefaults adds defaults to the Options.
func (o *Options) addDefaults() {
if o.DestWriter == nil && o.DestWritter == nil {
if o.DestWriter == nil {
o.DestWriter = os.Stderr
} else if o.DestWriter == nil && o.DestWritter != nil {
// while misspelled DestWritter is deprecated but still not removed
o.DestWriter = o.DestWritter
}
if o.Development {

View File

@@ -18,21 +18,19 @@ package manager
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
"net/http/pprof"
"sync"
"sync/atomic"
"time"
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
@@ -41,12 +39,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/config"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
@@ -59,7 +56,6 @@ const (
defaultReadinessEndpoint = "/readyz"
defaultLivenessEndpoint = "/healthz"
defaultMetricsEndpoint = "/metrics"
)
var _ Runnable = &controllerManager{}
@@ -86,11 +82,8 @@ type controllerManager struct {
// on shutdown
leaderElectionReleaseOnCancel bool
// metricsListener is used to serve prometheus metrics
metricsListener net.Listener
// metricsExtraHandlers contains extra handlers to register on http server that serves metrics.
metricsExtraHandlers map[string]http.Handler
// metricsServer is used to serve prometheus metrics
metricsServer metricsserver.Server
// healthProbeListener is used to serve liveness probe
healthProbeListener net.Listener
@@ -107,8 +100,11 @@ type controllerManager struct {
// Healthz probe handler
healthzHandler *healthz.Handler
// controllerOptions are the global controller options.
controllerOptions v1alpha1.ControllerConfigurationSpec
// pprofListener is used to serve pprof
pprofListener net.Listener
// controllerConfig are the global controller options.
controllerConfig config.Controller
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
@@ -128,18 +124,7 @@ type controllerManager struct {
// election was configured.
elected chan struct{}
// port is the port that the webhook server serves at.
port int
// host is the hostname that the webhook server binds to.
host string
// CertDir is the directory that contains the server key and certificate.
// if not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs
certDir string
// tlsOpts is used to allow configuring the TLS config used for the webhook server.
tlsOpts []func(*tls.Config)
webhookServer *webhook.Server
webhookServer webhook.Server
// webhookServerOnce will be called in GetWebhookServer() to optionally initialize
// webhookServer if unset, and Add() it to controllerManager.
webhookServerOnce sync.Once
@@ -191,53 +176,9 @@ func (cm *controllerManager) Add(r Runnable) error {
}
func (cm *controllerManager) add(r Runnable) error {
// Set dependencies on the object
if err := cm.SetFields(r); err != nil {
return err
}
return cm.runnables.Add(r)
}
// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10.
func (cm *controllerManager) SetFields(i interface{}) error {
if err := cm.cluster.SetFields(i); err != nil {
return err
}
if _, err := inject.InjectorInto(cm.SetFields, i); err != nil {
return err
}
if _, err := inject.StopChannelInto(cm.internalProceduresStop, i); err != nil {
return err
}
if _, err := inject.LoggerInto(cm.logger, i); err != nil {
return err
}
return nil
}
// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics.
func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error {
cm.Lock()
defer cm.Unlock()
if cm.started {
return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created")
}
if path == defaultMetricsEndpoint {
return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint)
}
if _, found := cm.metricsExtraHandlers[path]; found {
return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path)
}
cm.metricsExtraHandlers[path] = handler
cm.logger.V(2).Info("Registering metrics http server extra handler", "path", path)
return nil
}
// AddHealthzCheck allows you to add Healthz checker.
func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error {
cm.Lock()
@@ -272,6 +213,10 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker)
return nil
}
func (cm *controllerManager) GetHTTPClient() *http.Client {
return cm.cluster.GetHTTPClient()
}
func (cm *controllerManager) GetConfig() *rest.Config {
return cm.cluster.GetConfig()
}
@@ -304,15 +249,10 @@ func (cm *controllerManager) GetAPIReader() client.Reader {
return cm.cluster.GetAPIReader()
}
func (cm *controllerManager) GetWebhookServer() *webhook.Server {
func (cm *controllerManager) GetWebhookServer() webhook.Server {
cm.webhookServerOnce.Do(func() {
if cm.webhookServer == nil {
cm.webhookServer = &webhook.Server{
Port: cm.port,
Host: cm.host,
CertDir: cm.certDir,
TLSOpts: cm.tlsOpts,
}
panic("webhook should not be nil")
}
if err := cm.Add(cm.webhookServer); err != nil {
panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err))
@@ -325,28 +265,13 @@ func (cm *controllerManager) GetLogger() logr.Logger {
return cm.logger
}
func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec {
return cm.controllerOptions
func (cm *controllerManager) GetControllerOptions() config.Controller {
return cm.controllerConfig
}
func (cm *controllerManager) serveMetrics() {
handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
ErrorHandling: promhttp.HTTPErrorOnError,
})
// TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics
func (cm *controllerManager) addHealthProbeServer() error {
mux := http.NewServeMux()
mux.Handle(defaultMetricsEndpoint, handler)
for path, extraHandler := range cm.metricsExtraHandlers {
mux.Handle(path, extraHandler)
}
server := httpserver.New(mux)
go cm.httpServe("metrics", cm.logger.WithValues("path", defaultMetricsEndpoint), server, cm.metricsListener)
}
func (cm *controllerManager) serveHealthProbes() {
mux := http.NewServeMux()
server := httpserver.New(mux)
srv := httpserver.New(mux)
if cm.readyzHandler != nil {
mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler))
@@ -359,43 +284,30 @@ func (cm *controllerManager) serveHealthProbes() {
mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
}
go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener)
return cm.add(&server{
Kind: "health probe",
Log: cm.logger,
Server: srv,
Listener: cm.healthProbeListener,
})
}
func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) {
log = log.WithValues("kind", kind, "addr", ln.Addr())
func (cm *controllerManager) addPprofServer() error {
mux := http.NewServeMux()
srv := httpserver.New(mux)
go func() {
log.Info("Starting server")
if err := server.Serve(ln); err != nil {
if errors.Is(err, http.ErrServerClosed) {
return
}
if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
// There might be cases where connections are still open and we try to shutdown
// but not having enough time to close the connection causes an error in Serve
//
// In that case we want to avoid returning an error to the main error channel.
log.Error(err, "error on Serve after stop has been engaged")
return
}
cm.errChan <- err
}
}()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
// Shutdown the server when stop is closed.
<-cm.internalProceduresStop
if err := server.Shutdown(cm.shutdownCtx); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// Avoid logging context related errors.
return
}
if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 {
cm.logger.Error(err, "error on Shutdown after stop has been engaged")
return
}
cm.errChan <- err
}
return cm.add(&server{
Kind: "pprof",
Log: cm.logger,
Server: srv,
Listener: cm.pprofListener,
})
}
// Start starts the manager and waits indefinitely.
@@ -450,38 +362,61 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) {
// Metrics should be served whether the controller is leader or not.
// (If we don't serve metrics for non-leaders, prometheus will still scrape
// the pod but will get a connection refused).
if cm.metricsListener != nil {
cm.serveMetrics()
if cm.metricsServer != nil {
// Note: We are adding the metrics server directly to HTTPServers here as matching on the
// metricsserver.Server interface in cm.runnables.Add would be very brittle.
if err := cm.runnables.HTTPServers.Add(cm.metricsServer, nil); err != nil {
return fmt.Errorf("failed to add metrics server: %w", err)
}
}
// Serve health probes.
if cm.healthProbeListener != nil {
cm.serveHealthProbes()
if err := cm.addHealthProbeServer(); err != nil {
return fmt.Errorf("failed to add health probe server: %w", err)
}
}
// First start any webhook servers, which includes conversion, validation, and defaulting
// Add pprof server
if cm.pprofListener != nil {
if err := cm.addPprofServer(); err != nil {
return fmt.Errorf("failed to add pprof server: %w", err)
}
}
// First start any internal HTTP servers, which includes health probes, metrics and profiling if enabled.
//
// WARNING: Internal HTTP servers MUST start before any cache is populated, otherwise it would block
// conversion webhooks to be ready for serving which make the cache never get ready.
if err := cm.runnables.HTTPServers.Start(cm.internalCtx); err != nil {
if err != nil {
return fmt.Errorf("failed to start HTTP servers: %w", err)
}
}
// Start any webhook servers, which includes conversion, validation, and defaulting
// webhooks that are registered.
//
// WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition
// between conversion webhooks and the cache sync (usually initial list) which causes the webhooks
// to never start because no cache can be populated.
if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
if err != nil {
return fmt.Errorf("failed to start webhooks: %w", err)
}
}
// Start and wait for caches.
if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
if err != nil {
return fmt.Errorf("failed to start caches: %w", err)
}
}
// Start the non-leaderelection Runnables after the cache has synced.
if err := cm.runnables.Others.Start(cm.internalCtx); err != nil {
if !errors.Is(err, wait.ErrWaitTimeout) {
return err
if err != nil {
return fmt.Errorf("failed to start other runnables: %w", err)
}
}
@@ -591,10 +526,13 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e
cm.logger.Info("Stopping and waiting for caches")
cm.runnables.Caches.StopAndWait(cm.shutdownCtx)
// Webhooks should come last, as they might be still serving some requests.
// Webhooks and internal HTTP servers should come last, as they might be still serving some requests.
cm.logger.Info("Stopping and waiting for webhooks")
cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx)
cm.logger.Info("Stopping and waiting for HTTP servers")
cm.runnables.HTTPServers.StopAndWait(cm.shutdownCtx)
// Proceed to close the manager and overall shutdown context.
cm.logger.Info("Wait completed, proceeding to shutdown the manager")
shutdownCancel()

View File

@@ -18,7 +18,7 @@ package manager
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
@@ -26,6 +26,8 @@ import (
"time"
"github.com/go-logr/logr"
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -33,6 +35,8 @@ import (
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
@@ -42,9 +46,7 @@ import (
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/leaderelection"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/recorder"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
@@ -55,8 +57,7 @@ type Manager interface {
cluster.Cluster
// Add will set requested dependencies on the component, and cause the component to be
// started when Start is called. Add will inject any dependencies for which the argument
// implements the inject interface - e.g. inject.Client.
// started when Start is called.
// Depending on if a Runnable implements LeaderElectionRunnable interface, a Runnable can be run in either
// non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled).
Add(Runnable) error
@@ -66,13 +67,6 @@ type Manager interface {
// election was configured.
Elected() <-chan struct{}
// AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics.
// Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be
// sensitive and shouldn't be exposed publicly.
// If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as
// Runnable to the manager via Add method.
AddMetricsExtraHandler(path string, handler http.Handler) error
// AddHealthzCheck allows you to add Healthz checker
AddHealthzCheck(name string, check healthz.Checker) error
@@ -88,13 +82,13 @@ type Manager interface {
Start(ctx context.Context) error
// GetWebhookServer returns a webhook.Server
GetWebhookServer() *webhook.Server
GetWebhookServer() webhook.Server
// GetLogger returns this manager's logger.
GetLogger() logr.Logger
// GetControllerOptions returns controller global configuration options.
GetControllerOptions() v1alpha1.ControllerConfigurationSpec
GetControllerOptions() config.Controller
}
// Options are the arguments for creating a new Manager.
@@ -102,37 +96,44 @@ type Options struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources.
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// to pass your own scheme in. See the documentation in pkg/scheme for more information.
//
// If set, the Scheme will be used to create the default Client and Cache.
Scheme *runtime.Scheme
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
MapperProvider func(c *rest.Config) (meta.RESTMapper, error)
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs.
//
// If set, the RESTMapper returned by this function is used to create the RESTMapper
// used by the Client and Cache.
MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error)
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
// Cache is the cache.Options that will be used to create the default Cache.
// By default, the cache will watch and list requested objects in all namespaces.
Cache cache.Options
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
//
// This applies to all controllers.
// When using a custom NewCache, the Cache options will be passed to the
// NewCache function.
//
// A period sync happens for two reasons:
// 1. To insure against a bug in the controller that causes an object to not
// be requeued, when it otherwise should be requeued.
// 2. To insure against an unknown bug in controller-runtime, or its dependencies,
// that causes an object to not be requeued, when it otherwise should be
// requeued, or to be removed from the queue, when it otherwise should not
// be removed.
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewCache if you know what you are doing.
NewCache cache.NewCacheFunc
// Client is the client.Options that will be used to create the default Client.
// By default, the client will use the cache for reads and direct calls for writes.
Client client.Options
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create a Client backed by a Cache for read operations
// and a direct Client for write operations.
//
// If you want
// 1. to insure against missed watch events, or
// 2. to poll services that cannot be watched,
// then we recommend that, instead of changing the default period, the
// controller requeue, with a constant duration `t`, whenever the controller
// is "done" with an object, and would otherwise not requeue it, i.e., we
// recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`,
// instead of `reconcile.Result{}`.
SyncPeriod *time.Duration
// When using a custom NewClient, the Client options will be passed to the
// NewClient function.
//
// NOTE: LOW LEVEL PRIMITIVE!
// Only use a custom NewClient if you know what you are doing.
NewClient client.NewClientFunc
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
@@ -204,25 +205,17 @@ type Options struct {
// wait to force acquire leadership. This is measured against time of
// last observed ack. Default is 15 seconds.
LeaseDuration *time.Duration
// RenewDeadline is the duration that the acting controlplane will retry
// refreshing leadership before giving up. Default is 10 seconds.
RenewDeadline *time.Duration
// RetryPeriod is the duration the LeaderElector clients should wait
// between tries of actions. Default is 2 seconds.
RetryPeriod *time.Duration
// Namespace, if specified, restricts the manager's cache to watch objects in
// the desired namespace. Defaults to all namespaces.
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources, the cache
// will only hold objects from the desired namespace.
Namespace string
// MetricsBindAddress is the TCP address that the controller should bind to
// for serving prometheus metrics.
// It can be set to "0" to disable the metrics serving.
MetricsBindAddress string
// Metrics are the metricsserver.Options that will be used to create the metricsserver.Server.
Metrics metricsserver.Options
// HealthProbeBindAddress is the TCP address that the controller should bind to
// for serving health probes
@@ -235,52 +228,23 @@ type Options struct {
// Liveness probe endpoint name, defaults to "healthz"
LivenessEndpointName string
// Port is the port that the webhook server serves at.
// It is used to set webhook.Server.Port if WebhookServer is not set.
Port int
// Host is the hostname that the webhook server binds to.
// It is used to set webhook.Server.Host if WebhookServer is not set.
Host string
// CertDir is the directory that contains the server key and certificate.
// If not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
// must be named tls.key and tls.crt, respectively.
// It is used to set webhook.Server.CertDir if WebhookServer is not set.
CertDir string
// TLSOpts is used to allow configuring the TLS config used for the webhook server.
TLSOpts []func(*tls.Config)
// PprofBindAddress is the TCP address that the controller should bind to
// for serving pprof.
// It can be set to "" or "0" to disable the pprof serving.
// Since pprof may contain sensitive information, make sure to protect it
// before exposing it to public.
PprofBindAddress string
// WebhookServer is an externally configured webhook.Server. By default,
// a Manager will create a default server using Port, Host, and CertDir;
// if this is set, the Manager will use this server instead.
WebhookServer *webhook.Server
// Functions to allow for a user to customize values that will be injected.
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
NewCache cache.NewCacheFunc
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create the default DelegatingClient that will
// use the cache for reads and the client for writes.
NewClient cluster.NewClientFunc
// a Manager will create a server via webhook.NewServer with default settings.
// If this is set, the Manager will use this server instead.
WebhookServer webhook.Server
// BaseContext is the function that provides Context values to Runnables
// managed by the Manager. If a BaseContext function isn't provided, Runnables
// will receive a new Background Context instead.
BaseContext BaseContextFunc
// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
// for the given objects.
ClientDisableCacheFor []client.Object
// DryRunClient specifies whether the client should be configured to enforce
// dryRun mode.
DryRunClient bool
// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
// Use this to customize the event correlator and spam filter
//
@@ -297,7 +261,7 @@ type Options struct {
// Controller contains global configuration options for controllers
// registered within this manager.
// +optional
Controller v1alpha1.ControllerConfigurationSpec
Controller config.Controller
// makeBroadcaster allows deferring the creation of the broadcaster to
// avoid leaking goroutines if we never call Start on this manager. It also
@@ -306,10 +270,11 @@ type Options struct {
makeBroadcaster intrec.EventBroadcasterProducer
// Dependency injection for testing
newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error)
newMetricsListener func(addr string) (net.Listener, error)
newMetricsServer func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error)
newHealthProbeListener func(addr string) (net.Listener, error)
newPprofListener func(addr string) (net.Listener, error)
}
// BaseContextFunc is a function used to provide a base Context to Runnables
@@ -344,7 +309,13 @@ type LeaderElectionRunnable interface {
}
// New returns a new Manager for creating Controllers.
// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf"
// will be used for all built-in resources of Kubernetes, and "application/json" is for other types
// including all CRD resources.
func New(config *rest.Config, options Options) (Manager, error) {
if config == nil {
return nil, errors.New("must specify Config")
}
// Set default values for options fields
options = setOptionsDefaults(options)
@@ -352,22 +323,25 @@ func New(config *rest.Config, options Options) (Manager, error) {
clusterOptions.Scheme = options.Scheme
clusterOptions.MapperProvider = options.MapperProvider
clusterOptions.Logger = options.Logger
clusterOptions.SyncPeriod = options.SyncPeriod
clusterOptions.Namespace = options.Namespace
clusterOptions.NewCache = options.NewCache
clusterOptions.NewClient = options.NewClient
clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor
clusterOptions.DryRunClient = options.DryRunClient
clusterOptions.Cache = options.Cache
clusterOptions.Client = options.Client
clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck
})
if err != nil {
return nil, err
}
config = rest.CopyConfig(config)
if config.UserAgent == "" {
config.UserAgent = rest.DefaultKubernetesUserAgent()
}
// Create the recorder provider to inject event recorders for the components.
// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
// to the particular controller that it's being injected into, rather than a generic one like is here.
recorderProvider, err := options.newRecorderProvider(config, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
recorderProvider, err := options.newRecorderProvider(config, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
@@ -381,7 +355,20 @@ func New(config *rest.Config, options Options) (Manager, error) {
leaderRecorderProvider = recorderProvider
} else {
leaderConfig = rest.CopyConfig(options.LeaderElectionConfig)
leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster)
scheme := cluster.GetScheme()
err := corev1.AddToScheme(scheme)
if err != nil {
return nil, err
}
err = coordinationv1.AddToScheme(scheme)
if err != nil {
return nil, err
}
httpClient, err := rest.HTTPClientFor(options.LeaderElectionConfig)
if err != nil {
return nil, err
}
leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, httpClient, scheme, options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
@@ -402,16 +389,12 @@ func New(config *rest.Config, options Options) (Manager, error) {
}
}
// Create the metrics listener. This will throw an error if the metrics bind
// address is invalid or already in use.
metricsListener, err := options.newMetricsListener(options.MetricsBindAddress)
// Create the metrics server.
metricsServer, err := options.newMetricsServer(options.Metrics, config, cluster.GetHTTPClient())
if err != nil {
return nil, err
}
// By default we have no extra endpoints to expose on metrics http server.
metricsExtraHandlers := make(map[string]http.Handler)
// Create health probes listener. This will throw an error if the bind
// address is invalid or already in use.
healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress)
@@ -419,9 +402,15 @@ func New(config *rest.Config, options Options) (Manager, error) {
return nil, err
}
// Create pprof listener. This will throw an error if the bind
// address is invalid or already in use.
pprofListener, err := options.newPprofListener(options.PprofBindAddress)
if err != nil {
return nil, fmt.Errorf("failed to new pprof listener: %w", err)
}
errChan := make(chan error)
runnables := newRunnables(options.BaseContext, errChan)
return &controllerManager{
stopProcedureEngaged: pointer.Int64(0),
cluster: cluster,
@@ -429,15 +418,10 @@ func New(config *rest.Config, options Options) (Manager, error) {
errChan: errChan,
recorderProvider: recorderProvider,
resourceLock: resourceLock,
metricsListener: metricsListener,
metricsExtraHandlers: metricsExtraHandlers,
controllerOptions: options.Controller,
metricsServer: metricsServer,
controllerConfig: options.Controller,
logger: options.Logger,
elected: make(chan struct{}),
port: options.Port,
host: options.Host,
certDir: options.CertDir,
tlsOpts: options.TLSOpts,
webhookServer: options.WebhookServer,
leaderElectionID: options.LeaderElectionID,
leaseDuration: *options.LeaseDuration,
@@ -446,6 +430,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
healthProbeListener: healthProbeListener,
readinessEndpointName: options.ReadinessEndpointName,
livenessEndpointName: options.LivenessEndpointName,
pprofListener: pprofListener,
gracefulShutdownTimeout: *options.GracefulShutdownTimeout,
internalProceduresStop: make(chan struct{}),
leaderElectionStopped: make(chan struct{}),
@@ -456,14 +441,14 @@ func New(config *rest.Config, options Options) (Manager, error) {
// AndFrom will use a supplied type and convert to Options
// any options already set on Options will be ignored, this is used to allow
// cli flags to override anything specified in the config file.
//
// Deprecated: This function has been deprecated and will be removed in a future release,
// The Component Configuration package has been unmaintained for over a year and is no longer
// actively developed. Users should migrate to their own configuration format
// and configure Manager.Options directly.
// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
// for more information, feedback, and comments.
func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) {
if inj, wantsScheme := loader.(inject.Scheme); wantsScheme {
err := inj.InjectScheme(o.Scheme)
if err != nil {
return o, err
}
}
newObj, err := loader.Complete()
if err != nil {
return o, err
@@ -471,16 +456,16 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
o = o.setLeaderElectionConfig(newObj)
if o.SyncPeriod == nil && newObj.SyncPeriod != nil {
o.SyncPeriod = &newObj.SyncPeriod.Duration
if o.Cache.SyncPeriod == nil && newObj.SyncPeriod != nil {
o.Cache.SyncPeriod = &newObj.SyncPeriod.Duration
}
if o.Namespace == "" && newObj.CacheNamespace != "" {
o.Namespace = newObj.CacheNamespace
if len(o.Cache.DefaultNamespaces) == 0 && newObj.CacheNamespace != "" {
o.Cache.DefaultNamespaces = map[string]cache.Config{newObj.CacheNamespace: {}}
}
if o.MetricsBindAddress == "" && newObj.Metrics.BindAddress != "" {
o.MetricsBindAddress = newObj.Metrics.BindAddress
if o.Metrics.BindAddress == "" && newObj.Metrics.BindAddress != "" {
o.Metrics.BindAddress = newObj.Metrics.BindAddress
}
if o.HealthProbeBindAddress == "" && newObj.Health.HealthProbeBindAddress != "" {
@@ -495,21 +480,21 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
o.LivenessEndpointName = newObj.Health.LivenessEndpointName
}
if o.Port == 0 && newObj.Webhook.Port != nil {
o.Port = *newObj.Webhook.Port
}
if o.Host == "" && newObj.Webhook.Host != "" {
o.Host = newObj.Webhook.Host
}
if o.CertDir == "" && newObj.Webhook.CertDir != "" {
o.CertDir = newObj.Webhook.CertDir
if o.WebhookServer == nil {
port := 0
if newObj.Webhook.Port != nil {
port = *newObj.Webhook.Port
}
o.WebhookServer = webhook.NewServer(webhook.Options{
Port: port,
Host: newObj.Webhook.Host,
CertDir: newObj.Webhook.CertDir,
})
}
if newObj.Controller != nil {
if o.Controller.CacheSyncTimeout == nil && newObj.Controller.CacheSyncTimeout != nil {
o.Controller.CacheSyncTimeout = newObj.Controller.CacheSyncTimeout
if o.Controller.CacheSyncTimeout == 0 && newObj.Controller.CacheSyncTimeout != nil {
o.Controller.CacheSyncTimeout = *newObj.Controller.CacheSyncTimeout
}
if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 {
@@ -521,6 +506,13 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
}
// AndFromOrDie will use options.AndFrom() and will panic if there are errors.
//
// Deprecated: This function has been deprecated and will be removed in a future release,
// The Component Configuration package has been unmaintained for over a year and is no longer
// actively developed. Users should migrate to their own configuration format
// and configure Manager.Options directly.
// See https://github.com/kubernetes-sigs/controller-runtime/issues/895
// for more information, feedback, and comments.
func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options {
o, err := o.AndFrom(loader)
if err != nil {
@@ -579,6 +571,19 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) {
return ln, nil
}
// defaultPprofListener creates the default pprof listener bound to the given address.
func defaultPprofListener(addr string) (net.Listener, error) {
if addr == "" || addr == "0" {
return nil, nil
}
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("error listening on %s: %w", addr, err)
}
return ln, nil
}
// defaultBaseContext is used as the BaseContext value in Options if one
// has not already been set.
func defaultBaseContext() context.Context {
@@ -611,8 +616,8 @@ func setOptionsDefaults(options Options) Options {
}
}
if options.newMetricsListener == nil {
options.newMetricsListener = metrics.NewListener
if options.newMetricsServer == nil {
options.newMetricsServer = metricsserver.NewServer
}
leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod
if options.LeaseDuration == nil {
@@ -639,6 +644,10 @@ func setOptionsDefaults(options Options) Options {
options.newHealthProbeListener = defaultHealthProbeListener
}
if options.newPprofListener == nil {
options.newPprofListener = defaultPprofListener
}
if options.GracefulShutdownTimeout == nil {
gracefulShutdownTimeout := defaultGracefulShutdownPeriod
options.GracefulShutdownTimeout = &gracefulShutdownTimeout
@@ -652,5 +661,9 @@ func setOptionsDefaults(options Options) Options {
options.BaseContext = defaultBaseContext
}
if options.WebhookServer == nil {
options.WebhookServer = webhook.NewServer(webhook.Options{})
}
return options
}

View File

@@ -28,6 +28,7 @@ type runnableCheck func(ctx context.Context) bool
// runnables handles all the runnables for a manager by grouping them accordingly to their
// type (webhooks, caches etc.).
type runnables struct {
HTTPServers *runnableGroup
Webhooks *runnableGroup
Caches *runnableGroup
LeaderElection *runnableGroup
@@ -37,6 +38,7 @@ type runnables struct {
// newRunnables creates a new runnables object.
func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables {
return &runnables{
HTTPServers: newRunnableGroup(baseContext, errChan),
Webhooks: newRunnableGroup(baseContext, errChan),
Caches: newRunnableGroup(baseContext, errChan),
LeaderElection: newRunnableGroup(baseContext, errChan),
@@ -52,11 +54,13 @@ func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables {
// The runnables added after Start are started directly.
func (r *runnables) Add(fn Runnable) error {
switch runnable := fn.(type) {
case *server:
return r.HTTPServers.Add(fn, nil)
case hasCache:
return r.Caches.Add(fn, func(ctx context.Context) bool {
return runnable.GetCache().WaitForCacheSync(ctx)
})
case *webhook.Server:
case webhook.Server:
return r.Webhooks.Add(fn, nil)
case LeaderElectionRunnable:
if !runnable.NeedLeaderElection() {

View File

@@ -0,0 +1,61 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"context"
"errors"
"net"
"net/http"
"github.com/go-logr/logr"
)
// server is a general purpose HTTP server Runnable for a manager
// to serve some internal handlers such as health probes, metrics and profiling.
type server struct {
Kind string
Log logr.Logger
Server *http.Server
Listener net.Listener
}
func (s *server) Start(ctx context.Context) error {
log := s.Log.WithValues("kind", s.Kind, "addr", s.Listener.Addr())
serverShutdown := make(chan struct{})
go func() {
<-ctx.Done()
log.Info("shutting down server")
if err := s.Server.Shutdown(context.Background()); err != nil {
log.Error(err, "error shutting down server")
}
close(serverShutdown)
}()
log.Info("starting server")
if err := s.Server.Serve(s.Listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
<-serverShutdown
return nil
}
func (s *server) NeedLeaderElection() bool {
return false
}

View File

@@ -18,8 +18,6 @@ package metrics
import (
"context"
"net/url"
"time"
"github.com/prometheus/client_golang/prometheus"
clientmetrics "k8s.io/client-go/tools/metrics"
@@ -29,70 +27,9 @@ import (
// that client-go registers metrics. We copy the names and formats
// from Kubernetes so that we match the core controllers.
// Metrics subsystem and all of the keys used by the rest client.
const (
RestClientSubsystem = "rest_client"
LatencyKey = "request_latency_seconds"
ResultKey = "requests_total"
)
var (
// client metrics.
// RequestLatency reports the request latency in seconds per verb/URL.
// Deprecated: This metric is deprecated for removal in a future release: using the URL as a
// dimension results in cardinality explosion for some consumers. It was deprecated upstream
// in k8s v1.14 and hidden in v1.17 via https://github.com/kubernetes/kubernetes/pull/83836.
// It is not registered by default. To register:
// import (
// clientmetrics "k8s.io/client-go/tools/metrics"
// clmetrics "sigs.k8s.io/controller-runtime/metrics"
// )
//
// func init() {
// clmetrics.Registry.MustRegister(clmetrics.RequestLatency)
// clientmetrics.Register(clientmetrics.RegisterOpts{
// RequestLatency: clmetrics.LatencyAdapter
// })
// }
RequestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Subsystem: RestClientSubsystem,
Name: LatencyKey,
Help: "Request latency in seconds. Broken down by verb and URL.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 10),
}, []string{"verb", "url"})
// requestLatency is a Prometheus Histogram metric type partitioned by
// "verb", and "host" labels. It is used for the rest client latency metrics.
requestLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_request_duration_seconds",
Help: "Request latency in seconds. Broken down by verb, and host.",
Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0},
},
[]string{"verb", "host"},
)
requestSize = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_request_size_bytes",
Help: "Request size in bytes. Broken down by verb and host.",
// 64 bytes to 16MB
Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
},
[]string{"verb", "host"},
)
responseSize = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "rest_client_response_size_bytes",
Help: "Response size in bytes. Broken down by verb and host.",
// 64 bytes to 16MB
Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216},
},
[]string{"verb", "host"},
)
requestResult = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "rest_client_requests_total",
@@ -109,17 +46,11 @@ func init() {
// registerClientMetrics sets up the client latency metrics from client-go.
func registerClientMetrics() {
// register the metrics with our registry
Registry.MustRegister(requestLatency)
Registry.MustRegister(requestSize)
Registry.MustRegister(responseSize)
Registry.MustRegister(requestResult)
// register the metrics with client-go
clientmetrics.Register(clientmetrics.RegisterOpts{
RequestLatency: &LatencyAdapter{metric: requestLatency},
RequestSize: &sizeAdapter{metric: requestSize},
ResponseSize: &sizeAdapter{metric: responseSize},
RequestResult: &resultAdapter{metric: requestResult},
RequestResult: &resultAdapter{metric: requestResult},
})
}
@@ -131,24 +62,6 @@ func registerClientMetrics() {
// copied (more-or-less directly) from k8s.io/kubernetes setup code
// (which isn't anywhere in an easily-importable place).
// LatencyAdapter implements LatencyMetric.
type LatencyAdapter struct {
metric *prometheus.HistogramVec
}
// Observe increments the request latency metric for the given verb/URL.
func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds())
}
type sizeAdapter struct {
metric *prometheus.HistogramVec
}
func (s *sizeAdapter) Observe(ctx context.Context, verb string, host string, size float64) {
s.metric.WithLabelValues(verb, host).Observe(size)
}
type resultAdapter struct {
metric *prometheus.CounterVec
}

View File

@@ -1,52 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metrics
import (
"fmt"
"net"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("metrics")
// DefaultBindAddress sets the default bind address for the metrics listener
// The metrics is on by default.
var DefaultBindAddress = ":8080"
// NewListener creates a new TCP listener bound to the given address.
func NewListener(addr string) (net.Listener, error) {
if addr == "" {
// If the metrics bind address is empty, default to ":8080"
addr = DefaultBindAddress
}
// Add a case to disable metrics altogether
if addr == "0" {
return nil, nil
}
log.Info("Metrics server is starting to listen", "addr", addr)
ln, err := net.Listen("tcp", addr)
if err != nil {
er := fmt.Errorf("error listening on %s: %w", addr, err)
log.Error(er, "metrics server failed to listen. You may want to disable the metrics server or use another port if it is due to conflicts")
return nil, er
}
return ln, nil
}

View File

@@ -1,5 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -15,8 +15,12 @@ limitations under the License.
*/
/*
Package inject defines interfaces and functions for propagating dependencies from a ControllerManager to
the components registered with it. Dependencies are propagated to Reconciler, Source, EventHandler and Predicate
objects which implement the Injectable interfaces.
Package server provides the metrics server implementation.
*/
package inject
package server
import (
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("metrics")

View File

@@ -0,0 +1,312 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/client-go/rest"
certutil "k8s.io/client-go/util/cert"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)
const (
defaultMetricsEndpoint = "/metrics"
)
// DefaultBindAddress is the default bind address for the metrics server.
var DefaultBindAddress = ":8080"
// Server is a server that serves metrics.
type Server interface {
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the metrics server doesn't need leader election.
NeedLeaderElection() bool
// Start runs the server.
// It will install the metrics related resources depending on the server configuration.
Start(ctx context.Context) error
}
// Options are all available options for the metrics.Server
type Options struct {
// SecureServing enables serving metrics via https.
// Per default metrics will be served via http.
SecureServing bool
// BindAddress is the bind address for the metrics server.
// It will be defaulted to ":8080" if unspecified.
// Set this to "0" to disable the metrics server.
BindAddress string
// ExtraHandlers contains a map of handlers (by path) which will be added to the metrics server.
// This might be useful to register diagnostic endpoints e.g. pprof.
// Note that pprof endpoints are meant to be sensitive and shouldn't be exposed publicly.
// If the simple path -> handler mapping offered here is not enough, a new http
// server/listener should be added as Runnable to the manager via the Add method.
ExtraHandlers map[string]http.Handler
// FilterProvider provides a filter which is a func that is added around
// the metrics and the extra handlers on the metrics server.
// This can be e.g. used to enforce authentication and authorization on the handlers
// endpoint by setting this field to filters.WithAuthenticationAndAuthorization.
FilterProvider func(c *rest.Config, httpClient *http.Client) (Filter, error)
// CertDir is the directory that contains the server key and certificate. Defaults to
// <temp-dir>/k8s-metrics-server/serving-certs.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
// Note: If certificate or key doesn't exist a self-signed certificate will be used.
CertDir string
// CertName is the server certificate name. Defaults to tls.crt.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
// Note: If certificate or key doesn't exist a self-signed certificate will be used.
CertName string
// KeyName is the server key name. Defaults to tls.key.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
// Note: If certificate or key doesn't exist a self-signed certificate will be used.
KeyName string
// TLSOpts is used to allow configuring the TLS config used for the server.
// This also allows providing a certificate via GetCertificate.
TLSOpts []func(*tls.Config)
}
// Filter is a func that is added around metrics and extra handlers on the metrics server.
type Filter func(log logr.Logger, handler http.Handler) (http.Handler, error)
// NewServer constructs a new metrics.Server from the provided options.
func NewServer(o Options, config *rest.Config, httpClient *http.Client) (Server, error) {
o.setDefaults()
// Skip server creation if metrics are disabled.
if o.BindAddress == "0" {
return nil, nil
}
// Validate that ExtraHandlers is not overwriting the default /metrics endpoint.
if o.ExtraHandlers != nil {
if _, ok := o.ExtraHandlers[defaultMetricsEndpoint]; ok {
return nil, fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint)
}
}
// Create the metrics filter if a FilterProvider is set.
var metricsFilter Filter
if o.FilterProvider != nil {
var err error
metricsFilter, err = o.FilterProvider(config, httpClient)
if err != nil {
return nil, fmt.Errorf("filter provider failed to create filter for the metrics server: %w", err)
}
}
return &defaultServer{
metricsFilter: metricsFilter,
options: o,
}, nil
}
// defaultServer is the default implementation used for Server.
type defaultServer struct {
options Options
// metricsFilter is a filter which is added around
// the metrics and the extra handlers on the metrics server.
metricsFilter Filter
// mu protects access to the bindAddr field.
mu sync.RWMutex
// bindAddr is used to store the bindAddr after the listener has been created.
// This is used during testing to figure out the port that has been chosen randomly.
bindAddr string
}
// setDefaults does defaulting for the Server.
func (o *Options) setDefaults() {
if o.BindAddress == "" {
o.BindAddress = DefaultBindAddress
}
if len(o.CertDir) == 0 {
o.CertDir = filepath.Join(os.TempDir(), "k8s-metrics-server", "serving-certs")
}
if len(o.CertName) == 0 {
o.CertName = "tls.crt"
}
if len(o.KeyName) == 0 {
o.KeyName = "tls.key"
}
}
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the metrics server doesn't need leader election.
func (*defaultServer) NeedLeaderElection() bool {
return false
}
// Start runs the server.
// It will install the metrics related resources depend on the server configuration.
func (s *defaultServer) Start(ctx context.Context) error {
log.Info("Starting metrics server")
listener, err := s.createListener(ctx, log)
if err != nil {
return fmt.Errorf("failed to start metrics server: failed to create listener: %w", err)
}
// Storing bindAddr here so we can retrieve it during testing via GetBindAddr.
s.mu.Lock()
s.bindAddr = listener.Addr().String()
s.mu.Unlock()
mux := http.NewServeMux()
handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
ErrorHandling: promhttp.HTTPErrorOnError,
})
if s.metricsFilter != nil {
log := log.WithValues("path", defaultMetricsEndpoint)
var err error
handler, err = s.metricsFilter(log, handler)
if err != nil {
return fmt.Errorf("failed to start metrics server: failed to add metrics filter: %w", err)
}
}
// TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics
mux.Handle(defaultMetricsEndpoint, handler)
for path, extraHandler := range s.options.ExtraHandlers {
if s.metricsFilter != nil {
log := log.WithValues("path", path)
var err error
extraHandler, err = s.metricsFilter(log, extraHandler)
if err != nil {
return fmt.Errorf("failed to start metrics server: failed to add metrics filter to extra handler for path %s: %w", path, err)
}
}
mux.Handle(path, extraHandler)
}
log.Info("Serving metrics server", "bindAddress", s.options.BindAddress, "secure", s.options.SecureServing)
srv := httpserver.New(mux)
idleConnsClosed := make(chan struct{})
go func() {
<-ctx.Done()
log.Info("Shutting down metrics server with timeout of 1 minute")
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
// Error from closing listeners, or context timeout
log.Error(err, "error shutting down the HTTP server")
}
close(idleConnsClosed)
}()
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed {
return err
}
<-idleConnsClosed
return nil
}
func (s *defaultServer) createListener(ctx context.Context, log logr.Logger) (net.Listener, error) {
if !s.options.SecureServing {
return net.Listen("tcp", s.options.BindAddress)
}
cfg := &tls.Config{ //nolint:gosec
NextProtos: []string{"h2"},
}
// fallback TLS config ready, will now mutate if passer wants full control over it
for _, op := range s.options.TLSOpts {
op(cfg)
}
if cfg.GetCertificate == nil {
certPath := filepath.Join(s.options.CertDir, s.options.CertName)
keyPath := filepath.Join(s.options.CertDir, s.options.KeyName)
_, certErr := os.Stat(certPath)
certExists := !os.IsNotExist(certErr)
_, keyErr := os.Stat(keyPath)
keyExists := !os.IsNotExist(keyErr)
if certExists && keyExists {
// Create the certificate watcher and
// set the config's GetCertificate on the TLSConfig
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
return nil, err
}
cfg.GetCertificate = certWatcher.GetCertificate
go func() {
if err := certWatcher.Start(ctx); err != nil {
log.Error(err, "certificate watcher error")
}
}()
}
}
// If cfg.GetCertificate is still nil, i.e. we didn't configure a cert watcher, fallback to a self-signed certificate.
if cfg.GetCertificate == nil {
// Note: Using self-signed certificates here should be good enough. It's just important that we
// encrypt the communication. For example kube-controller-manager also uses a self-signed certificate
// for the metrics endpoint per default.
cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("localhost", []net.IP{{127, 0, 0, 1}}, nil, "")
if err != nil {
return nil, fmt.Errorf("failed to generate self-signed certificate for metrics server: %w", err)
}
keyPair, err := tls.X509KeyPair(cert, key)
if err != nil {
return nil, fmt.Errorf("failed to create self-signed key pair for metrics server: %w", err)
}
cfg.Certificates = []tls.Certificate{keyPair}
}
return tls.Listen("tcp", s.options.BindAddress, cfg)
}
func (s *defaultServer) GetBindAddr() string {
s.mu.RLock()
defer s.mu.RUnlock()
return s.bindAddr
}

View File

@@ -24,7 +24,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
var log = logf.RuntimeLog.WithName("predicate").WithName("eventFilters")
@@ -242,15 +241,6 @@ type and struct {
predicates []Predicate
}
func (a and) InjectFunc(f inject.Func) error {
for _, p := range a.predicates {
if err := f(p); err != nil {
return err
}
}
return nil
}
func (a and) Create(e event.CreateEvent) bool {
for _, p := range a.predicates {
if !p.Create(e) {
@@ -296,15 +286,6 @@ type or struct {
predicates []Predicate
}
func (o or) InjectFunc(f inject.Func) error {
for _, p := range o.predicates {
if err := f(p); err != nil {
return err
}
}
return nil
}
func (o or) Create(e event.CreateEvent) bool {
for _, p := range o.predicates {
if p.Create(e) {
@@ -350,10 +331,6 @@ type not struct {
predicate Predicate
}
func (n not) InjectFunc(f inject.Func) error {
return f(n.predicate)
}
func (n not) Create(e event.CreateEvent) bool {
return !n.predicate.Create(e)
}

View File

@@ -18,6 +18,7 @@ package reconcile
import (
"context"
"errors"
"time"
"k8s.io/apimachinery/pkg/types"
@@ -88,8 +89,16 @@ instead the reconcile function observes this when reading the cluster state and
*/
type Reconciler interface {
// Reconcile performs a full reconciliation for the object referred to by the Request.
// The Controller will requeue the Request to be processed again if an error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
//
// If the returned error is non-nil, the Result is ignored and the request will be
// requeued using exponential backoff. The only exception is if the error is a
// TerminalError in which case no requeuing happens.
//
// If the error is nil and the returned Result has a non-zero result.RequeueAfter, the request
// will be requeued after the specified duration.
//
// If the error is nil and result.RequeueAfter is zero and result.Reque is true, the request
// will be requeued using exponential backoff.
Reconcile(context.Context, Request) (Result, error)
}
@@ -100,3 +109,30 @@ var _ Reconciler = Func(nil)
// Reconcile implements Reconciler.
func (r Func) Reconcile(ctx context.Context, o Request) (Result, error) { return r(ctx, o) }
// TerminalError is an error that will not be retried but still be logged
// and recorded in metrics.
func TerminalError(wrapped error) error {
return &terminalError{err: wrapped}
}
type terminalError struct {
err error
}
// This function will return nil if te.err is nil.
func (te *terminalError) Unwrap() error {
return te.err
}
func (te *terminalError) Error() string {
if te.err == nil {
return "nil terminal error"
}
return "terminal error: " + te.err.Error()
}
func (te *terminalError) Is(target error) bool {
tp := &terminalError{}
return errors.As(target, &tp)
}

View File

@@ -1,164 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package inject is used by a Manager to inject types into Sources, EventHandlers, Predicates, and Reconciles.
// Deprecated: Use manager.Options fields directly. This package will be removed in v0.10.
package inject
import (
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// Cache is used by the ControllerManager to inject Cache into Sources, EventHandlers, Predicates, and
// Reconciles.
type Cache interface {
InjectCache(cache cache.Cache) error
}
// CacheInto will set informers on i and return the result if it implements Cache. Returns
// false if i does not implement Cache.
func CacheInto(c cache.Cache, i interface{}) (bool, error) {
if s, ok := i.(Cache); ok {
return true, s.InjectCache(c)
}
return false, nil
}
// APIReader is used by the Manager to inject the APIReader into necessary types.
type APIReader interface {
InjectAPIReader(client.Reader) error
}
// APIReaderInto will set APIReader on i and return the result if it implements APIReaderInto.
// Returns false if i does not implement APIReader.
func APIReaderInto(reader client.Reader, i interface{}) (bool, error) {
if s, ok := i.(APIReader); ok {
return true, s.InjectAPIReader(reader)
}
return false, nil
}
// Config is used by the ControllerManager to inject Config into Sources, EventHandlers, Predicates, and
// Reconciles.
type Config interface {
InjectConfig(*rest.Config) error
}
// ConfigInto will set config on i and return the result if it implements Config. Returns
// false if i does not implement Config.
func ConfigInto(config *rest.Config, i interface{}) (bool, error) {
if s, ok := i.(Config); ok {
return true, s.InjectConfig(config)
}
return false, nil
}
// Client is used by the ControllerManager to inject client into Sources, EventHandlers, Predicates, and
// Reconciles.
type Client interface {
InjectClient(client.Client) error
}
// ClientInto will set client on i and return the result if it implements Client. Returns
// false if i does not implement Client.
func ClientInto(client client.Client, i interface{}) (bool, error) {
if s, ok := i.(Client); ok {
return true, s.InjectClient(client)
}
return false, nil
}
// Scheme is used by the ControllerManager to inject Scheme into Sources, EventHandlers, Predicates, and
// Reconciles.
type Scheme interface {
InjectScheme(scheme *runtime.Scheme) error
}
// SchemeInto will set scheme and return the result on i if it implements Scheme. Returns
// false if i does not implement Scheme.
func SchemeInto(scheme *runtime.Scheme, i interface{}) (bool, error) {
if is, ok := i.(Scheme); ok {
return true, is.InjectScheme(scheme)
}
return false, nil
}
// Stoppable is used by the ControllerManager to inject stop channel into Sources,
// EventHandlers, Predicates, and Reconciles.
type Stoppable interface {
InjectStopChannel(<-chan struct{}) error
}
// StopChannelInto will set stop channel on i and return the result if it implements Stoppable.
// Returns false if i does not implement Stoppable.
func StopChannelInto(stop <-chan struct{}, i interface{}) (bool, error) {
if s, ok := i.(Stoppable); ok {
return true, s.InjectStopChannel(stop)
}
return false, nil
}
// Mapper is used to inject the rest mapper to components that may need it.
type Mapper interface {
InjectMapper(meta.RESTMapper) error
}
// MapperInto will set the rest mapper on i and return the result if it implements Mapper.
// Returns false if i does not implement Mapper.
func MapperInto(mapper meta.RESTMapper, i interface{}) (bool, error) {
if m, ok := i.(Mapper); ok {
return true, m.InjectMapper(mapper)
}
return false, nil
}
// Func injects dependencies into i.
type Func func(i interface{}) error
// Injector is used by the ControllerManager to inject Func into Controllers.
type Injector interface {
InjectFunc(f Func) error
}
// InjectorInto will set f and return the result on i if it implements Injector. Returns
// false if i does not implement Injector.
func InjectorInto(f Func, i interface{}) (bool, error) {
if ii, ok := i.(Injector); ok {
return true, ii.InjectFunc(f)
}
return false, nil
}
// Logger is used to inject Loggers into components that need them
// and don't otherwise have opinions.
type Logger interface {
InjectLogger(l logr.Logger) error
}
// LoggerInto will set the logger on the given object if it implements inject.Logger,
// returning true if a InjectLogger was called, and false otherwise.
func LoggerInto(l logr.Logger, i interface{}) (bool, error) {
if injectable, wantsLogger := i.(Logger); wantsLogger {
return true, injectable.InjectLogger(l)
}
return false, nil
}

View File

@@ -18,28 +18,19 @@ package source
import (
"context"
"errors"
"fmt"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/source/internal"
internal "sigs.k8s.io/controller-runtime/pkg/internal/source"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/predicate"
)
var log = logf.RuntimeLog.WithName("source")
const (
// defaultBufferSize is the default number of event notifications that can be buffered.
defaultBufferSize = 1024
@@ -52,8 +43,7 @@ const (
//
// * Use Channel for events originating outside the cluster (eh.g. GitHub Webhook callback, Polling external urls).
//
// Users may build their own Source implementations. If their implementations implement any of the inject package
// interfaces, the dependencies will be injected by the Controller when Watch is called.
// Users may build their own Source implementations.
type Source interface {
// Start is internal and should be called only by the Controller to register an EventHandler with the Informer
// to enqueue reconcile.Requests.
@@ -67,144 +57,9 @@ type SyncingSource interface {
WaitForSync(ctx context.Context) error
}
// NewKindWithCache creates a Source without InjectCache, so that it is assured that the given cache is used
// and not overwritten. It can be used to watch objects in a different cluster by passing the cache
// from that other cluster.
func NewKindWithCache(object client.Object, cache cache.Cache) SyncingSource {
return &kindWithCache{kind: Kind{Type: object, cache: cache}}
}
type kindWithCache struct {
kind Kind
}
func (ks *kindWithCache) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface,
prct ...predicate.Predicate) error {
return ks.kind.Start(ctx, handler, queue, prct...)
}
func (ks *kindWithCache) String() string {
return ks.kind.String()
}
func (ks *kindWithCache) WaitForSync(ctx context.Context) error {
return ks.kind.WaitForSync(ctx)
}
// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create).
type Kind struct {
// Type is the type of object to watch. e.g. &v1.Pod{}
Type client.Object
// cache used to watch APIs
cache cache.Cache
// started may contain an error if one was encountered during startup. If its closed and does not
// contain an error, startup and syncing finished.
started chan error
startCancel func()
}
var _ SyncingSource = &Kind{}
// Start is internal and should be called only by the Controller to register an EventHandler with the Informer
// to enqueue reconcile.Requests.
func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface,
prct ...predicate.Predicate) error {
// Type should have been specified by the user.
if ks.Type == nil {
return fmt.Errorf("must specify Kind.Type")
}
// cache should have been injected before Start was called
if ks.cache == nil {
return fmt.Errorf("must call CacheInto on Kind before calling Start")
}
// cache.GetInformer will block until its context is cancelled if the cache was already started and it can not
// sync that informer (most commonly due to RBAC issues).
ctx, ks.startCancel = context.WithCancel(ctx)
ks.started = make(chan error)
go func() {
var (
i cache.Informer
lastErr error
)
// Tries to get an informer until it returns true,
// an error or the specified context is cancelled or expired.
if err := wait.PollImmediateUntilWithContext(ctx, 10*time.Second, func(ctx context.Context) (bool, error) {
// Lookup the Informer from the Cache and add an EventHandler which populates the Queue
i, lastErr = ks.cache.GetInformer(ctx, ks.Type)
if lastErr != nil {
kindMatchErr := &meta.NoKindMatchError{}
switch {
case errors.As(lastErr, &kindMatchErr):
log.Error(lastErr, "if kind is a CRD, it should be installed before calling Start",
"kind", kindMatchErr.GroupKind)
case runtime.IsNotRegisteredError(lastErr):
log.Error(lastErr, "kind must be registered to the Scheme")
default:
log.Error(lastErr, "failed to get informer from cache")
}
return false, nil // Retry.
}
return true, nil
}); err != nil {
if lastErr != nil {
ks.started <- fmt.Errorf("failed to get informer from cache: %w", lastErr)
return
}
ks.started <- err
return
}
_, err := i.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct})
if err != nil {
ks.started <- err
return
}
if !ks.cache.WaitForCacheSync(ctx) {
// Would be great to return something more informative here
ks.started <- errors.New("cache did not sync")
}
close(ks.started)
}()
return nil
}
func (ks *Kind) String() string {
if ks.Type != nil {
return fmt.Sprintf("kind source: %T", ks.Type)
}
return "kind source: unknown type"
}
// WaitForSync implements SyncingSource to allow controllers to wait with starting
// workers until the cache is synced.
func (ks *Kind) WaitForSync(ctx context.Context) error {
select {
case err := <-ks.started:
return err
case <-ctx.Done():
ks.startCancel()
if errors.Is(ctx.Err(), context.Canceled) {
return nil
}
return errors.New("timed out waiting for cache to be synced")
}
}
var _ inject.Cache = &Kind{}
// InjectCache is internal should be called only by the Controller. InjectCache is used to inject
// the Cache dependency initialized by the ControllerManager.
func (ks *Kind) InjectCache(c cache.Cache) error {
if ks.cache == nil {
ks.cache = c
}
return nil
// Kind creates a KindSource with the given cache provider.
func Kind(cache cache.Cache, object client.Object) SyncingSource {
return &internal.Kind{Type: object, Cache: cache}
}
var _ Source = &Channel{}
@@ -219,9 +74,6 @@ type Channel struct {
// Source is the source channel to fetch GenericEvents
Source <-chan event.GenericEvent
// stop is to end ongoing goroutine, and close the channels
stop <-chan struct{}
// dest is the destination channels of the added event handlers
dest []chan event.GenericEvent
@@ -237,18 +89,6 @@ func (cs *Channel) String() string {
return fmt.Sprintf("channel source: %p", cs)
}
var _ inject.Stoppable = &Channel{}
// InjectStopChannel is internal should be called only by the Controller.
// It is used to inject the stop channel initialized by the ControllerManager.
func (cs *Channel) InjectStopChannel(stop <-chan struct{}) error {
if cs.stop == nil {
cs.stop = stop
}
return nil
}
// Start implements Source and should only be called by the Controller.
func (cs *Channel) Start(
ctx context.Context,
@@ -260,11 +100,6 @@ func (cs *Channel) Start(
return fmt.Errorf("must specify Channel.Source")
}
// stop should have been injected before Start was called
if cs.stop == nil {
return fmt.Errorf("must call InjectStop on Channel before calling Start")
}
// use default value if DestBufferSize not specified
if cs.DestBufferSize == 0 {
cs.DestBufferSize = defaultBufferSize
@@ -292,7 +127,11 @@ func (cs *Channel) Start(
}
if shouldHandle {
handler.Generic(evt, queue)
func() {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
handler.Generic(ctx, evt, queue)
}()
}
}
}()
@@ -359,7 +198,7 @@ func (is *Informer) Start(ctx context.Context, handler handler.EventHandler, que
return fmt.Errorf("must specify Informer.Informer")
}
_, err := is.Informer.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct})
_, err := is.Informer.AddEventHandler(internal.NewEventHandler(ctx, queue, handler, prct).HandlerFuncs())
if err != nil {
return err
}

View File

@@ -19,7 +19,6 @@ package admission
import (
"fmt"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/json"
@@ -32,8 +31,11 @@ type Decoder struct {
}
// NewDecoder creates a Decoder given the runtime.Scheme.
func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) {
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil
func NewDecoder(scheme *runtime.Scheme) *Decoder {
if scheme == nil {
panic("scheme should never be nil")
}
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}
}
// Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object.
@@ -62,9 +64,14 @@ func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) er
if len(rawObj.Raw) == 0 {
return fmt.Errorf("there is no content to decode")
}
if unstructuredInto, isUnstructured := into.(*unstructured.Unstructured); isUnstructured {
if unstructuredInto, isUnstructured := into.(runtime.Unstructured); isUnstructured {
// unmarshal into unstructured's underlying object to avoid calling the decoder
return json.Unmarshal(rawObj.Raw, &unstructuredInto.Object)
var object map[string]interface{}
if err := json.Unmarshal(rawObj.Raw, &object); err != nil {
return err
}
unstructuredInto.SetUnstructuredContent(object)
return nil
}
deserializer := d.codecs.UniversalDeserializer()

View File

@@ -33,9 +33,9 @@ type Defaulter interface {
}
// DefaultingWebhookFor creates a new Webhook for Defaulting the provided type.
func DefaultingWebhookFor(defaulter Defaulter) *Webhook {
func DefaultingWebhookFor(scheme *runtime.Scheme, defaulter Defaulter) *Webhook {
return &Webhook{
Handler: &mutatingHandler{defaulter: defaulter},
Handler: &mutatingHandler{defaulter: defaulter, decoder: NewDecoder(scheme)},
}
}
@@ -44,16 +44,11 @@ type mutatingHandler struct {
decoder *Decoder
}
var _ DecoderInjector = &mutatingHandler{}
// InjectDecoder injects the decoder into a mutatingHandler.
func (h *mutatingHandler) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *mutatingHandler) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.defaulter == nil {
panic("defaulter should never be nil")
}

View File

@@ -34,9 +34,9 @@ type CustomDefaulter interface {
}
// WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface.
func WithCustomDefaulter(obj runtime.Object, defaulter CustomDefaulter) *Webhook {
func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter) *Webhook {
return &Webhook{
Handler: &defaulterForType{object: obj, defaulter: defaulter},
Handler: &defaulterForType{object: obj, defaulter: defaulter, decoder: NewDecoder(scheme)},
}
}
@@ -46,15 +46,11 @@ type defaulterForType struct {
decoder *Decoder
}
var _ DecoderInjector = &defaulterForType{}
func (h *defaulterForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *defaulterForType) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.defaulter == nil {
panic("defaulter should never be nil")
}

View File

@@ -20,9 +20,3 @@ Package admission provides implementation for admission webhook and methods to i
See examples/mutatingwebhook.go and examples/validatingwebhook.go for examples of admission webhooks.
*/
package admission
import (
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
var log = logf.RuntimeLog.WithName("admission")

View File

@@ -52,7 +52,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var reviewResponse Response
if r.Body == nil {
err = errors.New("request body is empty")
wh.log.Error(err, "bad request")
wh.getLogger(nil).Error(err, "bad request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -60,7 +60,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
if body, err = io.ReadAll(r.Body); err != nil {
wh.log.Error(err, "unable to read the body from the incoming request")
wh.getLogger(nil).Error(err, "unable to read the body from the incoming request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -69,7 +69,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// verify the content type is accurate
if contentType := r.Header.Get("Content-Type"); contentType != "application/json" {
err = fmt.Errorf("contentType=%s, expected application/json", contentType)
wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType)
wh.getLogger(nil).Error(err, "unable to process a request with unknown content type")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
@@ -88,12 +88,12 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ar.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind("AdmissionReview"))
_, actualAdmRevGVK, err := admissionCodecs.UniversalDeserializer().Decode(body, nil, &ar)
if err != nil {
wh.log.Error(err, "unable to decode the request")
wh.getLogger(nil).Error(err, "unable to decode the request")
reviewResponse = Errored(http.StatusBadRequest, err)
wh.writeResponse(w, reviewResponse)
return
}
wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind, "resource", req.Resource)
wh.getLogger(&req).V(5).Info("received request")
reviewResponse = wh.Handle(ctx, req)
wh.writeResponseTyped(w, reviewResponse, actualAdmRevGVK)
@@ -124,7 +124,7 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, admRevGVK
// writeAdmissionResponse writes ar to w.
func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) {
if err := json.NewEncoder(w).Encode(ar); err != nil {
wh.log.Error(err, "unable to encode and write the response")
wh.getLogger(nil).Error(err, "unable to encode and write the response")
// Since the `ar v1.AdmissionReview` is a clear and legal object,
// it should not have problem to be marshalled into bytes.
// The error here is probably caused by the abnormal HTTP connection,
@@ -132,15 +132,15 @@ func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) {
// to avoid endless circular calling.
serverError := Errored(http.StatusInternalServerError, err)
if err = json.NewEncoder(w).Encode(v1.AdmissionReview{Response: &serverError.AdmissionResponse}); err != nil {
wh.log.Error(err, "still unable to encode and write the InternalServerError response")
wh.getLogger(nil).Error(err, "still unable to encode and write the InternalServerError response")
}
} else {
res := ar.Response
if log := wh.log; log.V(1).Enabled() {
if log := wh.getLogger(nil); log.V(5).Enabled() {
if res.Result != nil {
log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason)
log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason, "message", res.Result.Message)
}
log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed)
log.V(5).Info("wrote response", "requestID", res.UID, "allowed", res.Allowed)
}
}
}

View File

@@ -1,31 +0,0 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package admission
// DecoderInjector is used by the ControllerManager to inject decoder into webhook handlers.
type DecoderInjector interface {
InjectDecoder(*Decoder) error
}
// InjectDecoderInto will set decoder on i and return the result if it implements Decoder. Returns
// false if i does not implement Decoder.
func InjectDecoderInto(decoder *Decoder, i interface{}) (bool, error) {
if s, ok := i.(DecoderInjector); ok {
return true, s.InjectDecoder(decoder)
}
return false, nil
}

View File

@@ -25,8 +25,6 @@ import (
jsonpatch "gomodules.xyz/jsonpatch/v2"
admissionv1 "k8s.io/api/admission/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
type multiMutating []Handler
@@ -62,31 +60,6 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response {
}
}
// InjectFunc injects the field setter into the handlers.
func (hs multiMutating) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
for _, handler := range hs {
if err := f(handler); err != nil {
return err
}
}
return nil
}
// InjectDecoder injects the decoder into the handlers.
func (hs multiMutating) InjectDecoder(d *Decoder) error {
for _, handler := range hs {
if _, err := InjectDecoderInto(d, handler); err != nil {
return err
}
}
return nil
}
// MultiMutatingHandler combines multiple mutating webhook handlers into a single
// mutating webhook handler. Handlers are called in sequential order, and the first
// `allowed: false` response may short-circuit the rest. Users must take care to
@@ -120,28 +93,3 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response {
func MultiValidatingHandler(handlers ...Handler) Handler {
return multiValidating(handlers)
}
// InjectFunc injects the field setter into the handlers.
func (hs multiValidating) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
for _, handler := range hs {
if err := f(handler); err != nil {
return err
}
}
return nil
}
// InjectDecoder injects the decoder into the handlers.
func (hs multiValidating) InjectDecoder(d *Decoder) error {
for _, handler := range hs {
if _, err := InjectDecoderInto(d, handler); err != nil {
return err
}
}
return nil
}

View File

@@ -26,21 +26,21 @@ import (
// Allowed constructs a response indicating that the given operation
// is allowed (without any patches).
func Allowed(reason string) Response {
return ValidationResponse(true, reason)
func Allowed(message string) Response {
return ValidationResponse(true, message)
}
// Denied constructs a response indicating that the given operation
// is not allowed.
func Denied(reason string) Response {
return ValidationResponse(false, reason)
func Denied(message string) Response {
return ValidationResponse(false, message)
}
// Patched constructs a response indicating that the given operation is
// allowed, and that the target object should be modified by the given
// JSONPatch operations.
func Patched(reason string, patches ...jsonpatch.JsonPatchOperation) Response {
resp := Allowed(reason)
func Patched(message string, patches ...jsonpatch.JsonPatchOperation) Response {
resp := Allowed(message)
resp.Patches = patches
return resp
@@ -60,21 +60,24 @@ func Errored(code int32, err error) Response {
}
// ValidationResponse returns a response for admitting a request.
func ValidationResponse(allowed bool, reason string) Response {
func ValidationResponse(allowed bool, message string) Response {
code := http.StatusForbidden
reason := metav1.StatusReasonForbidden
if allowed {
code = http.StatusOK
reason = ""
}
resp := Response{
AdmissionResponse: admissionv1.AdmissionResponse{
Allowed: allowed,
Result: &metav1.Status{
Code: int32(code),
Code: int32(code),
Reason: reason,
},
},
}
if len(reason) > 0 {
resp.Result.Reason = metav1.StatusReason(reason)
if len(message) > 0 {
resp.Result.Message = message
}
return resp
}

View File

@@ -18,7 +18,8 @@ package admission
import (
"context"
goerrors "errors"
"errors"
"fmt"
"net/http"
v1 "k8s.io/api/admission/v1"
@@ -26,18 +27,35 @@ import (
"k8s.io/apimachinery/pkg/runtime"
)
// Warnings represents warning messages.
type Warnings []string
// Validator defines functions for validating an operation.
// The custom resource kind which implements this interface can validate itself.
// To validate the custom resource with another specific struct, use CustomValidator instead.
type Validator interface {
runtime.Object
ValidateCreate() error
ValidateUpdate(old runtime.Object) error
ValidateDelete() error
// ValidateCreate validates the object on creation.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateCreate() (warnings Warnings, err error)
// ValidateUpdate validates the object on update. The oldObj is the object before the update.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateUpdate(old runtime.Object) (warnings Warnings, err error)
// ValidateDelete validates the object on deletion.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateDelete() (warnings Warnings, err error)
}
// ValidatingWebhookFor creates a new Webhook for validating the provided type.
func ValidatingWebhookFor(validator Validator) *Webhook {
func ValidatingWebhookFor(scheme *runtime.Scheme, validator Validator) *Webhook {
return &Webhook{
Handler: &validatingHandler{validator: validator},
Handler: &validatingHandler{validator: validator, decoder: NewDecoder(scheme)},
}
}
@@ -46,42 +64,34 @@ type validatingHandler struct {
decoder *Decoder
}
var _ DecoderInjector = &validatingHandler{}
// InjectDecoder injects the decoder into a validatingHandler.
func (h *validatingHandler) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *validatingHandler) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.validator == nil {
panic("validator should never be nil")
}
// Get the object in the request
obj := h.validator.DeepCopyObject().(Validator)
if req.Operation == v1.Create {
err := h.decoder.Decode(req, obj)
if err != nil {
var err error
var warnings []string
switch req.Operation {
case v1.Connect:
// No validation for connect requests.
// TODO(vincepri): Should we validate CONNECT requests? In what cases?
case v1.Create:
if err = h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateCreate()
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
}
if req.Operation == v1.Update {
warnings, err = obj.ValidateCreate()
case v1.Update:
oldObj := obj.DeepCopyObject()
err := h.decoder.DecodeRaw(req.Object, obj)
err = h.decoder.DecodeRaw(req.Object, obj)
if err != nil {
return Errored(http.StatusBadRequest, err)
}
@@ -90,33 +100,26 @@ func (h *validatingHandler) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateUpdate(oldObj)
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
}
if req.Operation == v1.Delete {
warnings, err = obj.ValidateUpdate(oldObj)
case v1.Delete:
// In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346
// OldObject contains the object being deleted
err := h.decoder.DecodeRaw(req.OldObject, obj)
err = h.decoder.DecodeRaw(req.OldObject, obj)
if err != nil {
return Errored(http.StatusBadRequest, err)
}
err = obj.ValidateDelete()
if err != nil {
var apiStatus apierrors.APIStatus
if goerrors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
}
return Denied(err.Error())
}
warnings, err = obj.ValidateDelete()
default:
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation))
}
return Allowed("")
if err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...)
}
return Denied(err.Error()).WithWarnings(warnings...)
}
return Allowed("").WithWarnings(warnings...)
}

View File

@@ -28,16 +28,29 @@ import (
)
// CustomValidator defines functions for validating an operation.
// The object to be validated is passed into methods as a parameter.
type CustomValidator interface {
ValidateCreate(ctx context.Context, obj runtime.Object) error
ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error
ValidateDelete(ctx context.Context, obj runtime.Object) error
// ValidateCreate validates the object on creation.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateCreate(ctx context.Context, obj runtime.Object) (warnings Warnings, err error)
// ValidateUpdate validates the object on update.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings Warnings, err error)
// ValidateDelete validates the object on deletion.
// The optional warnings will be added to the response as warning messages.
// Return an error if the object is invalid.
ValidateDelete(ctx context.Context, obj runtime.Object) (warnings Warnings, err error)
}
// WithCustomValidator creates a new Webhook for validating the provided type.
func WithCustomValidator(obj runtime.Object, validator CustomValidator) *Webhook {
func WithCustomValidator(scheme *runtime.Scheme, obj runtime.Object, validator CustomValidator) *Webhook {
return &Webhook{
Handler: &validatorForType{object: obj, validator: validator},
Handler: &validatorForType{object: obj, validator: validator, decoder: NewDecoder(scheme)},
}
}
@@ -47,16 +60,11 @@ type validatorForType struct {
decoder *Decoder
}
var _ DecoderInjector = &validatorForType{}
// InjectDecoder injects the decoder into a validatingHandler.
func (h *validatorForType) InjectDecoder(d *Decoder) error {
h.decoder = d
return nil
}
// Handle handles admission requests.
func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
if h.decoder == nil {
panic("decoder should never be nil")
}
if h.validator == nil {
panic("validator should never be nil")
}
@@ -70,13 +78,18 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
obj := h.object.DeepCopyObject()
var err error
var warnings []string
switch req.Operation {
case v1.Connect:
// No validation for connect requests.
// TODO(vincepri): Should we validate CONNECT requests? In what cases?
case v1.Create:
if err := h.decoder.Decode(req, obj); err != nil {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateCreate(ctx, obj)
warnings, err = h.validator.ValidateCreate(ctx, obj)
case v1.Update:
oldObj := obj.DeepCopyObject()
if err := h.decoder.DecodeRaw(req.Object, obj); err != nil {
@@ -86,7 +99,7 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateUpdate(ctx, oldObj, obj)
warnings, err = h.validator.ValidateUpdate(ctx, oldObj, obj)
case v1.Delete:
// In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346
// OldObject contains the object being deleted
@@ -94,20 +107,20 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response {
return Errored(http.StatusBadRequest, err)
}
err = h.validator.ValidateDelete(ctx, obj)
warnings, err = h.validator.ValidateDelete(ctx, obj)
default:
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation))
return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation))
}
// Check the error message first.
if err != nil {
var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
return validationResponseFromStatus(false, apiStatus.Status())
return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...)
}
return Denied(err.Error())
return Denied(err.Error()).WithWarnings(warnings...)
}
// Return allowed if everything succeeded.
return Allowed("")
return Allowed("").WithWarnings(warnings...)
}

View File

@@ -21,18 +21,17 @@ import (
"errors"
"fmt"
"net/http"
"sync"
"github.com/go-logr/logr"
jsonpatch "gomodules.xyz/jsonpatch/v2"
"gomodules.xyz/jsonpatch/v2"
admissionv1 "k8s.io/api/admission/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/json"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog/v2"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
@@ -131,16 +130,14 @@ type Webhook struct {
// headers thus allowing you to read them from within the handler
WithContextFunc func(context.Context, *http.Request) context.Context
// decoder is constructed on receiving a scheme and passed down to then handler
decoder *Decoder
// LogConstructor is used to construct a logger for logging messages during webhook calls
// based on the given base logger (which might carry more values like the webhook's path).
// Note: LogConstructor has to be able to handle nil requests as we are also using it
// outside the context of requests.
LogConstructor func(base logr.Logger, req *Request) logr.Logger
log logr.Logger
}
// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook.
func (wh *Webhook) InjectLogger(l logr.Logger) error {
wh.log = l
return nil
setupLogOnce sync.Once
log logr.Logger
}
// WithRecoverPanic takes a bool flag which indicates whether the panic caused by webhook should be recovered.
@@ -166,79 +163,47 @@ func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response)
}()
}
reqLog := wh.getLogger(&req)
ctx = logf.IntoContext(ctx, reqLog)
resp := wh.Handler.Handle(ctx, req)
if err := resp.Complete(req); err != nil {
wh.log.Error(err, "unable to encode response")
reqLog.Error(err, "unable to encode response")
return Errored(http.StatusInternalServerError, errUnableToEncodeResponse)
}
return resp
}
// InjectScheme injects a scheme into the webhook, in order to construct a Decoder.
func (wh *Webhook) InjectScheme(s *runtime.Scheme) error {
// TODO(directxman12): we should have a better way to pass this down
var err error
wh.decoder, err = NewDecoder(s)
if err != nil {
return err
}
// inject the decoder here too, just in case the order of calling this is not
// scheme first, then inject func
if wh.Handler != nil {
if _, err := InjectDecoderInto(wh.GetDecoder(), wh.Handler); err != nil {
return err
// getLogger constructs a logger from the injected log and LogConstructor.
func (wh *Webhook) getLogger(req *Request) logr.Logger {
wh.setupLogOnce.Do(func() {
if wh.log.GetSink() == nil {
wh.log = logf.Log.WithName("admission")
}
}
})
return nil
logConstructor := wh.LogConstructor
if logConstructor == nil {
logConstructor = DefaultLogConstructor
}
return logConstructor(wh.log, req)
}
// GetDecoder returns a decoder to decode the objects embedded in admission requests.
// It may be nil if we haven't received a scheme to use to determine object types yet.
func (wh *Webhook) GetDecoder() *Decoder {
return wh.decoder
}
// InjectFunc injects the field setter into the webhook.
func (wh *Webhook) InjectFunc(f inject.Func) error {
// inject directly into the handlers. It would be more correct
// to do this in a sync.Once in Handle (since we don't have some
// other start/finalize-type method), but it's more efficient to
// do it here, presumably.
// also inject a decoder, and wrap this so that we get a setFields
// that injects a decoder (hopefully things don't ignore the duplicate
// InjectorInto call).
var setFields inject.Func
setFields = func(target interface{}) error {
if err := f(target); err != nil {
return err
}
if _, err := inject.InjectorInto(setFields, target); err != nil {
return err
}
if _, err := InjectDecoderInto(wh.GetDecoder(), target); err != nil {
return err
}
return nil
// DefaultLogConstructor adds some commonly interesting fields to the given logger.
func DefaultLogConstructor(base logr.Logger, req *Request) logr.Logger {
if req != nil {
return base.WithValues("object", klog.KRef(req.Namespace, req.Name),
"namespace", req.Namespace, "name", req.Name,
"resource", req.Resource, "user", req.UserInfo.Username,
"requestID", req.UID,
)
}
return setFields(wh.Handler)
return base
}
// StandaloneOptions let you configure a StandaloneWebhook.
type StandaloneOptions struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// idea to pass your own scheme in. See the documentation in pkg/scheme for more information.
Scheme *runtime.Scheme
// Logger to be used by the webhook.
// If none is set, it defaults to log.Log global logger.
Logger logr.Logger
@@ -258,19 +223,9 @@ type StandaloneOptions struct {
// in your own server/mux. In order to be accessed by a kubernetes cluster,
// all webhook servers require TLS.
func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) {
if opts.Scheme == nil {
opts.Scheme = scheme.Scheme
if opts.Logger.GetSink() != nil {
hook.log = opts.Logger
}
if err := hook.InjectScheme(opts.Scheme); err != nil {
return nil, err
}
if opts.Logger.GetSink() == nil {
opts.Logger = logf.RuntimeLog.WithName("webhook")
}
hook.log = opts.Logger
if opts.MetricsPath == "" {
return hook, nil
}

View File

@@ -39,28 +39,20 @@ var (
log = logf.Log.WithName("conversion-webhook")
)
// Webhook implements a CRD conversion webhook HTTP handler.
type Webhook struct {
func NewWebhookHandler(scheme *runtime.Scheme) http.Handler {
return &webhook{scheme: scheme, decoder: NewDecoder(scheme)}
}
// webhook implements a CRD conversion webhook HTTP handler.
type webhook struct {
scheme *runtime.Scheme
decoder *Decoder
}
// InjectScheme injects a scheme into the webhook, in order to construct a Decoder.
func (wh *Webhook) InjectScheme(s *runtime.Scheme) error {
var err error
wh.scheme = s
wh.decoder, err = NewDecoder(s)
if err != nil {
return err
}
return nil
}
// ensure Webhook implements http.Handler
var _ http.Handler = &Webhook{}
var _ http.Handler = &webhook{}
func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
convertReview := &apix.ConversionReview{}
err := json.NewDecoder(r.Body).Decode(convertReview)
if err != nil {
@@ -95,7 +87,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// handles a version conversion request.
func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) {
func (wh *webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) {
if req == nil {
return nil, fmt.Errorf("conversion request is nil")
}
@@ -128,7 +120,7 @@ func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.Conv
// convertObject will convert given a src object to dst object.
// Note(droot): couldn't find a way to reduce the cyclomatic complexity under 10
// without compromising readability, so disabling gocyclo linter
func (wh *Webhook) convertObject(src, dst runtime.Object) error {
func (wh *webhook) convertObject(src, dst runtime.Object) error {
srcGVK := src.GetObjectKind().GroupVersionKind()
dstGVK := dst.GetObjectKind().GroupVersionKind()
@@ -155,7 +147,7 @@ func (wh *Webhook) convertObject(src, dst runtime.Object) error {
}
}
func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error {
func (wh *webhook) convertViaHub(src, dst conversion.Convertible) error {
hub, err := wh.getHub(src)
if err != nil {
return err
@@ -179,7 +171,7 @@ func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error {
}
// getHub returns an instance of the Hub for passed-in object's group/kind.
func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
func (wh *webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
gvks, err := objectGVKs(wh.scheme, obj)
if err != nil {
return nil, err
@@ -207,7 +199,7 @@ func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) {
}
// allocateDstObject returns an instance for a given GVK.
func (wh *Webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) {
func (wh *webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) {
gvk := schema.FromAPIVersionAndKind(apiVersion, kind)
obj, err := wh.scheme.New(gvk)

View File

@@ -30,8 +30,11 @@ type Decoder struct {
}
// NewDecoder creates a Decoder given the runtime.Scheme
func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) {
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil
func NewDecoder(scheme *runtime.Scheme) *Decoder {
if scheme == nil {
panic("scheme should never be nil")
}
return &Decoder{codecs: serializer.NewCodecFactory(scheme)}
}
// Decode decodes the inlined object.

View File

@@ -29,12 +29,9 @@ import (
"sync"
"time"
"k8s.io/apimachinery/pkg/runtime"
kscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/certwatcher"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/internal/httpserver"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics"
)
@@ -49,7 +46,29 @@ var DefaultPort = 9443
// at the default locations (tls.crt and tls.key). If you do not
// want to configure TLS (i.e for testing purposes) run an
// admission.StandaloneWebhook in your own server.
type Server struct {
type Server interface {
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the webhook server doesn't need leader election.
NeedLeaderElection() bool
// Register marks the given webhook as being served at the given path.
// It panics if two hooks are registered on the same path.
Register(path string, hook http.Handler)
// Start runs the server.
// It will install the webhook related resources depend on the server configuration.
Start(ctx context.Context) error
// StartedChecker returns an healthz.Checker which is healthy after the
// server has been started.
StartedChecker() healthz.Checker
// WebhookMux returns the servers WebhookMux
WebhookMux() *http.ServeMux
}
// Options are all the available options for a webhook.Server
type Options struct {
// Host is the address that the server will listen on.
// Defaults to "" - all addresses.
Host string
@@ -58,38 +77,46 @@ type Server struct {
// It will be defaulted to 9443 if unspecified.
Port int
// CertDir is the directory that contains the server key and certificate. The
// server key and certificate.
// CertDir is the directory that contains the server key and certificate. Defaults to
// <temp-dir>/k8s-webhook-server/serving-certs.
CertDir string
// CertName is the server certificate name. Defaults to tls.crt.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
CertName string
// KeyName is the server key name. Defaults to tls.key.
//
// Note: This option is only used when TLSOpts does not set GetCertificate.
KeyName string
// ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate.
// Defaults to "", which means server does not verify client's certificate.
ClientCAName string
// TLSVersion is the minimum version of TLS supported. Accepts
// "", "1.0", "1.1", "1.2" and "1.3" only ("" is equivalent to "1.0" for backwards compatibility)
// Deprecated: Use TLSOpts instead.
TLSMinVersion string
// TLSOpts is used to allow configuring the TLS config used for the server
// TLSOpts is used to allow configuring the TLS config used for the server.
// This also allows providing a certificate via GetCertificate.
TLSOpts []func(*tls.Config)
// WebhookMux is the multiplexer that handles different webhooks.
WebhookMux *http.ServeMux
}
// webhooks keep track of all registered webhooks for dependency injection,
// and to provide better panic messages on duplicate webhook registration.
// NewServer constructs a new webhook.Server from the provided options.
func NewServer(o Options) Server {
return &DefaultServer{
Options: o,
}
}
// DefaultServer is the default implementation used for Server.
type DefaultServer struct {
Options Options
// webhooks keep track of all registered webhooks
webhooks map[string]http.Handler
// setFields allows injecting dependencies from an external source
setFields inject.Func
// defaultingOnce ensures that the default fields are only ever set once.
defaultingOnce sync.Once
@@ -99,41 +126,49 @@ type Server struct {
// mu protects access to the webhook map & setFields for Start, Register, etc
mu sync.Mutex
webhookMux *http.ServeMux
}
// setDefaults does defaulting for the Server.
func (s *Server) setDefaults() {
func (o *Options) setDefaults() {
if o.WebhookMux == nil {
o.WebhookMux = http.NewServeMux()
}
if o.Port <= 0 {
o.Port = DefaultPort
}
if len(o.CertDir) == 0 {
o.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
}
if len(o.CertName) == 0 {
o.CertName = "tls.crt"
}
if len(o.KeyName) == 0 {
o.KeyName = "tls.key"
}
}
func (s *DefaultServer) setDefaults() {
s.webhooks = map[string]http.Handler{}
if s.WebhookMux == nil {
s.WebhookMux = http.NewServeMux()
}
s.Options.setDefaults()
if s.Port <= 0 {
s.Port = DefaultPort
}
if len(s.CertDir) == 0 {
s.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs")
}
if len(s.CertName) == 0 {
s.CertName = "tls.crt"
}
if len(s.KeyName) == 0 {
s.KeyName = "tls.key"
}
s.webhookMux = s.Options.WebhookMux
}
// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates
// the webhook server doesn't need leader election.
func (*Server) NeedLeaderElection() bool {
func (*DefaultServer) NeedLeaderElection() bool {
return false
}
// Register marks the given webhook as being served at the given path.
// It panics if two hooks are registered on the same path.
func (s *Server) Register(path string, hook http.Handler) {
func (s *DefaultServer) Register(path string, hook http.Handler) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -141,110 +176,51 @@ func (s *Server) Register(path string, hook http.Handler) {
if _, found := s.webhooks[path]; found {
panic(fmt.Errorf("can't register duplicate path: %v", path))
}
// TODO(directxman12): call setfields if we've already started the server
s.webhooks[path] = hook
s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook))
s.webhookMux.Handle(path, metrics.InstrumentedHook(path, hook))
regLog := log.WithValues("path", path)
regLog.Info("Registering webhook")
// we've already been "started", inject dependencies here.
// Otherwise, InjectFunc will do this for us later.
if s.setFields != nil {
if err := s.setFields(hook); err != nil {
// TODO(directxman12): swallowing this error isn't great, but we'd have to
// change the signature to fix that
regLog.Error(err, "unable to inject fields into webhook during registration")
}
baseHookLog := log.WithName("webhooks")
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", path), hook); err != nil {
regLog.Error(err, "unable to logger into webhook during registration")
}
}
}
// StartStandalone runs a webhook server without
// a controller manager.
func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error {
// Use the Kubernetes client-go scheme if none is specified
if scheme == nil {
scheme = kscheme.Scheme
}
if err := s.InjectFunc(func(i interface{}) error {
if _, err := inject.SchemeInto(scheme, i); err != nil {
return err
}
return nil
}); err != nil {
return err
}
return s.Start(ctx)
}
// tlsVersion converts from human-readable TLS version (for example "1.1")
// to the values accepted by tls.Config (for example 0x301).
func tlsVersion(version string) (uint16, error) {
switch version {
// default is previous behaviour
case "":
return tls.VersionTLS10, nil
case "1.0":
return tls.VersionTLS10, nil
case "1.1":
return tls.VersionTLS11, nil
case "1.2":
return tls.VersionTLS12, nil
case "1.3":
return tls.VersionTLS13, nil
default:
return 0, fmt.Errorf("invalid TLSMinVersion %v: expects 1.0, 1.1, 1.2, 1.3 or empty", version)
}
}
// Start runs the server.
// It will install the webhook related resources depend on the server configuration.
func (s *Server) Start(ctx context.Context) error {
func (s *DefaultServer) Start(ctx context.Context) error {
s.defaultingOnce.Do(s.setDefaults)
baseHookLog := log.WithName("webhooks")
baseHookLog.Info("Starting webhook server")
certPath := filepath.Join(s.CertDir, s.CertName)
keyPath := filepath.Join(s.CertDir, s.KeyName)
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
return err
}
go func() {
if err := certWatcher.Start(ctx); err != nil {
log.Error(err, "certificate watcher error")
}
}()
tlsMinVersion, err := tlsVersion(s.TLSMinVersion)
if err != nil {
return err
}
log.Info("Starting webhook server")
cfg := &tls.Config{ //nolint:gosec
NextProtos: []string{"h2"},
GetCertificate: certWatcher.GetCertificate,
MinVersion: tlsMinVersion,
NextProtos: []string{"h2"},
}
// fallback TLS config ready, will now mutate if passer wants full control over it
for _, op := range s.Options.TLSOpts {
op(cfg)
}
// load CA to verify client certificate
if s.ClientCAName != "" {
if cfg.GetCertificate == nil {
certPath := filepath.Join(s.Options.CertDir, s.Options.CertName)
keyPath := filepath.Join(s.Options.CertDir, s.Options.KeyName)
// Create the certificate watcher and
// set the config's GetCertificate on the TLSConfig
certWatcher, err := certwatcher.New(certPath, keyPath)
if err != nil {
return err
}
cfg.GetCertificate = certWatcher.GetCertificate
go func() {
if err := certWatcher.Start(ctx); err != nil {
log.Error(err, "certificate watcher error")
}
}()
}
// Load CA to verify client certificate, if configured.
if s.Options.ClientCAName != "" {
certPool := x509.NewCertPool()
clientCABytes, err := os.ReadFile(filepath.Join(s.CertDir, s.ClientCAName))
clientCABytes, err := os.ReadFile(filepath.Join(s.Options.CertDir, s.Options.ClientCAName))
if err != nil {
return fmt.Errorf("failed to read client CA cert: %w", err)
}
@@ -258,27 +234,23 @@ func (s *Server) Start(ctx context.Context) error {
cfg.ClientAuth = tls.RequireAndVerifyClientCert
}
// fallback TLS config ready, will now mutate if passer wants full control over it
for _, op := range s.TLSOpts {
op(cfg)
}
listener, err := tls.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), cfg)
listener, err := tls.Listen("tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), cfg)
if err != nil {
return err
}
log.Info("Serving webhook server", "host", s.Host, "port", s.Port)
log.Info("Serving webhook server", "host", s.Options.Host, "port", s.Options.Port)
srv := httpserver.New(s.WebhookMux)
srv := httpserver.New(s.webhookMux)
idleConnsClosed := make(chan struct{})
go func() {
<-ctx.Done()
log.Info("shutting down webhook server")
log.Info("Shutting down webhook server with timeout of 1 minute")
// TODO: use a context with reasonable timeout
if err := srv.Shutdown(context.Background()); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
// Error from closing listeners, or context timeout
log.Error(err, "error shutting down the HTTP server")
}
@@ -298,7 +270,7 @@ func (s *Server) Start(ctx context.Context) error {
// StartedChecker returns an healthz.Checker which is healthy after the
// server has been started.
func (s *Server) StartedChecker() healthz.Checker {
func (s *DefaultServer) StartedChecker() healthz.Checker {
config := &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // config is used to connect to our own webhook port.
}
@@ -311,7 +283,7 @@ func (s *Server) StartedChecker() healthz.Checker {
}
d := &net.Dialer{Timeout: 10 * time.Second}
conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), config)
conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), config)
if err != nil {
return fmt.Errorf("webhook server is not reachable: %w", err)
}
@@ -324,23 +296,7 @@ func (s *Server) StartedChecker() healthz.Checker {
}
}
// InjectFunc injects the field setter into the server.
func (s *Server) InjectFunc(f inject.Func) error {
s.setFields = f
// inject fields here that weren't injected in Register because we didn't have setFields yet.
baseHookLog := log.WithName("webhooks")
for hookPath, webhook := range s.webhooks {
if err := s.setFields(webhook); err != nil {
return err
}
// NB(directxman12): we don't propagate this further by wrapping setFields because it's
// unclear if this is how we want to deal with log propagation. In this specific instance,
// we want to be able to pass a logger to webhooks because they don't know their own path.
if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil {
return err
}
}
return nil
// WebhookMux returns the servers WebhookMux
func (s *DefaultServer) WebhookMux() *http.ServeMux {
return s.webhookMux
}

View File

@@ -28,20 +28,15 @@ import (
// for PathElementSet and SetNodeMap, so we could probably share the
// code.
type PathElementValueMap struct {
members sortedPathElementValues
valueMap PathElementMap
}
func MakePathElementValueMap(size int) PathElementValueMap {
return PathElementValueMap{
members: make(sortedPathElementValues, 0, size),
valueMap: MakePathElementMap(size),
}
}
type pathElementValue struct {
PathElement PathElement
Value value.Value
}
type sortedPathElementValues []pathElementValue
// Implement the sort interface; this would permit bulk creation, which would
@@ -53,7 +48,40 @@ func (spev sortedPathElementValues) Less(i, j int) bool {
func (spev sortedPathElementValues) Swap(i, j int) { spev[i], spev[j] = spev[j], spev[i] }
// Insert adds the pathelement and associated value in the map.
// If insert is called twice with the same PathElement, the value is replaced.
func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) {
s.valueMap.Insert(pe, v)
}
// Get retrieves the value associated with the given PathElement from the map.
// (nil, false) is returned if there is no such PathElement.
func (s *PathElementValueMap) Get(pe PathElement) (value.Value, bool) {
v, ok := s.valueMap.Get(pe)
if !ok {
return nil, false
}
return v.(value.Value), true
}
// PathElementValueMap is a map from PathElement to interface{}.
type PathElementMap struct {
members sortedPathElementValues
}
type pathElementValue struct {
PathElement PathElement
Value interface{}
}
func MakePathElementMap(size int) PathElementMap {
return PathElementMap{
members: make(sortedPathElementValues, 0, size),
}
}
// Insert adds the pathelement and associated value in the map.
// If insert is called twice with the same PathElement, the value is replaced.
func (s *PathElementMap) Insert(pe PathElement, v interface{}) {
loc := sort.Search(len(s.members), func(i int) bool {
return !s.members[i].PathElement.Less(pe)
})
@@ -62,6 +90,7 @@ func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) {
return
}
if s.members[loc].PathElement.Equals(pe) {
s.members[loc].Value = v
return
}
s.members = append(s.members, pathElementValue{})
@@ -71,7 +100,7 @@ func (s *PathElementValueMap) Insert(pe PathElement, v value.Value) {
// Get retrieves the value associated with the given PathElement from the map.
// (nil, false) is returned if there is no such PathElement.
func (s *PathElementValueMap) Get(pe PathElement) (value.Value, bool) {
func (s *PathElementMap) Get(pe PathElement) (interface{}, bool) {
loc := sort.Search(len(s.members), func(i int) bool {
return !s.members[i].PathElement.Less(pe)
})

View File

@@ -0,0 +1,121 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package merge
import (
"fmt"
"sort"
"strings"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)
// Conflict is a conflict on a specific field with the current manager of
// that field. It does implement the error interface so that it can be
// used as an error.
type Conflict struct {
Manager string
Path fieldpath.Path
}
// Conflict is an error.
var _ error = Conflict{}
// Error formats the conflict as an error.
func (c Conflict) Error() string {
return fmt.Sprintf("conflict with %q: %v", c.Manager, c.Path)
}
// Equals returns true if c == c2
func (c Conflict) Equals(c2 Conflict) bool {
if c.Manager != c2.Manager {
return false
}
return c.Path.Equals(c2.Path)
}
// Conflicts accumulates multiple conflicts and aggregates them by managers.
type Conflicts []Conflict
var _ error = Conflicts{}
// Error prints the list of conflicts, grouped by sorted managers.
func (conflicts Conflicts) Error() string {
if len(conflicts) == 1 {
return conflicts[0].Error()
}
m := map[string][]fieldpath.Path{}
for _, conflict := range conflicts {
m[conflict.Manager] = append(m[conflict.Manager], conflict.Path)
}
managers := []string{}
for manager := range m {
managers = append(managers, manager)
}
// Print conflicts by sorted managers.
sort.Strings(managers)
messages := []string{}
for _, manager := range managers {
messages = append(messages, fmt.Sprintf("conflicts with %q:", manager))
for _, path := range m[manager] {
messages = append(messages, fmt.Sprintf("- %v", path))
}
}
return strings.Join(messages, "\n")
}
// Equals returns true if the lists of conflicts are the same.
func (c Conflicts) Equals(c2 Conflicts) bool {
if len(c) != len(c2) {
return false
}
for i := range c {
if !c[i].Equals(c2[i]) {
return false
}
}
return true
}
// ToSet aggregates conflicts for all managers into a single Set.
func (c Conflicts) ToSet() *fieldpath.Set {
set := fieldpath.NewSet()
for _, conflict := range []Conflict(c) {
set.Insert(conflict.Path)
}
return set
}
// ConflictsFromManagers creates a list of conflicts given Managers sets.
func ConflictsFromManagers(sets fieldpath.ManagedFields) Conflicts {
conflicts := []Conflict{}
for manager, set := range sets {
set.Set().Iterate(func(p fieldpath.Path) {
conflicts = append(conflicts, Conflict{
Manager: manager,
Path: p.Copy(),
})
})
}
return conflicts
}

View File

@@ -0,0 +1,358 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package merge
import (
"fmt"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
"sigs.k8s.io/structured-merge-diff/v4/typed"
"sigs.k8s.io/structured-merge-diff/v4/value"
)
// Converter is an interface to the conversion logic. The converter
// needs to be able to convert objects from one version to another.
type Converter interface {
Convert(object *typed.TypedValue, version fieldpath.APIVersion) (*typed.TypedValue, error)
IsMissingVersionError(error) bool
}
// UpdateBuilder allows you to create a new Updater by exposing all of
// the options and setting them once.
type UpdaterBuilder struct {
Converter Converter
IgnoredFields map[fieldpath.APIVersion]*fieldpath.Set
// Stop comparing the new object with old object after applying.
// This was initially used to avoid spurious etcd update, but
// since that's vastly inefficient, we've come-up with a better
// way of doing that. Create this flag to stop it.
// Comparing has become more expensive too now that we're not using
// `Compare` but `value.Equals` so this gives an option to avoid it.
ReturnInputOnNoop bool
}
func (u *UpdaterBuilder) BuildUpdater() *Updater {
return &Updater{
Converter: u.Converter,
IgnoredFields: u.IgnoredFields,
returnInputOnNoop: u.ReturnInputOnNoop,
}
}
// Updater is the object used to compute updated FieldSets and also
// merge the object on Apply.
type Updater struct {
// Deprecated: This will eventually become private.
Converter Converter
// Deprecated: This will eventually become private.
IgnoredFields map[fieldpath.APIVersion]*fieldpath.Set
returnInputOnNoop bool
}
func (s *Updater) update(oldObject, newObject *typed.TypedValue, version fieldpath.APIVersion, managers fieldpath.ManagedFields, workflow string, force bool) (fieldpath.ManagedFields, *typed.Comparison, error) {
conflicts := fieldpath.ManagedFields{}
removed := fieldpath.ManagedFields{}
compare, err := oldObject.Compare(newObject)
if err != nil {
return nil, nil, fmt.Errorf("failed to compare objects: %v", err)
}
versions := map[fieldpath.APIVersion]*typed.Comparison{
version: compare.ExcludeFields(s.IgnoredFields[version]),
}
for manager, managerSet := range managers {
if manager == workflow {
continue
}
compare, ok := versions[managerSet.APIVersion()]
if !ok {
var err error
versionedOldObject, err := s.Converter.Convert(oldObject, managerSet.APIVersion())
if err != nil {
if s.Converter.IsMissingVersionError(err) {
delete(managers, manager)
continue
}
return nil, nil, fmt.Errorf("failed to convert old object: %v", err)
}
versionedNewObject, err := s.Converter.Convert(newObject, managerSet.APIVersion())
if err != nil {
if s.Converter.IsMissingVersionError(err) {
delete(managers, manager)
continue
}
return nil, nil, fmt.Errorf("failed to convert new object: %v", err)
}
compare, err = versionedOldObject.Compare(versionedNewObject)
if err != nil {
return nil, nil, fmt.Errorf("failed to compare objects: %v", err)
}
versions[managerSet.APIVersion()] = compare.ExcludeFields(s.IgnoredFields[managerSet.APIVersion()])
}
conflictSet := managerSet.Set().Intersection(compare.Modified.Union(compare.Added))
if !conflictSet.Empty() {
conflicts[manager] = fieldpath.NewVersionedSet(conflictSet, managerSet.APIVersion(), false)
}
if !compare.Removed.Empty() {
removed[manager] = fieldpath.NewVersionedSet(compare.Removed, managerSet.APIVersion(), false)
}
}
if !force && len(conflicts) != 0 {
return nil, nil, ConflictsFromManagers(conflicts)
}
for manager, conflictSet := range conflicts {
managers[manager] = fieldpath.NewVersionedSet(managers[manager].Set().Difference(conflictSet.Set()), managers[manager].APIVersion(), managers[manager].Applied())
}
for manager, removedSet := range removed {
managers[manager] = fieldpath.NewVersionedSet(managers[manager].Set().Difference(removedSet.Set()), managers[manager].APIVersion(), managers[manager].Applied())
}
for manager := range managers {
if managers[manager].Set().Empty() {
delete(managers, manager)
}
}
return managers, compare, nil
}
// Update is the method you should call once you've merged your final
// object on CREATE/UPDATE/PATCH verbs. newObject must be the object
// that you intend to persist (after applying the patch if this is for a
// PATCH call), and liveObject must be the original object (empty if
// this is a CREATE call).
func (s *Updater) Update(liveObject, newObject *typed.TypedValue, version fieldpath.APIVersion, managers fieldpath.ManagedFields, manager string) (*typed.TypedValue, fieldpath.ManagedFields, error) {
var err error
managers, err = s.reconcileManagedFieldsWithSchemaChanges(liveObject, managers)
if err != nil {
return nil, fieldpath.ManagedFields{}, err
}
managers, compare, err := s.update(liveObject, newObject, version, managers, manager, true)
if err != nil {
return nil, fieldpath.ManagedFields{}, err
}
if _, ok := managers[manager]; !ok {
managers[manager] = fieldpath.NewVersionedSet(fieldpath.NewSet(), version, false)
}
ignored := s.IgnoredFields[version]
if ignored == nil {
ignored = fieldpath.NewSet()
}
managers[manager] = fieldpath.NewVersionedSet(
managers[manager].Set().Difference(compare.Removed).Union(compare.Modified).Union(compare.Added).RecursiveDifference(ignored),
version,
false,
)
if managers[manager].Set().Empty() {
delete(managers, manager)
}
return newObject, managers, nil
}
// Apply should be called when Apply is run, given the current object as
// well as the configuration that is applied. This will merge the object
// and return it.
func (s *Updater) Apply(liveObject, configObject *typed.TypedValue, version fieldpath.APIVersion, managers fieldpath.ManagedFields, manager string, force bool) (*typed.TypedValue, fieldpath.ManagedFields, error) {
var err error
managers, err = s.reconcileManagedFieldsWithSchemaChanges(liveObject, managers)
if err != nil {
return nil, fieldpath.ManagedFields{}, err
}
newObject, err := liveObject.Merge(configObject)
if err != nil {
return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to merge config: %v", err)
}
lastSet := managers[manager]
set, err := configObject.ToFieldSet()
if err != nil {
return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to get field set: %v", err)
}
ignored := s.IgnoredFields[version]
if ignored != nil {
set = set.RecursiveDifference(ignored)
// TODO: is this correct. If we don't remove from lastSet pruning might remove the fields?
if lastSet != nil {
lastSet.Set().RecursiveDifference(ignored)
}
}
managers[manager] = fieldpath.NewVersionedSet(set, version, true)
newObject, err = s.prune(newObject, managers, manager, lastSet)
if err != nil {
return nil, fieldpath.ManagedFields{}, fmt.Errorf("failed to prune fields: %v", err)
}
managers, _, err = s.update(liveObject, newObject, version, managers, manager, force)
if err != nil {
return nil, fieldpath.ManagedFields{}, err
}
if !s.returnInputOnNoop && value.EqualsUsing(value.NewFreelistAllocator(), liveObject.AsValue(), newObject.AsValue()) {
newObject = nil
}
return newObject, managers, nil
}
// prune will remove a field, list or map item, iff:
// * applyingManager applied it last time
// * applyingManager didn't apply it this time
// * no other applier claims to manage it
func (s *Updater) prune(merged *typed.TypedValue, managers fieldpath.ManagedFields, applyingManager string, lastSet fieldpath.VersionedSet) (*typed.TypedValue, error) {
if lastSet == nil || lastSet.Set().Empty() {
return merged, nil
}
version := lastSet.APIVersion()
convertedMerged, err := s.Converter.Convert(merged, version)
if err != nil {
if s.Converter.IsMissingVersionError(err) {
return merged, nil
}
return nil, fmt.Errorf("failed to convert merged object to last applied version: %v", err)
}
sc, tr := convertedMerged.Schema(), convertedMerged.TypeRef()
pruned := convertedMerged.RemoveItems(lastSet.Set().EnsureNamedFieldsAreMembers(sc, tr))
pruned, err = s.addBackOwnedItems(convertedMerged, pruned, version, managers, applyingManager)
if err != nil {
return nil, fmt.Errorf("failed add back owned items: %v", err)
}
pruned, err = s.addBackDanglingItems(convertedMerged, pruned, lastSet)
if err != nil {
return nil, fmt.Errorf("failed add back dangling items: %v", err)
}
return s.Converter.Convert(pruned, managers[applyingManager].APIVersion())
}
// addBackOwnedItems adds back any fields, list and map items that were removed by prune,
// but other appliers or updaters (or the current applier's new config) claim to own.
func (s *Updater) addBackOwnedItems(merged, pruned *typed.TypedValue, prunedVersion fieldpath.APIVersion, managedFields fieldpath.ManagedFields, applyingManager string) (*typed.TypedValue, error) {
var err error
managedAtVersion := map[fieldpath.APIVersion]*fieldpath.Set{}
for _, managerSet := range managedFields {
if _, ok := managedAtVersion[managerSet.APIVersion()]; !ok {
managedAtVersion[managerSet.APIVersion()] = fieldpath.NewSet()
}
managedAtVersion[managerSet.APIVersion()] = managedAtVersion[managerSet.APIVersion()].Union(managerSet.Set())
}
// Add back owned items at pruned version first to avoid conversion failure
// caused by pruned fields which are required for conversion.
if managed, ok := managedAtVersion[prunedVersion]; ok {
merged, pruned, err = s.addBackOwnedItemsForVersion(merged, pruned, prunedVersion, managed)
if err != nil {
return nil, err
}
delete(managedAtVersion, prunedVersion)
}
for version, managed := range managedAtVersion {
merged, pruned, err = s.addBackOwnedItemsForVersion(merged, pruned, version, managed)
if err != nil {
return nil, err
}
}
return pruned, nil
}
// addBackOwnedItemsForVersion adds back any fields, list and map items that were removed by prune with specific managed field path at a version.
// It is an extracted sub-function from addBackOwnedItems for code reuse.
func (s *Updater) addBackOwnedItemsForVersion(merged, pruned *typed.TypedValue, version fieldpath.APIVersion, managed *fieldpath.Set) (*typed.TypedValue, *typed.TypedValue, error) {
var err error
merged, err = s.Converter.Convert(merged, version)
if err != nil {
if s.Converter.IsMissingVersionError(err) {
return merged, pruned, nil
}
return nil, nil, fmt.Errorf("failed to convert merged object at version %v: %v", version, err)
}
pruned, err = s.Converter.Convert(pruned, version)
if err != nil {
if s.Converter.IsMissingVersionError(err) {
return merged, pruned, nil
}
return nil, nil, fmt.Errorf("failed to convert pruned object at version %v: %v", version, err)
}
mergedSet, err := merged.ToFieldSet()
if err != nil {
return nil, nil, fmt.Errorf("failed to create field set from merged object at version %v: %v", version, err)
}
prunedSet, err := pruned.ToFieldSet()
if err != nil {
return nil, nil, fmt.Errorf("failed to create field set from pruned object at version %v: %v", version, err)
}
sc, tr := merged.Schema(), merged.TypeRef()
pruned = merged.RemoveItems(mergedSet.EnsureNamedFieldsAreMembers(sc, tr).Difference(prunedSet.EnsureNamedFieldsAreMembers(sc, tr).Union(managed.EnsureNamedFieldsAreMembers(sc, tr))))
return merged, pruned, nil
}
// addBackDanglingItems makes sure that the fields list and map items removed by prune were
// previously owned by the currently applying manager. This will add back fields list and map items
// that are unowned or that are owned by Updaters and shouldn't be removed.
func (s *Updater) addBackDanglingItems(merged, pruned *typed.TypedValue, lastSet fieldpath.VersionedSet) (*typed.TypedValue, error) {
convertedPruned, err := s.Converter.Convert(pruned, lastSet.APIVersion())
if err != nil {
if s.Converter.IsMissingVersionError(err) {
return merged, nil
}
return nil, fmt.Errorf("failed to convert pruned object to last applied version: %v", err)
}
prunedSet, err := convertedPruned.ToFieldSet()
if err != nil {
return nil, fmt.Errorf("failed to create field set from pruned object in last applied version: %v", err)
}
mergedSet, err := merged.ToFieldSet()
if err != nil {
return nil, fmt.Errorf("failed to create field set from merged object in last applied version: %v", err)
}
sc, tr := merged.Schema(), merged.TypeRef()
prunedSet = prunedSet.EnsureNamedFieldsAreMembers(sc, tr)
mergedSet = mergedSet.EnsureNamedFieldsAreMembers(sc, tr)
last := lastSet.Set().EnsureNamedFieldsAreMembers(sc, tr)
return merged.RemoveItems(mergedSet.Difference(prunedSet).Intersection(last)), nil
}
// reconcileManagedFieldsWithSchemaChanges reconciles the managed fields with any changes to the
// object's schema since the managed fields were written.
//
// Supports:
// - changing types from atomic to granular
// - changing types from granular to atomic
func (s *Updater) reconcileManagedFieldsWithSchemaChanges(liveObject *typed.TypedValue, managers fieldpath.ManagedFields) (fieldpath.ManagedFields, error) {
result := fieldpath.ManagedFields{}
for manager, versionedSet := range managers {
tv, err := s.Converter.Convert(liveObject, versionedSet.APIVersion())
if s.Converter.IsMissingVersionError(err) { // okay to skip, obsolete versions will be deleted automatically anyway
continue
}
if err != nil {
return nil, err
}
reconciled, err := typed.ReconcileFieldSetWithSchema(versionedSet.Set(), tv)
if err != nil {
return nil, err
}
if reconciled != nil {
result[manager] = fieldpath.NewVersionedSet(reconciled, versionedSet.APIVersion(), versionedSet.Applied())
} else {
result[manager] = versionedSet
}
}
return result, nil
}

Some files were not shown because too many files have changed in this diff Show More