From 3798d08f49098688ad528d069bab8af1d6f5f8ca Mon Sep 17 00:00:00 2001 From: jillianwilson Date: Tue, 9 Nov 2021 16:23:42 -0400 Subject: [PATCH] addressing pr comments --- secret-injector/Dockerfile | 2 +- secret-injector/README.md | 42 +++--- secret-injector/app_example.yaml | 52 ------- secret-injector/cmd/main.go | 12 +- secret-injector/deploy/mutatingwebhook.yaml | 7 +- secret-injector/deploy/service.yaml | 2 +- .../deploy/webhook-create-signed-cert.sh | 131 ------------------ .../deploy/webhook-patch-ca-bundle.sh | 19 --- secret-injector/pkg/webhook/webhook.go | 85 +++++------- secret-injector/test/Dockerfile | 9 -- 10 files changed, 66 insertions(+), 295 deletions(-) delete mode 100644 secret-injector/app_example.yaml delete mode 100755 secret-injector/deploy/webhook-create-signed-cert.sh delete mode 100755 secret-injector/deploy/webhook-patch-ca-bundle.sh delete mode 100644 secret-injector/test/Dockerfile diff --git a/secret-injector/Dockerfile b/secret-injector/Dockerfile index aac7e97..f9d3ee7 100644 --- a/secret-injector/Dockerfile +++ b/secret-injector/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.13 as builder +FROM golang:1.17 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/secret-injector/README.md b/secret-injector/README.md index dd10bec..9a1e993 100644 --- a/secret-injector/README.md +++ b/secret-injector/README.md @@ -5,37 +5,31 @@ The 1Password Secrets Injector implements a mutating webhook to inject 1Password The 1Password Secrets Injector for Kubernetes can be used in conjuction with the 1Password Kubernetes Operator in order to provide automatic deployment restarts when a 1Password item being used by your deployment has been updated. -[Click here for more details on the 1Password Kubernetes Operator](operator/README.md) +[Click here for more details on the 1Password Kubernetes Operator](../operator/README.md) ## Setup and Deployment -The 1Password Secrets Injector for Kubernetes uses a webhook server in order to inject secrets into pods and deployments. Admission to the webhook server is needs to be s secure operation, thus communication with the webhook server requires a TLS certificate signed by a Kubernetes CA. +The 1Password Secrets Injector for Kubernetes uses a webhook server in order to inject secrets into pods and deployments. Admission to the webhook server must be a secure operation, thus communication with the webhook server requires a TLS certificate signed by a Kubernetes CA. -For a simple setup we suggest using s script by morvencao for [creating a signed cert for the webook](https://github.com/morvencao/kube-mutating-webhook-tutorial/blob/master/deploy/webhook-create-signed-cert.sh). A copy of this script can also be found in this repo [here](secret-injector/deploy/webhook-create-signed-cert.sh). +For managing TLS certifcates for your cluster please see the [official documentation](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/). The certificate and key generated in the offical documentation must be set in the [deployment](deploy/deployment.yaml) arguments (`tlsCertFile` and `tlsKeyFile` respectively) for the Secret injector. -Run the script with the following: -``` -./deploy/webhook-create-signed-cert.sh \ - --service \ - --secret \ - --namespace -``` -This will genrate a Kubernetes Secret with your signed certificate. +In additon to setting the tlsCert and tlsKey for the Secret Injector service, we must also create a webhook configuration for the service. An example of the confiugration can be found [here](deploy/mutatingwebhook.yaml). In the provided example you may notice that the caBundle is not set. Please replace this value with your caBundle. This can be generated with the Kubernetes apiserver's default caBundle with the following command -Next we must set the webhook configuration. An example of this configuration can be found [here](secret-injector/deploy/mutatingwebhook.sh). If you choose to use this example, replace `${CA_BUNDLE}` file's with the value stored for `client-ca-file` in the Kubernetes Secret you generated in the previous step. +```export CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n')``` ``` apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: - name: op-secret-injector-webhook-cfg + name: op-secret-injector-webhook-config labels: app: op-secret-injector webhooks: -- name: op-secret-injector.morven.me +- name: op-secret-injector.1password + failurePolicy: Fail clientConfig: service: - name: op-secret-injector-webhook-svc + name: op-secret-injector-webhook-service namespace: op-secret-injector path: "/inject" caBundle: ${CA_BUNDLE} //replace this with your own CA Bundle @@ -75,6 +69,8 @@ kubectl label namespace op-secret-injection=enabled To inject a 1Password secret as an environment variable, your pod or deployment you must add an environment variable to the resource with a value referencing your 1Password item in the format `op:///[/section]/`. You must also annotate your pod/deployment spec with `operator.1password.io/inject` which expects a comma separated list of the names of the containers to that will be mutated and have secrets injected. +Note: You must also include the command needed to run the container as the secret injector prepends a script to this command in order to allow for secret injection. + ``` #example @@ -96,6 +92,7 @@ spec: containers: - name: app-example image: my-image + command: ["./example"] env: - name: DB_USERNAME value: op://my-vault/my-item/sql/username @@ -116,16 +113,11 @@ spec: - name: DB_PASSWORD value: op://my-vault/my-item/sql/password ``` - -## Attributions - -This project is based on and heavily inspired by [morvencao's Kubernetes Mutating Webhook for Sidecar Injection tutorial](https://github.com/morvencao/kube-mutating-webhook-tutorial). - ## Troubleshooting -Sometimes you may find that pod is injected with sidecar container as expected, check the following items: +If you are trouble getting secrets injected in your pod, check the following: -1. The sidecar-injector webhook is in running state and no error logs. -2. The namespace in which application pod is deployed has the correct labels as configured in `mutatingwebhookconfiguration`. -3. Check the `caBundle` is patched to `mutatingwebhookconfiguration` object by checking if `caBundle` fields is empty. -4. Check if the application pod has annotation `sidecar-injector-webhook.morven.me/inject":"yes"`. +1. Check that that the namespace of your pod has the `op-secret-injection=enabled` label +2. Check that the `caBundle` in `mutatingwebhookconfiguration.yaml` is set with a correct value +3. Ensure that the 1Password Secret Injector webhook is running (`op-secret-injector` by default). +4. Check that your container has a `command` field specifying the command to run the app in your container diff --git a/secret-injector/app_example.yaml b/secret-injector/app_example.yaml deleted file mode 100644 index e66ece0..0000000 --- a/secret-injector/app_example.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: app-example -spec: - type: NodePort - selector: - app: app-example - ports: - - port: 5000 - name: app-example ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-example -spec: - selector: - matchLabels: - app: app-example - template: - metadata: - annotations: - operator.1password.io/inject: "app-example" - labels: - app: app-example - spec: - containers: - - name: app-example - command: ["./example"] - image: connect-app-example:latest - imagePullPolicy: Never - resources: - limits: - memory: "128Mi" - cpu: "0.2" - ports: - - containerPort: 5000 - env: - - name: OP_VAULT - value: ApplicationConfiguration - - name: APP_TITLE - value: op://ApplicationConfiguration/Webapp/title - - name: BUTTON_TEXT - value: op://ApplicationConfiguration/Webapp/action - - name: OP_CONNECT_HOST - value: http://onepassword-connect:8080/ - - name: OP_CONNECT_TOKEN - valueFrom: - secretKeyRef: - name: onepassword-token - key: token diff --git a/secret-injector/cmd/main.go b/secret-injector/cmd/main.go index 288af3b..acb01b2 100644 --- a/secret-injector/cmd/main.go +++ b/secret-injector/cmd/main.go @@ -21,7 +21,7 @@ const ( ) func main() { - var parameters webhook.WebhookServerParameters + var parameters webhook.SecretInjectorParameters glog.Info("Starting webhook") // get command line parameters @@ -56,7 +56,7 @@ func main() { ConnectTokenName: connectTokenName, ConnectTokenKey: connectTokenKey, } - webhookServer := &webhook.WebhookServer{ + secretInjector := &webhook.SecretInjector{ Config: webhookConfig, Server: &http.Server{ Addr: fmt.Sprintf(":%v", parameters.Port), @@ -66,12 +66,12 @@ func main() { // define http server and server handler mux := http.NewServeMux() - mux.HandleFunc("/inject", webhookServer.Serve) - webhookServer.Server.Handler = mux + mux.HandleFunc("/inject", secretInjector.Serve) + secretInjector.Server.Handler = mux // start webhook server in new rountine go func() { - if err := webhookServer.Server.ListenAndServeTLS("", ""); err != nil { + if err := secretInjector.Server.ListenAndServeTLS("", ""); err != nil { glog.Errorf("Failed to listen and serve webhook server: %v", err) os.Exit(1) } @@ -83,5 +83,5 @@ func main() { <-signalChan glog.Infof("Got OS shutdown signal, shutting down webhook server gracefully...") - webhookServer.Server.Shutdown(context.Background()) + secretInjector.Server.Shutdown(context.Background()) } diff --git a/secret-injector/deploy/mutatingwebhook.yaml b/secret-injector/deploy/mutatingwebhook.yaml index 10defc7..898e7ef 100644 --- a/secret-injector/deploy/mutatingwebhook.yaml +++ b/secret-injector/deploy/mutatingwebhook.yaml @@ -1,14 +1,15 @@ apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: - name: op-secret-injector-webhook-cfg + name: op-secret-injector-webhook-config labels: app: op-secret-injector webhooks: -- name: op-secret-injector.morven.me +- name: op-secret-injector.1password + failurePolicy: Fail clientConfig: service: - name: op-secret-injector-webhook-svc + name: op-secret-injector-webhook-service namespace: op-secret-injector path: "/inject" caBundle: ${CA_BUNDLE} diff --git a/secret-injector/deploy/service.yaml b/secret-injector/deploy/service.yaml index 46ce14b..90b28fd 100644 --- a/secret-injector/deploy/service.yaml +++ b/secret-injector/deploy/service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: op-secret-injector-webhook-svc + name: op-secret-injector-webhook-service namespace: op-secret-injector labels: app: op-secret-injector diff --git a/secret-injector/deploy/webhook-create-signed-cert.sh b/secret-injector/deploy/webhook-create-signed-cert.sh deleted file mode 100755 index d0aadc5..0000000 --- a/secret-injector/deploy/webhook-create-signed-cert.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -set -e - -usage() { - cat <> "${tmpdir}"/csr.conf -[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -extendedKeyUsage = serverAuth -subjectAltName = @alt_names -[alt_names] -DNS.1 = ${service} -DNS.2 = ${service}.${namespace} -DNS.3 = ${service}.${namespace}.svc -EOF - -openssl genrsa -out "${tmpdir}"/server-key.pem 2048 -openssl req -new -key "${tmpdir}"/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out "${tmpdir}"/server.csr -config "${tmpdir}"/csr.conf - -# clean-up any previously created CSR for our service. Ignore errors if not present. -kubectl delete csr ${csrName} 2>/dev/null || true - -# create server cert/key CSR and send to k8s API -cat <&2 - exit 1 -fi -echo "${serverCert}" | openssl base64 -d -A -out "${tmpdir}"/server-cert.pem - - -# create the secret with CA cert and server cert/key -kubectl create secret generic ${secret} \ - --from-file=key.pem="${tmpdir}"/server-key.pem \ - --from-file=cert.pem="${tmpdir}"/server-cert.pem \ - --dry-run -o yaml | - kubectl -n ${namespace} apply -f - diff --git a/secret-injector/deploy/webhook-patch-ca-bundle.sh b/secret-injector/deploy/webhook-patch-ca-bundle.sh deleted file mode 100755 index bc7690c..0000000 --- a/secret-injector/deploy/webhook-patch-ca-bundle.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -CA_BUNDLE=$(kubectl config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}') - -if [ -z "${CA_BUNDLE}" ]; then - CA_BUNDLE=$(kubectl get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='default')].data.ca\.crt}") -fi - -export CA_BUNDLE - -if command -v envsubst >/dev/null 2>&1; then - envsubst -else - sed -e "s|\${CA_BUNDLE}|${CA_BUNDLE}|g" -fi diff --git a/secret-injector/pkg/webhook/webhook.go b/secret-injector/pkg/webhook/webhook.go index 117f1a7..18f01c7 100644 --- a/secret-injector/pkg/webhook/webhook.go +++ b/secret-injector/pkg/webhook/webhook.go @@ -55,33 +55,28 @@ var ( defaulter = runtime.ObjectDefaulter(runtimeScheme) ) -var ignoredNamespaces = []string{ - metav1.NamespaceSystem, - metav1.NamespacePublic, -} - const ( injectionStatus = "operator.1password.io/status" injectAnnotation = "operator.1password.io/inject" versionAnnotation = "operator.1password.io/version" ) -type WebhookServer struct { +type SecretInjector struct { Config Config Server *http.Server } -// Webhook Server parameters -type WebhookServerParameters struct { +// the command line parameters for configuraing the webhook +type SecretInjectorParameters struct { Port int // webhook server port CertFile string // path to the x509 certificate for https KeyFile string // path to the x509 private key matching `CertFile` } type Config struct { - ConnectHost string - ConnectTokenName string - ConnectTokenKey string + ConnectHost string // the host in which a connect server is running + ConnectTokenName string // the token name of the secret that stores the connect token + ConnectTokenKey string // the name of the data field in the secret the stores the connect token } type patchOperation struct { @@ -105,15 +100,8 @@ func applyDefaultsWorkaround(containers []corev1.Container, volumes []corev1.Vol }) } -// Check whether the target resoured need to be mutated -func mutationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { - // skip special kubernete system namespaces - for _, namespace := range ignoredList { - if metadata.Namespace == namespace { - glog.Infof("Skip mutation for %v for it's in special namespace:%v", metadata.Name, metadata.Namespace) - return false - } - } +// Check if the pod should have secrets injected +func mutationRequired(metadata *metav1.ObjectMeta) bool { annotations := metadata.GetAnnotations() if annotations == nil { @@ -123,13 +111,13 @@ func mutationRequired(ignoredList []string, metadata *metav1.ObjectMeta) bool { status := annotations[injectionStatus] _, enabled := annotations[injectAnnotation] - // determine whether to perform mutation based on annotation for the target resource + // if pod has not already been injected and injection has been enabled mark the pod for injection required := false if strings.ToLower(status) != "injected" && enabled { required = true } - glog.Infof("Mutation policy for %v/%v: status: %q required:%v", metadata.Namespace, metadata.Name, status, required) + glog.Infof("Pod %v at namepspace %v. Secret injection status: %v Secret Injection Enabled:%v", metadata.Name, metadata.Namespace, status, required) return required } @@ -197,8 +185,8 @@ func updateAnnotation(target map[string]string, added map[string]string) (patch return patch } -// main mutation process -func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { +// mutation process for injecting secrets into pods +func (s *SecretInjector) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { ctx := context.Background() req := ar.Request var pod corev1.Pod @@ -211,12 +199,12 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi } } - glog.Infof("AdmissionReview for Kind=%v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%v", - req.Kind, req.Namespace, req.Name, pod.Name, req.UID, req.Operation, req.UserInfo) + glog.Infof("Checking if secret injection is needed for %v %s at namespace %v", + req.Kind, pod.Name, req.Namespace) - // determine whether to perform mutation - if !mutationRequired(ignoredNamespaces, &pod.ObjectMeta) { - glog.Infof("Skipping mutation for %s/%s due to policy check", pod.Namespace, pod.Name) + // determine whether to inject secrets + if !mutationRequired(&pod.ObjectMeta) { + glog.Infof("Secret injection not required for %s at namespace %s", pod.Name, pod.Namespace) return &v1beta1.AdmissionResponse{ Allowed: true, } @@ -227,7 +215,7 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi containers := map[string]struct{}{} if containersStr == "" { - glog.Infof("No mutations made for %s/%s", pod.Namespace, pod.Name) + glog.Infof("No containers set for secret injection for %s/%s", pod.Namespace, pod.Name) return &v1beta1.AdmissionResponse{ Allowed: true, } @@ -238,7 +226,7 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi version, ok := pod.Annotations[versionAnnotation] if !ok { - version = "latest" + version = "2.0.0-beta.4" } mutated := false @@ -249,7 +237,7 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi if !mutate { continue } - c, didMutate, initContainerPatch, err := whsvr.mutateContainer(ctx, &c, i) + c, didMutate, initContainerPatch, err := s.mutateContainer(ctx, &c, i) if err != nil { return &v1beta1.AdmissionResponse{ Result: &metav1.Status{ @@ -270,9 +258,9 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi continue } - c, didMutate, containerPatch, err := whsvr.mutateContainer(ctx, &c, i) + c, didMutate, containerPatch, err := s.mutateContainer(ctx, &c, i) if err != nil { - glog.Error("Error occured mutating container: ", err) + glog.Error("Error occured mutating container for secret injection: ", err) return &v1beta1.AdmissionResponse{ Result: &metav1.Status{ Message: err.Error(), @@ -287,7 +275,7 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi } if !mutated { - glog.Infof("No mutations made for %s/%s", pod.Namespace, pod.Name) + glog.Infof("No containers set for secret injection for %s/%s", pod.Namespace, pod.Name) return &v1beta1.AdmissionResponse{ Allowed: true, } @@ -309,8 +297,7 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi }, } - annotations := map[string]string{injectionStatus: "injected"} - patchBytes, err := createOPCLIPatch(&pod, annotations, []corev1.Container{binInitContainer}, patch) + patchBytes, err := createOPCLIPatch(&pod, []corev1.Container{binInitContainer}, patch) if err != nil { return &v1beta1.AdmissionResponse{ Result: &metav1.Status{ @@ -331,8 +318,9 @@ func (whsvr *WebhookServer) mutate(ar *v1beta1.AdmissionReview) *v1beta1.Admissi } // create mutation patch for resoures -func createOPCLIPatch(pod *corev1.Pod, annotations map[string]string, containers []corev1.Container, patch []patchOperation) ([]byte, error) { +func createOPCLIPatch(pod *corev1.Pod, containers []corev1.Container, patch []patchOperation) ([]byte, error) { + annotations := map[string]string{injectionStatus: "injected"} patch = append(patch, addVolume(pod.Spec.Volumes, []corev1.Volume{binVolume}, "/spec/volumes")...) patch = append(patch, addContainers(pod.Spec.InitContainers, containers, "/spec/initContainers")...) patch = append(patch, updateAnnotation(pod.Annotations, annotations)...) @@ -344,9 +332,10 @@ func createOPConnectPatch(container *corev1.Container, containerIndex int, host, var patch []patchOperation envs := []corev1.EnvVar{} + // if connect configuration is already set in the container do not overwrite it hostConfig, tokenConfig := isConnectConfigurationSet(container) - if hostConfig { + if !hostConfig { connectHostEnvVar := corev1.EnvVar{ Name: "OP_CONNECT_HOST", Value: host, @@ -354,7 +343,7 @@ func createOPConnectPatch(container *corev1.Container, containerIndex int, host, envs = append(envs, connectHostEnvVar) } - if tokenConfig { + if !tokenConfig { connectTokenEnvVar := corev1.EnvVar{ Name: "OP_CONNECT_TOKEN", ValueFrom: &corev1.EnvVarSource{ @@ -395,9 +384,9 @@ func isConnectConfigurationSet(container *corev1.Container) (bool, bool) { return hostConfig, tokenConfig } -func (whsvr *WebhookServer) mutateContainer(_ context.Context, container *corev1.Container, containerIndex int) (*corev1.Container, bool, []patchOperation, error) { - // Because we are running a command in the pod before starting the container app, - // we need to prepend the pod comand with the op run command +// mutates the container to allow for secrets to be injected into the container via the op cli +func (s *SecretInjector) mutateContainer(_ context.Context, container *corev1.Container, containerIndex int) (*corev1.Container, bool, []patchOperation, error) { + // prepending op run command to the container command so that secrets are injected before the main process is started if len(container.Command) == 0 { return container, false, nil, fmt.Errorf("not attaching OP to the container %s: the podspec does not define a command", container.Name) } @@ -423,8 +412,8 @@ func (whsvr *WebhookServer) mutateContainer(_ context.Context, container *corev1 Value: container.Command, }) - //creating patch for adding conenct environment variables to container - patch = append(patch, createOPConnectPatch(container, containerIndex, whsvr.Config.ConnectHost, whsvr.Config.ConnectTokenName, whsvr.Config.ConnectTokenKey)...) + //creating patch for adding connect environment variables to container. If they are already set in the container then this will be skipped + patch = append(patch, createOPConnectPatch(container, containerIndex, s.Config.ConnectHost, s.Config.ConnectTokenName, s.Config.ConnectTokenKey)...) return container, true, patch, nil } @@ -449,8 +438,8 @@ func setEnvironment(container corev1.Container, containerIndex int, addedEnv []c return patch } -// Serve method for webhook server -func (whsvr *WebhookServer) Serve(w http.ResponseWriter, r *http.Request) { +// Serve method for secrets injector webhook +func (s *SecretInjector) Serve(w http.ResponseWriter, r *http.Request) { var body []byte if r.Body != nil { if data, err := ioutil.ReadAll(r.Body); err == nil { @@ -481,7 +470,7 @@ func (whsvr *WebhookServer) Serve(w http.ResponseWriter, r *http.Request) { }, } } else { - admissionResponse = whsvr.mutate(&ar) + admissionResponse = s.mutate(&ar) } admissionReview := v1beta1.AdmissionReview{} diff --git a/secret-injector/test/Dockerfile b/secret-injector/test/Dockerfile deleted file mode 100644 index 98e812b..0000000 --- a/secret-injector/test/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ubuntu:latest -ARG VERSION - -RUN apt-get update && apt-get install -y curl unzip jq && \ - curl -o 1password.zip https://bucket.agilebits.com/cli-private-beta/v2/op_linux_amd64_v2-alpha2.zip && \ - unzip 1password.zip -d /usr/local/bin && \ - rm 1password.zip - -CMD ["op"] \ No newline at end of file