Upgrade the operator to use Operator SDK v1.33.0 (#182)

* Move controller package inside internal directory

Based on the go/v4 project structure, the following changed:
- Pakcage `controllers` is now named `controller`
- Package `controller` now lives inside new `internal` directory

* Move main.go in cmd directory

Based on the new go/v4 project structure, `main.go` now lives in the `cmd` directory.

* Change package import in main.go

* Update go mod dependencies

Update the dependencies based on the versions obtained by creating a new operator project using `kubebuilder init --domain onepassword.com --plugins=go/v4`.

This is based on the migration steps provided to go from go/v3 to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update vendor

* Adjust code for breaking changes from pkg update

sigs.k8s.io/controller-runtime package had breaking changes from v0.14.5 to v0.16.3. This commit brings the changes needed to achieve the same things using the new functionality avaialble.

* Adjust paths to connect yaml files

Since `main.go` is now in `cmd` directory, the paths to the files for deploying Connect have to be adjusted based on the new location `main.go` is executed from.

* Update files based on new structure and scaffolding

These changes are made based on the new project structure and scaffolding obtained when using the new go/v4 project structure.

These were done based on the migration steps mentioned when migrating to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4).

* Update config files

These updates are made based on the Kustomize v4 syntax.

This is part of the upgrate to go/v4 (https://book.kubebuilder.io/migration/migration_guide_gov3_to_gov4)

* Update dependencies and GO version

* Update vendor

* Update Kubernetes tools versions

* Update operator version in Makefile

Now the version in the Makefile matches the version of the operator

* Update Operator SDK version in version.go

* Adjust generated deepcopy

It seems that the +build tag is no longer needed based on the latest generated scaffolding, therefore it's removed.

* Update copyright year

* Bring back missing changes from migration

Some customization in Makefile was lost during the migration process. Specifically, the namespace customization for `make deploy` command.

Also, we push changes to kustomization.yaml for making the deploy process smoother.

* Add RBAC perms for coordination.k8s.io

It seems that with the latest changes to Kubernetes and Kustomize, we need to add additional RBAC to the service account used so that it can properly access the `leases` resource.

* Optimize Dockerfile

Dockerfile had a step for caching dependencies (go mod download). However, this is already done by the vendor directory, which we include. Therefore, this step can be removed to make the image build time faster.

vendor/github.com/google/gnostic-models/compiler/README.md

@@ -0,0 +1,4 @@
+# Compiler support code
+
+This directory contains compiler support code used by Gnostic and Gnostic
+extensions.

vendor/github.com/google/gnostic-models/compiler/context.go

@@ -0,0 +1,49 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	yaml "gopkg.in/yaml.v3"
+)
+
+// Context contains state of the compiler as it traverses a document.
+type Context struct {
+	Parent            *Context
+	Name              string
+	Node              *yaml.Node
+	ExtensionHandlers *[]ExtensionHandler
+}
+
+// NewContextWithExtensions returns a new object representing the compiler state
+func NewContextWithExtensions(name string, node *yaml.Node, parent *Context, extensionHandlers *[]ExtensionHandler) *Context {
+	return &Context{Name: name, Node: node, Parent: parent, ExtensionHandlers: extensionHandlers}
+}
+
+// NewContext returns a new object representing the compiler state
+func NewContext(name string, node *yaml.Node, parent *Context) *Context {
+	if parent != nil {
+		return &Context{Name: name, Node: node, Parent: parent, ExtensionHandlers: parent.ExtensionHandlers}
+	}
+	return &Context{Name: name, Parent: parent, ExtensionHandlers: nil}
+}
+
+// Description returns a text description of the compiler state
+func (context *Context) Description() string {
+	name := context.Name
+	if context.Parent != nil {
+		name = context.Parent.Description() + "." + name
+	}
+	return name
+}

vendor/github.com/google/gnostic-models/compiler/error.go

@@ -0,0 +1,70 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import "fmt"
+
+// Error represents compiler errors and their location in the document.
+type Error struct {
+	Context *Context
+	Message string
+}
+
+// NewError creates an Error.
+func NewError(context *Context, message string) *Error {
+	return &Error{Context: context, Message: message}
+}
+
+func (err *Error) locationDescription() string {
+	if err.Context.Node != nil {
+		return fmt.Sprintf("[%d,%d] %s", err.Context.Node.Line, err.Context.Node.Column, err.Context.Description())
+	}
+	return err.Context.Description()
+}
+
+// Error returns the string value of an Error.
+func (err *Error) Error() string {
+	if err.Context == nil {
+		return err.Message
+	}
+	return err.locationDescription() + " " + err.Message
+}
+
+// ErrorGroup is a container for groups of Error values.
+type ErrorGroup struct {
+	Errors []error
+}
+
+// NewErrorGroupOrNil returns a new ErrorGroup for a slice of errors or nil if the slice is empty.
+func NewErrorGroupOrNil(errors []error) error {
+	if len(errors) == 0 {
+		return nil
+	} else if len(errors) == 1 {
+		return errors[0]
+	} else {
+		return &ErrorGroup{Errors: errors}
+	}
+}
+
+func (group *ErrorGroup) Error() string {
+	result := ""
+	for i, err := range group.Errors {
+		if i > 0 {
+			result += "\n"
+		}
+		result += err.Error()
+	}
+	return result
+}

vendor/github.com/google/gnostic-models/compiler/extensions.go

@@ -0,0 +1,86 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes/any"
+	yaml "gopkg.in/yaml.v3"
+
+	extensions "github.com/google/gnostic-models/extensions"
+)
+
+// ExtensionHandler describes a binary that is called by the compiler to handle specification extensions.
+type ExtensionHandler struct {
+	Name string
+}
+
+// CallExtension calls a binary extension handler.
+func CallExtension(context *Context, in *yaml.Node, extensionName string) (handled bool, response *any.Any, err error) {
+	if context == nil || context.ExtensionHandlers == nil {
+		return false, nil, nil
+	}
+	handled = false
+	for _, handler := range *(context.ExtensionHandlers) {
+		response, err = handler.handle(in, extensionName)
+		if response == nil {
+			continue
+		} else {
+			handled = true
+			break
+		}
+	}
+	return handled, response, err
+}
+
+func (extensionHandlers *ExtensionHandler) handle(in *yaml.Node, extensionName string) (*any.Any, error) {
+	if extensionHandlers.Name != "" {
+		yamlData, _ := yaml.Marshal(in)
+		request := &extensions.ExtensionHandlerRequest{
+			CompilerVersion: &extensions.Version{
+				Major: 0,
+				Minor: 1,
+				Patch: 0,
+			},
+			Wrapper: &extensions.Wrapper{
+				Version:       "unknown", // TODO: set this to the type/version of spec being parsed.
+				Yaml:          string(yamlData),
+				ExtensionName: extensionName,
+			},
+		}
+		requestBytes, _ := proto.Marshal(request)
+		cmd := exec.Command(extensionHandlers.Name)
+		cmd.Stdin = bytes.NewReader(requestBytes)
+		output, err := cmd.Output()
+		if err != nil {
+			return nil, err
+		}
+		response := &extensions.ExtensionHandlerResponse{}
+		err = proto.Unmarshal(output, response)
+		if err != nil || !response.Handled {
+			return nil, err
+		}
+		if len(response.Errors) != 0 {
+			return nil, fmt.Errorf("Errors when parsing: %+v for field %s by vendor extension handler %s. Details %+v", in, extensionName, extensionHandlers.Name, strings.Join(response.Errors, ","))
+		}
+		return response.Value, nil
+	}
+	return nil, nil
+}

vendor/github.com/google/gnostic-models/compiler/helpers.go

@@ -0,0 +1,397 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/google/gnostic-models/jsonschema"
+)
+
+// compiler helper functions, usually called from generated code
+
+// UnpackMap gets a *yaml.Node if possible.
+func UnpackMap(in *yaml.Node) (*yaml.Node, bool) {
+	if in == nil {
+		return nil, false
+	}
+	return in, true
+}
+
+// SortedKeysForMap returns the sorted keys of a yamlv2.MapSlice.
+func SortedKeysForMap(m *yaml.Node) []string {
+	keys := make([]string, 0)
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			keys = append(keys, m.Content[i].Value)
+		}
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// MapHasKey returns true if a yamlv2.MapSlice contains a specified key.
+func MapHasKey(m *yaml.Node, key string) bool {
+	if m == nil {
+		return false
+	}
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			itemKey := m.Content[i].Value
+			if key == itemKey {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// MapValueForKey gets the value of a map value for a specified key.
+func MapValueForKey(m *yaml.Node, key string) *yaml.Node {
+	if m == nil {
+		return nil
+	}
+	if m.Kind == yaml.MappingNode {
+		for i := 0; i < len(m.Content); i += 2 {
+			itemKey := m.Content[i].Value
+			if key == itemKey {
+				return m.Content[i+1]
+			}
+		}
+	}
+	return nil
+}
+
+// ConvertInterfaceArrayToStringArray converts an array of interfaces to an array of strings, if possible.
+func ConvertInterfaceArrayToStringArray(interfaceArray []interface{}) []string {
+	stringArray := make([]string, 0)
+	for _, item := range interfaceArray {
+		v, ok := item.(string)
+		if ok {
+			stringArray = append(stringArray, v)
+		}
+	}
+	return stringArray
+}
+
+// SequenceNodeForNode returns a node if it is a SequenceNode.
+func SequenceNodeForNode(node *yaml.Node) (*yaml.Node, bool) {
+	if node.Kind != yaml.SequenceNode {
+		return nil, false
+	}
+	return node, true
+}
+
+// BoolForScalarNode returns the bool value of a node.
+func BoolForScalarNode(node *yaml.Node) (bool, bool) {
+	if node == nil {
+		return false, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return BoolForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return false, false
+	}
+	if node.Tag != "!!bool" {
+		return false, false
+	}
+	v, err := strconv.ParseBool(node.Value)
+	if err != nil {
+		return false, false
+	}
+	return v, true
+}
+
+// IntForScalarNode returns the integer value of a node.
+func IntForScalarNode(node *yaml.Node) (int64, bool) {
+	if node == nil {
+		return 0, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return IntForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return 0, false
+	}
+	if node.Tag != "!!int" {
+		return 0, false
+	}
+	v, err := strconv.ParseInt(node.Value, 10, 64)
+	if err != nil {
+		return 0, false
+	}
+	return v, true
+}
+
+// FloatForScalarNode returns the float value of a node.
+func FloatForScalarNode(node *yaml.Node) (float64, bool) {
+	if node == nil {
+		return 0.0, false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return FloatForScalarNode(node.Content[0])
+	}
+	if node.Kind != yaml.ScalarNode {
+		return 0.0, false
+	}
+	if (node.Tag != "!!int") && (node.Tag != "!!float") {
+		return 0.0, false
+	}
+	v, err := strconv.ParseFloat(node.Value, 64)
+	if err != nil {
+		return 0.0, false
+	}
+	return v, true
+}
+
+// StringForScalarNode returns the string value of a node.
+func StringForScalarNode(node *yaml.Node) (string, bool) {
+	if node == nil {
+		return "", false
+	}
+	if node.Kind == yaml.DocumentNode {
+		return StringForScalarNode(node.Content[0])
+	}
+	switch node.Kind {
+	case yaml.ScalarNode:
+		switch node.Tag {
+		case "!!int":
+			return node.Value, true
+		case "!!str":
+			return node.Value, true
+		case "!!timestamp":
+			return node.Value, true
+		case "!!null":
+			return "", true
+		default:
+			return "", false
+		}
+	default:
+		return "", false
+	}
+}
+
+// StringArrayForSequenceNode converts a sequence node to an array of strings, if possible.
+func StringArrayForSequenceNode(node *yaml.Node) []string {
+	stringArray := make([]string, 0)
+	for _, item := range node.Content {
+		v, ok := StringForScalarNode(item)
+		if ok {
+			stringArray = append(stringArray, v)
+		}
+	}
+	return stringArray
+}
+
+// MissingKeysInMap identifies which keys from a list of required keys are not in a map.
+func MissingKeysInMap(m *yaml.Node, requiredKeys []string) []string {
+	missingKeys := make([]string, 0)
+	for _, k := range requiredKeys {
+		if !MapHasKey(m, k) {
+			missingKeys = append(missingKeys, k)
+		}
+	}
+	return missingKeys
+}
+
+// InvalidKeysInMap returns keys in a map that don't match a list of allowed keys and patterns.
+func InvalidKeysInMap(m *yaml.Node, allowedKeys []string, allowedPatterns []*regexp.Regexp) []string {
+	invalidKeys := make([]string, 0)
+	if m == nil || m.Kind != yaml.MappingNode {
+		return invalidKeys
+	}
+	for i := 0; i < len(m.Content); i += 2 {
+		key := m.Content[i].Value
+		found := false
+		// does the key match an allowed key?
+		for _, allowedKey := range allowedKeys {
+			if key == allowedKey {
+				found = true
+				break
+			}
+		}
+		if !found {
+			// does the key match an allowed pattern?
+			for _, allowedPattern := range allowedPatterns {
+				if allowedPattern.MatchString(key) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				invalidKeys = append(invalidKeys, key)
+			}
+		}
+	}
+	return invalidKeys
+}
+
+// NewNullNode creates a new Null node.
+func NewNullNode() *yaml.Node {
+	node := &yaml.Node{
+		Kind: yaml.ScalarNode,
+		Tag:  "!!null",
+	}
+	return node
+}
+
+// NewMappingNode creates a new Mapping node.
+func NewMappingNode() *yaml.Node {
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Content: make([]*yaml.Node, 0),
+	}
+}
+
+// NewSequenceNode creates a new Sequence node.
+func NewSequenceNode() *yaml.Node {
+	node := &yaml.Node{
+		Kind:    yaml.SequenceNode,
+		Content: make([]*yaml.Node, 0),
+	}
+	return node
+}
+
+// NewScalarNodeForString creates a new node to hold a string.
+func NewScalarNodeForString(s string) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!str",
+		Value: s,
+	}
+}
+
+// NewSequenceNodeForStringArray creates a new node to hold an array of strings.
+func NewSequenceNodeForStringArray(strings []string) *yaml.Node {
+	node := &yaml.Node{
+		Kind:    yaml.SequenceNode,
+		Content: make([]*yaml.Node, 0),
+	}
+	for _, s := range strings {
+		node.Content = append(node.Content, NewScalarNodeForString(s))
+	}
+	return node
+}
+
+// NewScalarNodeForBool creates a new node to hold a bool.
+func NewScalarNodeForBool(b bool) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!bool",
+		Value: fmt.Sprintf("%t", b),
+	}
+}
+
+// NewScalarNodeForFloat creates a new node to hold a float.
+func NewScalarNodeForFloat(f float64) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!float",
+		Value: fmt.Sprintf("%g", f),
+	}
+}
+
+// NewScalarNodeForInt creates a new node to hold an integer.
+func NewScalarNodeForInt(i int64) *yaml.Node {
+	return &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!int",
+		Value: fmt.Sprintf("%d", i),
+	}
+}
+
+// PluralProperties returns the string "properties" pluralized.
+func PluralProperties(count int) string {
+	if count == 1 {
+		return "property"
+	}
+	return "properties"
+}
+
+// StringArrayContainsValue returns true if a string array contains a specified value.
+func StringArrayContainsValue(array []string, value string) bool {
+	for _, item := range array {
+		if item == value {
+			return true
+		}
+	}
+	return false
+}
+
+// StringArrayContainsValues returns true if a string array contains all of a list of specified values.
+func StringArrayContainsValues(array []string, values []string) bool {
+	for _, value := range values {
+		if !StringArrayContainsValue(array, value) {
+			return false
+		}
+	}
+	return true
+}
+
+// StringValue returns the string value of an item.
+func StringValue(item interface{}) (value string, ok bool) {
+	value, ok = item.(string)
+	if ok {
+		return value, ok
+	}
+	intValue, ok := item.(int)
+	if ok {
+		return strconv.Itoa(intValue), true
+	}
+	return "", false
+}
+
+// Description returns a human-readable represention of an item.
+func Description(item interface{}) string {
+	value, ok := item.(*yaml.Node)
+	if ok {
+		return jsonschema.Render(value)
+	}
+	return fmt.Sprintf("%+v", item)
+}
+
+// Display returns a description of a node for use in error messages.
+func Display(node *yaml.Node) string {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		switch node.Tag {
+		case "!!str":
+			return fmt.Sprintf("%s (string)", node.Value)
+		}
+	}
+	return fmt.Sprintf("%+v (%T)", node, node)
+}
+
+// Marshal creates a yaml version of a structure in our preferred style
+func Marshal(in *yaml.Node) []byte {
+	clearStyle(in)
+	//bytes, _ := yaml.Marshal(&yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{in}})
+	bytes, _ := yaml.Marshal(in)
+
+	return bytes
+}
+
+func clearStyle(node *yaml.Node) {
+	node.Style = 0
+	for _, c := range node.Content {
+		clearStyle(c)
+	}
+}

vendor/github.com/google/gnostic-models/compiler/main.go

@@ -0,0 +1,16 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package compiler provides support functions to generated compiler code.
+package compiler

vendor/github.com/google/gnostic-models/compiler/reader.go

@@ -0,0 +1,307 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compiler
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+var verboseReader = false
+
+var fileCache map[string][]byte
+var infoCache map[string]*yaml.Node
+
+var fileCacheEnable = true
+var infoCacheEnable = true
+
+// These locks are used to synchronize accesses to the fileCache and infoCache
+// maps (above). They are global state and can throw thread-related errors
+// when modified from separate goroutines. The general strategy is to protect
+// all public functions in this file with mutex Lock() calls. As a result, to
+// avoid deadlock, these public functions should not call other public
+// functions, so some public functions have private equivalents.
+// In the future, we might consider replacing the maps with sync.Map and
+// eliminating these mutexes.
+var fileCacheMutex sync.Mutex
+var infoCacheMutex sync.Mutex
+
+func initializeFileCache() {
+	if fileCache == nil {
+		fileCache = make(map[string][]byte, 0)
+	}
+}
+
+func initializeInfoCache() {
+	if infoCache == nil {
+		infoCache = make(map[string]*yaml.Node, 0)
+	}
+}
+
+// EnableFileCache turns on file caching.
+func EnableFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCacheEnable = true
+}
+
+// EnableInfoCache turns on parsed info caching.
+func EnableInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCacheEnable = true
+}
+
+// DisableFileCache turns off file caching.
+func DisableFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCacheEnable = false
+}
+
+// DisableInfoCache turns off parsed info caching.
+func DisableInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCacheEnable = false
+}
+
+// RemoveFromFileCache removes an entry from the file cache.
+func RemoveFromFileCache(fileurl string) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	if !fileCacheEnable {
+		return
+	}
+	initializeFileCache()
+	delete(fileCache, fileurl)
+}
+
+// RemoveFromInfoCache removes an entry from the info cache.
+func RemoveFromInfoCache(filename string) {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	if !infoCacheEnable {
+		return
+	}
+	initializeInfoCache()
+	delete(infoCache, filename)
+}
+
+// GetInfoCache returns the info cache map.
+func GetInfoCache() map[string]*yaml.Node {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	if infoCache == nil {
+		initializeInfoCache()
+	}
+	return infoCache
+}
+
+// ClearFileCache clears the file cache.
+func ClearFileCache() {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	fileCache = make(map[string][]byte, 0)
+}
+
+// ClearInfoCache clears the info cache.
+func ClearInfoCache() {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	infoCache = make(map[string]*yaml.Node)
+}
+
+// ClearCaches clears all caches.
+func ClearCaches() {
+	ClearFileCache()
+	ClearInfoCache()
+}
+
+// FetchFile gets a specified file from the local filesystem or a remote location.
+func FetchFile(fileurl string) ([]byte, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	return fetchFile(fileurl)
+}
+
+func fetchFile(fileurl string) ([]byte, error) {
+	var bytes []byte
+	initializeFileCache()
+	if fileCacheEnable {
+		bytes, ok := fileCache[fileurl]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit %s", fileurl)
+			}
+			return bytes, nil
+		}
+		if verboseReader {
+			log.Printf("Fetching %s", fileurl)
+		}
+	}
+	response, err := http.Get(fileurl)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 200 {
+		return nil, fmt.Errorf("Error downloading %s: %s", fileurl, response.Status)
+	}
+	bytes, err = ioutil.ReadAll(response.Body)
+	if fileCacheEnable && err == nil {
+		fileCache[fileurl] = bytes
+	}
+	return bytes, err
+}
+
+// ReadBytesForFile reads the bytes of a file.
+func ReadBytesForFile(filename string) ([]byte, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	return readBytesForFile(filename)
+}
+
+func readBytesForFile(filename string) ([]byte, error) {
+	// is the filename a url?
+	fileurl, _ := url.Parse(filename)
+	if fileurl.Scheme != "" {
+		// yes, fetch it
+		bytes, err := fetchFile(filename)
+		if err != nil {
+			return nil, err
+		}
+		return bytes, nil
+	}
+	// no, it's a local filename
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, nil
+}
+
+// ReadInfoFromBytes unmarshals a file as a *yaml.Node.
+func ReadInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	return readInfoFromBytes(filename, bytes)
+}
+
+func readInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) {
+	initializeInfoCache()
+	if infoCacheEnable {
+		cachedInfo, ok := infoCache[filename]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit info for file %s", filename)
+			}
+			return cachedInfo, nil
+		}
+		if verboseReader {
+			log.Printf("Reading info for file %s", filename)
+		}
+	}
+	var info yaml.Node
+	err := yaml.Unmarshal(bytes, &info)
+	if err != nil {
+		return nil, err
+	}
+	if infoCacheEnable && len(filename) > 0 {
+		infoCache[filename] = &info
+	}
+	return &info, nil
+}
+
+// ReadInfoForRef reads a file and return the fragment needed to resolve a $ref.
+func ReadInfoForRef(basefile string, ref string) (*yaml.Node, error) {
+	fileCacheMutex.Lock()
+	defer fileCacheMutex.Unlock()
+	infoCacheMutex.Lock()
+	defer infoCacheMutex.Unlock()
+	initializeInfoCache()
+	if infoCacheEnable {
+		info, ok := infoCache[ref]
+		if ok {
+			if verboseReader {
+				log.Printf("Cache hit for ref %s#%s", basefile, ref)
+			}
+			return info, nil
+		}
+		if verboseReader {
+			log.Printf("Reading info for ref %s#%s", basefile, ref)
+		}
+	}
+	basedir, _ := filepath.Split(basefile)
+	parts := strings.Split(ref, "#")
+	var filename string
+	if parts[0] != "" {
+		filename = parts[0]
+		if _, err := url.ParseRequestURI(parts[0]); err != nil {
+			// It is not an URL, so the file is local
+			filename = basedir + parts[0]
+		}
+	} else {
+		filename = basefile
+	}
+	bytes, err := readBytesForFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	info, err := readInfoFromBytes(filename, bytes)
+	if info != nil && info.Kind == yaml.DocumentNode {
+		info = info.Content[0]
+	}
+	if err != nil {
+		log.Printf("File error: %v\n", err)
+	} else {
+		if info == nil {
+			return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
+		}
+		if len(parts) > 1 {
+			path := strings.Split(parts[1], "/")
+			for i, key := range path {
+				if i > 0 {
+					m := info
+					if true {
+						found := false
+						for i := 0; i < len(m.Content); i += 2 {
+							if m.Content[i].Value == key {
+								info = m.Content[i+1]
+								found = true
+							}
+						}
+						if !found {
+							infoCache[ref] = nil
+							return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
+						}
+					}
+				}
+			}
+		}
+	}
+	if infoCacheEnable {
+		infoCache[ref] = info
+	}
+	return info, nil
+}