Skip to content

cluster: migrate registry from host container to in-cluster deployment #3856

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lkingland wants to merge 1 commit into
base: main
Choose a base branch
from
+102 −187
Open

cluster: migrate registry from host container to in-cluster deployment #3856

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 18 additions & 26 deletions pkg/cluster/delete.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,29 @@ import (
"path/filepath"
)

// Delete removes a single func-managed dev cluster. The shared registry
// container and the host's insecure-registries entry are removed only when
// the *last* func-managed cluster is being torn down — other surviving
// clusters keep using the shared registry.
// Delete removes a func-managed dev cluster. The in-cluster registry is
// destroyed automatically with the Kind cluster. Host-side trust config
// (insecure-registries) is only reverted when this is the last func cluster,
// since other surviving clusters share the same host entry.
func Delete(ctx context.Context, cfg ClusterConfig, out io.Writer) error {
// Set KUBECONFIG for child processes; restore the caller's value on return.
defer setKubeconfig(cfg.Kubeconfig())()

status(out, "Deleting Cluster")

if err := run(ctx, out, "",
cfg.kind(), "delete", "cluster",
"--name="+cfg.Name,
"--kubeconfig="+cfg.Kubeconfig()); err != nil {
warnf(out, "failed to delete cluster %q: %v", cfg.Name, err)
if _, err := os.Stat(cfg.Kubeconfig()); err == nil {
status(out, "Deleting Cluster")
if err := run(ctx, out, "",
cfg.kind(), "delete", "cluster",
"--name="+cfg.Name,
"--kubeconfig="+cfg.Kubeconfig()); err != nil {
warnf(out, "failed to delete cluster %q: %v", cfg.Name, err)
}
_ = os.RemoveAll(filepath.Dir(cfg.Kubeconfig()))
}

// Remove this cluster's kubeconfig dir so the "last cluster?" check
// below reflects the post-delete state.
_ = os.RemoveAll(filepath.Dir(cfg.Kubeconfig()))

remaining := List()
if len(remaining) == 0 {
status(out, "Last func cluster removed; tearing down shared registry")
teardownRegistry(ctx, cfg, out)
if !cfg.SkipRegistryConfig {
revertHostRegistry(out)
}
} else {
fmt.Fprintf(out, "Registry left running; shared with %d other func-managed cluster(s): %v\n",
len(remaining), remaining)
if remaining := List(); len(remaining) > 0 {
fmt.Fprintf(out, "Other func-managed cluster(s) still running: %v; leaving host registry config in place.\n",
remaining)
} else if !cfg.SkipRegistryConfig {
revertHostRegistry(out)
}

fmt.Fprintf(out, "%s Downloaded container images are not automatically removed.\n", red("NOTE:"))
Expand Down
17 changes: 8 additions & 9 deletions pkg/cluster/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ const kindConfigTemplate = `kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
image: kindest/node:%[1]s
image: kindest/node:%s
extraPortMappings:
- containerPort: 80
hostPort: 80
Expand All @@ -29,14 +29,14 @@ nodes:
listenAddress: "127.0.0.1"
containerdConfigPatches:
- |-
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%[2]d"]
endpoint = ["http://%[3]s:%[4]d"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:%[4]d"]
endpoint = ["http://%[3]s:%[4]d"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.localtest.me"]
endpoint = ["http://localhost:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
endpoint = ["http://localhost:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."ghcr.io"]
endpoint = ["http://%[3]s:%[4]d"]
endpoint = ["http://localhost:5000"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
endpoint = ["http://%[3]s:%[4]d"]
endpoint = ["http://localhost:5000"]
`

const metalLBPoolTemplate = `apiVersion: metallb.io/v1beta1
Expand All @@ -59,8 +59,7 @@ func installKubernetes(ctx context.Context, cfg ClusterConfig, out io.Writer) er
start := time.Now()
status(out, "Allocating")

kindConfig := fmt.Sprintf(kindConfigTemplate,
kindNodeVersion, registryHostPort, registryContainerName, registryContainerPort)
kindConfig := fmt.Sprintf(kindConfigTemplate, kindNodeVersion)

err := run(ctx, out, kindConfig,
cfg.kind(), "create", "cluster",
Expand Down
228 changes: 76 additions & 152 deletions pkg/cluster/registry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,34 +15,84 @@ import (
"time"
)

const (
// registryContainerName is the fixed name of the shared local registry
// container. All func-managed dev clusters on the host share this
// single registry.
registryContainerName = "func-registry"
// registryHostPort is the TCP port the registry is published on to the
// host; it also appears in the host container engine's
// insecure-registries list.
registryHostPort = 50000
// registryContainerPort is the port the `registry:2` image listens on
// inside the container.
registryContainerPort = 5000
)

// registryAddr is the host-side address used in daemon.json /
// registries.conf and in the in-cluster `local-registry-hosting` ConfigMap.
// Derived from registryHostPort so the two can't drift apart.
var registryAddr = fmt.Sprintf("localhost:%d", registryHostPort)
const registryAddr = "registry.localtest.me"

// installRegistry starts the shared local container registry, configures
// host-side trust for it, and applies the in-cluster ConfigMap + Service
// the kind cluster uses to reach it.
// installRegistry deploys the container registry as in-cluster Kubernetes
// resources (Deployment + ClusterIP Service + Contour Ingress), configures
// host-side trust, and applies the local-registry-hosting ConfigMap.
func installRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) error {

@matejvasek matejvasek May 26, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure when/how installRegistry() is called but it preferably should be called after Contour installation (or in parallel to it). But it's not necessary.

@matejvasek matejvasek May 26, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it's not necessary.

Because we use generic k8s ingress not some specific Contour resource.

start := time.Now()
status(out, "Creating Registry")

if err := ensureRegistry(ctx, cfg, out); err != nil {
return err
registryManifest := `apiVersion: apps/v1

@matejvasek matejvasek May 26, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

possible improvement: make this Go struct instead of string literal.

kind: Deployment
metadata:
name: registry
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: registry
template:
metadata:
labels:
app: registry
spec:
containers:
- name: registry
image: registry:2
ports:
- containerPort: 5000
hostPort: 5000
volumeMounts:
- name: registry-data
mountPath: /var/lib/registry
volumes:
- name: registry-data
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: registry
namespace: default
spec:
selector:
app: registry
ports:
- port: 5000
targetPort: 5000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: registry
namespace: default
spec:
ingressClassName: contour-external
rules:
- host: registry.localtest.me
http:
paths:
- backend:
service:
name: registry
port:
number: 5000
pathType: Prefix
path: /
`

if err := applyManifest(ctx, out, cfg, registryManifest); err != nil {
return fmt.Errorf("applying registry resources: %w", err)
}

if err := run(ctx, out, "",
cfg.kubectl(), "wait",
"--for=condition=Available", "deployment/registry",
"-n", "default", "--timeout=5m"); err != nil {
return fmt.Errorf("waiting for registry deployment: %w", err)
}

if !cfg.SkipRegistryConfig {
Expand All @@ -51,114 +101,25 @@ func installRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) erro
}
}

// ConfigMap for local registry hosting
registryConfigMap := fmt.Sprintf(`apiVersion: v1
registryConfigMap := `apiVersion: v1
kind: ConfigMap
metadata:
name: local-registry-hosting
namespace: kube-public
data:
localRegistryHosting.v1: |
host: "localhost:%d"
host: "registry.localtest.me"
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
`, registryHostPort)
`

if err := applyManifest(ctx, out, cfg, registryConfigMap); err != nil {
return fmt.Errorf("applying registry configmap: %w", err)
}

// ExternalName service for in-cluster access
registrySvc := fmt.Sprintf(`apiVersion: v1
kind: Service
metadata:
name: registry
namespace: default
spec:
type: ExternalName
externalName: %s
`, registryContainerName)

if err := applyManifest(ctx, out, cfg, registrySvc); err != nil {
return fmt.Errorf("applying registry service: %w", err)
}

success(out, "Registry", time.Since(start))

@matejvasek matejvasek May 26, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

potential improvement: we could wait until http://registry.localtest.me/v2/ returns 200.

return nil
}

// ensureRegistry makes sure the shared func-registry container exists, is
// running, and is attached to the `kind` docker network. Idempotent; safe
// to call whether or not another func-managed cluster has already
// provisioned it.
//
// Scope is intentionally just the container lifecycle — host-side trust
// config is the orchestrator's responsibility (see installRegistry).
// TODO: should we rename the kind network to "kind-func" to avoid collision
// with a developer's own kind usage?
func ensureRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) error {
exists, running, networked, err := registryStatus(ctx, cfg)
if err != nil {
return err
}
if !exists {
portMap := fmt.Sprintf("127.0.0.1:%d:%d", registryHostPort, registryContainerPort)
// --net=kind attaches at creation time, so no separate network
// connect is needed on this path.
return run(ctx, out, "",
cfg.ContainerEngine(), "run",
"-d",
"--restart=always",
"-p", portMap,
"--net=kind",
"--name", registryContainerName,
"registry:2")
}
if !running {
if err := run(ctx, out, "", cfg.ContainerEngine(), "start", registryContainerName); err != nil {
return fmt.Errorf("starting registry: %w", err)
}
}
if !networked {
if err := run(ctx, out, "", cfg.ContainerEngine(), "network", "connect", "kind", registryContainerName); err != nil {
return fmt.Errorf("connecting registry to kind network: %w", err)
}
}
return nil
}

// registryStatus inspects the shared registry container. A non-nil err
// means the engine itself errored in a way that isn't "no such object";
// callers should surface it. A (false, false, false, nil) return means
// "container is absent" or the inspect was unparseable — either way,
// treated as fresh state.
func registryStatus(ctx context.Context, cfg ClusterConfig) (exists, running, networked bool, err error) {
output, inspectErr := runOutput(ctx, cfg.ContainerEngine(), "container", "inspect", registryContainerName)
if inspectErr != nil {
// `container inspect <missing>` exits non-zero; so does any real
// engine failure. Treat both as "not present" — a real failure
// resurfaces on the next engine command.
return false, false, false, nil
}
var results []struct {
State struct {
Running bool `json:"Running"`
} `json:"State"`
NetworkSettings struct {
Networks map[string]json.RawMessage `json:"Networks"`
} `json:"NetworkSettings"`
}
if err := json.Unmarshal([]byte(output), &results); err != nil {
return false, false, false, fmt.Errorf("parsing inspect output: %w", err)
}
if len(results) == 0 {
return false, false, false, nil
}
exists = true
running = results[0].State.Running
_, networked = results[0].NetworkSettings.Networks["kind"]
return
}

// configureHostRegistry configures the host's container engine(s) to
// trust the shared local registry. Mirror of revertHostRegistry; called
// at most once per installRegistry (the caller gates on
Expand Down Expand Up @@ -265,7 +226,6 @@ func configurePodmanHTTP(out io.Writer) error {
return fmt.Errorf("writing config: %w", err)
}
fmt.Fprintf(out, "Successfully created Podman registry configuration for %s\n", registryAddr)
setupPodmanMacOSForwarding(out)
return nil
}

Expand Down Expand Up @@ -304,34 +264,9 @@ func configurePodmanHTTP(out io.Writer) error {
return err
}

setupPodmanMacOSForwarding(out)
return nil
}

// setupPodmanMacOSForwarding sets up SSH port forwarding on macOS so the
// Podman VM can access the host's local registry. Idempotent: detects an
// existing backgrounded ssh forwarder and skips rather than spawning
// another (which would leak or fail to bind).
func setupPodmanMacOSForwarding(out io.Writer) {
if runtime.GOOS != "darwin" {
return
}
forward := fmt.Sprintf("-L %d:localhost:%d", registryHostPort, registryHostPort)
if err := exec.Command("pgrep", "-f", forward).Run(); err == nil {
fmt.Fprintln(out, "Podman VM port forwarding already active; skipping")
return
}
fmt.Fprintln(out, "Setting up port forwarding for Podman VM to access registry...")
port := fmt.Sprintf("%d", registryHostPort)
cmd := exec.Command("podman", "machine", "ssh", "--",
"-L", port+":localhost:"+port, "-N", "-f")
cmd.Stdout = out
cmd.Stderr = out
if err := cmd.Run(); err != nil {
fmt.Fprintf(out, "Warning: port forwarding setup failed: %v\n", err)
}
}

// warnNix detects Nix and emits configuration guidance.
func warnNix(out io.Writer) {
if !hasCommand("nix") && !hasCommand("nixos-rebuild") {
Expand Down Expand Up @@ -366,17 +301,6 @@ The configuration required is adding the following to registries.conf:
}
}

// Teardowns
// ---------

// teardownRegistry stops and removes the shared registry container. Called
// from Delete when the last func-managed cluster is being removed.
func teardownRegistry(ctx context.Context, cfg ClusterConfig, out io.Writer) {
if err := run(ctx, out, "", cfg.ContainerEngine(), "rm", "-f", registryContainerName); err != nil {
fmt.Fprintf(out, "Warning: failed to remove registry container %q: %v\n", registryContainerName, err)
}
}

// revertHostRegistry removes the insecure-registries entry we added at
// create time and the matching podman stanza. Best-effort: per-engine
// failures warn but don't abort the delete.
Expand Down
Loading