Network services: load balance across the locations an application runs in

PROJECT

@@ -203,4 +203,12 @@ resources:
   kind: NetworkInterfaceClaim
   path: go.datum.net/network-services-operator/api/v1alpha
   version: v1alpha
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: datumapis.com
+  group: networking
+  kind: NetworkService
+  path: go.datum.net/network-services-operator/api/v1alpha
+  version: v1alpha
 version: "3"

Taskfile.test-infra.yml

@@ -198,6 +198,9 @@ tasks:
       - task: billing-usage-collector
       - task: external-dns
       - task: karmada-up
+      # Same shape as ipam-kubeconfig above: the manager and the cell both mount
+      # this and neither can federate without it, so both wait in
+      # ContainerCreating until it lands here.
       - task: karmada-hub-secret
       - task: location-fixtures
       - task: link-clusters
@@ -411,6 +414,7 @@ tasks:
     desc: "Install the NSO CRDs the replicator mirrors into the downstream cluster (the Gateway-API/EG CRDs come from eg-crds)."
     cmds:
       - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_connectors.yaml
+      - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_edgereachabilities.yaml
       - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml
       - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml
       - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_servinglocations.yaml
@@ -1090,7 +1094,7 @@ tasks:
     vars:
       # Scenarios authored against this env's cluster names and downstream path.
       # Older fixtures targeting the previous cluster names are excluded here.
-      DEFAULT_SCENARIOS: extension-server-smoke waf-enforcement branded-error-page connector-offline-503 atomic-reject-isolation
+      DEFAULT_SCENARIOS: extension-server-smoke waf-enforcement branded-error-page connector-offline-503 atomic-reject-isolation networkservice-endpoints
       # CLI_ARGS (after --) wins; else SCENARIOS env; else the default set.
       SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}'
     deps:

api/v1alpha/edgereachability_types.go

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package v1alpha
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EdgeReachabilityName is the name of the single record a namespace holds.
+// One record answers for the whole namespace, so a reader gets its answer with
+// a get rather than a list it has to decide is complete.
+const EdgeReachabilityName = "default"
+
+// EdgeReachabilitySpec is the set of workload addresses in one project
+// namespace that an edge is expected to reach.
+type EdgeReachabilitySpec struct {
+	// addresses are the workload addresses currently behind a proxy, one entry
+	// per address, with no prefix length. An empty list is a real answer: it
+	// says the project publishes nothing, which is different from no record at
+	// all.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:MaxItems=8192
+	Addresses []string `json:"addresses,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// EdgeReachability records which of a project's workload addresses are behind
+// an HTTPProxy, so the platform carries a workload's location to the edges that
+// serve it and stops carrying it everywhere else.
+//
+// It is written by the control plane onto the federation hub and read by the
+// cells publishing into it. No consumer creates or edits one, and nothing in a
+// project control plane holds one.
+//
+// Absence of the record means the control plane has not answered for this
+// namespace yet, and a reader must keep publishing rather than treat silence as
+// a withdrawal. An empty list is the answer that withdraws.
+// +kubebuilder:printcolumn:name="Addresses",type=integer,JSONPath=".spec.addresses.length()"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+type EdgeReachability struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	Spec EdgeReachabilitySpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// EdgeReachabilityList contains a list of EdgeReachability.
+type EdgeReachabilityList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []EdgeReachability `json:"items"`
+}

api/v1alpha/groupversion_info.go

@@ -26,6 +26,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(GroupVersion,
 		&Domain{},
 		&DomainList{},
+		&EdgeReachability{},
+		&EdgeReachabilityList{},
 		&HTTPProxy{},
 		&HTTPProxyList{},
 		&Location{},
@@ -44,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&NetworkInterfaceClaimList{},
 		&NetworkPolicy{},
 		&NetworkPolicyList{},
+		&NetworkService{},
+		&NetworkServiceList{},
 		&ServingLocation{},
 		&ServingLocationList{},
 		&Subnet{},

api/v1alpha/httpproxy_types.go

@@ -112,7 +112,8 @@ type HTTPProxyRule struct {
 	Backends []HTTPProxyRuleBackend `json:"backends,omitempty"`
 }
 
-// +kubebuilder:validation:XValidation:message="endpoint is required unless instance is set, and instance is mutually exclusive with endpoint and connector",rule="has(self.instance) ? (!has(self.endpoint) && !has(self.connector)) : has(self.endpoint)"
+// +kubebuilder:validation:XValidation:message="endpoint is required unless instance or networkService is set; instance and networkService are mutually exclusive with each other and with endpoint and connector",rule="has(self.instance) ? (!has(self.endpoint) && !has(self.connector) && !has(self.networkService)) : (has(self.networkService) ? (!has(self.endpoint) && !has(self.connector)) : has(self.endpoint))"
+// +kubebuilder:validation:XValidation:message="backend TLS is not supported for networkService backends",rule="has(self.networkService) ? !has(self.tls) : true"
 type HTTPProxyRuleBackend struct {
 	// Endpoint for the backend. Must be a valid URL.
 	//
@@ -147,11 +148,24 @@ type HTTPProxyRuleBackend struct {
 	// +kubebuilder:validation:Optional
 	Instance *InstanceBackendRef `json:"instance,omitempty"`
 
+	// NetworkService references a NetworkService in the same namespace, and one
+	// of the ports it declares. Every member the service resolves to becomes an
+	// endpoint of this backend, so instances appearing, disappearing, and moving
+	// between locations need no edit here.
+	//
+	// Mutually exclusive with endpoint, connector and instance.
+	//
+	// +kubebuilder:validation:Optional
+	NetworkService *NetworkServiceBackendRef `json:"networkService,omitempty"`
+
 	// TLS contains backend TLS configuration.
 	//
 	// When the backend endpoint uses HTTPS with an IP address, the Hostname field
 	// must be specified for TLS certificate validation.
 	//
+	// Not supported for networkService backends, which are always reached over
+	// plaintext HTTP.
+	//
 	// +kubebuilder:validation:Optional
 	TLS *HTTPProxyBackendTLS `json:"tls,omitempty"`
 
@@ -209,6 +223,28 @@ type InstanceBackendRef struct {
 	Port int32 `json:"port"`
 }
 
+// NetworkServiceBackendRef references a NetworkService, and one of the ports it
+// declares, as the backend of a rule.
+type NetworkServiceBackendRef struct {
+	// Name of the referenced NetworkService. Must exist in the same namespace as
+	// this HTTPProxy.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=253
+	Name string `json:"name"`
+
+	// Port names a port declared in the referenced service's spec.ports, rather
+	// than giving a number, so the reference survives a change to the port the
+	// members answer on.
+	//
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
+	Port string `json:"port"`
+}
+
 // ConnectorReference references a Connector by name.
 type ConnectorReference struct {
 	// Name of the referenced Connector.
@@ -457,6 +493,17 @@ const (
 	// references an EndpointSlice that does not exist.
 	HTTPProxyReasonInstanceBackendNotFound = "InstanceBackendNotFound"
 
+	// HTTPProxyReasonNetworkServiceBackendNotFound indicates that a
+	// networkService backend references a NetworkService that does not exist, or
+	// a port name that service does not declare.
+	HTTPProxyReasonNetworkServiceBackendNotFound = "NetworkServiceBackendNotFound"
+
+	// HTTPProxyReasonNetworkServiceMembersUnreferenced indicates that a
+	// networkService backend resolved more members than a single EndpointSlice
+	// holds. Every member is published, but only the members in the referenced
+	// slice are being served.
+	HTTPProxyReasonNetworkServiceMembersUnreferenced = "NetworkServiceMembersUnreferenced"
+
 	// This reason is used with the "Accepted" and "Programmed"
 	// conditions when the status is "Unknown" and no controller has reconciled
 	// the HTTPProxy.

api/v1alpha/networkinterface_types.go

@@ -86,6 +86,40 @@ const (
 	// created, so anything that withholds the workload until it is true waits for
 	// something its own waiting prevents.
 	NetworkInterfaceProgrammed = "Programmed"
+
+	// NetworkInterfaceHolderAvailable reports that whatever holds this interface
+	// says it is available to serve. It is written by the holder named in the
+	// held-by label, and the networking operator has no idea what a holder is.
+	//
+	// The operator only ever writes it Unknown: it seeds it when it creates the
+	// interface, and takes the departed holder's word back when a retained
+	// interface is unbound. True and False are the holder's alone.
+	//
+	// This is the condition a service reads to decide whether a member takes
+	// traffic. It says nothing about the interface: an interface with every
+	// address allocated and programmed still carries this false while whatever
+	// is behind it is starting, failing, or shutting down.
+	//
+	// Unrelated to status.phase, which reports whether a claim holds the
+	// interface at all. A phase of Available means no claim holds it, which is
+	// the opposite of anything being available to serve.
+	NetworkInterfaceHolderAvailable = "HolderAvailable"
+)
+
+const (
+	// NetworkInterfaceReasonHolderAvailable is what a holder reports on
+	// HolderAvailable once it is serving.
+	NetworkInterfaceReasonHolderAvailable = "HolderAvailable"
+
+	// NetworkInterfaceReasonHolderUnavailable is what a holder reports on
+	// HolderAvailable while it is starting, failing, or shutting down.
+	NetworkInterfaceReasonHolderUnavailable = "HolderUnavailable"
+
+	// NetworkInterfaceReasonHolderReleased is the operator taking back a
+	// departed holder's word when a retained interface is unbound. A retained
+	// interface outlives its holder, and the next claim of its name is a
+	// different holder that has said nothing yet.
+	NetworkInterfaceReasonHolderReleased = "HolderReleased"
 )
 
 // NetworkInterfaceAddress is an address the interface holds inside its network.
@@ -327,6 +361,8 @@ type NetworkInterfaceStatus struct {
 	// conditions report the current state of the interface. Allocated means every
 	// address is held. Prepared means the data plane is ready for a workload to
 	// consume it. Programmed means the data plane carries the addresses.
+	// HolderAvailable means whatever holds the interface reports itself available
+	// to serve, and it is the only one of the four a service reads.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
 }
 
@@ -351,14 +387,15 @@ type NetworkInterfaceStatus struct {
 // +kubebuilder:printcolumn:name="Allocated",type=string,JSONPath=`.status.conditions[?(@.type=="Allocated")].status`
 // +kubebuilder:printcolumn:name="Prepared",type=string,JSONPath=`.status.conditions[?(@.type=="Prepared")].status`
 // +kubebuilder:printcolumn:name="Programmed",type=string,JSONPath=`.status.conditions[?(@.type=="Programmed")].status`
+// +kubebuilder:printcolumn:name="HolderAvailable",type=string,JSONPath=`.status.conditions[?(@.type=="HolderAvailable")].status`
 type NetworkInterface struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
 	// +kubebuilder:validation:Required
 	Spec NetworkInterfaceSpec `json:"spec,omitempty"`
 
-	// +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Prepared",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}}
+	// +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Prepared",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"HolderAvailable",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}}
 	Status NetworkInterfaceStatus `json:"status,omitempty"`
 }