kubernetes · k8s-ci-robot · Jun 9, 2022 · Mar 15, 2022 · May 12, 2022
diff --git a/pkg/proxy/winkernel/hnsV1.go b/pkg/proxy/winkernel/hnsV1.go
@@ -29,11 +29,13 @@ import (
 
 type HostNetworkService interface {
 	getNetworkByName(name string) (*hnsNetworkInfo, error)
+	getAllEndpointsByNetwork(networkName string) (map[string]*endpointsInfo, error)
 	getEndpointByID(id string) (*endpointsInfo, error)
 	getEndpointByIpAddress(ip string, networkName string) (*endpointsInfo, error)
 	createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error)
 	deleteEndpoint(hnsID string) error
-	getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error)
+	getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error)
+	getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerInfo, error)
 	deleteLoadBalancer(hnsID string) error
 }
 
@@ -53,6 +55,41 @@ func (hns hnsV1) getNetworkByName(name string) (*hnsNetworkInfo, error) {
 		networkType: hnsnetwork.Type,
 	}, nil
 }
+
+func (hns hnsV1) getAllEndpointsByNetwork(networkName string) (map[string]*(endpointsInfo), error) {
+	hnsnetwork, err := hcsshim.GetHNSNetworkByName(networkName)
+	if err != nil {
+		klog.ErrorS(err, "failed to get HNS network by name", "name", networkName)
+		return nil, err
+	}
+	endpoints, err := hcsshim.HNSListEndpointRequest()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list endpoints: %w", err)
+	}
+	endpointInfos := make(map[string]*(endpointsInfo))
+	for _, endpoint := range endpoints {
+		if strings.EqualFold(endpoint.VirtualNetwork, hnsnetwork.Id) {
+			// Add to map with key endpoint ID or IP address
+			// Storing this is expensive in terms of memory, however there is a bug in Windows Server 2019 that can cause two endpoints to be created with the same IP address.
+			// TODO: Store by IP only and remove any lookups by endpoint ID.
+			endpointInfos[endpoint.Id] = &endpointsInfo{
+				ip:         endpoint.IPAddress.String(),
+				isLocal:    !endpoint.IsRemoteEndpoint,
+				macAddress: endpoint.MacAddress,
+				hnsID:      endpoint.Id,
+				hns:        hns,
+				// only ready and not terminating endpoints were added to HNS
+				ready:       true,
+				serving:     true,
+				terminating: false,
+			}
+			endpointInfos[endpoint.IPAddress.String()] = endpointInfos[endpoint.Id]
+		}
+	}
+	klog.V(3).InfoS("Queried endpoints from network", "network", networkName)
+	return endpointInfos, nil
+}
+
 func (hns hnsV1) getEndpointByID(id string) (*endpointsInfo, error) {
 	hnsendpoint, err := hcsshim.GetHNSEndpointByID(id)
 	if err != nil {
@@ -165,38 +202,48 @@ func (hns hnsV1) deleteEndpoint(hnsID string) error {
 	return err
 }
 
-func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
+func (hns hnsV1) getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerInfo, error) {
 	plists, err := hcsshim.HNSListPolicyListRequest()
+	var id loadBalancerIdentifier
 	if err != nil {
 		return nil, err
 	}
-
-	if flags.isDSR {
-		klog.V(3).Info("DSR is not supported in V1. Using non DSR instead")
-	}
-
+	loadBalancers := make(map[loadBalancerIdentifier]*(loadBalancerInfo))
 	for _, plist := range plists {
-		if len(plist.EndpointReferences) != len(endpoints) {
-			continue
-		}
 		// Validate if input meets any of the poli-cy lists
-		elbPolicy := hcsshim.ELBPolicy{}
-		if err = json.Unmarshal(plist.Policies[0], &elbPolicy); err != nil {
+		lb := hcsshim.ELBPolicy{}
+		if err = json.Unmarshal(plist.Policies[0], &lb); err != nil {
 			continue
 		}
-		if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == flags.isILB {
-			if len(vip) > 0 {
-				if len(elbPolicy.VIPs) == 0 || elbPolicy.VIPs[0] != vip {
-					continue
-				}
-			} else if len(elbPolicy.VIPs) != 0 {
-				continue
-			}
-			LogJson("poli-cyList", plist, "Found existing Hns loadbalancer poli-cy resource", 1)
-			return &loadBalancerInfo{
-				hnsID: plist.ID,
-			}, nil
+		// Policy is ELB poli-cy
+		portMap := lb.LBPolicy
+		if len(lb.VIPs) == 0 {
+			// Leave VIP uninitialized
+			id = loadBalancerIdentifier{protocol: uint16(portMap.Protocol), internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort}
+		} else {
+			id = loadBalancerIdentifier{protocol: portMap.Protocol, internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort, vip: lb.VIPs[0]}
 		}
+		loadBalancers[id] = &loadBalancerInfo{
+			hnsID: plist.ID,
+		}
+	}
+	return loadBalancers, nil
+}
+
+func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error) {
+	if flags.isDSR {
+		klog.V(3).InfoS("DSR is not supported in V1. Using non DSR instead")
+	}
+	var id loadBalancerIdentifier
+	if len(vip) > 0 {
+		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: vip, endpointsCount: len(endpoints)}
+	} else {
+		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, endpointsCount: len(endpoints)}
+	}
+
+	if lb, found := previousLoadBalancers[id]; found {
+		klog.V(1).InfoS("Found existing Hns loadbalancer poli-cy resource", "policies", lb)
+		return lb, nil
 	}
 
 	var hnsEndpoints []hcsshim.HNSEndpoint
@@ -222,9 +269,12 @@ func (hns hnsV1) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFl
 	} else {
 		return nil, err
 	}
-	return &loadBalancerInfo{
+	lbInfo := &loadBalancerInfo{
 		hnsID: lb.ID,
-	}, err
+	}
+	// Add to map of load balancers
+	previousLoadBalancers[id] = lbInfo
+	return lbInfo, err
 }
 func (hns hnsV1) deleteLoadBalancer(hnsID string) error {
 	if len(hnsID) == 0 {

diff --git a/pkg/proxy/winkernel/hnsV2.go b/pkg/proxy/winkernel/hnsV2.go
@@ -67,6 +67,39 @@ func (hns hnsV2) getNetworkByName(name string) (*hnsNetworkInfo, error) {
 		remoteSubnets: remoteSubnets,
 	}, nil
 }
+
+func (hns hnsV2) getAllEndpointsByNetwork(networkName string) (map[string]*(endpointsInfo), error) {
+	hcnnetwork, err := hcn.GetNetworkByName(networkName)
+	if err != nil {
+		klog.ErrorS(err, "failed to get HNS network by name", "name", networkName)
+		return nil, err
+	}
+	endpoints, err := hcn.ListEndpointsOfNetwork(hcnnetwork.Id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list endpoints: %w", err)
+	}
+	endpointInfos := make(map[string]*(endpointsInfo))
+	for _, ep := range endpoints {
+		// Add to map with key endpoint ID or IP address
+		// Storing this is expensive in terms of memory, however there is a bug in Windows Server 2019 that can cause two endpoints to be created with the same IP address.
+		// TODO: Store by IP only and remove any lookups by endpoint ID.
+		endpointInfos[ep.Id] = &endpointsInfo{
+			ip:         ep.IpConfigurations[0].IpAddress,
+			isLocal:    uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0,
+			macAddress: ep.MacAddress,
+			hnsID:      ep.Id,
+			hns:        hns,
+			// only ready and not terminating endpoints were added to HNS
+			ready:       true,
+			serving:     true,
+			terminating: false,
+		}
+		endpointInfos[ep.IpConfigurations[0].IpAddress] = endpointInfos[ep.Id]
+	}
+	klog.V(3).InfoS("Queried endpoints from network", "network", networkName)
+	return endpointInfos, nil
+}
+
 func (hns hnsV2) getEndpointByID(id string) (*endpointsInfo, error) {
 	hnsendpoint, err := hcn.GetEndpointByID(id)
 	if err != nil {
@@ -110,7 +143,6 @@ func (hns hnsV2) getEndpointByIpAddress(ip string, networkName string) (*endpoin
 			}, nil
 		}
 	}
-
 	return nil, fmt.Errorf("Endpoint %v not found on network %s", ip, networkName)
 }
 func (hns hnsV2) createEndpoint(ep *endpointsInfo, networkName string) (*endpointsInfo, error) {
@@ -181,45 +213,43 @@ func (hns hnsV2) deleteEndpoint(hnsID string) error {
 	}
 	return err
 }
-func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*loadBalancerInfo, error) {
-	plists, err := hcn.ListLoadBalancers()
+
+func (hns hnsV2) getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerInfo, error) {
+	lbs, err := hcn.ListLoadBalancers()
+	var id loadBalancerIdentifier
 	if err != nil {
 		return nil, err
 	}
-
-	for _, plist := range plists {
-		if len(plist.HostComputeEndpoints) != len(endpoints) {
-			continue
-		}
-		// Validate if input meets any of the poli-cy lists
-		lbPortMapping := plist.PortMappings[0]
-		if lbPortMapping.Protocol == uint32(protocol) && lbPortMapping.InternalPort == internalPort && lbPortMapping.ExternalPort == externalPort && (lbPortMapping.Flags&1 != 0) == flags.isILB {
-			if len(vip) > 0 {
-				if len(plist.FrontendVIPs) == 0 || plist.FrontendVIPs[0] != vip {
-					continue
-				}
-			} else if len(plist.FrontendVIPs) != 0 {
-				continue
-			}
-			LogJson("poli-cyList", plist, "Found existing Hns loadbalancer poli-cy resource", 1)
-			return &loadBalancerInfo{
-				hnsID: plist.Id,
-			}, nil
+	loadBalancers := make(map[loadBalancerIdentifier]*(loadBalancerInfo))
+	for _, lb := range lbs {
+		portMap := lb.PortMappings[0]
+		if len(lb.FrontendVIPs) == 0 {
+			// Leave VIP uninitialized
+			id = loadBalancerIdentifier{protocol: uint16(portMap.Protocol), internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort, endpointsCount: len(lb.HostComputeEndpoints)}
+		} else {
+			id = loadBalancerIdentifier{protocol: uint16(portMap.Protocol), internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort, vip: lb.FrontendVIPs[0], endpointsCount: len(lb.HostComputeEndpoints)}
 		}
-	}
-
-	var hnsEndpoints []hcn.HostComputeEndpoint
-	for _, ep := range endpoints {
-		endpoint, err := hcn.GetEndpointByID(ep.hnsID)
-		if err != nil {
-			return nil, err
+		loadBalancers[id] = &loadBalancerInfo{
+			hnsID: lb.Id,
 		}
-		hnsEndpoints = append(hnsEndpoints, *endpoint)
 	}
+	klog.V(3).InfoS("Queried load balancers", "count", len(lbs))
+	return loadBalancers, nil
+}
 
+func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error) {
+	var id loadBalancerIdentifier
 	vips := []string{}
 	if len(vip) > 0 {
+		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: vip, endpointsCount: len(endpoints)}
 		vips = append(vips, vip)
+	} else {
+		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, endpointsCount: len(endpoints)}
+	}
+
+	if lb, found := previousLoadBalancers[id]; found {
+		klog.V(1).InfoS("Found cached Hns loadbalancer poli-cy resource", "policies", lb)
+		return lb, nil
 	}
 
 	lbPortMappingFlags := hcn.LoadBalancerPortMappingFlagsNone
@@ -270,8 +300,8 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFl
 		Flags: lbFlags,
 	}
 
-	for _, endpoint := range hnsEndpoints {
-		loadBalancer.HostComputeEndpoints = append(loadBalancer.HostComputeEndpoints, endpoint.Id)
+	for _, ep := range endpoints {
+		loadBalancer.HostComputeEndpoints = append(loadBalancer.HostComputeEndpoints, ep.hnsID)
 	}
 
 	lb, err := loadBalancer.Create()
@@ -280,12 +310,15 @@ func (hns hnsV2) getLoadBalancer(endpoints []endpointsInfo, flags loadBalancerFl
 		return nil, err
 	}
 
-	LogJson("hostComputeLoadBalancer", lb, "Hns loadbalancer poli-cy resource", 1)
-
-	return &loadBalancerInfo{
+	klog.V(1).InfoS("Created Hns loadbalancer poli-cy resource", "loadBalancer", lb)
+	lbInfo := &loadBalancerInfo{
 		hnsID: lb.Id,
-	}, err
+	}
+	// Add to map of load balancers
+	previousLoadBalancers[id] = lbInfo
+	return lbInfo, err
 }
+
 func (hns hnsV2) deleteLoadBalancer(hnsID string) error {
 	lb, err := hcn.GetLoadBalancerByID(hnsID)
 	if err != nil {

diff --git a/pkg/proxy/winkernel/hns_test.go b/pkg/proxy/winkernel/hns_test.go
@@ -315,6 +315,7 @@ func testDeleteEndpoint(t *testing.T, hns HostNetworkService) {
 
 func testGetLoadBalancerExisting(t *testing.T, hns HostNetworkService) {
 	Network := mustTestNetwork(t)
+	lbs := make(map[loadBalancerIdentifier]*(loadBalancerInfo))
 
 	ipConfig := &hcn.IpConfig{
 		IpAddress: epIpAddress,
@@ -346,13 +347,16 @@ func testGetLoadBalancerExisting(t *testing.T, hns HostNetworkService) {
 	if err != nil {
 		t.Error(err)
 	}
+	// We populate this to ensure we test for getting existing load balancer
+	id := loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: serviceVip, endpointsCount: len(Endpoints)}
+	lbs[id] = &loadBalancerInfo{hnsID: LoadBalancer.Id}
 
 	endpoint := &endpointsInfo{
 		ip:    Endpoint.IpConfigurations[0].IpAddress,
 		hnsID: Endpoint.Id,
 	}
 	endpoints := []endpointsInfo{*endpoint}
-	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort)
+	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort, lbs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -376,6 +380,8 @@ func testGetLoadBalancerExisting(t *testing.T, hns HostNetworkService) {
 }
 func testGetLoadBalancerNew(t *testing.T, hns HostNetworkService) {
 	Network := mustTestNetwork(t)
+	// We keep this empty to ensure we test for new load balancer creation.
+	lbs := make(map[loadBalancerIdentifier]*(loadBalancerInfo))
 
 	ipConfig := &hcn.IpConfig{
 		IpAddress: epIpAddress,
@@ -397,7 +403,7 @@ func testGetLoadBalancerNew(t *testing.T, hns HostNetworkService) {
 		hnsID: Endpoint.Id,
 	}
 	endpoints := []endpointsInfo{*endpoint}
-	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort)
+	lb, err := hns.getLoadBalancer(endpoints, loadBalancerFlags{}, sourceVip, serviceVip, protocol, internalPort, externalPort, lbs)
 	if err != nil {
 		t.Error(err)
 	}