diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 82747537..a4e0830b 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -51,6 +51,9 @@ var Options struct { NodeBalancerBackendIPv4Subnet string GlobalStopChannel chan<- struct{} EnableIPv6ForLoadBalancers bool + EnableNodeCIDRAllocation bool + ClusterIPv4CIDR string + NodeCIDRMaskSizeIPv4 int } type linodeCloud struct { @@ -187,6 +190,10 @@ func (c *linodeCloud) Initialize(clientBuilder cloudprovider.ControllerClientBui serviceInformer := sharedInformer.Core().V1().Services() nodeInformer := sharedInformer.Core().V1().Nodes() + if _, err := startNodeIpamController(stopCh, c, nodeInformer, kubeclient); err != nil { + klog.Fatal("starting of node ipam controller failed", err) + } + if c.linodeTokenHealthChecker != nil { go c.linodeTokenHealthChecker.Run(stopCh) } diff --git a/cloud/linode/nodeipamcontroller.go b/cloud/linode/nodeipamcontroller.go new file mode 100644 index 00000000..32fdc791 --- /dev/null +++ b/cloud/linode/nodeipamcontroller.go @@ -0,0 +1,114 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file holds the code related with the sample nodeipamcontroller +// which demonstrates how cloud providers add external controllers to cloud-controller-manager + +package linode + +import ( + "fmt" + "net" + "strings" + + v1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + cloudprovider "k8s.io/cloud-provider" + netutils "k8s.io/utils/net" + + nodeipamcontroller "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam" + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam" +) + +const ( + // defaultNodeMaskCIDRIPv4 is default mask size for IPv4 node cidr + defaultNodeMaskCIDRIPv4 = 24 + // defaultNodeMaskCIDRIPv6 is default mask size for IPv6 node cidr + defaultNodeMaskCIDRIPv6 = 64 +) + +func startNodeIpamController(stopCh <-chan struct{}, cloud cloudprovider.Interface, nodeInformer v1.NodeInformer, kubeclient kubernetes.Interface) (bool, error) { + var serviceCIDR *net.IPNet + var secondaryServiceCIDR *net.IPNet + + // should we start nodeIPAM + if !Options.EnableNodeCIDRAllocation { + return false, nil + } + + // failure: bad cidrs in config + clusterCIDRs, dualStack, err := processCIDRs(Options.ClusterIPv4CIDR) + if err != nil { + return false, fmt.Errorf("processCIDRs failed: %v", err) + } + + // failure: more than one cidr but they are not configured as dual stack + if len(clusterCIDRs) > 1 && !dualStack { + return false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs)) + } + + // failure: more than cidrs is not allowed even with dual stack + if len(clusterCIDRs) > 2 { + return false, fmt.Errorf("len of clusters is:%v > more than max allowed of 2", len(clusterCIDRs)) + } + + nodeCIDRMaskSizes, err := setNodeCIDRMaskSizes(clusterCIDRs) + if err != nil { + return false, fmt.Errorf("setNodeCIDRMaskSizes failed: %v", err) + } + + nodeIpamController, err := nodeipamcontroller.NewNodeIpamController( + nodeInformer, + cloud, + kubeclient, + clusterCIDRs, + serviceCIDR, + secondaryServiceCIDR, + nodeCIDRMaskSizes, + ipam.CIDRAllocatorType(ipam.RangeAllocatorType), + ) + if err != nil { + return true, err + } + go nodeIpamController.Run(stopCh) + return true, nil +} + +// processCIDRs is a helper function that works on a comma separated cidrs and returns +// a list of typed cidrs +// a flag if cidrs represents a dual stack +// error if failed to parse any of the cidrs +func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) { + cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",") + + cidrs, err := netutils.ParseCIDRs(cidrsSplit) + if err != nil { + return nil, false, err + } + + // if cidrs has an error then the previous call will fail + // safe to ignore error checking on next call + dualstack, _ := netutils.IsDualStackCIDRs(cidrs) + + return cidrs, dualstack, nil +} + +func setNodeCIDRMaskSizes(clusterCIDRs []*net.IPNet) ([]int, error) { + if Options.NodeCIDRMaskSizeIPv4 != 0 { + return []int{Options.NodeCIDRMaskSizeIPv4, defaultNodeMaskCIDRIPv6}, nil + } + return []int{defaultNodeMaskCIDRIPv4, defaultNodeMaskCIDRIPv6}, nil +} diff --git a/cloud/nodeipam/config/doc.go b/cloud/nodeipam/config/doc.go new file mode 100644 index 00000000..164ec715 --- /dev/null +++ b/cloud/nodeipam/config/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +package config diff --git a/cloud/nodeipam/config/types.go b/cloud/nodeipam/config/types.go new file mode 100644 index 00000000..8c923e19 --- /dev/null +++ b/cloud/nodeipam/config/types.go @@ -0,0 +1,34 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// NodeIPAMControllerConfiguration contains elements describing NodeIPAMController. +type NodeIPAMControllerConfiguration struct { + // serviceCIDR is CIDR Range for Services in cluster. + ServiceCIDR string + // secondaryServiceCIDR is CIDR Range for Services in cluster. This is used in dual stack clusters. SecondaryServiceCIDR must be of different IP family than ServiceCIDR + SecondaryServiceCIDR string + // NodeCIDRMaskSize is the mask size for node cidr in single-stack cluster. + // This can be used only with single stack clusters and is incompatible with dual stack clusters. + NodeCIDRMaskSize int32 + // NodeCIDRMaskSizeIPv4 is the mask size for IPv4 node cidr in dual-stack cluster. + // This can be used only with dual stack clusters and is incompatible with single stack clusters. + NodeCIDRMaskSizeIPv4 int32 + // NodeCIDRMaskSizeIPv6 is the mask size for IPv6 node cidr in dual-stack cluster. + // This can be used only with dual stack clusters and is incompatible with single stack clusters. + NodeCIDRMaskSizeIPv6 int32 +} diff --git a/cloud/nodeipam/doc.go b/cloud/nodeipam/doc.go new file mode 100644 index 00000000..af4cd15c --- /dev/null +++ b/cloud/nodeipam/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package nodeipam contains code for syncing cloud instances with +// node registry +package nodeipam diff --git a/cloud/nodeipam/ipam/cidr_allocator.go b/cloud/nodeipam/ipam/cidr_allocator.go new file mode 100644 index 00000000..52f56d85 --- /dev/null +++ b/cloud/nodeipam/ipam/cidr_allocator.go @@ -0,0 +1,155 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipam + +import ( + "context" + "fmt" + "net" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + informers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog/v2" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/cidrset" +) + +// CIDRAllocatorType is the type of the allocator to use. +type CIDRAllocatorType string + +const ( + // RangeAllocatorType is the allocator that uses an internal CIDR + // range allocator to do node CIDR range allocations. + RangeAllocatorType CIDRAllocatorType = "RangeAllocator" + // CloudAllocatorType is the allocator that uses cloud platform + // support to do node CIDR range allocations. + CloudAllocatorType CIDRAllocatorType = "CloudAllocator" + // IPAMFromClusterAllocatorType uses the ipam controller sync'ing the node + // CIDR range allocations from the cluster to the cloud. + IPAMFromClusterAllocatorType = "IPAMFromCluster" + // IPAMFromCloudAllocatorType uses the ipam controller sync'ing the node + // CIDR range allocations from the cloud to the cluster. + IPAMFromCloudAllocatorType = "IPAMFromCloud" +) + +// TODO: figure out the good setting for those constants. +const ( + // The amount of time the nodecontroller polls on the list nodes endpoint. + apiserverStartupGracePeriod = 10 * time.Minute + + // The no. of NodeSpec updates NC can process concurrently. + cidrUpdateWorkers = 30 + + // cidrUpdateRetries is the no. of times a NodeSpec update will be retried before dropping it. + cidrUpdateRetries = 3 +) + +// nodePollInterval is used in listing node +var nodePollInterval = 10 * time.Second + +// CIDRAllocator is an interface implemented by things that know how +// to allocate/occupy/recycle CIDR for nodes. +type CIDRAllocator interface { + // AllocateOrOccupyCIDR looks at the given node, assigns it a valid + // CIDR if it doesn't currently have one or mark the CIDR as used if + // the node already have one. + AllocateOrOccupyCIDR(ctx context.Context, node *v1.Node) error + // ReleaseCIDR releases the CIDR of the removed node. + ReleaseCIDR(node *v1.Node) error + // Run starts all the working logic of the allocator. + Run(stopCh <-chan struct{}) +} + +// CIDRAllocatorParams is parameters that's required for creating new +// cidr range allocator. +type CIDRAllocatorParams struct { + // ClusterCIDRs is list of cluster cidrs. + ClusterCIDRs []*net.IPNet + // ServiceCIDR is primary service cidr for cluster. + ServiceCIDR *net.IPNet + // SecondaryServiceCIDR is secondary service cidr for cluster. + SecondaryServiceCIDR *net.IPNet + // NodeCIDRMaskSizes is list of node cidr mask sizes. + NodeCIDRMaskSizes []int +} + +// New creates a new CIDR range allocator. +func New(kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeInformer informers.NodeInformer, allocatorType CIDRAllocatorType, allocatorParams CIDRAllocatorParams) (CIDRAllocator, error) { + nodeList, err := listNodes(kubeClient) + if err != nil { + return nil, err + } + + switch allocatorType { + case RangeAllocatorType: + return NewCIDRRangeAllocator(kubeClient, nodeInformer, allocatorParams, nodeList) + default: + return nil, fmt.Errorf("invalid CIDR allocator type: %v", allocatorType) + } +} + +func listNodes(kubeClient clientset.Interface) (*v1.NodeList, error) { + var nodeList *v1.NodeList + ctx, cancel := context.WithTimeout(context.Background(), apiserverStartupGracePeriod) + defer cancel() + + // We must poll because apiserver might not be up. This error causes + // controller manager to restart. + if pollErr := wait.PollUntilContextTimeout(ctx, nodePollInterval, apiserverStartupGracePeriod, true, func(ctx context.Context) (bool, error) { + var err error + nodeList, err = kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + FieldSelector: fields.Everything().String(), + LabelSelector: labels.Everything().String(), + }) + if err != nil { + klog.Error(err, "Failed to list all nodes") + return false, nil + } + return true, nil + }); pollErr != nil { + return nil, fmt.Errorf("failed to list all nodes in %v, cannot proceed without updating CIDR map", + apiserverStartupGracePeriod) + } + return nodeList, nil +} + +// ipnetToStringList converts a slice of net.IPNet into a list of CIDR in string format +func ipnetToStringList(inCIDRs []*net.IPNet) []string { + outCIDRs := make([]string, len(inCIDRs)) + for idx, inCIDR := range inCIDRs { + outCIDRs[idx] = inCIDR.String() + } + return outCIDRs +} + +// occupyServiceCIDR removes the service CIDR range from the cluster CIDR if it +// intersects. +func occupyServiceCIDR(set *cidrset.CidrSet, clusterCIDR, serviceCIDR *net.IPNet) error { + if clusterCIDR.Contains(serviceCIDR.IP) || serviceCIDR.Contains(clusterCIDR.IP) { + if err := set.Occupy(serviceCIDR); err != nil { + return err + } + } + return nil +} diff --git a/cloud/nodeipam/ipam/cidrset/cidr_set.go b/cloud/nodeipam/ipam/cidrset/cidr_set.go new file mode 100644 index 00000000..7614db75 --- /dev/null +++ b/cloud/nodeipam/ipam/cidrset/cidr_set.go @@ -0,0 +1,293 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cidrset + +import ( + "encoding/binary" + "errors" + "fmt" + "math/big" + "math/bits" + "net" + "sync" +) + +// CidrSet manages a set of CIDR ranges from which blocks of IPs can +// be allocated from. +type CidrSet struct { + sync.Mutex + // clusterCIDR is the CIDR assigned to the cluster + clusterCIDR *net.IPNet + // clusterMaskSize is the mask size, in bits, assigned to the cluster + // caches the mask size to avoid the penalty of calling clusterCIDR.Mask.Size() + clusterMaskSize int + // nodeMask is the network mask assigned to the nodes + nodeMask net.IPMask + // nodeMaskSize is the mask size, in bits,assigned to the nodes + // caches the mask size to avoid the penalty of calling nodeMask.Size() + nodeMaskSize int + // maxCIDRs is the maximum number of CIDRs that can be allocated + maxCIDRs int + // allocatedCIDRs counts the number of CIDRs allocated + allocatedCIDRs int + // nextCandidate points to the next CIDR that should be free + nextCandidate int + // used is a bitmap used to track the CIDRs allocated + used big.Int + // label is used to identify the metrics + label string +} + +const ( + // The subnet mask size cannot be greater than 16 more than the cluster mask size + // TODO: https://github.com/kubernetes/kubernetes/issues/44918 + // clusterSubnetMaxDiff limited to 16 due to the uncompressed bitmap + // Due to this limitation the subnet mask for IPv6 cluster cidr needs to be >= 48 + // as default mask size for IPv6 is 64. + clusterSubnetMaxDiff = 16 + // halfIPv6Len is the half of the IPv6 length + halfIPv6Len = net.IPv6len / 2 +) + +var ( + // ErrCIDRRangeNoCIDRsRemaining occurs when there is no more space + // to allocate CIDR ranges. + ErrCIDRRangeNoCIDRsRemaining = errors.New( + "CIDR allocation failed; there are no remaining CIDRs left to allocate in the accepted range") + // ErrCIDRSetSubNetTooBig occurs when the subnet mask size is too + // big compared to the CIDR mask size. + ErrCIDRSetSubNetTooBig = errors.New( + "New CIDR set failed; the node CIDR size is too big") +) + +// NewCIDRSet creates a new CidrSet. +func NewCIDRSet(clusterCIDR *net.IPNet, subNetMaskSize int) (*CidrSet, error) { + clusterMask := clusterCIDR.Mask + clusterMaskSize, bits := clusterMask.Size() + + if (clusterCIDR.IP.To4() == nil) && (subNetMaskSize-clusterMaskSize > clusterSubnetMaxDiff) { + return nil, ErrCIDRSetSubNetTooBig + } + + // register CidrSet metrics + registerCidrsetMetrics() + + maxCIDRs := getMaxCIDRs(subNetMaskSize, clusterMaskSize) + cidrSet := &CidrSet{ + clusterCIDR: clusterCIDR, + nodeMask: net.CIDRMask(subNetMaskSize, bits), + clusterMaskSize: clusterMaskSize, + maxCIDRs: maxCIDRs, + nodeMaskSize: subNetMaskSize, + label: clusterCIDR.String(), + } + cidrSetMaxCidrs.WithLabelValues(cidrSet.label).Set(float64(maxCIDRs)) + + return cidrSet, nil +} + +func (s *CidrSet) indexToCIDRBlock(index int) *net.IPNet { + var ip []byte + switch /*v4 or v6*/ { + case s.clusterCIDR.IP.To4() != nil: + { + j := uint32(index) << uint32(32-s.nodeMaskSize) + ipInt := (binary.BigEndian.Uint32(s.clusterCIDR.IP)) | j + ip = make([]byte, net.IPv4len) + binary.BigEndian.PutUint32(ip, ipInt) + } + case s.clusterCIDR.IP.To16() != nil: + { + // leftClusterIP | rightClusterIP + // 2001:0DB8:1234:0000:0000:0000:0000:0000 + const v6NBits = 128 + const halfV6NBits = v6NBits / 2 + leftClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[:halfIPv6Len]) + rightClusterIP := binary.BigEndian.Uint64(s.clusterCIDR.IP[halfIPv6Len:]) + + ip = make([]byte, net.IPv6len) + + if s.nodeMaskSize <= halfV6NBits { + // We only care about left side IP + leftClusterIP |= uint64(index) << uint(halfV6NBits-s.nodeMaskSize) + } else { + if s.clusterMaskSize < halfV6NBits { + // see how many bits are needed to reach the left side + btl := uint(s.nodeMaskSize - halfV6NBits) + indexMaxBit := uint(64 - bits.LeadingZeros64(uint64(index))) + if indexMaxBit > btl { + leftClusterIP |= uint64(index) >> btl + } + } + // the right side will be calculated the same way either the + // subNetMaskSize affects both left and right sides + rightClusterIP |= uint64(index) << uint(v6NBits-s.nodeMaskSize) + } + binary.BigEndian.PutUint64(ip[:halfIPv6Len], leftClusterIP) + binary.BigEndian.PutUint64(ip[halfIPv6Len:], rightClusterIP) + } + } + return &net.IPNet{ + IP: ip, + Mask: s.nodeMask, + } +} + +// AllocateNext allocates the next free CIDR range. This will set the range +// as occupied and return the allocated range. +func (s *CidrSet) AllocateNext() (*net.IPNet, error) { + s.Lock() + defer s.Unlock() + + if s.allocatedCIDRs == s.maxCIDRs { + return nil, ErrCIDRRangeNoCIDRsRemaining + } + candidate := s.nextCandidate + var i int + for i = 0; i < s.maxCIDRs; i++ { + if s.used.Bit(candidate) == 0 { + break + } + candidate = (candidate + 1) % s.maxCIDRs + } + + s.nextCandidate = (candidate + 1) % s.maxCIDRs + s.used.SetBit(&s.used, candidate, 1) + s.allocatedCIDRs++ + // Update metrics + cidrSetAllocations.WithLabelValues(s.label).Inc() + cidrSetAllocationTriesPerRequest.WithLabelValues(s.label).Observe(float64(i)) + cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs)) + + return s.indexToCIDRBlock(candidate), nil +} + +func (s *CidrSet) getBeginningAndEndIndices(cidr *net.IPNet) (begin, end int, err error) { + if cidr == nil { + return -1, -1, fmt.Errorf("error getting indices for cluster cidr %v, cidr is nil", s.clusterCIDR) + } + begin, end = 0, s.maxCIDRs-1 + cidrMask := cidr.Mask + maskSize, _ := cidrMask.Size() + var ipSize int + + if !s.clusterCIDR.Contains(cidr.IP.Mask(s.clusterCIDR.Mask)) && !cidr.Contains(s.clusterCIDR.IP.Mask(cidr.Mask)) { + return -1, -1, fmt.Errorf("cidr %v is out the range of cluster cidr %v", cidr, s.clusterCIDR) + } + + if s.clusterMaskSize < maskSize { + + ipSize = net.IPv4len + if cidr.IP.To4() == nil { + ipSize = net.IPv6len + } + begin, err = s.getIndexForIP(cidr.IP.Mask(s.nodeMask)) + if err != nil { + return -1, -1, err + } + ip := make([]byte, ipSize) + if cidr.IP.To4() != nil { + ipInt := binary.BigEndian.Uint32(cidr.IP) | (^binary.BigEndian.Uint32(cidr.Mask)) + binary.BigEndian.PutUint32(ip, ipInt) + } else { + // ipIntLeft | ipIntRight + // 2001:0DB8:1234:0000:0000:0000:0000:0000 + ipIntLeft := binary.BigEndian.Uint64(cidr.IP[:net.IPv6len/2]) | (^binary.BigEndian.Uint64(cidr.Mask[:net.IPv6len/2])) + ipIntRight := binary.BigEndian.Uint64(cidr.IP[net.IPv6len/2:]) | (^binary.BigEndian.Uint64(cidr.Mask[net.IPv6len/2:])) + binary.BigEndian.PutUint64(ip[:net.IPv6len/2], ipIntLeft) + binary.BigEndian.PutUint64(ip[net.IPv6len/2:], ipIntRight) + } + end, err = s.getIndexForIP(net.IP(ip).Mask(s.nodeMask)) + if err != nil { + return -1, -1, err + } + } + return begin, end, nil +} + +// Release releases the given CIDR range. +func (s *CidrSet) Release(cidr *net.IPNet) error { + begin, end, err := s.getBeginningAndEndIndices(cidr) + if err != nil { + return err + } + s.Lock() + defer s.Unlock() + for i := begin; i <= end; i++ { + // Only change the counters if we change the bit to prevent + // double counting. + if s.used.Bit(i) != 0 { + s.used.SetBit(&s.used, i, 0) + s.allocatedCIDRs-- + cidrSetReleases.WithLabelValues(s.label).Inc() + } + } + + cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs)) + return nil +} + +// Occupy marks the given CIDR range as used. Occupy succeeds even if the CIDR +// range was previously used. +func (s *CidrSet) Occupy(cidr *net.IPNet) (err error) { + begin, end, err := s.getBeginningAndEndIndices(cidr) + if err != nil { + return err + } + s.Lock() + defer s.Unlock() + for i := begin; i <= end; i++ { + // Only change the counters if we change the bit to prevent + // double counting. + if s.used.Bit(i) == 0 { + s.used.SetBit(&s.used, i, 1) + s.allocatedCIDRs++ + cidrSetAllocations.WithLabelValues(s.label).Inc() + } + } + + cidrSetUsage.WithLabelValues(s.label).Set(float64(s.allocatedCIDRs) / float64(s.maxCIDRs)) + return nil +} + +func (s *CidrSet) getIndexForIP(ip net.IP) (int, error) { + if ip.To4() != nil { + cidrIndex := (binary.BigEndian.Uint32(s.clusterCIDR.IP) ^ binary.BigEndian.Uint32(ip.To4())) >> uint32(32-s.nodeMaskSize) + if cidrIndex >= uint32(s.maxCIDRs) { + return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize) + } + return int(cidrIndex), nil + } + if ip.To16() != nil { + bigIP := big.NewInt(0).SetBytes(s.clusterCIDR.IP) + bigIP = bigIP.Xor(bigIP, big.NewInt(0).SetBytes(ip)) + cidrIndexBig := bigIP.Rsh(bigIP, uint(net.IPv6len*8-s.nodeMaskSize)) + cidrIndex := cidrIndexBig.Uint64() + if cidrIndex >= uint64(s.maxCIDRs) { + return 0, fmt.Errorf("CIDR: %v/%v is out of the range of CIDR allocator", ip, s.nodeMaskSize) + } + return int(cidrIndex), nil + } + + return 0, fmt.Errorf("invalid IP: %v", ip) +} + +// getMaxCIDRs returns the max number of CIDRs that can be obtained by subdividing a mask of size `clusterMaskSize` +// into subnets with mask of size `subNetMaskSize`. +func getMaxCIDRs(subNetMaskSize, clusterMaskSize int) int { + return 1 << uint32(subNetMaskSize-clusterMaskSize) +} diff --git a/cloud/nodeipam/ipam/cidrset/cidr_set_test.go b/cloud/nodeipam/ipam/cidrset/cidr_set_test.go new file mode 100644 index 00000000..9f34e3cd --- /dev/null +++ b/cloud/nodeipam/ipam/cidrset/cidr_set_test.go @@ -0,0 +1,1140 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cidrset + +import ( + "math/big" + "net" + "reflect" + "testing" + + "k8s.io/component-base/metrics/testutil" + netutils "k8s.io/utils/net" +) + +func TestCIDRSetFullyAllocated(t *testing.T) { + cases := []struct { + clusterCIDRStr string + subNetMaskSize int + expectedCIDR string + description string + }{ + { + clusterCIDRStr: "127.123.234.0/30", + subNetMaskSize: 30, + expectedCIDR: "127.123.234.0/30", + description: "Fully allocated CIDR with IPv4", + }, + { + clusterCIDRStr: "beef:1234::/30", + subNetMaskSize: 30, + expectedCIDR: "beef:1234::/30", + description: "Fully allocated CIDR with IPv6", + }, + } + for _, tc := range cases { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, tc.subNetMaskSize) + if err != nil { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + p, err := a.AllocateNext() + if err != nil { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + if p.String() != tc.expectedCIDR { + t.Fatalf("unexpected allocated cidr: %v, expecting %v for %v", + p.String(), tc.expectedCIDR, tc.description) + } + + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + + a.Release(p) + + p, err = a.AllocateNext() + if err != nil { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + if p.String() != tc.expectedCIDR { + t.Fatalf("unexpected allocated cidr: %v, expecting %v for %v", + p.String(), tc.expectedCIDR, tc.description) + } + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + } +} + +func TestIndexToCIDRBlock(t *testing.T) { + cases := []struct { + clusterCIDRStr string + subnetMaskSize int + index int + CIDRBlock string + description string + }{ + { + clusterCIDRStr: "127.123.3.0/16", + subnetMaskSize: 24, + index: 0, + CIDRBlock: "127.123.0.0/24", + description: "1st IP address indexed with IPv4", + }, + { + clusterCIDRStr: "127.123.0.0/16", + subnetMaskSize: 24, + index: 15, + CIDRBlock: "127.123.15.0/24", + description: "16th IP address indexed with IPv4", + }, + { + clusterCIDRStr: "192.168.5.219/28", + subnetMaskSize: 32, + index: 5, + CIDRBlock: "192.168.5.213/32", + description: "5th IP address indexed with IPv4", + }, + { + clusterCIDRStr: "2001:0db8:1234:3::/48", + subnetMaskSize: 64, + index: 0, + CIDRBlock: "2001:db8:1234::/64", + description: "1st IP address indexed with IPv6 /64", + }, + { + clusterCIDRStr: "2001:0db8:1234::/48", + subnetMaskSize: 64, + index: 15, + CIDRBlock: "2001:db8:1234:f::/64", + description: "16th IP address indexed with IPv6 /64", + }, + { + clusterCIDRStr: "2001:0db8:85a3::8a2e:0370:7334/50", + subnetMaskSize: 63, + index: 6425, + CIDRBlock: "2001:db8:85a3:3232::/63", + description: "6426th IP address indexed with IPv6 /63", + }, + { + clusterCIDRStr: "2001:0db8::/32", + subnetMaskSize: 48, + index: 0, + CIDRBlock: "2001:db8::/48", + description: "1st IP address indexed with IPv6 /48", + }, + { + clusterCIDRStr: "2001:0db8::/32", + subnetMaskSize: 48, + index: 15, + CIDRBlock: "2001:db8:f::/48", + description: "16th IP address indexed with IPv6 /48", + }, + { + clusterCIDRStr: "2001:0db8:85a3::8a2e:0370:7334/32", + subnetMaskSize: 48, + index: 6425, + CIDRBlock: "2001:db8:1919::/48", + description: "6426th IP address indexed with IPv6 /48", + }, + { + clusterCIDRStr: "2001:0db8:1234:ff00::/56", + subnetMaskSize: 72, + index: 0, + CIDRBlock: "2001:db8:1234:ff00::/72", + description: "1st IP address indexed with IPv6 /72", + }, + { + clusterCIDRStr: "2001:0db8:1234:ff00::/56", + subnetMaskSize: 72, + index: 15, + CIDRBlock: "2001:db8:1234:ff00:f00::/72", + description: "16th IP address indexed with IPv6 /72", + }, + { + clusterCIDRStr: "2001:0db8:1234:ff00::0370:7334/56", + subnetMaskSize: 72, + index: 6425, + CIDRBlock: "2001:db8:1234:ff19:1900::/72", + description: "6426th IP address indexed with IPv6 /72", + }, + { + clusterCIDRStr: "2001:0db8:1234:0:1234::/80", + subnetMaskSize: 96, + index: 0, + CIDRBlock: "2001:db8:1234:0:1234::/96", + description: "1st IP address indexed with IPv6 /96", + }, + { + clusterCIDRStr: "2001:0db8:1234:0:1234::/80", + subnetMaskSize: 96, + index: 15, + CIDRBlock: "2001:db8:1234:0:1234:f::/96", + description: "16th IP address indexed with IPv6 /96", + }, + { + clusterCIDRStr: "2001:0db8:1234:ff00::0370:7334/80", + subnetMaskSize: 96, + index: 6425, + CIDRBlock: "2001:db8:1234:ff00:0:1919::/96", + description: "6426th IP address indexed with IPv6 /96", + }, + } + for _, tc := range cases { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, tc.subnetMaskSize) + if err != nil { + t.Fatalf("error for %v ", tc.description) + } + cidr := a.indexToCIDRBlock(tc.index) + if cidr.String() != tc.CIDRBlock { + t.Fatalf("error for %v index %d %s", tc.description, tc.index, cidr.String()) + } + } +} + +func TestCIDRSet_RandomishAllocation(t *testing.T) { + cases := []struct { + clusterCIDRStr string + description string + }{ + { + clusterCIDRStr: "127.123.234.0/16", + description: "RandomishAllocation with IPv4", + }, + { + clusterCIDRStr: "beef:1234::/16", + description: "RandomishAllocation with IPv6", + }, + } + for _, tc := range cases { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, 24) + if err != nil { + t.Fatalf("Error allocating CIDRSet for %v", tc.description) + } + // allocate all the CIDRs + var cidrs []*net.IPNet + + for i := 0; i < 256; i++ { + if c, err := a.AllocateNext(); err == nil { + cidrs = append(cidrs, c) + } else { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + } + + //var err error + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + // release them all + for i := 0; i < len(cidrs); i++ { + a.Release(cidrs[i]) + } + + // allocate the CIDRs again + var rcidrs []*net.IPNet + for i := 0; i < 256; i++ { + if c, err := a.AllocateNext(); err == nil { + rcidrs = append(rcidrs, c) + } else { + t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description) + } + } + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + + if !reflect.DeepEqual(cidrs, rcidrs) { + t.Fatalf("expected re-allocated cidrs are the same collection for %v", tc.description) + } + } +} + +func TestCIDRSet_AllocationOccupied(t *testing.T) { + cases := []struct { + clusterCIDRStr string + description string + }{ + { + clusterCIDRStr: "127.123.234.0/16", + description: "AllocationOccupied with IPv4", + }, + { + clusterCIDRStr: "beef:1234::/16", + description: "AllocationOccupied with IPv6", + }, + } + for _, tc := range cases { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, 24) + if err != nil { + t.Fatalf("Error allocating CIDRSet for %v", tc.description) + } + // allocate all the CIDRs + var cidrs []*net.IPNet + var numCIDRs = 256 + + for i := 0; i < numCIDRs; i++ { + if c, err := a.AllocateNext(); err == nil { + cidrs = append(cidrs, c) + } else { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + } + + //var err error + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + // release them all + for i := 0; i < len(cidrs); i++ { + a.Release(cidrs[i]) + } + // occupy the last 128 CIDRs + for i := numCIDRs / 2; i < numCIDRs; i++ { + a.Occupy(cidrs[i]) + } + // occupy the first of the last 128 again + a.Occupy(cidrs[numCIDRs/2]) + + // allocate the first 128 CIDRs again + var rcidrs []*net.IPNet + for i := 0; i < numCIDRs/2; i++ { + if c, err := a.AllocateNext(); err == nil { + rcidrs = append(rcidrs, c) + } else { + t.Fatalf("unexpected error: %d, %v for %v", i, err, tc.description) + } + } + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("expected error because of fully-allocated range for %v", tc.description) + } + + // check Occupy() work properly + for i := numCIDRs / 2; i < numCIDRs; i++ { + rcidrs = append(rcidrs, cidrs[i]) + } + if !reflect.DeepEqual(cidrs, rcidrs) { + t.Fatalf("expected re-allocated cidrs are the same collection for %v", tc.description) + } + } +} + +func TestDoubleOccupyRelease(t *testing.T) { + // Run a sequence of operations and check the number of occupied CIDRs + // after each one. + clusterCIDRStr := "10.42.0.0/16" + operations := []struct { + cidrStr string + operation string + numOccupied int + }{ + // Occupy 1 element: +1 + { + cidrStr: "10.42.5.0/24", + operation: "occupy", + numOccupied: 1, + }, + // Occupy 1 more element: +1 + { + cidrStr: "10.42.9.0/24", + operation: "occupy", + numOccupied: 2, + }, + // Occupy 4 elements overlapping with one from the above: +3 + { + cidrStr: "10.42.8.0/22", + operation: "occupy", + numOccupied: 5, + }, + // Occupy an already-coccupied element: no change + { + cidrStr: "10.42.9.0/24", + operation: "occupy", + numOccupied: 5, + }, + // Release an coccupied element: -1 + { + cidrStr: "10.42.9.0/24", + operation: "release", + numOccupied: 4, + }, + // Release an unoccupied element: no change + { + cidrStr: "10.42.9.0/24", + operation: "release", + numOccupied: 4, + }, + // Release 4 elements, only one of which is occupied: -1 + { + cidrStr: "10.42.4.0/22", + operation: "release", + numOccupied: 3, + }, + } + // Check that there are exactly that many allocatable CIDRs after all + // operations have been executed. + numAllocatable24s := (1 << 8) - 3 + + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, 24) + if err != nil { + t.Fatalf("Error allocating CIDRSet") + } + + // Execute the operations + for _, op := range operations { + _, cidr, _ := netutils.ParseCIDRSloppy(op.cidrStr) + switch op.operation { + case "occupy": + a.Occupy(cidr) + case "release": + a.Release(cidr) + default: + t.Fatalf("test error: unknown operation %v", op.operation) + } + if a.allocatedCIDRs != op.numOccupied { + t.Fatalf("Expected %d occupied CIDRS, got %d", op.numOccupied, a.allocatedCIDRs) + } + } + + // Make sure that we can allocate exactly `numAllocatable24s` elements. + for i := 0; i < numAllocatable24s; i++ { + _, err := a.AllocateNext() + if err != nil { + t.Fatalf("Expected to be able to allocate %d CIDRS, failed after %d", numAllocatable24s, i) + } + } + + _, err = a.AllocateNext() + if err == nil { + t.Fatalf("Expected to be able to allocate exactly %d CIDRS, got one more", numAllocatable24s) + } +} + +func TestGetBitforCIDR(t *testing.T) { + cases := []struct { + clusterCIDRStr string + subNetMaskSize int + subNetCIDRStr string + expectedBit int + expectErr bool + description string + }{ + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.0.0.0/16", + expectedBit: 0, + expectErr: false, + description: "Get 0 Bit with IPv4", + }, + { + clusterCIDRStr: "be00::/8", + subNetMaskSize: 16, + subNetCIDRStr: "be00::/16", + expectedBit: 0, + expectErr: false, + description: "Get 0 Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.123.0.0/16", + expectedBit: 123, + expectErr: false, + description: "Get 123rd Bit with IPv4", + }, + { + clusterCIDRStr: "be00::/8", + subNetMaskSize: 16, + subNetCIDRStr: "beef::/16", + expectedBit: 0xef, + expectErr: false, + description: "Get xef Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.168.0.0/16", + expectedBit: 168, + expectErr: false, + description: "Get 168th Bit with IPv4", + }, + { + clusterCIDRStr: "be00::/8", + subNetMaskSize: 16, + subNetCIDRStr: "be68::/16", + expectedBit: 0x68, + expectErr: false, + description: "Get x68th Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.224.0.0/16", + expectedBit: 224, + expectErr: false, + description: "Get 224th Bit with IPv4", + }, + { + clusterCIDRStr: "be00::/8", + subNetMaskSize: 16, + subNetCIDRStr: "be24::/16", + expectedBit: 0x24, + expectErr: false, + description: "Get x24th Bit with IPv6", + }, + { + clusterCIDRStr: "192.168.0.0/16", + subNetMaskSize: 24, + subNetCIDRStr: "192.168.12.0/24", + expectedBit: 12, + expectErr: false, + description: "Get 12th Bit with IPv4", + }, + { + clusterCIDRStr: "beef::/16", + subNetMaskSize: 24, + subNetCIDRStr: "beef:1200::/24", + expectedBit: 0x12, + expectErr: false, + description: "Get x12th Bit with IPv6", + }, + { + clusterCIDRStr: "192.168.0.0/16", + subNetMaskSize: 24, + subNetCIDRStr: "192.168.151.0/24", + expectedBit: 151, + expectErr: false, + description: "Get 151st Bit with IPv4", + }, + { + clusterCIDRStr: "beef::/16", + subNetMaskSize: 24, + subNetCIDRStr: "beef:9700::/24", + expectedBit: 0x97, + expectErr: false, + description: "Get x97st Bit with IPv6", + }, + { + clusterCIDRStr: "192.168.0.0/16", + subNetMaskSize: 24, + subNetCIDRStr: "127.168.224.0/24", + expectErr: true, + description: "Get error with IPv4", + }, + { + clusterCIDRStr: "beef::/16", + subNetMaskSize: 24, + subNetCIDRStr: "2001:db00::/24", + expectErr: true, + description: "Get error with IPv6", + }, + } + + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + _, clusterCIDR, err := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + cs, err := NewCIDRSet(clusterCIDR, tc.subNetMaskSize) + if err != nil { + t.Fatalf("Error allocating CIDRSet") + } + _, subnetCIDR, err := netutils.ParseCIDRSloppy(tc.subNetCIDRStr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got, err := cs.getIndexForIP(subnetCIDR.IP) + if err == nil && tc.expectErr { + t.Errorf("expected error but got null") + return + } + + if err != nil && !tc.expectErr { + t.Errorf("unexpected error: %v", err) + return + } + + if got != tc.expectedBit { + t.Errorf("expected %v, but got %v", tc.expectedBit, got) + } + }) + } +} + +func TestOccupy(t *testing.T) { + cases := []struct { + clusterCIDRStr string + subNetMaskSize int + subNetCIDRStr string + expectedUsedBegin int + expectedUsedEnd int + expectErr bool + description string + }{ + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.0.0.0/8", + expectedUsedBegin: 0, + expectedUsedEnd: 255, + expectErr: false, + description: "Occupy all Bits with IPv4", + }, + { + clusterCIDRStr: "2001:beef:1200::/40", + subNetMaskSize: 48, + subNetCIDRStr: "2001:beef:1200::/40", + expectedUsedBegin: 0, + expectedUsedEnd: 255, + expectErr: false, + description: "Occupy all Bits with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.0.0.0/2", + expectedUsedBegin: 0, + expectedUsedEnd: 255, + expectErr: false, + description: "Occupy every Bit with IPv4", + }, + { + clusterCIDRStr: "2001:beef:1200::/40", + subNetMaskSize: 48, + subNetCIDRStr: "2001:beef:1234::/34", + expectedUsedBegin: 0, + expectedUsedEnd: 255, + expectErr: false, + description: "Occupy every Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 16, + subNetCIDRStr: "127.0.0.0/16", + expectedUsedBegin: 0, + expectedUsedEnd: 0, + expectErr: false, + description: "Occupy 1st Bit with IPv4", + }, + { + clusterCIDRStr: "2001:beef:1200::/40", + subNetMaskSize: 48, + subNetCIDRStr: "2001:beef:1200::/48", + expectedUsedBegin: 0, + expectedUsedEnd: 0, + expectErr: false, + description: "Occupy 1st Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 32, + subNetCIDRStr: "127.0.0.0/16", + expectedUsedBegin: 0, + expectedUsedEnd: 65535, + expectErr: false, + description: "Occupy 65535 Bits with IPv4", + }, + { + clusterCIDRStr: "2001:beef:1200::/48", + subNetMaskSize: 64, + subNetCIDRStr: "2001:beef:1200::/48", + expectedUsedBegin: 0, + expectedUsedEnd: 65535, + expectErr: false, + description: "Occupy 65535 Bits with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/7", + subNetMaskSize: 16, + subNetCIDRStr: "127.0.0.0/15", + expectedUsedBegin: 256, + expectedUsedEnd: 257, + expectErr: false, + description: "Occupy 257th Bit with IPv4", + }, + { + clusterCIDRStr: "2001:beef:7f00::/39", + subNetMaskSize: 48, + subNetCIDRStr: "2001:beef:7f00::/47", + expectedUsedBegin: 256, + expectedUsedEnd: 257, + expectErr: false, + description: "Occupy 257th Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/7", + subNetMaskSize: 15, + subNetCIDRStr: "127.0.0.0/15", + expectedUsedBegin: 128, + expectedUsedEnd: 128, + expectErr: false, + description: "Occupy 128th Bit with IPv4", + }, + { + clusterCIDRStr: "2001:beef:7f00::/39", + subNetMaskSize: 47, + subNetCIDRStr: "2001:beef:7f00::/47", + expectedUsedBegin: 128, + expectedUsedEnd: 128, + expectErr: false, + description: "Occupy 128th Bit with IPv6", + }, + { + clusterCIDRStr: "127.0.0.0/7", + subNetMaskSize: 18, + subNetCIDRStr: "127.0.0.0/15", + expectedUsedBegin: 1024, + expectedUsedEnd: 1031, + expectErr: false, + description: "Occupy 1031st Bit with IPv4", + }, + { + clusterCIDRStr: "2001:beef:7f00::/39", + subNetMaskSize: 50, + subNetCIDRStr: "2001:beef:7f00::/47", + expectedUsedBegin: 1024, + expectedUsedEnd: 1031, + expectErr: false, + description: "Occupy 1031st Bit with IPv6", + }, + } + + for _, tc := range cases { + _, clusterCIDR, err := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + if err != nil { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + + cs, err := NewCIDRSet(clusterCIDR, tc.subNetMaskSize) + if err != nil { + t.Fatalf("Error allocating CIDRSet for %v", tc.description) + } + + _, subnetCIDR, err := netutils.ParseCIDRSloppy(tc.subNetCIDRStr) + if err != nil { + t.Fatalf("unexpected error: %v for %v", err, tc.description) + } + + err = cs.Occupy(subnetCIDR) + if err == nil && tc.expectErr { + t.Errorf("expected error but got none for %v", tc.description) + continue + } + if err != nil && !tc.expectErr { + t.Errorf("unexpected error: %v for %v", err, tc.description) + continue + } + + expectedUsed := big.Int{} + for i := tc.expectedUsedBegin; i <= tc.expectedUsedEnd; i++ { + expectedUsed.SetBit(&expectedUsed, i, 1) + } + if expectedUsed.Cmp(&cs.used) != 0 { + t.Errorf("error for %v", tc.description) + } + } +} + +func TestCIDRSetv6(t *testing.T) { + cases := []struct { + clusterCIDRStr string + subNetMaskSize int + expectedCIDR string + expectedCIDR2 string + expectErr bool + description string + }{ + { + clusterCIDRStr: "127.0.0.0/8", + subNetMaskSize: 32, + expectErr: false, + expectedCIDR: "127.0.0.0/32", + expectedCIDR2: "127.0.0.1/32", + description: "Max cluster subnet size with IPv4", + }, + { + clusterCIDRStr: "beef:1234::/32", + subNetMaskSize: 49, + expectErr: true, + description: "Max cluster subnet size with IPv6", + }, + { + clusterCIDRStr: "2001:beef:1234:369b::/60", + subNetMaskSize: 64, + expectedCIDR: "2001:beef:1234:3690::/64", + expectedCIDR2: "2001:beef:1234:3691::/64", + expectErr: false, + description: "Allocate a few IPv6", + }, + } + for _, tc := range cases { + t.Run(tc.description, func(t *testing.T) { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(tc.clusterCIDRStr) + a, err := NewCIDRSet(clusterCIDR, tc.subNetMaskSize) + if gotErr := err != nil; gotErr != tc.expectErr { + t.Fatalf("NewCIDRSet(%v, %v) = %v, %v; gotErr = %t, want %t", clusterCIDR, tc.subNetMaskSize, a, err, gotErr, tc.expectErr) + } + if a == nil { + return + } + p, err := a.AllocateNext() + if err == nil && tc.expectErr { + t.Errorf("a.AllocateNext() = nil, want error") + } + if err != nil && !tc.expectErr { + t.Errorf("a.AllocateNext() = %+v, want no error", err) + } + if !tc.expectErr { + if p != nil && p.String() != tc.expectedCIDR { + t.Fatalf("a.AllocateNext() got %+v, want %+v", p.String(), tc.expectedCIDR) + } + } + p2, err := a.AllocateNext() + if err == nil && tc.expectErr { + t.Errorf("a.AllocateNext() = nil, want error") + } + if err != nil && !tc.expectErr { + t.Errorf("a.AllocateNext() = %+v, want no error", err) + } + if !tc.expectErr { + if p2 != nil && p2.String() != tc.expectedCIDR2 { + t.Fatalf("a.AllocateNext() got %+v, want %+v", p2.String(), tc.expectedCIDR) + } + } + }) + } +} + +func TestCidrSetMetrics(t *testing.T) { + cidr := "10.0.0.0/16" + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + + // We have 256 free cidrs + a, err := NewCIDRSet(clusterCIDR, 24) + if err != nil { + t.Fatalf("unexpected error creating CidrSet: %v", err) + } + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) + + // Allocate next all + for i := 1; i <= 256; i++ { + _, err := a.AllocateNext() + if err != nil { + t.Fatalf("unexpected error allocating a new CIDR: %v", err) + } + em := testMetrics{ + usage: float64(i) / float64(256), + allocs: float64(i), + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) + } + // Release all + a.Release(clusterCIDR) + em = testMetrics{ + usage: 0, + allocs: 256, + releases: 256, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) + + // Allocate all + a.Occupy(clusterCIDR) + em = testMetrics{ + usage: 1, + allocs: 512, + releases: 256, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) +} + +func TestCidrSetMetricsHistogram(t *testing.T) { + cidr := "10.0.0.0/16" + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(cidr) + clearMetrics(map[string]string{"clusterCIDR": cidr}) + + // We have 256 free cidrs + a, err := NewCIDRSet(clusterCIDR, 24) + if err != nil { + t.Fatalf("unexpected error creating CidrSet: %v", err) + } + + clusterMaskSize, _ := clusterCIDR.Mask.Size() + max := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) + + // Allocate half of the range + // Occupy does not update the nextCandidate + _, halfClusterCIDR, _ := netutils.ParseCIDRSloppy("10.0.0.0/17") + a.Occupy(halfClusterCIDR) + em = testMetrics{ + usage: 0.5, + allocs: 128, + releases: 0, + allocTries: 0, + max: float64(max), + } + expectMetrics(t, cidr, em) + // Allocate next should iterate until the next free cidr + // that is exactly the same number we allocated previously + _, err = a.AllocateNext() + if err != nil { + t.Fatalf("unexpected error allocating a new CIDR: %v", err) + } + em = testMetrics{ + usage: float64(129) / float64(256), + allocs: 129, + releases: 0, + allocTries: 128, + max: float64(max), + } + expectMetrics(t, cidr, em) +} + +func TestCidrSetMetricsDual(t *testing.T) { + // create IPv4 cidrSet + cidrIPv4 := "10.0.0.0/16" + _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv4}) + + a, err := NewCIDRSet(clusterCIDRv4, 24) + if err != nil { + t.Fatalf("unexpected error creating CidrSet: %v", err) + } + + clusterMaskSize, _ := clusterCIDRv4.Mask.Size() + maxIPv4 := getMaxCIDRs(24, clusterMaskSize) + em := testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv4), + } + expectMetrics(t, cidrIPv4, em) + + // create IPv6 cidrSet + cidrIPv6 := "2001:db8::/48" + _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy(cidrIPv6) + clearMetrics(map[string]string{"clusterCIDR": cidrIPv6}) + + b, err := NewCIDRSet(clusterCIDRv6, 64) + if err != nil { + t.Fatalf("unexpected error creating CidrSet: %v", err) + } + + clusterMaskSize, _ = clusterCIDRv6.Mask.Size() + maxIPv6 := getMaxCIDRs(64, clusterMaskSize) + em = testMetrics{ + usage: 0, + allocs: 0, + releases: 0, + allocTries: 0, + max: float64(maxIPv6), + } + expectMetrics(t, cidrIPv6, em) + + // Allocate all + a.Occupy(clusterCIDRv4) + em = testMetrics{ + usage: 1, + allocs: 256, + releases: 0, + allocTries: 0, + max: float64(maxIPv4), + } + expectMetrics(t, cidrIPv4, em) + + b.Occupy(clusterCIDRv6) + em = testMetrics{ + usage: 1, + allocs: 65536, + releases: 0, + allocTries: 0, + max: float64(maxIPv6), + } + expectMetrics(t, cidrIPv6, em) + + // Release all + a.Release(clusterCIDRv4) + em = testMetrics{ + usage: 0, + allocs: 256, + releases: 256, + allocTries: 0, + max: float64(maxIPv4), + } + expectMetrics(t, cidrIPv4, em) + b.Release(clusterCIDRv6) + em = testMetrics{ + usage: 0, + allocs: 65536, + releases: 65536, + allocTries: 0, + max: float64(maxIPv6), + } + expectMetrics(t, cidrIPv6, em) +} + +func Test_getMaxCIDRs(t *testing.T) { + cidrIPv4 := "10.0.0.0/16" + _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) + + cidrIPv6 := "2001:db8::/48" + _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy(cidrIPv6) + + tests := []struct { + name string + subNetMaskSize int + clusterCIDR *net.IPNet + expectedMaxCIDRs int + }{ + { + name: "IPv4", + subNetMaskSize: 24, + clusterCIDR: clusterCIDRv4, + expectedMaxCIDRs: 256, + }, + { + name: "IPv6", + subNetMaskSize: 64, + clusterCIDR: clusterCIDRv6, + expectedMaxCIDRs: 65536, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clusterMaskSize, _ := test.clusterCIDR.Mask.Size() + maxCIDRs := getMaxCIDRs(test.subNetMaskSize, clusterMaskSize) + if test.expectedMaxCIDRs != maxCIDRs { + t.Errorf("incorrect maxCIDRs, expected: %d, got: %d", test.expectedMaxCIDRs, maxCIDRs) + } + }) + } +} + +// Metrics helpers +func clearMetrics(labels map[string]string) { + cidrSetAllocations.Delete(labels) + cidrSetReleases.Delete(labels) + cidrSetUsage.Delete(labels) + cidrSetAllocationTriesPerRequest.Delete(labels) + cidrSetMaxCidrs.Delete(labels) +} + +type testMetrics struct { + usage float64 + allocs float64 + releases float64 + allocTries float64 + max float64 +} + +func expectMetrics(t *testing.T, label string, em testMetrics) { + var m testMetrics + var err error + m.usage, err = testutil.GetGaugeMetricValue(cidrSetUsage.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetUsage.Name, err) + } + m.allocs, err = testutil.GetCounterMetricValue(cidrSetAllocations.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetAllocations.Name, err) + } + m.releases, err = testutil.GetCounterMetricValue(cidrSetReleases.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetReleases.Name, err) + } + m.allocTries, err = testutil.GetHistogramMetricValue(cidrSetAllocationTriesPerRequest.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetAllocationTriesPerRequest.Name, err) + } + m.max, err = testutil.GetGaugeMetricValue(cidrSetMaxCidrs.WithLabelValues(label)) + if err != nil { + t.Errorf("failed to get %s value, err: %v", cidrSetMaxCidrs.Name, err) + } + + if m != em { + t.Fatalf("metrics error: expected %v, received %v", em, m) + } +} + +// Benchmarks +func benchmarkAllocateAllIPv6(cidr string, subnetMaskSize int, b *testing.B) { + _, clusterCIDR, _ := netutils.ParseCIDRSloppy(cidr) + a, _ := NewCIDRSet(clusterCIDR, subnetMaskSize) + for n := 0; n < b.N; n++ { + // Allocate the whole range + 1 + for i := 0; i <= a.maxCIDRs; i++ { + a.AllocateNext() + } + // Release all + a.Release(clusterCIDR) + } +} + +func BenchmarkAllocateAll_48_52(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/48", 52, b) } +func BenchmarkAllocateAll_48_56(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/48", 56, b) } + +func BenchmarkAllocateAll_48_60(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/48", 60, b) } +func BenchmarkAllocateAll_48_64(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/48", 64, b) } + +func BenchmarkAllocateAll_64_68(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/64", 68, b) } + +func BenchmarkAllocateAll_64_72(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/64", 72, b) } +func BenchmarkAllocateAll_64_76(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/64", 76, b) } + +func BenchmarkAllocateAll_64_80(b *testing.B) { benchmarkAllocateAllIPv6("2001:db8::/64", 80, b) } diff --git a/cloud/nodeipam/ipam/cidrset/metrics.go b/cloud/nodeipam/ipam/cidrset/metrics.go new file mode 100644 index 00000000..b32c94ca --- /dev/null +++ b/cloud/nodeipam/ipam/cidrset/metrics.go @@ -0,0 +1,89 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cidrset + +import ( + "sync" + + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const nodeIpamSubsystem = "node_ipam_controller" + +var ( + cidrSetAllocations = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cidrset_cidrs_allocations_total", + Help: "Counter measuring total number of CIDR allocations.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) + cidrSetReleases = metrics.NewCounterVec( + &metrics.CounterOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cidrset_cidrs_releases_total", + Help: "Counter measuring total number of CIDR releases.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) + // This is a gauge, as in theory, a limit can increase or decrease. + cidrSetMaxCidrs = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cirdset_max_cidrs", + Help: "Maximum number of CIDRs that can be allocated.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) + cidrSetUsage = metrics.NewGaugeVec( + &metrics.GaugeOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cidrset_usage_cidrs", + Help: "Gauge measuring percentage of allocated CIDRs.", + StabilityLevel: metrics.ALPHA, + }, + []string{"clusterCIDR"}, + ) + cidrSetAllocationTriesPerRequest = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: nodeIpamSubsystem, + Name: "cidrset_allocation_tries_per_request", + Help: "Number of endpoints added on each Service sync", + StabilityLevel: metrics.ALPHA, + Buckets: metrics.ExponentialBuckets(1, 5, 5), + }, + []string{"clusterCIDR"}, + ) +) + +var registerMetrics sync.Once + +// registerCidrsetMetrics the metrics that are to be monitored. +func registerCidrsetMetrics() { + registerMetrics.Do(func() { + legacyregistry.MustRegister(cidrSetAllocations) + legacyregistry.MustRegister(cidrSetReleases) + legacyregistry.MustRegister(cidrSetMaxCidrs) + legacyregistry.MustRegister(cidrSetUsage) + legacyregistry.MustRegister(cidrSetAllocationTriesPerRequest) + }) +} diff --git a/cloud/nodeipam/ipam/controller_test.go b/cloud/nodeipam/ipam/controller_test.go new file mode 100644 index 00000000..fa766ab4 --- /dev/null +++ b/cloud/nodeipam/ipam/controller_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipam + +import ( + "net" + "testing" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/cidrset" + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/test" +) + +func TestOccupyServiceCIDR(t *testing.T) { + const clusterCIDR = "10.1.0.0/16" + +TestCase: + for _, tc := range []struct { + serviceCIDR string + }{ + {"10.0.255.0/24"}, + {"10.1.0.0/24"}, + {"10.1.255.0/24"}, + {"10.2.0.0/24"}, + } { + serviceCIDR := test.MustParseCIDR(tc.serviceCIDR) + set, err := cidrset.NewCIDRSet(test.MustParseCIDR(clusterCIDR), 24) + if err != nil { + t.Errorf("test case %+v: NewCIDRSet() = %v, want nil", tc, err) + } + if err := occupyServiceCIDR(set, test.MustParseCIDR(clusterCIDR), serviceCIDR); err != nil { + t.Errorf("test case %+v: occupyServiceCIDR() = %v, want nil", tc, err) + } + // Allocate until full. + var cidrs []*net.IPNet + for { + cidr, err := set.AllocateNext() + if err != nil { + if err == cidrset.ErrCIDRRangeNoCIDRsRemaining { + break + } + t.Errorf("set.AllocateNext() = %v, want %v", err, cidrset.ErrCIDRRangeNoCIDRsRemaining) + continue TestCase + } + cidrs = append(cidrs, cidr) + } + // No allocated CIDR range should intersect with serviceCIDR. + for _, c := range cidrs { + if c.Contains(serviceCIDR.IP) || serviceCIDR.Contains(c.IP) { + t.Errorf("test case %+v: allocated CIDR %v from service range", tc, c) + } + } + } +} diff --git a/cloud/nodeipam/ipam/doc.go b/cloud/nodeipam/ipam/doc.go new file mode 100644 index 00000000..eab481f9 --- /dev/null +++ b/cloud/nodeipam/ipam/doc.go @@ -0,0 +1,30 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ipam provides different allocators for assigning IP ranges to nodes. +// We currently support several kinds of IPAM allocators (these are denoted by +// the CIDRAllocatorType): +// - RangeAllocator is an allocator that assigns PodCIDRs to nodes and works +// in conjunction with the RouteController to configure the network to get +// connectivity. +// - CloudAllocator is an allocator that synchronizes PodCIDRs from IP +// ranges assignments from the underlying cloud platform. +// - (Alpha only) IPAMFromCluster is an allocator that has the similar +// functionality as the RangeAllocator but also synchronizes cluster-managed +// ranges into the cloud platform. +// - (Alpha only) IPAMFromCloud is the same as CloudAllocator (synchronizes +// from cloud into the cluster.) +package ipam diff --git a/cloud/nodeipam/ipam/range_allocator.go b/cloud/nodeipam/ipam/range_allocator.go new file mode 100644 index 00000000..ac7ba933 --- /dev/null +++ b/cloud/nodeipam/ipam/range_allocator.go @@ -0,0 +1,454 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipam + +import ( + "context" + "fmt" + "net" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + netutils "k8s.io/utils/net" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/cidrset" + controllerutil "github.com/linode/linode-cloud-controller-manager/cloud/util/controller/node" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + informers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + nodeutil "k8s.io/component-helpers/node/util" +) + +type rangeAllocator struct { + client clientset.Interface + // cluster cidrs as passed in during controller creation + clusterCIDRs []*net.IPNet + // for each entry in clusterCIDRs we maintain a list of what is used and what is not + cidrSets []*cidrset.CidrSet + // nodeLister is able to list/get nodes and is populated by the shared informer passed to controller + nodeLister corelisters.NodeLister + // nodesSynced returns true if the node shared informer has been synced at least once. + nodesSynced cache.InformerSynced + broadcaster record.EventBroadcaster + recorder record.EventRecorder + + // queues are where incoming work is placed to de-dup and to allow "easy" + // rate limited requeues on errors + queue workqueue.RateLimitingInterface +} + +var _ CIDRAllocator = &rangeAllocator{} + +// NewCIDRRangeAllocator returns a CIDRAllocator to allocate CIDRs for node (one from each of clusterCIDRs) +// Caller must ensure subNetMaskSize is not less than cluster CIDR mask size. +// Caller must always pass in a list of existing nodes so the new allocator. +// Caller must ensure that ClusterCIDRs are semantically correct e.g (1 for non DualStack, 2 for DualStack etc..) +// can initialize its CIDR map. NodeList is only nil in testing. +func NewCIDRRangeAllocator(client clientset.Interface, nodeInformer informers.NodeInformer, allocatorParams CIDRAllocatorParams, nodeList *v1.NodeList) (CIDRAllocator, error) { + if client == nil { + klog.Fatal("kubeClient is nil when starting CIDRRangeAllocator") + } + + eventBroadcaster := record.NewBroadcaster() + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "cidrAllocator"}) + + // create a cidrSet for each cidr we operate on + // cidrSet are mapped to clusterCIDR by index + cidrSets := make([]*cidrset.CidrSet, len(allocatorParams.ClusterCIDRs)) + for idx, cidr := range allocatorParams.ClusterCIDRs { + cidrSet, err := cidrset.NewCIDRSet(cidr, allocatorParams.NodeCIDRMaskSizes[idx]) + if err != nil { + return nil, err + } + cidrSets[idx] = cidrSet + } + + ra := &rangeAllocator{ + client: client, + clusterCIDRs: allocatorParams.ClusterCIDRs, + cidrSets: cidrSets, + nodeLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + broadcaster: eventBroadcaster, + recorder: recorder, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "cidrallocator_node"), + } + + if allocatorParams.ServiceCIDR != nil { + ra.filterOutServiceRange(allocatorParams.ServiceCIDR) + } else { + klog.Info("No Service CIDR provided. Skipping filtering out service addresses") + } + + if allocatorParams.SecondaryServiceCIDR != nil { + ra.filterOutServiceRange(allocatorParams.SecondaryServiceCIDR) + } else { + klog.Info("No Secondary Service CIDR provided. Skipping filtering out secondary service addresses") + } + + if nodeList != nil { + for _, node := range nodeList.Items { + if len(node.Spec.PodCIDRs) == 0 { + klog.V(4).Info("Node has no CIDR, ignoring", "node", klog.KObj(&node)) + continue + } + klog.V(4).Info("Node has CIDR, occupying it in CIDR map", "node", klog.KObj(&node), "podCIDR", node.Spec.PodCIDR) + if err := ra.occupyCIDRs(&node); err != nil { + // This will happen if: + // 1. We find garbage in the podCIDRs field. Retrying is useless. + // 2. CIDR out of range: This means a node CIDR has changed. + // This error will keep crashing controller-manager. + return nil, err + } + } + } + + nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err == nil { + ra.queue.Add(key) + } + }, + UpdateFunc: func(old, new interface{}) { + key, err := cache.MetaNamespaceKeyFunc(new) + if err == nil { + ra.queue.Add(key) + } + }, + DeleteFunc: func(obj interface{}) { + // The informer cache no longer has the object, and since Node doesn't have a finalizer, + // we don't see the Update with DeletionTimestamp != 0. + node, ok := obj.(*v1.Node) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + utilruntime.HandleError(fmt.Errorf("unexpected object type: %v", obj)) + return + } + node, ok = tombstone.Obj.(*v1.Node) + if !ok { + utilruntime.HandleError(fmt.Errorf("unexpected object types: %v", obj)) + return + } + } + if err := ra.ReleaseCIDR(node); err != nil { + utilruntime.HandleError(fmt.Errorf("error while processing CIDR Release: %w", err)) + + } + }, + }) + + return ra, nil +} + +func (r *rangeAllocator) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + + // Start event processing pipeline. + r.broadcaster.StartStructuredLogging(3) + klog.Info("Sending events to api server") + r.broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: r.client.CoreV1().Events("")}) + defer r.broadcaster.Shutdown() + + defer r.queue.ShutDown() + + klog.Info("Starting range CIDR allocator") + defer klog.Info("Shutting down range CIDR allocator") + + if !cache.WaitForNamedCacheSync("cidrallocator", stopCh, r.nodesSynced) { + return + } + + for i := 0; i < cidrUpdateWorkers; i++ { + go r.runWorker(stopCh) + } + + <-stopCh +} + +// runWorker is a long-running function that will continually call the +// processNextWorkItem function in order to read and process a message on the +// queue. +func (r *rangeAllocator) runWorker(stopCh <-chan struct{}) { + for { + select { + case <-stopCh: + return + default: + r.processNextNodeWorkItem() + } + } +} + +// processNextWorkItem will read a single work item off the queue and +// attempt to process it, by calling the syncHandler. +func (r *rangeAllocator) processNextNodeWorkItem() bool { + obj, shutdown := r.queue.Get() + if shutdown { + return false + } + + logger := klog.Background() + ctx, cancel := context.WithTimeout(context.Background(), nodePollInterval) + defer cancel() + ctx = klog.NewContext(ctx, logger) + + // We wrap this block in a func so we can defer r.queue.Done. + err := func(logger klog.Logger, obj interface{}) error { + // We call Done here so the workNodeQueue knows we have finished + // processing this item. We also must remember to call Forget if we + // do not want this work item being re-queued. For example, we do + // not call Forget if a transient error occurs, instead the item is + // put back on the queue and attempted again after a back-off + // period. + defer r.queue.Done(obj) + var key string + var ok bool + // We expect strings to come off the workNodeQueue. These are of the + // form namespace/name. We do this as the delayed nature of the + // workNodeQueue means the items in the informer cache may actually be + // more up to date that when the item was initially put onto the + // workNodeQueue. + if key, ok = obj.(string); !ok { + // As the item in the workNodeQueue is actually invalid, we call + // Forget here else we'd go into a loop of attempting to + // process a work item that is invalid. + r.queue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in workNodeQueue but got %#v", obj)) + return nil + } + // Run the syncHandler, passing it the namespace/name string of the + // Foo resource to be synced. + if err := r.syncNode(ctx, key); err != nil { + // Put the item back on the queue to handle any transient errors. + r.queue.AddRateLimited(key) + return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) + } + // Finally, if no error occurs we Forget this item so it does not + // get queue again until another change happens. + r.queue.Forget(obj) + logger.V(4).Info("Successfully synced", "key", key) + return nil + }(klog.FromContext(ctx), obj) + + if err != nil { + utilruntime.HandleError(err) + return true + } + + return true +} + +func (r *rangeAllocator) syncNode(ctx context.Context, key string) error { + logger := klog.FromContext(ctx) + startTime := time.Now() + defer func() { + logger.V(4).Info("Finished syncing Node request", "node", key, "elapsed", time.Since(startTime)) + }() + + node, err := r.nodeLister.Get(key) + if apierrors.IsNotFound(err) { + logger.V(3).Info("node has been deleted", "node", key) + // TODO: obtain the node object information to call ReleaseCIDR from here + // and retry if there is an error. + return nil + } + if err != nil { + return err + } + // Check the DeletionTimestamp to determine if object is under deletion. + if !node.DeletionTimestamp.IsZero() { + logger.V(3).Info("node is being deleted", "node", key) + return nil + } + return r.AllocateOrOccupyCIDR(ctx, node) +} + +// marks node.PodCIDRs[...] as used in allocator's tracked cidrSet +func (r *rangeAllocator) occupyCIDRs(node *v1.Node) error { + if len(node.Spec.PodCIDRs) == 0 { + return nil + } + for idx, cidr := range node.Spec.PodCIDRs { + _, podCIDR, err := netutils.ParseCIDRSloppy(cidr) + if err != nil { + return fmt.Errorf("failed to parse node %s, CIDR %s", node.Name, node.Spec.PodCIDR) + } + // If node has a pre allocate cidr that does not exist in our cidrs. + // This will happen if cluster went from dualstack(multi cidrs) to non-dualstack + // then we have now way of locking it + if idx >= len(r.cidrSets) { + return fmt.Errorf("node:%s has an allocated cidr: %v at index:%v that does not exist in cluster cidrs configuration", node.Name, cidr, idx) + } + + if err := r.cidrSets[idx].Occupy(podCIDR); err != nil { + return fmt.Errorf("failed to mark cidr[%v] at idx [%v] as occupied for node: %v: %v", podCIDR, idx, node.Name, err) + } + } + return nil +} + +// WARNING: If you're adding any return calls or defer any more work from this +// function you have to make sure to update nodesInProcessing properly with the +// disposition of the node when the work is done. +func (r *rangeAllocator) AllocateOrOccupyCIDR(ctx context.Context, node *v1.Node) error { + if node == nil { + return nil + } + + if len(node.Spec.PodCIDRs) > 0 { + return r.occupyCIDRs(node) + } + + logger := klog.FromContext(ctx) + allocatedCIDRs := make([]*net.IPNet, len(r.cidrSets)) + + for idx := range r.cidrSets { + podCIDR, err := r.cidrSets[idx].AllocateNext() + if err != nil { + controllerutil.RecordNodeStatusChange(logger, r.recorder, node, "CIDRNotAvailable") + return fmt.Errorf("failed to allocate cidr from cluster cidr at idx:%v: %v", idx, err) + } + allocatedCIDRs[idx] = podCIDR + } + + //queue the assignment + logger.V(4).Info("Putting node with CIDR into the work queue", "node", klog.KObj(node), "CIDRs", allocatedCIDRs) + return r.updateCIDRsAllocation(ctx, node.Name, allocatedCIDRs) +} + +// ReleaseCIDR marks node.podCIDRs[...] as unused in our tracked cidrSets +func (r *rangeAllocator) ReleaseCIDR(node *v1.Node) error { + if node == nil || len(node.Spec.PodCIDRs) == 0 { + return nil + } + + for idx, cidr := range node.Spec.PodCIDRs { + _, podCIDR, err := netutils.ParseCIDRSloppy(cidr) + if err != nil { + return fmt.Errorf("failed to parse CIDR %s on Node %v: %v", cidr, node.Name, err) + } + + // If node has a pre allocate cidr that does not exist in our cidrs. + // This will happen if cluster went from dualstack(multi cidrs) to non-dualstack + // then we have now way of locking it + if idx >= len(r.cidrSets) { + return fmt.Errorf("node:%s has an allocated cidr: %v at index:%v that does not exist in cluster cidrs configuration", node.Name, cidr, idx) + } + + klog.V(4).Info("Release CIDR for node", "CIDR", cidr, "node", klog.KObj(node)) + if err = r.cidrSets[idx].Release(podCIDR); err != nil { + return fmt.Errorf("error when releasing CIDR %v: %v", cidr, err) + } + } + return nil +} + +// Marks all CIDRs with subNetMaskSize that belongs to serviceCIDR as used across all cidrs +// so that they won't be assignable. +func (r *rangeAllocator) filterOutServiceRange(serviceCIDR *net.IPNet) { + // Checks if service CIDR has a nonempty intersection with cluster + // CIDR. It is the case if either clusterCIDR contains serviceCIDR with + // clusterCIDR's Mask applied (this means that clusterCIDR contains + // serviceCIDR) or vice versa (which means that serviceCIDR contains + // clusterCIDR). + for idx, cidr := range r.clusterCIDRs { + // if they don't overlap then ignore the filtering + if !cidr.Contains(serviceCIDR.IP.Mask(cidr.Mask)) && !serviceCIDR.Contains(cidr.IP.Mask(serviceCIDR.Mask)) { + continue + } + + // at this point, len(cidrSet) == len(clusterCidr) + if err := r.cidrSets[idx].Occupy(serviceCIDR); err != nil { + klog.Error(err, "Error filtering out service cidr out cluster cidr", "CIDR", cidr, "index", idx, "serviceCIDR", serviceCIDR) + } + } +} + +// updateCIDRsAllocation assigns CIDR to Node and sends an update to the API server. +func (r *rangeAllocator) updateCIDRsAllocation(ctx context.Context, nodeName string, allocatedCIDRs []*net.IPNet) error { + var err error + var node *v1.Node + logger := klog.FromContext(ctx) + cidrsString := ipnetToStringList(allocatedCIDRs) + node, err = r.nodeLister.Get(nodeName) + if err != nil { + logger.Error(err, "Failed while getting node for updating Node.Spec.PodCIDRs", "node", klog.KRef("", nodeName)) + return err + } + + // if cidr list matches the proposed. + // then we possibly updated this node + // and just failed to ack the success. + if len(node.Spec.PodCIDRs) == len(allocatedCIDRs) { + match := true + for idx, cidr := range cidrsString { + if node.Spec.PodCIDRs[idx] != cidr { + match = false + break + } + } + if match { + logger.V(4).Info("Node already has allocated CIDR. It matches the proposed one", "node", klog.KObj(node), "CIDRs", allocatedCIDRs) + return nil + } + } + + // node has cidrs, release the reserved + if len(node.Spec.PodCIDRs) != 0 { + logger.Error(nil, "Node already has a CIDR allocated. Releasing the new one", "node", klog.KObj(node), "podCIDRs", node.Spec.PodCIDRs) + for idx, cidr := range allocatedCIDRs { + if releaseErr := r.cidrSets[idx].Release(cidr); releaseErr != nil { + logger.Error(releaseErr, "Error when releasing CIDR", "index", idx, "CIDR", cidr) + } + } + return nil + } + + // If we reached here, it means that the node has no CIDR currently assigned. So we set it. + for i := 0; i < cidrUpdateRetries; i++ { + if err = nodeutil.PatchNodeCIDRs(ctx, r.client, types.NodeName(node.Name), cidrsString); err == nil { + logger.Info("Set node PodCIDR", "node", klog.KObj(node), "podCIDRs", cidrsString) + return nil + } + } + // failed release back to the pool + logger.Error(err, "Failed to update node PodCIDR after multiple attempts", "node", klog.KObj(node), "podCIDRs", cidrsString) + controllerutil.RecordNodeStatusChange(logger, r.recorder, node, "CIDRAssignmentFailed") + // We accept the fact that we may leak CIDRs here. This is safer than releasing + // them in case when we don't know if request went through. + // NodeController restart will return all falsely allocated CIDRs to the pool. + if !apierrors.IsServerTimeout(err) { + logger.Error(err, "CIDR assignment for node failed. Releasing allocated CIDR", "node", klog.KObj(node)) + for idx, cidr := range allocatedCIDRs { + if releaseErr := r.cidrSets[idx].Release(cidr); releaseErr != nil { + logger.Error(releaseErr, "Error releasing allocated CIDR for node", "node", klog.KObj(node)) + } + } + } + return err +} diff --git a/cloud/nodeipam/ipam/sync/sync.go b/cloud/nodeipam/ipam/sync/sync.go new file mode 100644 index 00000000..80a565e2 --- /dev/null +++ b/cloud/nodeipam/ipam/sync/sync.go @@ -0,0 +1,381 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sync + +import ( + "context" + "fmt" + "net" + "time" + + "k8s.io/klog/v2" + netutils "k8s.io/utils/net" + + v1 "k8s.io/api/core/v1" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/cidrset" +) + +const ( + // InvalidPodCIDR is the event recorded when a node is found with an + // invalid PodCIDR. + InvalidPodCIDR = "CloudCIDRAllocatorInvalidPodCIDR" + // InvalidModeEvent is the event recorded when the CIDR range cannot be + // sync'd due to the cluster running in the wrong mode. + InvalidModeEvent = "CloudCIDRAllocatorInvalidMode" + // MismatchEvent is the event recorded when the CIDR range allocated in the + // node spec does not match what has been allocated in the cloud. + MismatchEvent = "CloudCIDRAllocatorMismatch" +) + +// cloudAlias is the interface to the cloud platform APIs. +type cloudAlias interface { + // Alias returns the IP alias for the node. + Alias(ctx context.Context, node *v1.Node) (*net.IPNet, error) + // AddAlias adds an alias to the node. + AddAlias(ctx context.Context, node *v1.Node, cidrRange *net.IPNet) error +} + +// kubeAPI is the interface to the Kubernetes APIs. +type kubeAPI interface { + // Node returns the spec for the Node object. + Node(ctx context.Context, name string) (*v1.Node, error) + // UpdateNodePodCIDR updates the PodCIDR in the Node spec. + UpdateNodePodCIDR(ctx context.Context, node *v1.Node, cidrRange *net.IPNet) error + // UpdateNodeNetworkUnavailable updates the network unavailable status for the node. + UpdateNodeNetworkUnavailable(nodeName string, unavailable bool) error + // EmitNodeWarningEvent emits an event for the given node. + EmitNodeWarningEvent(nodeName, reason, fmt string, args ...interface{}) +} + +// controller is the interface to the controller. +type controller interface { + // ReportResult updates the controller with the result of the latest + // sync operation. + ReportResult(err error) + // ResyncTimeout returns the amount of time to wait before retrying + // a sync with a node. + ResyncTimeout() time.Duration +} + +// NodeSyncMode is the mode the cloud CIDR allocator runs in. +type NodeSyncMode string + +var ( + // SyncFromCloud is the mode that synchronizes the IP allocation from the cloud + // platform to the node. + SyncFromCloud NodeSyncMode = "SyncFromCloud" + // SyncFromCluster is the mode that synchronizes the IP allocation determined + // by the k8s controller to the cloud provider. + SyncFromCluster NodeSyncMode = "SyncFromCluster" +) + +// IsValidMode returns true if the given mode is valid. +func IsValidMode(m NodeSyncMode) bool { + switch m { + case SyncFromCloud: + case SyncFromCluster: + default: + return false + } + return true +} + +// NodeSync synchronizes the state for a single node in the cluster. +type NodeSync struct { + c controller + cloudAlias cloudAlias + kubeAPI kubeAPI + mode NodeSyncMode + nodeName string + opChan chan syncOp + set *cidrset.CidrSet +} + +// New returns a new syncer for a given node. +func New(c controller, cloudAlias cloudAlias, kubeAPI kubeAPI, mode NodeSyncMode, nodeName string, set *cidrset.CidrSet) *NodeSync { + return &NodeSync{ + c: c, + cloudAlias: cloudAlias, + kubeAPI: kubeAPI, + mode: mode, + nodeName: nodeName, + opChan: make(chan syncOp, 1), + set: set, + } +} + +// Loop runs the sync loop for a given node. done is an optional channel that +// is closed when the Loop() returns. +func (sync *NodeSync) Loop(logger klog.Logger, done chan struct{}) { + logger.V(2).Info("Starting sync loop", "node", klog.KRef("", sync.nodeName)) + + defer func() { + if done != nil { + close(done) + } + }() + + timeout := sync.c.ResyncTimeout() + delayTimer := time.NewTimer(timeout) + logger.V(4).Info("Try to resync node later", "node", klog.KRef("", sync.nodeName), "resyncTime", timeout) + + for { + select { + case op, more := <-sync.opChan: + if !more { + logger.V(2).Info("Stopping sync loop") + return + } + sync.c.ReportResult(op.run(logger, sync)) + if !delayTimer.Stop() { + <-delayTimer.C + } + case <-delayTimer.C: + logger.V(4).Info("Running resync", "node", klog.KRef("", sync.nodeName)) + sync.c.ReportResult((&updateOp{}).run(logger, sync)) + } + + timeout := sync.c.ResyncTimeout() + delayTimer.Reset(timeout) + logger.V(4).Info("Try to resync node later", "node", klog.KRef("", sync.nodeName), "resyncTime", timeout) + } +} + +// Update causes an update operation on the given node. If node is nil, then +// the syncer will fetch the node spec from the API server before syncing. +// +// This method is safe to call from multiple goroutines. +func (sync *NodeSync) Update(node *v1.Node) { + sync.opChan <- &updateOp{node} +} + +// Delete performs the sync operations necessary to remove the node from the +// IPAM state. +// +// This method is safe to call from multiple goroutines. +func (sync *NodeSync) Delete(node *v1.Node) { + sync.opChan <- &deleteOp{node} + close(sync.opChan) +} + +// syncOp is the interface for generic sync operation. +type syncOp interface { + // run the requested sync operation. + run(logger klog.Logger, sync *NodeSync) error +} + +// updateOp handles creation and updates of a node. +type updateOp struct { + node *v1.Node +} + +func (op *updateOp) String() string { + if op.node == nil { + return fmt.Sprintf("updateOp(nil)") + } + return fmt.Sprintf("updateOp(%q,%v)", op.node.Name, op.node.Spec.PodCIDR) +} + +func (op *updateOp) run(logger klog.Logger, sync *NodeSync) error { + logger.V(3).Info("Running updateOp", "updateOp", op) + + ctx := context.Background() + + if op.node == nil { + logger.V(3).Info("Getting node spec", "node", klog.KRef("", sync.nodeName)) + node, err := sync.kubeAPI.Node(ctx, sync.nodeName) + if err != nil { + logger.Error(err, "Error getting node pec", "node", klog.KRef("", sync.nodeName)) + return err + } + op.node = node + } + + aliasRange, err := sync.cloudAlias.Alias(ctx, op.node) + if err != nil { + logger.Error(err, "Error getting cloud alias for node", "node", klog.KRef("", sync.nodeName)) + return err + } + + switch { + case op.node.Spec.PodCIDR == "" && aliasRange == nil: + err = op.allocateRange(ctx, sync, op.node) + case op.node.Spec.PodCIDR == "" && aliasRange != nil: + err = op.updateNodeFromAlias(ctx, sync, op.node, aliasRange) + case op.node.Spec.PodCIDR != "" && aliasRange == nil: + err = op.updateAliasFromNode(ctx, sync, op.node) + case op.node.Spec.PodCIDR != "" && aliasRange != nil: + err = op.validateRange(ctx, sync, op.node, aliasRange) + } + + return err +} + +// validateRange checks that the allocated range and the alias range +// match. +func (op *updateOp) validateRange(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error { + if node.Spec.PodCIDR != aliasRange.String() { + klog.FromContext(ctx).Error(nil, "Inconsistency detected between node PodCIDR and node alias", "podCIDR", node.Spec.PodCIDR, "alias", aliasRange) + sync.kubeAPI.EmitNodeWarningEvent(node.Name, MismatchEvent, + "Node.Spec.PodCIDR != cloud alias (%v != %v)", node.Spec.PodCIDR, aliasRange) + // User intervention is required in this case, as this is most likely due + // to the user mucking around with their VM aliases on the side. + } else { + klog.FromContext(ctx).V(4).Info("Node CIDR range is matches cloud assignment", "node", klog.KObj(node), "podCIDR", node.Spec.PodCIDR) + } + return nil +} + +// updateNodeFromAlias updates the node from the cloud allocated +// alias. +func (op *updateOp) updateNodeFromAlias(ctx context.Context, sync *NodeSync, node *v1.Node, aliasRange *net.IPNet) error { + if sync.mode != SyncFromCloud { + sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent, + "Cannot sync from cloud in mode %q", sync.mode) + return fmt.Errorf("cannot sync from cloud in mode %q", sync.mode) + } + logger := klog.FromContext(ctx) + logger.V(2).Info("Updating node spec with alias range", "podCIDR", aliasRange) + + if err := sync.set.Occupy(aliasRange); err != nil { + logger.Error(nil, "Error occupying range for node", "node", klog.KRef("", sync.nodeName), "alias", aliasRange) + return err + } + + if err := sync.kubeAPI.UpdateNodePodCIDR(ctx, node, aliasRange); err != nil { + logger.Error(err, "Could not update node PodCIDR", "node", klog.KObj(node), "podCIDR", aliasRange) + return err + } + + logger.V(2).Info("Node PodCIDR updated", "node", klog.KObj(node), "podCIDR", aliasRange) + + if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { + logger.Error(err, "Could not update node NetworkUnavailable status to false") + return err + } + + logger.V(2).Info("Updated node PodCIDR from cloud alias", "node", klog.KObj(node), "alias", aliasRange) + + return nil +} + +// updateAliasFromNode updates the cloud alias given the node allocation. +func (op *updateOp) updateAliasFromNode(ctx context.Context, sync *NodeSync, node *v1.Node) error { + if sync.mode != SyncFromCluster { + sync.kubeAPI.EmitNodeWarningEvent( + node.Name, InvalidModeEvent, "Cannot sync to cloud in mode %q", sync.mode) + return fmt.Errorf("cannot sync to cloud in mode %q", sync.mode) + } + + _, aliasRange, err := netutils.ParseCIDRSloppy(node.Spec.PodCIDR) + + logger := klog.FromContext(ctx) + if err != nil { + logger.Error(err, "Could not parse PodCIDR for node", "node", klog.KObj(node), "podCIDR", node.Spec.PodCIDR) + return err + } + + if err := sync.set.Occupy(aliasRange); err != nil { + logger.Error(nil, "Error occupying range for node", "node", klog.KRef("", sync.nodeName), "alias", aliasRange) + return err + } + + if err := sync.cloudAlias.AddAlias(ctx, node, aliasRange); err != nil { + logger.Error(err, "Could not add alias for node", "node", klog.KObj(node), "alias", aliasRange) + return err + } + + if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { + logger.Error(err, "Could not update node NetworkUnavailable status to false") + return err + } + + logger.V(2).Info("Updated node cloud alias with node spec", "node", klog.KObj(node), "podCIDR", node.Spec.PodCIDR) + + return nil +} + +// allocateRange allocates a new range and updates both the cloud +// platform and the node allocation. +func (op *updateOp) allocateRange(ctx context.Context, sync *NodeSync, node *v1.Node) error { + if sync.mode != SyncFromCluster { + sync.kubeAPI.EmitNodeWarningEvent(node.Name, InvalidModeEvent, + "Cannot allocate CIDRs in mode %q", sync.mode) + return fmt.Errorf("controller cannot allocate CIDRS in mode %q", sync.mode) + } + + cidrRange, err := sync.set.AllocateNext() + if err != nil { + return err + } + // If addAlias returns a hard error, cidrRange will be leaked as there + // is no durable record of the range. The missing space will be + // recovered on the next restart of the controller. + logger := klog.FromContext(ctx) + if err := sync.cloudAlias.AddAlias(ctx, node, cidrRange); err != nil { + logger.Error(err, "Could not add alias for node", "node", klog.KObj(node), "alias", cidrRange) + return err + } + + if err := sync.kubeAPI.UpdateNodePodCIDR(ctx, node, cidrRange); err != nil { + logger.Error(err, "Could not update node PodCIDR", "node", klog.KObj(node), "podCIDR", cidrRange) + return err + } + + if err := sync.kubeAPI.UpdateNodeNetworkUnavailable(node.Name, false); err != nil { + logger.Error(err, "Could not update node NetworkUnavailable status to false") + return err + } + + logger.V(2).Info("Allocated PodCIDR for node", "node", klog.KObj(node), "podCIDR", cidrRange) + + return nil +} + +// deleteOp handles deletion of a node. +type deleteOp struct { + node *v1.Node +} + +func (op *deleteOp) String() string { + if op.node == nil { + return fmt.Sprintf("deleteOp(nil)") + } + return fmt.Sprintf("deleteOp(%q,%v)", op.node.Name, op.node.Spec.PodCIDR) +} + +func (op *deleteOp) run(logger klog.Logger, sync *NodeSync) error { + logger.V(3).Info("Running deleteOp", "deleteOp", op) + if op.node.Spec.PodCIDR == "" { + logger.V(2).Info("Node was deleted, node had no PodCIDR range assigned", "node", klog.KObj(op.node)) + return nil + } + + _, cidrRange, err := netutils.ParseCIDRSloppy(op.node.Spec.PodCIDR) + if err != nil { + logger.Error(err, "Deleted node has an invalid podCIDR", "node", klog.KObj(op.node), "podCIDR", op.node.Spec.PodCIDR) + sync.kubeAPI.EmitNodeWarningEvent(op.node.Name, InvalidPodCIDR, + "Node %q has an invalid PodCIDR: %q", op.node.Name, op.node.Spec.PodCIDR) + return nil + } + + sync.set.Release(cidrRange) + logger.V(2).Info("Node was deleted, releasing CIDR range", "node", klog.KObj(op.node), "podCIDR", op.node.Spec.PodCIDR) + + return nil +} diff --git a/cloud/nodeipam/ipam/sync/sync_test.go b/cloud/nodeipam/ipam/sync/sync_test.go new file mode 100644 index 00000000..36d357d7 --- /dev/null +++ b/cloud/nodeipam/ipam/sync/sync_test.go @@ -0,0 +1,307 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sync + +import ( + "context" + "fmt" + "net" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + "k8s.io/klog/v2/ktesting" + netutils "k8s.io/utils/net" + + v1 "k8s.io/api/core/v1" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/cidrset" + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam/test" +) + +var ( + _, clusterCIDRRange, _ = netutils.ParseCIDRSloppy("10.1.0.0/16") +) + +type fakeEvent struct { + nodeName string + reason string +} + +type fakeAPIs struct { + aliasRange *net.IPNet + aliasErr error + addAliasErr error + nodeRet *v1.Node + nodeErr error + updateNodeErr error + resyncTimeout time.Duration + reportChan chan struct{} + + updateNodeNetworkUnavailableErr error + + calls []string + events []fakeEvent + results []error + + logger klog.Logger +} + +func (f *fakeAPIs) Alias(ctx context.Context, node *v1.Node) (*net.IPNet, error) { + f.calls = append(f.calls, fmt.Sprintf("alias %v", node.Name)) + return f.aliasRange, f.aliasErr +} + +func (f *fakeAPIs) AddAlias(ctx context.Context, node *v1.Node, cidrRange *net.IPNet) error { + f.calls = append(f.calls, fmt.Sprintf("addAlias %v %v", node.Name, cidrRange)) + return f.addAliasErr +} + +func (f *fakeAPIs) Node(ctx context.Context, name string) (*v1.Node, error) { + f.calls = append(f.calls, fmt.Sprintf("node %v", name)) + return f.nodeRet, f.nodeErr +} + +func (f *fakeAPIs) UpdateNodePodCIDR(ctx context.Context, node *v1.Node, cidrRange *net.IPNet) error { + f.calls = append(f.calls, fmt.Sprintf("updateNode %v", node)) + return f.updateNodeErr +} + +func (f *fakeAPIs) UpdateNodeNetworkUnavailable(nodeName string, unavailable bool) error { + f.calls = append(f.calls, fmt.Sprintf("updateNodeNetworkUnavailable %v %v", nodeName, unavailable)) + return f.updateNodeNetworkUnavailableErr +} + +func (f *fakeAPIs) EmitNodeWarningEvent(nodeName, reason, fmtStr string, args ...interface{}) { + f.events = append(f.events, fakeEvent{nodeName, reason}) +} + +func (f *fakeAPIs) ReportResult(err error) { + f.logger.V(2).Info("ReportResult", "err", err) + f.results = append(f.results, err) + if f.reportChan != nil { + f.reportChan <- struct{}{} + } +} + +func (f *fakeAPIs) ResyncTimeout() time.Duration { + if f.resyncTimeout == 0 { + return time.Second * 10000 + } + return f.resyncTimeout +} + +func (f *fakeAPIs) dumpTrace() { + for i, x := range f.calls { + f.logger.Info("trace", "index", i, "call", x) + } +} + +var nodeWithoutCIDRRange = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, +} + +var nodeWithCIDRRange = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Spec: v1.NodeSpec{PodCIDR: "10.1.1.0/24"}, +} + +func TestNodeSyncUpdate(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + desc string + mode NodeSyncMode + node *v1.Node + fake fakeAPIs + + events []fakeEvent + wantError bool + }{ + { + desc: "validate range ==", + mode: SyncFromCloud, + node: nodeWithCIDRRange, + fake: fakeAPIs{ + aliasRange: test.MustParseCIDR(nodeWithCIDRRange.Spec.PodCIDR), + }, + }, + { + desc: "validate range !=", + mode: SyncFromCloud, + node: nodeWithCIDRRange, + fake: fakeAPIs{aliasRange: test.MustParseCIDR("192.168.0.0/24")}, + events: []fakeEvent{{"node1", "CloudCIDRAllocatorMismatch"}}, + }, + { + desc: "update alias from node", + mode: SyncFromCloud, + node: nodeWithCIDRRange, + events: []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}}, + wantError: true, + }, + { + desc: "update alias from node", + mode: SyncFromCluster, + node: nodeWithCIDRRange, + // XXX/bowei -- validation + }, + { + desc: "update node from alias", + mode: SyncFromCloud, + node: nodeWithoutCIDRRange, + fake: fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")}, + // XXX/bowei -- validation + }, + { + desc: "update node from alias", + mode: SyncFromCluster, + node: nodeWithoutCIDRRange, + fake: fakeAPIs{aliasRange: test.MustParseCIDR("10.1.2.3/16")}, + events: []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}}, + wantError: true, + }, + { + desc: "allocate range", + mode: SyncFromCloud, + node: nodeWithoutCIDRRange, + events: []fakeEvent{{"node1", "CloudCIDRAllocatorInvalidMode"}}, + wantError: true, + }, + { + desc: "allocate range", + mode: SyncFromCluster, + node: nodeWithoutCIDRRange, + }, + { + desc: "update with node==nil", + mode: SyncFromCluster, + node: nil, + fake: fakeAPIs{ + nodeRet: nodeWithCIDRRange, + }, + wantError: false, + }, + } { + logger, _ := ktesting.NewTestContext(t) + cidr, _ := cidrset.NewCIDRSet(clusterCIDRRange, 24) + tc.fake.logger = logger + sync := New(&tc.fake, &tc.fake, &tc.fake, tc.mode, "node1", cidr) + doneChan := make(chan struct{}) + + // Do a single step of the loop. + go sync.Loop(logger, doneChan) + sync.Update(tc.node) + close(sync.opChan) + <-doneChan + tc.fake.dumpTrace() + + if !reflect.DeepEqual(tc.fake.events, tc.events) { + t.Errorf("%v, %v; fake.events = %#v, want %#v", tc.desc, tc.mode, tc.fake.events, tc.events) + } + + var hasError bool + for _, r := range tc.fake.results { + hasError = hasError || (r != nil) + } + if hasError != tc.wantError { + t.Errorf("%v, %v; hasError = %t, errors = %v, want %t", + tc.desc, tc.mode, hasError, tc.fake.events, tc.wantError) + } + } +} + +func TestNodeSyncResync(t *testing.T) { + logger, _ := ktesting.NewTestContext(t) + fake := &fakeAPIs{ + nodeRet: nodeWithCIDRRange, + resyncTimeout: time.Millisecond, + reportChan: make(chan struct{}), + logger: logger, + } + cidr, _ := cidrset.NewCIDRSet(clusterCIDRRange, 24) + sync := New(fake, fake, fake, SyncFromCluster, "node1", cidr) + doneChan := make(chan struct{}) + go sync.Loop(logger, doneChan) + <-fake.reportChan + close(sync.opChan) + // Unblock loop(). + go func() { + <-fake.reportChan + }() + <-doneChan + fake.dumpTrace() +} + +func TestNodeSyncDelete(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + desc string + mode NodeSyncMode + node *v1.Node + fake fakeAPIs + }{ + { + desc: "delete", + mode: SyncFromCluster, + node: nodeWithCIDRRange, + }, + { + desc: "delete without CIDR range", + mode: SyncFromCluster, + node: nodeWithoutCIDRRange, + }, + { + desc: "delete with invalid CIDR range", + mode: SyncFromCluster, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Spec: v1.NodeSpec{PodCIDR: "invalid"}, + }, + }, + } { + logger, _ := ktesting.NewTestContext(t) + cidr, _ := cidrset.NewCIDRSet(clusterCIDRRange, 24) + tc.fake.logger = logger + sync := New(&tc.fake, &tc.fake, &tc.fake, tc.mode, "node1", cidr) + doneChan := make(chan struct{}) + + // Do a single step of the loop. + go sync.Loop(logger, doneChan) + sync.Delete(tc.node) + <-doneChan + tc.fake.dumpTrace() + + /* + if !reflect.DeepEqual(tc.fake.events, tc.events) { + t.Errorf("%v, %v; fake.events = %#v, want %#v", tc.desc, tc.mode, tc.fake.events, tc.events) + } + + var hasError bool + for _, r := range tc.fake.results { + hasError = hasError || (r != nil) + } + if hasError != tc.wantError { + t.Errorf("%v, %v; hasError = %t, errors = %v, want %t", + tc.desc, tc.mode, hasError, tc.fake.events, tc.wantError) + } + */ + } +} diff --git a/cloud/nodeipam/ipam/test/utils.go b/cloud/nodeipam/ipam/test/utils.go new file mode 100644 index 00000000..d37cfc59 --- /dev/null +++ b/cloud/nodeipam/ipam/test/utils.go @@ -0,0 +1,418 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "sync" + "time" + + jsonpatch "github.com/evanphx/json-patch" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/watch" + v1apply "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/kubernetes/fake" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog/v2" + + netutils "k8s.io/utils/net" +) + +const NodePollInterval = 10 * time.Millisecond + +var AlwaysReady = func() bool { return true } + +// MustParseCIDR returns the CIDR range parsed from s or panics if the string +// cannot be parsed. +func MustParseCIDR(s string) *net.IPNet { + _, ret, err := netutils.ParseCIDRSloppy(s) + if err != nil { + panic(err) + } + return ret +} + +// FakeNodeHandler is a fake implementation of NodesInterface and NodeInterface. It +// allows test cases to have fine-grained control over mock behaviors. We also need +// PodsInterface and PodInterface to test list & delete pods, which is implemented in +// the embedded client.Fake field. +type FakeNodeHandler struct { + *fake.Clientset + + // Input: Hooks determine if request is valid or not + CreateHook func(*FakeNodeHandler, *v1.Node) bool + Existing []*v1.Node + AsyncCalls []func(*FakeNodeHandler) + + // Output + CreatedNodes []*v1.Node + DeletedNodes []*v1.Node + UpdatedNodes []*v1.Node + UpdatedNodeStatuses []*v1.Node + RequestCount int + + // Synchronization + lock sync.Mutex + DeleteWaitChan chan struct{} + PatchWaitChan chan struct{} +} + +// FakeLegacyHandler is a fake implementation of CoreV1Interface. +type FakeLegacyHandler struct { + v1core.CoreV1Interface + n *FakeNodeHandler +} + +// GetUpdatedNodesCopy returns a slice of Nodes with updates applied. +func (m *FakeNodeHandler) GetUpdatedNodesCopy() []*v1.Node { + m.lock.Lock() + defer m.lock.Unlock() + updatedNodesCopy := make([]*v1.Node, len(m.UpdatedNodes)) + copy(updatedNodesCopy, m.UpdatedNodes) + return updatedNodesCopy +} + +// Core returns fake CoreInterface. +func (m *FakeNodeHandler) Core() v1core.CoreV1Interface { + return &FakeLegacyHandler{m.Clientset.CoreV1(), m} +} + +// CoreV1 returns fake CoreV1Interface. +func (m *FakeNodeHandler) CoreV1() v1core.CoreV1Interface { + return &FakeLegacyHandler{m.Clientset.CoreV1(), m} +} + +// Nodes return fake NodeInterfaces. +func (m *FakeLegacyHandler) Nodes() v1core.NodeInterface { + return m.n +} + +// Create adds a new Node to the fake store. +func (m *FakeNodeHandler) Create(_ context.Context, node *v1.Node, _ metav1.CreateOptions) (*v1.Node, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + m.lock.Unlock() + }() + for _, n := range m.Existing { + if n.Name == node.Name { + return nil, apierrors.NewAlreadyExists(v1.Resource("nodes"), node.Name) + } + } + if m.CreateHook == nil || m.CreateHook(m, node) { + nodeCopy := *node + m.CreatedNodes = append(m.CreatedNodes, &nodeCopy) + return node, nil + } + return nil, errors.New("create error") +} + +// Get returns a Node from the fake store. +func (m *FakeNodeHandler) Get(_ context.Context, name string, _ metav1.GetOptions) (*v1.Node, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + m.runAsyncCalls() + m.lock.Unlock() + }() + for i := range m.UpdatedNodes { + if m.UpdatedNodes[i].Name == name { + nodeCopy := *m.UpdatedNodes[i] + return &nodeCopy, nil + } + } + for i := range m.Existing { + if m.Existing[i].Name == name { + nodeCopy := *m.Existing[i] + return &nodeCopy, nil + } + } + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "nodes"}, name) +} + +func (m *FakeNodeHandler) runAsyncCalls() { + for _, a := range m.AsyncCalls { + a(m) + } +} + +// List returns a list of Nodes from the fake store. +func (m *FakeNodeHandler) List(_ context.Context, _ metav1.ListOptions) (*v1.NodeList, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + m.lock.Unlock() + }() + var nodes []*v1.Node + for i := 0; i < len(m.UpdatedNodes); i++ { + if !contains(m.UpdatedNodes[i], m.DeletedNodes) { + nodes = append(nodes, m.UpdatedNodes[i]) + } + } + for i := 0; i < len(m.Existing); i++ { + if !contains(m.Existing[i], m.DeletedNodes) && !contains(m.Existing[i], nodes) { + nodes = append(nodes, m.Existing[i]) + } + } + for i := 0; i < len(m.CreatedNodes); i++ { + if !contains(m.CreatedNodes[i], m.DeletedNodes) && !contains(m.CreatedNodes[i], nodes) { + nodes = append(nodes, m.CreatedNodes[i]) + } + } + nodeList := &v1.NodeList{} + for _, node := range nodes { + nodeList.Items = append(nodeList.Items, *node) + } + return nodeList, nil +} + +// Delete deletes a Node from the fake store. +func (m *FakeNodeHandler) Delete(_ context.Context, id string, _ metav1.DeleteOptions) error { + m.lock.Lock() + defer func() { + m.RequestCount++ + if m.DeleteWaitChan != nil { + m.DeleteWaitChan <- struct{}{} + } + m.lock.Unlock() + }() + m.DeletedNodes = append(m.DeletedNodes, NewNode(id)) + return nil +} + +// DeleteCollection deletes a collection of Nodes from the fake store. +func (m *FakeNodeHandler) DeleteCollection(_ context.Context, _ metav1.DeleteOptions, _ metav1.ListOptions) error { + return nil +} + +// Update updates a Node in the fake store. +func (m *FakeNodeHandler) Update(_ context.Context, node *v1.Node, _ metav1.UpdateOptions) (*v1.Node, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + m.lock.Unlock() + }() + + nodeCopy := *node + for i, updateNode := range m.UpdatedNodes { + if updateNode.Name == nodeCopy.Name { + if updateNode.GetObjectMeta().GetResourceVersion() != nodeCopy.GetObjectMeta().GetResourceVersion() { + return nil, apierrors.NewConflict(schema.GroupResource{}, "fake conflict", nil) + } + m.UpdatedNodes[i] = &nodeCopy + return node, nil + } + } + m.UpdatedNodes = append(m.UpdatedNodes, &nodeCopy) + return node, nil +} + +// UpdateStatus updates a status of a Node in the fake store. +func (m *FakeNodeHandler) UpdateStatus(_ context.Context, node *v1.Node, _ metav1.UpdateOptions) (*v1.Node, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + m.lock.Unlock() + }() + + var origNodeCopy v1.Node + found := false + for i := range m.Existing { + if m.Existing[i].Name == node.Name { + origNodeCopy = *m.Existing[i] + found = true + break + } + } + updatedNodeIndex := -1 + for i := range m.UpdatedNodes { + if m.UpdatedNodes[i].Name == node.Name { + origNodeCopy = *m.UpdatedNodes[i] + updatedNodeIndex = i + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("not found node %v", node) + } + + origNodeCopy.Status = node.Status + if updatedNodeIndex < 0 { + m.UpdatedNodes = append(m.UpdatedNodes, &origNodeCopy) + } else { + m.UpdatedNodes[updatedNodeIndex] = &origNodeCopy + } + + nodeCopy := *node + m.UpdatedNodeStatuses = append(m.UpdatedNodeStatuses, &nodeCopy) + return node, nil +} + +// PatchStatus patches a status of a Node in the fake store. +func (m *FakeNodeHandler) PatchStatus(ctx context.Context, nodeName string, data []byte) (*v1.Node, error) { + m.RequestCount++ + return m.Patch(ctx, nodeName, types.StrategicMergePatchType, data, metav1.PatchOptions{}, "status") +} + +// Watch watches Nodes in a fake store. +func (m *FakeNodeHandler) Watch(_ context.Context, _ metav1.ListOptions) (watch.Interface, error) { + return watch.NewFake(), nil +} + +// Patch patches a Node in the fake store. +func (m *FakeNodeHandler) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, _ metav1.PatchOptions, _ ...string) (*v1.Node, error) { + m.lock.Lock() + defer func() { + m.RequestCount++ + if m.PatchWaitChan != nil { + m.PatchWaitChan <- struct{}{} + } + m.lock.Unlock() + }() + var nodeCopy v1.Node + for i := range m.Existing { + if m.Existing[i].Name == name { + nodeCopy = *m.Existing[i] + } + } + updatedNodeIndex := -1 + for i := range m.UpdatedNodes { + if m.UpdatedNodes[i].Name == name { + nodeCopy = *m.UpdatedNodes[i] + updatedNodeIndex = i + } + } + + originalObjJS, err := json.Marshal(nodeCopy) + if err != nil { + klog.FromContext(ctx).Error(nil, "Failed to marshal", "node", klog.KObj(&nodeCopy)) + return nil, err + } + var originalNode v1.Node + if err = json.Unmarshal(originalObjJS, &originalNode); err != nil { + klog.FromContext(ctx).Error(err, "Failed to unmarshal original object") + return nil, nil + } + + var patchedObjJS []byte + switch pt { + case types.JSONPatchType: + patchObj, err := jsonpatch.DecodePatch(data) + if err != nil { + klog.FromContext(ctx).Error(err, "") + return nil, nil + } + if patchedObjJS, err = patchObj.Apply(originalObjJS); err != nil { + klog.FromContext(ctx).Error(err, "") + return nil, nil + } + case types.MergePatchType: + if patchedObjJS, err = jsonpatch.MergePatch(originalObjJS, data); err != nil { + klog.FromContext(ctx).Error(err, "") + return nil, nil + } + case types.StrategicMergePatchType: + if patchedObjJS, err = strategicpatch.StrategicMergePatch(originalObjJS, data, originalNode); err != nil { + klog.FromContext(ctx).Error(err, "") + return nil, nil + } + default: + klog.FromContext(ctx).Error(nil, "Unknown Content-Type header", "patch", pt) + return nil, nil + } + + var updatedNode v1.Node + if err = json.Unmarshal(patchedObjJS, &updatedNode); err != nil { + klog.FromContext(ctx).Error(err, "Failed to unmarshal patched object") + return nil, nil + } + + if updatedNodeIndex < 0 { + m.UpdatedNodes = append(m.UpdatedNodes, &updatedNode) + } else { + if updatedNode.GetObjectMeta().GetResourceVersion() != m.UpdatedNodes[updatedNodeIndex].GetObjectMeta().GetResourceVersion() { + return nil, apierrors.NewConflict(schema.GroupResource{}, "fake conflict", nil) + } + m.UpdatedNodes[updatedNodeIndex] = &updatedNode + } + + return &updatedNode, nil +} + +// Apply applies a NodeApplyConfiguration to a Node in the fake store. +func (m *FakeNodeHandler) Apply(ctx context.Context, node *v1apply.NodeApplyConfiguration, opts metav1.ApplyOptions) (*v1.Node, error) { + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(node) + if err != nil { + return nil, err + } + name := node.Name + if name == nil { + return nil, fmt.Errorf("deployment.Name must be provided to Apply") + } + + return m.Patch(ctx, *name, types.ApplyPatchType, data, patchOpts) +} + +// ApplyStatus applies a status of a Node in the fake store. +func (m *FakeNodeHandler) ApplyStatus(ctx context.Context, node *v1apply.NodeApplyConfiguration, opts metav1.ApplyOptions) (*v1.Node, error) { + patchOpts := opts.ToPatchOptions() + data, err := json.Marshal(node) + if err != nil { + return nil, err + } + name := node.Name + if name == nil { + return nil, fmt.Errorf("deployment.Name must be provided to Apply") + } + + return m.Patch(ctx, *name, types.ApplyPatchType, data, patchOpts, "status") +} + +func contains(node *v1.Node, nodes []*v1.Node) bool { + for i := 0; i < len(nodes); i++ { + if node.Name == nodes[i].Name { + return true + } + } + return false +} + +// NewNode is a helper function for creating Nodes for testing. +func NewNode(name string) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10G"), + }, + }, + } +} diff --git a/cloud/nodeipam/ipam/timeout.go b/cloud/nodeipam/ipam/timeout.go new file mode 100644 index 00000000..f1160b4c --- /dev/null +++ b/cloud/nodeipam/ipam/timeout.go @@ -0,0 +1,67 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipam + +import ( + "time" +) + +// Timeout manages the resync loop timing for a given node sync operation. The +// timeout changes depending on whether or not there was an error reported for +// the operation. Consecutive errors will result in exponential backoff to a +// maxBackoff timeout. +type Timeout struct { + // Resync is the default timeout duration when there are no errors. + Resync time.Duration + // MaxBackoff is the maximum timeout when in a error backoff state. + MaxBackoff time.Duration + // InitialRetry is the initial retry interval when an error is reported. + InitialRetry time.Duration + + // errs is the count of consecutive errors that have occurred. + errs int + // current is the current backoff timeout. + current time.Duration +} + +// Update the timeout with the current error state. +func (b *Timeout) Update(ok bool) { + if ok { + b.errs = 0 + b.current = b.Resync + return + } + + b.errs++ + if b.errs == 1 { + b.current = b.InitialRetry + return + } + + b.current *= 2 + if b.current >= b.MaxBackoff { + b.current = b.MaxBackoff + } +} + +// Next returns the next operation timeout given the disposition of err. +func (b *Timeout) Next() time.Duration { + if b.errs == 0 { + return b.Resync + } + return b.current +} diff --git a/cloud/nodeipam/ipam/timeout_test.go b/cloud/nodeipam/ipam/timeout_test.go new file mode 100644 index 00000000..0da314e6 --- /dev/null +++ b/cloud/nodeipam/ipam/timeout_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ipam + +import ( + "errors" + "testing" + "time" +) + +func TestTimeout(t *testing.T) { + time10s := 10 * time.Second + time5s := 5 * time.Second + timeout := &Timeout{ + Resync: time10s, + MaxBackoff: time5s, + InitialRetry: time.Second, + } + + for _, testStep := range []struct { + err error + want time.Duration + }{ + {nil, time10s}, + {nil, time10s}, + {errors.New("x"), time.Second}, + {errors.New("x"), 2 * time.Second}, + {errors.New("x"), 4 * time.Second}, + {errors.New("x"), 5 * time.Second}, + {errors.New("x"), 5 * time.Second}, + {nil, time10s}, + {nil, time10s}, + {errors.New("x"), time.Second}, + {errors.New("x"), 2 * time.Second}, + {nil, time10s}, + } { + timeout.Update(testStep.err == nil) + next := timeout.Next() + if next != testStep.want { + t.Errorf("timeout.next(%v) = %v, want %v", testStep.err, next, testStep.want) + } + } +} diff --git a/cloud/nodeipam/node_ipam_controller.go b/cloud/nodeipam/node_ipam_controller.go new file mode 100644 index 00000000..865b1b67 --- /dev/null +++ b/cloud/nodeipam/node_ipam_controller.go @@ -0,0 +1,133 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeipam + +import ( + "fmt" + "net" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + v1core "k8s.io/client-go/kubernetes/typed/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog/v2" + + "github.com/linode/linode-cloud-controller-manager/cloud/nodeipam/ipam" +) + +// Controller is the controller that manages node ipam state. +type Controller struct { + allocatorType ipam.CIDRAllocatorType + + cloud cloudprovider.Interface + clusterCIDRs []*net.IPNet + serviceCIDR *net.IPNet + secondaryServiceCIDR *net.IPNet + kubeClient clientset.Interface + eventBroadcaster record.EventBroadcaster + + nodeLister corelisters.NodeLister + nodeInformerSynced cache.InformerSynced + + cidrAllocator ipam.CIDRAllocator +} + +// NewNodeIpamController returns a new node IP Address Management controller to +// sync instances from cloudprovider. +// This method returns an error if it is unable to initialize the CIDR bitmap with +// podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes +// currently, this should be handled as a fatal error. +func NewNodeIpamController( + nodeInformer coreinformers.NodeInformer, + cloud cloudprovider.Interface, + kubeClient clientset.Interface, + clusterCIDRs []*net.IPNet, + serviceCIDR *net.IPNet, + secondaryServiceCIDR *net.IPNet, + nodeCIDRMaskSizes []int, + allocatorType ipam.CIDRAllocatorType) (*Controller, error) { + + if kubeClient == nil { + return nil, fmt.Errorf("kubeClient is nil when starting Controller") + } + + // Cloud CIDR allocator does not rely on clusterCIDR or nodeCIDRMaskSize for allocation. + if allocatorType != ipam.CloudAllocatorType { + if len(clusterCIDRs) == 0 { + return nil, fmt.Errorf("Controller: Must specify --cluster-cidr if --allocate-node-cidrs is set") + } + + for idx, cidr := range clusterCIDRs { + mask := cidr.Mask + if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSizes[idx] { + return nil, fmt.Errorf("Controller: Invalid --cluster-cidr, mask size of cluster CIDR must be less than or equal to --node-cidr-mask-size configured for CIDR family") + } + } + } + + ic := &Controller{ + cloud: cloud, + kubeClient: kubeClient, + eventBroadcaster: record.NewBroadcaster(), + clusterCIDRs: clusterCIDRs, + serviceCIDR: serviceCIDR, + secondaryServiceCIDR: secondaryServiceCIDR, + allocatorType: allocatorType, + } + + var err error + + allocatorParams := ipam.CIDRAllocatorParams{ + ClusterCIDRs: clusterCIDRs, + ServiceCIDR: ic.serviceCIDR, + SecondaryServiceCIDR: ic.secondaryServiceCIDR, + NodeCIDRMaskSizes: nodeCIDRMaskSizes, + } + + ic.cidrAllocator, err = ipam.New(kubeClient, cloud, nodeInformer, ic.allocatorType, allocatorParams) + if err != nil { + return nil, err + } + + ic.nodeLister = nodeInformer.Lister() + ic.nodeInformerSynced = nodeInformer.Informer().HasSynced + + return ic, nil +} + +// Run starts an asynchronous loop that monitors the status of cluster nodes. +func (nc *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + + // Start event processing pipeline. + nc.eventBroadcaster.StartStructuredLogging(3) + nc.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: nc.kubeClient.CoreV1().Events("")}) + defer nc.eventBroadcaster.Shutdown() + klog.Info("Starting ipam controller") + defer klog.Info("Shutting down ipam controller") + + if !cache.WaitForNamedCacheSync("node", stopCh, nc.nodeInformerSynced) { + return + } + + go nc.cidrAllocator.Run(stopCh) + <-stopCh +} diff --git a/cloud/util/controller/node/controller_utils.go b/cloud/util/controller/node/controller_utils.go new file mode 100644 index 00000000..01cac63e --- /dev/null +++ b/cloud/util/controller/node/controller_utils.go @@ -0,0 +1,178 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + + "k8s.io/klog/v2" + + controller "github.com/linode/linode-cloud-controller-manager/cloud/util/node" +) + +// RecordNodeEvent records a event related to a node. +func RecordNodeEvent(ctx context.Context, recorder record.EventRecorder, nodeName, nodeUID, eventtype, reason, event string) { + logger := klog.FromContext(ctx) + ref := &v1.ObjectReference{ + APIVersion: "v1", + Kind: "Node", + Name: nodeName, + UID: types.UID(nodeUID), + Namespace: "", + } + logger.V(2).Info("Recording event message for node", "event", event, "node", klog.KRef("", nodeName)) + recorder.Eventf(ref, eventtype, reason, "Node %s event: %s", nodeName, event) +} + +// RecordNodeStatusChange records a event related to a node status change. (Common to lifecycle and ipam) +func RecordNodeStatusChange(logger klog.Logger, recorder record.EventRecorder, node *v1.Node, newStatus string) { + ref := &v1.ObjectReference{ + APIVersion: "v1", + Kind: "Node", + Name: node.Name, + UID: node.UID, + Namespace: "", + } + logger.V(2).Info("Recording status change event message for node", "status", newStatus, "node", node.Name) + // TODO: This requires a transaction, either both node status is updated + // and event is recorded or neither should happen, see issue #6055. + recorder.Eventf(ref, v1.EventTypeNormal, newStatus, "Node %s status is now: %s", node.Name, newStatus) +} + +// SwapNodeControllerTaint returns true in case of success and false +// otherwise. +func SwapNodeControllerTaint(ctx context.Context, kubeClient clientset.Interface, taintsToAdd, taintsToRemove []*v1.Taint, node *v1.Node) bool { + logger := klog.FromContext(ctx) + for _, taintToAdd := range taintsToAdd { + now := metav1.Now() + taintToAdd.TimeAdded = &now + } + + err := controller.AddOrUpdateTaintOnNode(ctx, kubeClient, node.Name, taintsToAdd...) + if err != nil { + utilruntime.HandleError( + fmt.Errorf( + "unable to taint %+v unresponsive Node %q: %v", + taintsToAdd, + node.Name, + err)) + return false + } + logger.V(4).Info("Added taint to node", "taint", taintsToAdd, "node", klog.KRef("", node.Name)) + + err = controller.RemoveTaintOffNode(ctx, kubeClient, node.Name, node, taintsToRemove...) + if err != nil { + utilruntime.HandleError( + fmt.Errorf( + "unable to remove %+v unneeded taint from unresponsive Node %q: %v", + taintsToRemove, + node.Name, + err)) + return false + } + logger.V(4).Info("Made sure that node has no taint", "node", klog.KRef("", node.Name), "taint", taintsToRemove) + + return true +} + +// AddOrUpdateLabelsOnNode updates the labels on the node and returns true on +// success and false on failure. +func AddOrUpdateLabelsOnNode(ctx context.Context, kubeClient clientset.Interface, labelsToUpdate map[string]string, node *v1.Node) bool { + logger := klog.FromContext(ctx) + if err := controller.AddOrUpdateLabelsOnNode(kubeClient, node.Name, labelsToUpdate); err != nil { + utilruntime.HandleError( + fmt.Errorf( + "unable to update labels %+v for Node %q: %v", + labelsToUpdate, + node.Name, + err)) + return false + } + logger.V(4).Info("Updated labels to node", "label", labelsToUpdate, "node", klog.KRef("", node.Name)) + return true +} + +// CreateAddNodeHandler creates an add node handler. +func CreateAddNodeHandler(f func(node *v1.Node) error) func(obj interface{}) { + return func(originalObj interface{}) { + node := originalObj.(*v1.Node).DeepCopy() + if err := f(node); err != nil { + utilruntime.HandleError(fmt.Errorf("Error while processing Node Add: %v", err)) + } + } +} + +// CreateUpdateNodeHandler creates a node update handler. (Common to lifecycle and ipam) +func CreateUpdateNodeHandler(f func(oldNode, newNode *v1.Node) error) func(oldObj, newObj interface{}) { + return func(origOldObj, origNewObj interface{}) { + node := origNewObj.(*v1.Node).DeepCopy() + prevNode := origOldObj.(*v1.Node).DeepCopy() + + if err := f(prevNode, node); err != nil { + utilruntime.HandleError(fmt.Errorf("Error while processing Node Add/Delete: %v", err)) + } + } +} + +// CreateDeleteNodeHandler creates a delete node handler. (Common to lifecycle and ipam) +func CreateDeleteNodeHandler(logger klog.Logger, f func(node *v1.Node) error) func(obj interface{}) { + return func(originalObj interface{}) { + originalNode, isNode := originalObj.(*v1.Node) + // We can get DeletedFinalStateUnknown instead of *v1.Node here and + // we need to handle that correctly. #34692 + if !isNode { + deletedState, ok := originalObj.(cache.DeletedFinalStateUnknown) + if !ok { + logger.Error(nil, "Received unexpected object", "object", originalObj) + return + } + originalNode, ok = deletedState.Obj.(*v1.Node) + if !ok { + logger.Error(nil, "DeletedFinalStateUnknown contained non-Node object", "object", deletedState.Obj) + return + } + } + node := originalNode.DeepCopy() + if err := f(node); err != nil { + utilruntime.HandleError(fmt.Errorf("Error while processing Node Add/Delete: %v", err)) + } + } +} + +// GetNodeCondition extracts the provided condition from the given status and returns that. +// Returns nil and -1 if the condition is not present, and the index of the located condition. +func GetNodeCondition(status *v1.NodeStatus, conditionType v1.NodeConditionType) (int, *v1.NodeCondition) { + if status == nil { + return -1, nil + } + for i := range status.Conditions { + if status.Conditions[i].Type == conditionType { + return i, &status.Conditions[i] + } + } + return -1, nil +} diff --git a/cloud/util/node/node.go b/cloud/util/node/node.go new file mode 100644 index 00000000..79d5cd6c --- /dev/null +++ b/cloud/util/node/node.go @@ -0,0 +1,206 @@ +package node + +import ( + "context" + "encoding/json" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + clientretry "k8s.io/client-go/util/retry" +) + +var UpdateTaintBackoff = wait.Backoff{ + Steps: 5, + Duration: 100 * time.Millisecond, + Jitter: 1.0, +} + +var UpdateLabelBackoff = wait.Backoff{ + Steps: 5, + Duration: 100 * time.Millisecond, + Jitter: 1.0, +} + +// AddOrUpdateTaintOnNode add taints to the node. If taint was added into node, it'll issue API calls +// to update nodes; otherwise, no API calls. Return error if any. +func AddOrUpdateTaintOnNode(ctx context.Context, c clientset.Interface, nodeName string, taints ...*v1.Taint) error { + if len(taints) == 0 { + return nil + } + firstTry := true + return clientretry.RetryOnConflict(UpdateTaintBackoff, func() error { + var err error + var oldNode *v1.Node + // First we try getting node from the API server cache, as it's cheaper. If it fails + // we get it from etcd to be sure to have fresh data. + option := metav1.GetOptions{} + if firstTry { + option.ResourceVersion = "0" + firstTry = false + } + oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, option) + if err != nil { + return err + } + + var newNode *v1.Node + updated := false + /* + oldNodeCopy := oldNode + for _, taint := range taints { + curNewNode, ok, err := taintutils.AddOrUpdateTaint(oldNodeCopy, taint) + if err != nil { + return fmt.Errorf("failed to update taint of node") + } + updated = updated || ok + newNode = curNewNode + oldNodeCopy = curNewNode + } + */ + if !updated { + return nil + } + return PatchNodeTaints(ctx, c, nodeName, oldNode, newNode) + }) +} + +// PatchNodeTaints patches node's taints. +func PatchNodeTaints(ctx context.Context, c clientset.Interface, nodeName string, oldNode *v1.Node, newNode *v1.Node) error { + // Strip base diff node from RV to ensure that our Patch request will set RV to check for conflicts over .spec.taints. + // This is needed because .spec.taints does not specify patchMergeKey and patchStrategy and adding them is no longer an option for compatibility reasons. + // Using other Patch strategy works for adding new taints, however will not resolve problem with taint removal. + oldNodeNoRV := oldNode.DeepCopy() + oldNodeNoRV.ResourceVersion = "" + oldDataNoRV, err := json.Marshal(&oldNodeNoRV) + if err != nil { + return fmt.Errorf("failed to marshal old node %#v for node %q: %v", oldNodeNoRV, nodeName, err) + } + + newTaints := newNode.Spec.Taints + newNodeClone := oldNode.DeepCopy() + newNodeClone.Spec.Taints = newTaints + newData, err := json.Marshal(newNodeClone) + if err != nil { + return fmt.Errorf("failed to marshal new node %#v for node %q: %v", newNodeClone, nodeName, err) + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldDataNoRV, newData, v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create patch for node %q: %v", nodeName, err) + } + + _, err = c.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) + return err +} + +// RemoveTaintOffNode is for cleaning up taints temporarily added to node, +// won't fail if target taint doesn't exist or has been removed. +// If passed a node it'll check if there's anything to be done, if taint is not present it won't issue +// any API calls. +func RemoveTaintOffNode(ctx context.Context, c clientset.Interface, nodeName string, node *v1.Node, taints ...*v1.Taint) error { + if len(taints) == 0 { + return nil + } + // Short circuit for limiting amount of API calls. + if node != nil { + match := false + /* + for _, taint := range taints { + if taintutils.TaintExists(node.Spec.Taints, taint) { + match = true + break + } + } + */ + if !match { + return nil + } + } + + firstTry := true + return clientretry.RetryOnConflict(UpdateTaintBackoff, func() error { + var err error + var oldNode *v1.Node + // First we try getting node from the API server cache, as it's cheaper. If it fails + // we get it from etcd to be sure to have fresh data. + option := metav1.GetOptions{} + if firstTry { + option.ResourceVersion = "0" + firstTry = false + } + oldNode, err = c.CoreV1().Nodes().Get(ctx, nodeName, option) + if err != nil { + return err + } + + var newNode *v1.Node + updated := false + /* + oldNodeCopy := oldNode + for _, taint := range taints { + curNewNode, ok, err := taintutils.RemoveTaint(oldNodeCopy, taint) + if err != nil { + return fmt.Errorf("failed to remove taint of node") + } + updated = updated || ok + newNode = curNewNode + oldNodeCopy = curNewNode + } + */ + if !updated { + return nil + } + return PatchNodeTaints(ctx, c, nodeName, oldNode, newNode) + }) +} + +func AddOrUpdateLabelsOnNode(kubeClient clientset.Interface, nodeName string, labelsToUpdate map[string]string) error { + firstTry := true + return clientretry.RetryOnConflict(UpdateLabelBackoff, func() error { + var err error + var node *v1.Node + // First we try getting node from the API server cache, as it's cheaper. If it fails + // we get it from etcd to be sure to have fresh data. + option := metav1.GetOptions{} + if firstTry { + option.ResourceVersion = "0" + firstTry = false + } + node, err = kubeClient.CoreV1().Nodes().Get(context.TODO(), nodeName, option) + if err != nil { + return err + } + + // Make a copy of the node and update the labels. + newNode := node.DeepCopy() + if newNode.Labels == nil { + newNode.Labels = make(map[string]string) + } + for key, value := range labelsToUpdate { + newNode.Labels[key] = value + } + + oldData, err := json.Marshal(node) + if err != nil { + return fmt.Errorf("failed to marshal the existing node %#v: %v", node, err) + } + newData, err := json.Marshal(newNode) + if err != nil { + return fmt.Errorf("failed to marshal the new node %#v: %v", newNode, err) + } + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, &v1.Node{}) + if err != nil { + return fmt.Errorf("failed to create a two-way merge patch: %v", err) + } + if _, err := kubeClient.CoreV1().Nodes().Patch(context.TODO(), node.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to patch the node: %v", err) + } + return nil + }) +} diff --git a/devbox.lock b/devbox.lock index f172e44c..cc5f9975 100644 --- a/devbox.lock +++ b/devbox.lock @@ -2,104 +2,104 @@ "lockfile_version": "1", "packages": { "clusterctl@latest": { - "last_modified": "2025-02-23T09:42:26Z", - "resolved": "github:NixOS/nixpkgs/2d068ae5c6516b2d04562de50a58c682540de9bf#clusterctl", + "last_modified": "2025-03-20T07:39:01Z", + "resolved": "github:NixOS/nixpkgs/7344a3b78128f7b1765dba89060b015fb75431a7#clusterctl", "source": "devbox-search", - "version": "1.9.5", + "version": "1.9.6", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/rhz6ibkai0lk1hfjc9cplmplgg2birvj-clusterctl-1.9.5", + "path": "/nix/store/i3784y410i7b0zr3qlxb7r9h3gb2fh1l-clusterctl-1.9.6", "default": true } ], - "store_path": "/nix/store/rhz6ibkai0lk1hfjc9cplmplgg2birvj-clusterctl-1.9.5" + "store_path": "/nix/store/i3784y410i7b0zr3qlxb7r9h3gb2fh1l-clusterctl-1.9.6" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/fxbllswc2ajmfacrj3455a9lqwqp6rah-clusterctl-1.9.5", + "path": "/nix/store/qmajsfli07c8gc3iznlkcb60g76bqwpa-clusterctl-1.9.6", "default": true } ], - "store_path": "/nix/store/fxbllswc2ajmfacrj3455a9lqwqp6rah-clusterctl-1.9.5" + "store_path": "/nix/store/qmajsfli07c8gc3iznlkcb60g76bqwpa-clusterctl-1.9.6" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/fgxhqa2ap491rc5hhrxh3y9av7ysv4nx-clusterctl-1.9.5", + "path": "/nix/store/ydi3sbhj7q3hssm1nl2487hg606hf96x-clusterctl-1.9.6", "default": true } ], - "store_path": "/nix/store/fgxhqa2ap491rc5hhrxh3y9av7ysv4nx-clusterctl-1.9.5" + "store_path": "/nix/store/ydi3sbhj7q3hssm1nl2487hg606hf96x-clusterctl-1.9.6" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/s3fsmnxsiz1gv5gk81igbw46ijkidv43-clusterctl-1.9.5", + "path": "/nix/store/wjyzcn9h4idp6z18gdz16z2wmkw66nm1-clusterctl-1.9.6", "default": true } ], - "store_path": "/nix/store/s3fsmnxsiz1gv5gk81igbw46ijkidv43-clusterctl-1.9.5" + "store_path": "/nix/store/wjyzcn9h4idp6z18gdz16z2wmkw66nm1-clusterctl-1.9.6" } } }, "ctlptl@latest": { - "last_modified": "2025-02-27T15:48:43Z", - "resolved": "github:NixOS/nixpkgs/6c5c5f5100281f8f4ff23f13edd17d645178c87c#ctlptl", + "last_modified": "2025-03-23T05:31:05Z", + "resolved": "github:NixOS/nixpkgs/dd613136ee91f67e5dba3f3f41ac99ae89c5406b#ctlptl", "source": "devbox-search", - "version": "0.8.39", + "version": "0.8.40", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/s403dlw5syd2mvwk655zjc2dza0r79ad-ctlptl-0.8.39", + "path": "/nix/store/yh2kn77gr4lr318w9n58566i9s222m87-ctlptl-0.8.40", "default": true } ], - "store_path": "/nix/store/s403dlw5syd2mvwk655zjc2dza0r79ad-ctlptl-0.8.39" + "store_path": "/nix/store/yh2kn77gr4lr318w9n58566i9s222m87-ctlptl-0.8.40" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/cjxia3v18wmhmrngy77dq7l7gbpxzh6w-ctlptl-0.8.39", + "path": "/nix/store/ym5y398pycdlw05zw6639w8nx6nahz81-ctlptl-0.8.40", "default": true } ], - "store_path": "/nix/store/cjxia3v18wmhmrngy77dq7l7gbpxzh6w-ctlptl-0.8.39" + "store_path": "/nix/store/ym5y398pycdlw05zw6639w8nx6nahz81-ctlptl-0.8.40" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/kzz611ikpys2994zma71y8xi7kjszrbl-ctlptl-0.8.39", + "path": "/nix/store/60fwxkhgkd24n4hn9mh0gjhd4vkhbvs3-ctlptl-0.8.40", "default": true } ], - "store_path": "/nix/store/kzz611ikpys2994zma71y8xi7kjszrbl-ctlptl-0.8.39" + "store_path": "/nix/store/60fwxkhgkd24n4hn9mh0gjhd4vkhbvs3-ctlptl-0.8.40" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/c4q76lxk08v6csh46v841wi6jv4vycqw-ctlptl-0.8.39", + "path": "/nix/store/9c935wx8x3s43yxnc0npr04wfk3x5b83-ctlptl-0.8.40", "default": true } ], - "store_path": "/nix/store/c4q76lxk08v6csh46v841wi6jv4vycqw-ctlptl-0.8.39" + "store_path": "/nix/store/9c935wx8x3s43yxnc0npr04wfk3x5b83-ctlptl-0.8.40" } } }, "docker@latest": { - "last_modified": "2025-03-11T17:52:14Z", - "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#docker", + "last_modified": "2025-03-23T05:31:05Z", + "resolved": "github:NixOS/nixpkgs/dd613136ee91f67e5dba3f3f41ac99ae89c5406b#docker", "source": "devbox-search", "version": "27.5.1", "systems": { @@ -107,41 +107,41 @@ "outputs": [ { "name": "out", - "path": "/nix/store/r3bnfs9i0fs8f7zvws56ff7jvxa7l6dw-docker-27.5.1", + "path": "/nix/store/i9h5shp9wsfg8a1zq77m7sa5ym6gynx7-docker-27.5.1", "default": true } ], - "store_path": "/nix/store/r3bnfs9i0fs8f7zvws56ff7jvxa7l6dw-docker-27.5.1" + "store_path": "/nix/store/i9h5shp9wsfg8a1zq77m7sa5ym6gynx7-docker-27.5.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/g44x3mhdkwiqlxyxynwc28qzl0d7mcx2-docker-27.5.1", + "path": "/nix/store/rmbzg6a5lg44qyy4ivwyqwzfgg4mwyiy-docker-27.5.1", "default": true } ], - "store_path": "/nix/store/g44x3mhdkwiqlxyxynwc28qzl0d7mcx2-docker-27.5.1" + "store_path": "/nix/store/rmbzg6a5lg44qyy4ivwyqwzfgg4mwyiy-docker-27.5.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/w8xv8gg4vbcdavb54aji5jna0ljl25z3-docker-27.5.1", + "path": "/nix/store/qfrfjbcvah6p3pnvg297a8agv56prvzs-docker-27.5.1", "default": true } ], - "store_path": "/nix/store/w8xv8gg4vbcdavb54aji5jna0ljl25z3-docker-27.5.1" + "store_path": "/nix/store/qfrfjbcvah6p3pnvg297a8agv56prvzs-docker-27.5.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/pkc753xpjd6kycn8q637d6s7x2aqs1a4-docker-27.5.1", + "path": "/nix/store/5q4lf21k0rwk7i90h4314wdr3nisglqg-docker-27.5.1", "default": true } ], - "store_path": "/nix/store/pkc753xpjd6kycn8q637d6s7x2aqs1a4-docker-27.5.1" + "store_path": "/nix/store/5q4lf21k0rwk7i90h4314wdr3nisglqg-docker-27.5.1" } } }, @@ -242,56 +242,56 @@ } }, "golangci-lint@latest": { - "last_modified": "2025-03-11T17:52:14Z", - "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#golangci-lint", + "last_modified": "2025-03-23T14:04:58Z", + "resolved": "github:NixOS/nixpkgs/f3a2a0601e9669a6e38af25b46ce6c4563bcb6da#golangci-lint", "source": "devbox-search", - "version": "1.64.6", + "version": "1.64.8", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/w7v0fzxk2y99h5h92rbbn5q4cg3khsk7-golangci-lint-1.64.6", + "path": "/nix/store/8142ildg0bp11j0ssdc77il5rxv25sm8-golangci-lint-1.64.8", "default": true } ], - "store_path": "/nix/store/w7v0fzxk2y99h5h92rbbn5q4cg3khsk7-golangci-lint-1.64.6" + "store_path": "/nix/store/8142ildg0bp11j0ssdc77il5rxv25sm8-golangci-lint-1.64.8" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/945ji0smgfq62ngq8hyp987khy1194c0-golangci-lint-1.64.6", + "path": "/nix/store/pl31v3jjbrf8l0ll7xg40dw3s2xgxbaa-golangci-lint-1.64.8", "default": true } ], - "store_path": "/nix/store/945ji0smgfq62ngq8hyp987khy1194c0-golangci-lint-1.64.6" + "store_path": "/nix/store/pl31v3jjbrf8l0ll7xg40dw3s2xgxbaa-golangci-lint-1.64.8" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/gvr1ngpwj9azzayxjvbiczflh75njk3j-golangci-lint-1.64.6", + "path": "/nix/store/0lz3b95c0y9mh95dl0jyiyn76fjxq3yf-golangci-lint-1.64.8", "default": true } ], - "store_path": "/nix/store/gvr1ngpwj9azzayxjvbiczflh75njk3j-golangci-lint-1.64.6" + "store_path": "/nix/store/0lz3b95c0y9mh95dl0jyiyn76fjxq3yf-golangci-lint-1.64.8" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/0mmm8g5pwwk7j86zwb10x53bkdm0k174-golangci-lint-1.64.6", + "path": "/nix/store/9xz2x85fpbblk2mpwj54azimxx4cvkb1-golangci-lint-1.64.8", "default": true } ], - "store_path": "/nix/store/0mmm8g5pwwk7j86zwb10x53bkdm0k174-golangci-lint-1.64.6" + "store_path": "/nix/store/9xz2x85fpbblk2mpwj54azimxx4cvkb1-golangci-lint-1.64.8" } } }, "jq@latest": { - "last_modified": "2025-03-13T11:38:39Z", - "resolved": "github:NixOS/nixpkgs/573c650e8a14b2faa0041645ab18aed7e60f0c9a#jq", + "last_modified": "2025-03-24T07:07:41Z", + "resolved": "github:NixOS/nixpkgs/1750f3c1c89488e2ffdd47cab9d05454dddfb734#jq", "source": "devbox-search", "version": "1.7.1", "systems": { @@ -299,109 +299,109 @@ "outputs": [ { "name": "bin", - "path": "/nix/store/a0d5735pbpxr817gjzvv6bdy2ycj38nf-jq-1.7.1-bin", + "path": "/nix/store/6njxc4ma96r9qc88scwsmnh64a086yrn-jq-1.7.1-bin", "default": true }, { "name": "man", - "path": "/nix/store/id3vbyr5wqz7ljylcdqjvrpdsiv4a02v-jq-1.7.1-man", + "path": "/nix/store/806zwhmy9qkp695x9y4c373fpcz8ql7i-jq-1.7.1-man", "default": true }, { - "name": "dev", - "path": "/nix/store/yhykv5iafas7lp327w3xkj1gxkhifcbk-jq-1.7.1-dev" + "name": "out", + "path": "/nix/store/7h4z72g5bpfwcnmc0a77x3aqp8vg0h0f-jq-1.7.1" }, { - "name": "doc", - "path": "/nix/store/rqjr3mln9jmidha6izp07fq9fp30y4sx-jq-1.7.1-doc" + "name": "dev", + "path": "/nix/store/haxb6i05p1ganzinkzk3xib7d8zg2zzd-jq-1.7.1-dev" }, { - "name": "out", - "path": "/nix/store/m0kqs2n6b4idvhpzcqad9jfz1cbwib2n-jq-1.7.1" + "name": "doc", + "path": "/nix/store/d1ij699i2mbwix1my3b8k7ckfykhrp1m-jq-1.7.1-doc" } ], - "store_path": "/nix/store/a0d5735pbpxr817gjzvv6bdy2ycj38nf-jq-1.7.1-bin" + "store_path": "/nix/store/6njxc4ma96r9qc88scwsmnh64a086yrn-jq-1.7.1-bin" }, "aarch64-linux": { "outputs": [ { "name": "bin", - "path": "/nix/store/64k31kawylkwbjh4r1bwxfq8zb0yzqyi-jq-1.7.1-bin", + "path": "/nix/store/xv6l5qpmjm1cm1v519d3zzx72givc912-jq-1.7.1-bin", "default": true }, { "name": "man", - "path": "/nix/store/w29k8y3v6rfyy4kvx95dc79lyfcq7ikf-jq-1.7.1-man", + "path": "/nix/store/pbq5gvwvgbfx0njjvx7zaijhn9jxgv9j-jq-1.7.1-man", "default": true }, - { - "name": "doc", - "path": "/nix/store/mhfyqmwvnqihqs5av778m4lic0p15hyf-jq-1.7.1-doc" - }, { "name": "out", - "path": "/nix/store/y3sdsgvnpmwp42d2kd9rq3d5anraccpy-jq-1.7.1" + "path": "/nix/store/dr03sim3pqyxh2aakwzmp19l54alic6c-jq-1.7.1" }, { "name": "dev", - "path": "/nix/store/r5wikmfi7na32gcx46p3ncxmzns1f5xz-jq-1.7.1-dev" + "path": "/nix/store/79rh194hlm4y033ry74mhm5c1gzhicxh-jq-1.7.1-dev" + }, + { + "name": "doc", + "path": "/nix/store/58k6ajhi376snpa2qx9nmismqwr23fi6-jq-1.7.1-doc" } ], - "store_path": "/nix/store/64k31kawylkwbjh4r1bwxfq8zb0yzqyi-jq-1.7.1-bin" + "store_path": "/nix/store/xv6l5qpmjm1cm1v519d3zzx72givc912-jq-1.7.1-bin" }, "x86_64-darwin": { "outputs": [ { "name": "bin", - "path": "/nix/store/p6ph78v4kf373vnsigwaqh7j3dlapnny-jq-1.7.1-bin", + "path": "/nix/store/8xh7an43y0xkawgf0pb7mxvcr20l17qh-jq-1.7.1-bin", "default": true }, { "name": "man", - "path": "/nix/store/83f3x5240k5kqc8bw758nq4q6d3j7l5p-jq-1.7.1-man", + "path": "/nix/store/72iz7d48dslq7dlba92f72gcqiwy1bnv-jq-1.7.1-man", "default": true }, { "name": "doc", - "path": "/nix/store/5md80wrj52gklrxwsf68dkqc9igmq928-jq-1.7.1-doc" + "path": "/nix/store/y7riwf6jr799jvd9v8aj5626ix6w4s6v-jq-1.7.1-doc" }, { "name": "out", - "path": "/nix/store/fbv7k9csv3m2af8chfblxzlv2mgaz8i0-jq-1.7.1" + "path": "/nix/store/yv3i7vxi014xhly4dv8lkgh05p2n5wsy-jq-1.7.1" }, { "name": "dev", - "path": "/nix/store/6lhl23njgh2pjh3sypxv4s0g6n0zdmqk-jq-1.7.1-dev" + "path": "/nix/store/x1rvxbvad0bkmrb849likb36kp44p6k3-jq-1.7.1-dev" } ], - "store_path": "/nix/store/p6ph78v4kf373vnsigwaqh7j3dlapnny-jq-1.7.1-bin" + "store_path": "/nix/store/8xh7an43y0xkawgf0pb7mxvcr20l17qh-jq-1.7.1-bin" }, "x86_64-linux": { "outputs": [ { "name": "bin", - "path": "/nix/store/134m2q047vsr9miwh5l227j7sh9jb130-jq-1.7.1-bin", + "path": "/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin", "default": true }, { "name": "man", - "path": "/nix/store/8vlqxykb9959nn5bl5vph99d2c4np9wg-jq-1.7.1-man", + "path": "/nix/store/0h33kdarki6bp6pwf7xc5rzjnx8ky4j4-jq-1.7.1-man", "default": true }, - { - "name": "doc", - "path": "/nix/store/byqqs2ayp67zvmd6dqi4jahwm1f779cj-jq-1.7.1-doc" - }, { "name": "out", - "path": "/nix/store/6zy1hkwlwq9r97swjy616vf096dccrsr-jq-1.7.1" + "path": "/nix/store/ma6hybrgazb471pd53mwnvh1vsv87cxb-jq-1.7.1" }, { "name": "dev", - "path": "/nix/store/94yz5x26ssx2pqn7gamlngp306s8mz60-jq-1.7.1-dev" + "path": "/nix/store/ld8ys81ch5fj94j0jrwbkgwwjxmbwdxh-jq-1.7.1-dev" + }, + { + "name": "doc", + "path": "/nix/store/nhg3x57268kb3q1021w3m002xxqpj50w-jq-1.7.1-doc" } ], - "store_path": "/nix/store/134m2q047vsr9miwh5l227j7sh9jb130-jq-1.7.1-bin" + "store_path": "/nix/store/gpxsdrrd4x93fs75395vr2dfys1ki9mq-jq-1.7.1-bin" } } }, @@ -454,86 +454,86 @@ } }, "kubectl@latest": { - "last_modified": "2025-03-11T17:52:14Z", - "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#kubectl", + "last_modified": "2025-03-22T01:19:59Z", + "resolved": "github:NixOS/nixpkgs/0740f6f238767d4caf9afe774d3e88105766dfc6#kubectl", "source": "devbox-search", - "version": "1.32.2", + "version": "1.32.3", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/cr69x3lwm2bvicn21xijyswvvfvckb78-kubectl-1.32.2", + "path": "/nix/store/7b2rpxkv6v4a29bk9dp4gqs57zhf9v9p-kubectl-1.32.3", "default": true }, { "name": "man", - "path": "/nix/store/h827jr60kqihjv4kgasfibhsrg4hbxmd-kubectl-1.32.2-man", + "path": "/nix/store/rm3753bbn2kxmwyxy7rir8z80g71lrrx-kubectl-1.32.3-man", "default": true }, { "name": "convert", - "path": "/nix/store/554dpvy977n9871yvc21nqhsnqkdg5fr-kubectl-1.32.2-convert" + "path": "/nix/store/fi6g9qi63p7ivvz3qsipfaxmym7qi623-kubectl-1.32.3-convert" } ], - "store_path": "/nix/store/cr69x3lwm2bvicn21xijyswvvfvckb78-kubectl-1.32.2" + "store_path": "/nix/store/7b2rpxkv6v4a29bk9dp4gqs57zhf9v9p-kubectl-1.32.3" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/ln7cfszvc6gjjrarbv0ipf1h8yzgkl46-kubectl-1.32.2", + "path": "/nix/store/xxngczzyxq22m1i8pc9a57w16kzir33f-kubectl-1.32.3", "default": true }, { "name": "man", - "path": "/nix/store/m79ivh1p7158mycrxiqslsdwqz0a2y15-kubectl-1.32.2-man", + "path": "/nix/store/n547iajzrqypmy89s5j2iyrdjn56fd5v-kubectl-1.32.3-man", "default": true }, { "name": "convert", - "path": "/nix/store/x9c450w9l5l9hdb4sqvysi6wlsg522x7-kubectl-1.32.2-convert" + "path": "/nix/store/0frwvbmd2zxx2n1s7vw15cbdj7iw95js-kubectl-1.32.3-convert" } ], - "store_path": "/nix/store/ln7cfszvc6gjjrarbv0ipf1h8yzgkl46-kubectl-1.32.2" + "store_path": "/nix/store/xxngczzyxq22m1i8pc9a57w16kzir33f-kubectl-1.32.3" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/i31rmk5c0cj83i961kamvnz6wns61rr5-kubectl-1.32.2", + "path": "/nix/store/95lpgw52n8x18ams6bz9nm68sml870rs-kubectl-1.32.3", "default": true }, { "name": "man", - "path": "/nix/store/bxxybbhglqz5dcrqfziil75kfp0xgbsf-kubectl-1.32.2-man", + "path": "/nix/store/3ilj3sca004bmzi0kp2lwihbqb2j2j2l-kubectl-1.32.3-man", "default": true }, { "name": "convert", - "path": "/nix/store/4pma212svdc8gvicxrs9dwv1d070jz6x-kubectl-1.32.2-convert" + "path": "/nix/store/k2wr9b02pgyj7r5fz959vxys2w3s4j5v-kubectl-1.32.3-convert" } ], - "store_path": "/nix/store/i31rmk5c0cj83i961kamvnz6wns61rr5-kubectl-1.32.2" + "store_path": "/nix/store/95lpgw52n8x18ams6bz9nm68sml870rs-kubectl-1.32.3" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/z71m7l32ll43nb5cpfs5fb3acaz3rpx2-kubectl-1.32.2", + "path": "/nix/store/fjcd0hsivc5h56ndpg357i3rwvygxr7i-kubectl-1.32.3", "default": true }, { "name": "man", - "path": "/nix/store/0nwm2ksngri57gvgn2vvpcvy2gcz79bb-kubectl-1.32.2-man", + "path": "/nix/store/ly41y2ja108cp9jpqc77s2sayvbahliw-kubectl-1.32.3-man", "default": true }, { "name": "convert", - "path": "/nix/store/52cf0bmr4pqpid2nid4l1b9i1zr8zkgn-kubectl-1.32.2-convert" + "path": "/nix/store/1ph5mjp12w3snxc2clc24akc5mwlzb47-kubectl-1.32.3-convert" } ], - "store_path": "/nix/store/z71m7l32ll43nb5cpfs5fb3acaz3rpx2-kubectl-1.32.2" + "store_path": "/nix/store/fjcd0hsivc5h56ndpg357i3rwvygxr7i-kubectl-1.32.3" } } }, @@ -586,8 +586,8 @@ } }, "kyverno-chainsaw@latest": { - "last_modified": "2025-02-23T09:42:26Z", - "resolved": "github:NixOS/nixpkgs/2d068ae5c6516b2d04562de50a58c682540de9bf#kyverno-chainsaw", + "last_modified": "2025-03-11T17:52:14Z", + "resolved": "github:NixOS/nixpkgs/0d534853a55b5d02a4ababa1d71921ce8f0aee4c#kyverno-chainsaw", "source": "devbox-search", "version": "0.2.12", "systems": { @@ -595,41 +595,41 @@ "outputs": [ { "name": "out", - "path": "/nix/store/xq8y9rk2mv3g0bh5j12v5qwwavc60cbq-kyverno-chainsaw-0.2.12", + "path": "/nix/store/yny87aqq3np9lgvmgllr9fn4mglnkk8d-kyverno-chainsaw-0.2.12", "default": true } ], - "store_path": "/nix/store/xq8y9rk2mv3g0bh5j12v5qwwavc60cbq-kyverno-chainsaw-0.2.12" + "store_path": "/nix/store/yny87aqq3np9lgvmgllr9fn4mglnkk8d-kyverno-chainsaw-0.2.12" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/awgv9ca78l625y31gm5jib535dkg6lv6-kyverno-chainsaw-0.2.12", + "path": "/nix/store/nz7c95v7pvpbimdqyggqsj3hi82v0q36-kyverno-chainsaw-0.2.12", "default": true } ], - "store_path": "/nix/store/awgv9ca78l625y31gm5jib535dkg6lv6-kyverno-chainsaw-0.2.12" + "store_path": "/nix/store/nz7c95v7pvpbimdqyggqsj3hi82v0q36-kyverno-chainsaw-0.2.12" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/i6s9gmqrfyp4mnn6qqwchh9lk832xf3b-kyverno-chainsaw-0.2.12", + "path": "/nix/store/1z3kh2g81mfkrp7r2n33dssrykw5pdyx-kyverno-chainsaw-0.2.12", "default": true } ], - "store_path": "/nix/store/i6s9gmqrfyp4mnn6qqwchh9lk832xf3b-kyverno-chainsaw-0.2.12" + "store_path": "/nix/store/1z3kh2g81mfkrp7r2n33dssrykw5pdyx-kyverno-chainsaw-0.2.12" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/7fhp7vnvchql72za4vz40cz0a53zcz75-kyverno-chainsaw-0.2.12", + "path": "/nix/store/a6g0wm8iv9by75dj2mdgpm12f2zm4nwg-kyverno-chainsaw-0.2.12", "default": true } ], - "store_path": "/nix/store/7fhp7vnvchql72za4vz40cz0a53zcz75-kyverno-chainsaw-0.2.12" + "store_path": "/nix/store/a6g0wm8iv9by75dj2mdgpm12f2zm4nwg-kyverno-chainsaw-0.2.12" } } }, @@ -663,11 +663,11 @@ "outputs": [ { "name": "out", - "path": "/nix/store/dvg2ic642cv35k3bm6qyd9azhdlnp1cg-mockgen-0.5.0", + "path": "/nix/store/zr9h7x8b02sbp79xh0xsawcfmlfq7pw2-mockgen-0.5.0", "default": true } ], - "store_path": "/nix/store/dvg2ic642cv35k3bm6qyd9azhdlnp1cg-mockgen-0.5.0" + "store_path": "/nix/store/zr9h7x8b02sbp79xh0xsawcfmlfq7pw2-mockgen-0.5.0" }, "x86_64-linux": { "outputs": [ @@ -701,11 +701,11 @@ "outputs": [ { "name": "out", - "path": "/nix/store/kn5sgwgs94gnsifzlcysb2xgiyvm3aqz-yq-go-4.45.1", + "path": "/nix/store/j3n9gm4f277ys5annkc9r8ds2mn1760d-yq-go-4.45.1", "default": true } ], - "store_path": "/nix/store/kn5sgwgs94gnsifzlcysb2xgiyvm3aqz-yq-go-4.45.1" + "store_path": "/nix/store/j3n9gm4f277ys5annkc9r8ds2mn1760d-yq-go-4.45.1" }, "x86_64-darwin": { "outputs": [ diff --git a/go.mod b/go.mod index 1ed11434..5814915b 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v5.9.11+incompatible github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -157,7 +158,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.32.0 // indirect k8s.io/apiserver v0.32.3 // indirect - k8s.io/component-helpers v0.32.3 // indirect + k8s.io/component-helpers v0.32.3 k8s.io/controller-manager v0.32.3 // indirect k8s.io/kms v0.32.3 // indirect k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect diff --git a/go.sum b/go.sum index a4533348..7cde690d 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= diff --git a/main.go b/main.go index f11a2614..9c504359 100644 --- a/main.go +++ b/main.go @@ -93,7 +93,9 @@ func main() { command.Flags().StringVar(&linode.Options.NodeBalancerBackendIPv4Subnet, "nodebalancer-backend-ipv4-subnet", "", "ipv4 subnet to use for NodeBalancer backends") command.Flags().StringSliceVar(&linode.Options.NodeBalancerTags, "nodebalancer-tags", []string{}, "Linode tags to apply to all NodeBalancers") command.Flags().BoolVar(&linode.Options.EnableIPv6ForLoadBalancers, "enable-ipv6-for-loadbalancers", false, "set both IPv4 and IPv6 addresses for all LoadBalancer services (when disabled, only IPv4 is used)") - + command.Flags().BoolVar(&linode.Options.EnableNodeCIDRAllocation, "enable-node-cidr-allocation", false, "allocate cidrs to nodes") + command.Flags().StringVar(&linode.Options.ClusterIPv4CIDR, "cluster-ipv4-cidr", "", "ipv4 cidr range for cluster ips") + command.Flags().IntVar(&linode.Options.NodeCIDRMaskSizeIPv4, "node-cidr-mask-size-ipv4", 0, "ipv4 cidr mask size for node ips") // Set static flags command.Flags().VisitAll(func(fl *pflag.Flag) { var err error