Skip to content

Commit b9a06a9

Browse files
hrefhref
authored
Merge pull request #16 from cloudscale-ch/denis/lb-status-hostname
Add option to prevent cluster-traffic from bypassing loadbalancers
2 parents 0a82dba + 0c0afd8 commit b9a06a9

File tree

8 files changed

+432
-26
lines changed

8 files changed

+432
-26
lines changed

‎.github/workflows/ccm-integration-tests.yml

+1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ jobs:
129129

130130
env:
131131
CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }}
132+
HTTP_ECHO_BRANCH: ${{ vars.HTTP_ECHO_BRANCH }}
132133
KUBERNETES: '${{ matrix.kubernetes }}'
133134
SUBNET: '${{ matrix.subnet }}'
134135
CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}'

‎cmd/http-echo/go.mod

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/cloudscale-ch/cloudscale-cloud-controller-manager/cmd/http-echo
2+
3+
go 1.23.0
4+
5+
require github.com/pires/go-proxyproto v0.7.0

‎cmd/http-echo/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs=
2+
github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4=

‎cmd/http-echo/main.go

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// A http echo server to get information about connections made to it.
2+
package main
3+
4+
import (
5+
"context"
6+
"flag"
7+
"fmt"
8+
"net"
9+
"net/http"
10+
"time"
11+
12+
proxyproto "github.com/pires/go-proxyproto"
13+
)
14+
15+
func main() {
16+
host := flag.String("host", "127.0.0.1", "Host to connect to")
17+
port := flag.Int("port", 8080, "Port to connect to")
18+
19+
flag.Parse()
20+
21+
serve(*host, *port)
22+
}
23+
24+
// log http requests in basic fashion
25+
func log(h http.Handler) http.Handler {
26+
return http.HandlerFunc(
27+
func(w http.ResponseWriter, r *http.Request) {
28+
h.ServeHTTP(w, r)
29+
fmt.Printf("%s %s (%s)\n", r.Method, r.RequestURI, r.RemoteAddr)
30+
})
31+
}
32+
33+
// serve HTTP API on the given host and port
34+
func serve(host string, port int) {
35+
router := http.NewServeMux()
36+
37+
// Returns 'true' if the PROXY protocol was used for the given connection
38+
router.HandleFunc("GET /proxy-protocol/used",
39+
func(w http.ResponseWriter, r *http.Request) {
40+
fmt.Fprintln(w, r.Context().Value("HasProxyHeader"))
41+
})
42+
43+
addr := fmt.Sprintf("%s:%d", host, port)
44+
45+
server := http.Server{
46+
Addr: addr,
47+
Handler: log(router),
48+
ConnContext: func(ctx context.Context, c net.Conn) context.Context {
49+
hasProxyHeader := false
50+
51+
if c, ok := c.(*proxyproto.Conn); ok {
52+
hasProxyHeader = c.ProxyHeader() != nil
53+
}
54+
55+
return context.WithValue(ctx, "HasProxyHeader", hasProxyHeader)
56+
},
57+
}
58+
59+
listener, err := net.Listen("tcp", server.Addr)
60+
if err != nil {
61+
panic(err)
62+
}
63+
64+
fmt.Printf("Listening on %s\n", addr)
65+
66+
proxyListener := &proxyproto.Listener{
67+
Listener: listener,
68+
ReadHeaderTimeout: 10 * time.Second,
69+
}
70+
defer proxyListener.Close()
71+
72+
server.Serve(proxyListener)
73+
}

‎pkg/cloudscale_ccm/loadbalancer.go

+116-16
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1313
"k8s.io/client-go/kubernetes"
1414
"k8s.io/klog/v2"
15+
"k8s.io/utils/ptr"
1516
)
1617

1718
// Annotations used by the loadbalancer integration of cloudscale_ccm. Those
@@ -133,6 +134,60 @@ const (
133134
// as all pools have to be recreated.
134135
LoadBalancerPoolProtocol = "k8s.cloudscale.ch/loadbalancer-pool-protocol"
135136

137+
// LoadBalancerForceHostname forces the CCM to report a specific hostname
138+
// to Kubernetes when returning the loadbalancer status, instead of
139+
// reporting the IP address(es).
140+
//
141+
// The hostname used should point to the same IP address that would
142+
// otherwise be reported. This is used as a workaround for clusters that
143+
// do not support status.loadBalancer.ingress.ipMode, and use `proxy` or
144+
// `proxyv2` protocol.
145+
//
146+
// For newer clusters, .status.loadBalancer.ingress.ipMode is automatically
147+
// set to "Proxy", unless LoadBalancerIPMode is set to "VIP"
148+
//
149+
// For more information about this workaround see
150+
// https://kubernetes.io/blog/2023/12/18/kubernetes-1-29-feature-loadbalancer-ip-mode-alpha/
151+
//
152+
// To illustrate, here's an example of a load balancer status shown on
153+
// a Kubernetes 1.29 service with default settings:
154+
//
155+
// apiVersion: v1
156+
// kind: Service
157+
// ...
158+
// status:
159+
// loadBalancer:
160+
// ingress:
161+
// - ip: 45.81.71.1
162+
// - ip: 2a06:c00::1
163+
//
164+
// Using the annotation causes the status to use the given value instead:
165+
//
166+
// apiVersion: v1
167+
// kind: Service
168+
// metadata:
169+
// annotations:
170+
// k8s.cloudscale.ch/loadbalancer-force-hostname: example.org
171+
// status:
172+
// loadBalancer:
173+
// ingress:
174+
// - hostname: example.org
175+
//
176+
// If you are not using the `proxy` or `proxyv2` protocol, or if you are
177+
// on Kubernetes 1.30 or newer, you probly do not need this setting.
178+
//
179+
// See `LoadBalancerIPMode` below.
180+
LoadBalancerForceHostname = "k8s.cloudscale.ch/loadbalancer-force-hostname"
181+
182+
// LoadBalancerIPMode defines the IP mode reported to Kubernetes for the
183+
// loadbalancers managed by this CCM. It defaults to "Proxy", where all
184+
// traffic destined to the load balancer is sent through the load balancer,
185+
// even if coming from inside the cluster.
186+
//
187+
// The older behavior, where traffic inside the cluster is directly
188+
// sent to the backend service, can be activated by using "VIP" instead.
189+
LoadBalancerIPMode = "k8s.cloudscale.ch/loadbalancer-ip-mode"
190+
136191
// LoadBalancerHealthMonitorDelayS is the delay between two successive
137192
// checks, in seconds. Defaults to 2.
138193
//
@@ -269,7 +324,13 @@ func (l *loadbalancer) GetLoadBalancer(
269324
return nil, false, nil
270325
}
271326

272-
return loadBalancerStatus(instance), true, nil
327+
result, err := l.loadBalancerStatus(serviceInfo, instance)
328+
if err != nil {
329+
return nil, true, fmt.Errorf(
330+
"unable to get load balancer state for %s: %w", service.Name, err)
331+
}
332+
333+
return result, true, nil
273334
}
274335

275336
// GetLoadBalancerName returns the name of the load balancer. Implementations
@@ -361,7 +422,13 @@ func (l *loadbalancer) EnsureLoadBalancer(
361422
"unable to annotate service %s: %w", service.Name, err)
362423
}
363424

364-
return loadBalancerStatus(actual.lb), nil
425+
result, err := l.loadBalancerStatus(serviceInfo, actual.lb)
426+
if err != nil {
427+
return nil, fmt.Errorf(
428+
"unable to get load balancer state for %s: %w", service.Name, err)
429+
}
430+
431+
return result, nil
365432
}
366433

367434
// UpdateLoadBalancer updates hosts under the specified load balancer.
@@ -432,6 +499,53 @@ func (l *loadbalancer) EnsureLoadBalancerDeleted(
432499
})
433500
}
434501

502+
// loadBalancerStatus generates the v1.LoadBalancerStatus for the given
503+
// loadbalancer, as required by Kubernetes.
504+
func (l *loadbalancer) loadBalancerStatus(
505+
serviceInfo *serviceInfo,
506+
lb *cloudscale.LoadBalancer,
507+
) (*v1.LoadBalancerStatus, error) {
508+
509+
status := v1.LoadBalancerStatus{}
510+
511+
// When forcing the use of a hostname, there's exactly one ingress item
512+
hostname := serviceInfo.annotation(LoadBalancerForceHostname)
513+
if len(hostname) > 0 {
514+
status.Ingress = []v1.LoadBalancerIngress{{Hostname: hostname}}
515+
return &status, nil
516+
}
517+
518+
// Otherwise there as many items as there are addresses
519+
status.Ingress = make([]v1.LoadBalancerIngress, len(lb.VIPAddresses))
520+
521+
var ipmode *v1.LoadBalancerIPMode
522+
switch serviceInfo.annotation(LoadBalancerIPMode) {
523+
case "Proxy":
524+
ipmode = ptr.To(v1.LoadBalancerIPModeProxy)
525+
case "VIP":
526+
ipmode = ptr.To(v1.LoadBalancerIPModeVIP)
527+
default:
528+
return nil, fmt.Errorf(
529+
"unsupported IP mode: '%s', must be 'Proxy' or 'VIP'", *ipmode)
530+
}
531+
532+
// On newer releases, we explicitly configure the IP mode
533+
supportsIPMode, err := kubeutil.IsKubernetesReleaseOrNewer(l.k8s, 1, 30)
534+
if err != nil {
535+
return nil, fmt.Errorf("failed to get load balancer status: %w", err)
536+
}
537+
538+
for i, address := range lb.VIPAddresses {
539+
status.Ingress[i].IP = address.Address
540+
541+
if supportsIPMode {
542+
status.Ingress[i].IPMode = ipmode
543+
}
544+
}
545+
546+
return &status, nil
547+
}
548+
435549
// ensureValidConfig ensures that the configuration can be applied at all,
436550
// acting as a gate that ensures certain invariants before any changes are
437551
// made.
@@ -545,17 +659,3 @@ func (l *loadbalancer) findIPsAssignedElsewhere(
545659

546660
return conflicts, nil
547661
}
548-
549-
// loadBalancerStatus generates the v1.LoadBalancerStatus for the given
550-
// loadbalancer, as required by Kubernetes.
551-
func loadBalancerStatus(lb *cloudscale.LoadBalancer) *v1.LoadBalancerStatus {
552-
553-
status := v1.LoadBalancerStatus{}
554-
status.Ingress = make([]v1.LoadBalancerIngress, len(lb.VIPAddresses))
555-
556-
for i, address := range lb.VIPAddresses {
557-
status.Ingress[i].IP = address.Address
558-
}
559-
560-
return &status
561-
}

‎pkg/cloudscale_ccm/service_info.go

+4
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ func (s serviceInfo) annotation(key string) string {
8282
return s.annotationOrDefault(key, "")
8383
case LoadBalancerPoolProtocol:
8484
return s.annotationOrDefault(key, "tcp")
85+
case LoadBalancerForceHostname:
86+
return s.annotationOrDefault(key, "")
87+
case LoadBalancerIPMode:
88+
return s.annotationOrDefault(key, "Proxy")
8589
case LoadBalancerFlavor:
8690
return s.annotationOrDefault(key, "lb-standard")
8791
case LoadBalancerVIPAddresses:

0 commit comments

Comments
 (0)