Skip to content

Commit f97122c

Browse files
pleshakovciarams87
pleshakov
and
ciarams87
committed
Automate scale test
Problem: Non-functional scale test needs to be run manually. Solution: - Automate scale test. - Use in-cluster Prometheus to collect CPU, memory and NGF metrics. - Use Kubernetes API server to get NGF logs. For development and troubleshooting, it is possible to run scale test locally in Kind cluster. However, it is necessary to bring down the number of HTTPRoutes to 50 or less (roughly). Testing: - Ran this test locally with 64 listeners, 50 routes and 50 upstreams. - Ran this test on GKE with the default configuration. Out of scope: ensuring this test runs successfully via GitHub pipeline. Closes nginx#1368 Largely based on work by Ciara in nginx#1804 Co-authored-by: Ciara Stacke <[email protected]>
1 parent 8505f8b commit f97122c

20 files changed

+1536
-435
lines changed

‎tests/scale/generate_manifests.go renamed to ‎tests/framework/generate_manifests.go

+111-72
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
//go:build scale
2-
// +build scale
3-
4-
package scale
1+
package framework
52

63
import (
74
"bytes"
5+
"errors"
86
"fmt"
9-
"os"
10-
"path/filepath"
7+
"io"
118
"text/template"
9+
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/apimachinery/pkg/util/yaml"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
1213
)
1314

14-
var gwTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
15+
const gwTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
1516
kind: Gateway
1617
metadata:
1718
name: gateway
@@ -33,7 +34,7 @@ spec:
3334
{{- end -}}
3435
{{- end -}}`
3536

36-
var hrTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
37+
const hrTmplTxt = `apiVersion: gateway.networking.k8s.io/v1
3738
kind: HTTPRoute
3839
metadata:
3940
name: {{ .Name }}
@@ -53,7 +54,7 @@ spec:
5354
port: 80`
5455

5556
// nolint:all
56-
var secretTmplTxt = `apiVersion: v1
57+
const secretTmplTxt = `apiVersion: v1
5758
kind: Secret
5859
metadata:
5960
name: {{ . }}
@@ -63,8 +64,7 @@ data:
6364
tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzZtTnJSdUZ2WXZoSE4KbXI3c1FvNUtKSUVDN3N6TFVrNExFeklSNS9yMEVaUjQ2RnRTaGJQd0ZuaXAwMFBxekhpVkhKYy92TjdkQTVLeApQS1VmdFJuQ1J6YldVaTZBZzJpRU93bXF6WUhGbVNpZkFlVjk0RlAxOGtSbjl1ckV3OEpiRXJIUncrVW51L25tCmFMRHF1eGpFTVBweGhuRklCSnYwK1R3djNEVGx6TjNwUlV6dnpidGZvZCtEVTZBSmR6N3Rid1dTNmR6MHc1Z2kKbW9RelZnbFpnVDBJek9FZkV3NVpWMnRMZllHZWRlRVJ1VjhtR041c09va3R2aGxsMU1udHRaMkZNVHgySmVjUQo3K0xBRm9YVnBTS2NjbUFVZ1JBM0xOOHdVZXBVTHZZdFhiUm1QTFc4SjFINmhFeHJHTHBiTERZNmpzbGxBNlZpCk0xMjVjU0hsQWdNQkFBRUNnZ0VBQnpaRE50bmVTdWxGdk9HZlFYaHRFWGFKdWZoSzJBenRVVVpEcUNlRUxvekQKWlV6dHdxbkNRNlJLczUyandWNTN4cU9kUU94bTNMbjNvSHdNa2NZcEliWW82MjJ2dUczYnkwaVEzaFlsVHVMVgpqQmZCcS9UUXFlL2NMdngvSkczQWhFNmJxdFRjZFlXeGFmTmY2eUtpR1dzZk11WVVXTWs4MGVJVUxuRmZaZ1pOCklYNTlSOHlqdE9CVm9Sa3hjYTVoMW1ZTDFsSlJNM3ZqVHNHTHFybmpOTjNBdWZ3ZGRpK1VDbGZVL2l0K1EvZkUKV216aFFoTlRpNVFkRWJLVStOTnYvNnYvb2JvandNb25HVVBCdEFTUE05cmxFemIralQ1WHdWQjgvLzRGY3VoSwoyVzNpcjhtNHVlQ1JHSVlrbGxlLzhuQmZ0eVhiVkNocVRyZFBlaGlPM1FLQmdRRGlrR3JTOTc3cjg3Y1JPOCtQClpoeXltNXo4NVIzTHVVbFNTazJiOTI1QlhvakpZL2RRZDVTdFVsSWE4OUZKZnNWc1JRcEhHaTFCYzBMaTY1YjIKazR0cE5xcVFoUmZ1UVh0UG9GYXRuQzlPRnJVTXJXbDVJN0ZFejZnNkNQMVBXMEg5d2hPemFKZUdpZVpNYjlYTQoybDdSSFZOcC9jTDlYbmhNMnN0Q1lua2Iwd0tCZ1FEUzF4K0crakEyUVNtRVFWNXA1RnRONGcyamsyZEFjMEhNClRIQ2tTazFDRjhkR0Z2UWtsWm5ZbUt0dXFYeXNtekJGcnZKdmt2eUhqbUNYYTducXlpajBEdDZtODViN3BGcVAKQWxtajdtbXI3Z1pUeG1ZMXBhRWFLMXY4SDNINGtRNVl3MWdrTWRybVJHcVAvaTBGaDVpaGtSZS9DOUtGTFVkSQpDcnJjTzhkUVp3S0JnSHA1MzRXVWNCMVZibzFlYStIMUxXWlFRUmxsTWlwRFM2TzBqeWZWSmtFb1BZSEJESnp2ClIrdzZLREJ4eFoyWmJsZ05LblV0YlhHSVFZd3lGelhNcFB5SGxNVHpiZkJhYmJLcDFyR2JVT2RCMXpXM09PRkgKcmppb21TUm1YNmxhaDk0SjRHU0lFZ0drNGw1SHhxZ3JGRDZ2UDd4NGRjUktJWFpLZ0w2dVJSSUpBb0dCQU1CVApaL2p5WStRNTBLdEtEZHUrYU9ORW4zaGxUN3hrNXRKN3NBek5rbWdGMU10RXlQUk9Xd1pQVGFJbWpRbk9qbHdpCldCZ2JGcXg0M2ZlQ1Z4ZXJ6V3ZEM0txaWJVbWpCTkNMTGtYeGh3ZEVteFQwVit2NzZGYzgwaTNNYVdSNnZZR08KditwVVovL0F6UXdJcWZ6dlVmV2ZxdStrMHlhVXhQOGNlcFBIRyt0bEFvR0FmQUtVVWhqeFU0Ym5vVzVwVUhKegpwWWZXZXZ5TW54NWZyT2VsSmRmNzlvNGMvMHhVSjh1eFBFWDFkRmNrZW96dHNpaVFTNkN6MENRY09XVWxtSkRwCnVrdERvVzM3VmNSQU1BVjY3NlgxQVZlM0UwNm5aL2g2Tkd4Z28rT042Q3pwL0lkMkJPUm9IMFAxa2RjY1NLT3kKMUtFZlNnb1B0c1N1eEpBZXdUZmxDMXc9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
6465
`
6566

66-
var appTmplTxt = `apiVersion: v1
67-
apiVersion: apps/v1
67+
const appTmplTxt = `apiVersion: apps/v1
6868
kind: Deployment
6969
metadata:
7070
name: {{ . }}
@@ -105,25 +105,55 @@ var (
105105
appTmpl = template.Must(template.New("app").Parse(appTmplTxt))
106106
)
107107

108-
type Listener struct {
108+
type listener struct {
109109
Name string
110110
HostnamePrefix string
111111
SecretName string
112112
}
113113

114-
type Route struct {
114+
type route struct {
115115
Name string
116116
ListenerName string
117117
HostnamePrefix string
118118
BackendName string
119119
}
120120

121-
func getPrereqDirName(manifestDir string) string {
122-
return filepath.Join(manifestDir, "prereqs")
121+
// ScaleObjects contains objects for scale testing.
122+
type ScaleObjects struct {
123+
// BaseObjects contains objects that are common to all scale iterations.
124+
BaseObjects []client.Object
125+
// ScaleIterationGroups contains objects for each scale iteration.
126+
ScaleIterationGroups [][]client.Object
123127
}
124128

125-
func generateScaleListenerManifests(numListeners int, manifestDir string, tls bool) error {
126-
listeners := make([]Listener, 0)
129+
func decodeObjects(reader io.Reader) ([]client.Object, error) {
130+
var objects []client.Object
131+
132+
decoder := yaml.NewYAMLOrJSONDecoder(reader, 4096)
133+
for {
134+
obj := unstructured.Unstructured{}
135+
if err := decoder.Decode(&obj); err != nil {
136+
if errors.Is(err, io.EOF) {
137+
break
138+
}
139+
return nil, fmt.Errorf("error decoding resource: %w", err)
140+
}
141+
142+
if len(obj.Object) == 0 {
143+
continue
144+
}
145+
146+
objects = append(objects, &obj)
147+
}
148+
149+
return objects, nil
150+
}
151+
152+
// GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test.
153+
func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) {
154+
var result ScaleObjects
155+
156+
listeners := make([]listener, 0)
127157
backends := make([]string, 0)
128158
secrets := make([]string, 0)
129159

@@ -138,13 +168,13 @@ func generateScaleListenerManifests(numListeners int, manifestDir string, tls bo
138168
secrets = append(secrets, secretName)
139169
}
140170

141-
listeners = append(listeners, Listener{
171+
listeners = append(listeners, listener{
142172
Name: listenerName,
143173
HostnamePrefix: hostnamePrefix,
144174
SecretName: secretName,
145175
})
146176

147-
route := Route{
177+
r := route{
148178
Name: fmt.Sprintf("route-%d", i),
149179
ListenerName: listenerName,
150180
HostnamePrefix: hostnamePrefix,
@@ -153,80 +183,101 @@ func generateScaleListenerManifests(numListeners int, manifestDir string, tls bo
153183

154184
backends = append(backends, backendName)
155185

156-
if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil {
157-
return err
186+
objects, err := generateManifests(listeners, []route{r})
187+
if err != nil {
188+
return ScaleObjects{}, err
158189
}
190+
191+
result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
159192
}
160193

161-
if err := generateSecrets(getPrereqDirName(manifestDir), secrets); err != nil {
162-
return err
194+
secretObjects, err := generateSecrets(secrets)
195+
if err != nil {
196+
return ScaleObjects{}, err
163197
}
164198

165-
return generateBackendAppManifests(getPrereqDirName(manifestDir), backends)
166-
}
199+
result.BaseObjects = append(result.BaseObjects, secretObjects...)
167200

168-
func generateSecrets(secretsDir string, secrets []string) error {
169-
err := os.Mkdir(secretsDir, 0o750)
170-
if err != nil && !os.IsExist(err) {
171-
return err
201+
backendObjects, err := generateBackendAppObjects(backends)
202+
if err != nil {
203+
return ScaleObjects{}, err
172204
}
173205

206+
result.BaseObjects = append(result.BaseObjects, backendObjects...)
207+
208+
return result, nil
209+
}
210+
211+
func generateSecrets(secrets []string) ([]client.Object, error) {
212+
objects := make([]client.Object, 0, len(secrets))
213+
174214
for _, secret := range secrets {
175215
var buf bytes.Buffer
176216

177-
if err = secretTmpl.Execute(&buf, secret); err != nil {
178-
return err
217+
if err := secretTmpl.Execute(&buf, secret); err != nil {
218+
return nil, err
179219
}
180220

181-
path := filepath.Join(secretsDir, fmt.Sprintf("%s.yaml", secret))
182-
183-
fmt.Println("Writing", path)
184-
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
185-
return err
221+
objs, err := decodeObjects(&buf)
222+
if err != nil {
223+
return nil, err
186224
}
225+
226+
objects = append(objects, objs...)
187227
}
188228

189-
return nil
229+
return objects, nil
190230
}
191231

192-
func generateScaleHTTPRouteManifests(numRoutes int, manifestDir string) error {
193-
l := Listener{
232+
// GenerateScaleHTTPRouteObjects generates objects for a given number of routes for the scale test.
233+
func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) {
234+
var result ScaleObjects
235+
236+
l := listener{
194237
Name: "listener",
195238
HostnamePrefix: "*",
196239
}
197240

198241
backendName := "backend"
199242

200243
for i := 0; i < numRoutes; i++ {
201-
202-
route := Route{
244+
r := route{
203245
Name: fmt.Sprintf("route-%d", i),
204246
HostnamePrefix: fmt.Sprintf("%d", i),
205247
ListenerName: "listener",
206248
BackendName: backendName,
207249
}
208250

209-
var listeners []Listener
251+
var listeners []listener
210252
if i == 0 {
211253
// only generate a Gateway on the first iteration
212-
listeners = []Listener{l}
254+
listeners = []listener{l}
213255
}
214256

215-
if err := generateManifests(manifestDir, i, listeners, []Route{route}); err != nil {
216-
return err
257+
objects, err := generateManifests(listeners, []route{r})
258+
if err != nil {
259+
return ScaleObjects{}, err
217260
}
218261

262+
result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
263+
}
264+
265+
backendObjects, err := generateBackendAppObjects([]string{backendName})
266+
if err != nil {
267+
return ScaleObjects{}, err
219268
}
220269

221-
return generateBackendAppManifests(getPrereqDirName(manifestDir), []string{backendName})
270+
result.BaseObjects = backendObjects
271+
272+
return result, nil
222273
}
223274

224-
func generateManifests(outDir string, version int, listeners []Listener, routes []Route) error {
275+
func generateManifests(listeners []listener, routes []route) ([]client.Object, error) {
225276
var buf bytes.Buffer
226277

227278
if len(listeners) > 0 {
228279
if err := gwTmpl.Execute(&buf, listeners); err != nil {
229-
return err
280+
return nil, err
230281
}
231282
}
232283

@@ -236,42 +287,30 @@ func generateManifests(outDir string, version int, listeners []Listener, routes
236287
}
237288

238289
if err := hrTmpl.Execute(&buf, r); err != nil {
239-
return err
290+
return nil, err
240291
}
241292
}
242293

243-
err := os.Mkdir(outDir, 0o750)
244-
if err != nil && !os.IsExist(err) {
245-
return err
246-
}
247-
248-
filename := fmt.Sprintf("manifest-%d.yaml", version)
249-
path := filepath.Join(outDir, filename)
250-
251-
fmt.Println("Writing", path)
252-
return os.WriteFile(path, buf.Bytes(), 0o600)
294+
return decodeObjects(&buf)
253295
}
254296

255-
func generateBackendAppManifests(outDir string, backends []string) error {
256-
err := os.Mkdir(outDir, 0o750)
257-
if err != nil && !os.IsExist(err) {
258-
return err
259-
}
297+
func generateBackendAppObjects(backends []string) ([]client.Object, error) {
298+
objects := make([]client.Object, 0, 2*len(backends))
260299

261300
for _, backend := range backends {
262301
var buf bytes.Buffer
263302

264-
if err = appTmpl.Execute(&buf, backend); err != nil {
265-
return err
303+
if err := appTmpl.Execute(&buf, backend); err != nil {
304+
return nil, err
266305
}
267306

268-
path := filepath.Join(outDir, fmt.Sprintf("%s.yaml", backend))
269-
270-
fmt.Println("Writing", path)
271-
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
272-
return err
307+
objs, err := decodeObjects(&buf)
308+
if err != nil {
309+
return nil, err
273310
}
311+
312+
objects = append(objects, objs...)
274313
}
275314

276-
return nil
315+
return objects, nil
277316
}

‎tests/framework/portforward.go

+29-17
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@ import (
66
"net/http"
77
"net/url"
88
"path"
9+
"time"
10+
11+
"log/slog"
912

1013
"k8s.io/client-go/rest"
1114
"k8s.io/client-go/tools/portforward"
1215
"k8s.io/client-go/transport/spdy"
1316
)
1417

15-
// PortForward starts a port-forward to the specified Pod and returns the local port being forwarded.
16-
func PortForward(config *rest.Config, namespace, podName string, stopCh chan struct{}) (int, error) {
18+
// PortForward starts a port-forward to the specified Pod.
19+
func PortForward(config *rest.Config, namespace, podName string, ports []string, stopCh <-chan struct{}) error {
1720
roundTripper, upgrader, err := spdy.RoundTripperFor(config)
1821
if err != nil {
19-
return 0, fmt.Errorf("error creating roundtripper: %w", err)
22+
return fmt.Errorf("error creating roundtripper: %w", err)
2023
}
2124

2225
serverURL, err := url.Parse(config.Host)
2326
if err != nil {
24-
return 0, fmt.Errorf("error parsing rest config host: %w", err)
27+
return fmt.Errorf("error parsing rest config host: %w", err)
2528
}
2629

2730
serverURL.Path = path.Join(
@@ -33,25 +36,34 @@ func PortForward(config *rest.Config, namespace, podName string, stopCh chan str
3336

3437
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL)
3538

36-
readyCh := make(chan struct{}, 1)
3739
out, errOut := new(bytes.Buffer), new(bytes.Buffer)
3840

39-
forwarder, err := portforward.New(dialer, []string{":80"}, stopCh, readyCh, out, errOut)
40-
if err != nil {
41-
return 0, fmt.Errorf("error creating port forwarder: %w", err)
41+
forward := func() error {
42+
readyCh := make(chan struct{}, 1)
43+
44+
forwarder, err := portforward.New(dialer, ports, stopCh, readyCh, out, errOut)
45+
if err != nil {
46+
return fmt.Errorf("error creating port forwarder: %w", err)
47+
}
48+
49+
return forwarder.ForwardPorts()
4250
}
4351

4452
go func() {
45-
if err := forwarder.ForwardPorts(); err != nil {
46-
panic(err)
53+
for {
54+
if err := forward(); err != nil {
55+
slog.Error("error forwarding ports", "error", err)
56+
slog.Info("retrying port forward in 100ms...")
57+
}
58+
59+
select {
60+
case <-stopCh:
61+
return
62+
case <-time.After(100 * time.Millisecond):
63+
// retrying
64+
}
4765
}
4866
}()
4967

50-
<-readyCh
51-
ports, err := forwarder.GetPorts()
52-
if err != nil {
53-
return 0, fmt.Errorf("error getting ports being forwarded: %w", err)
54-
}
55-
56-
return int(ports[0].Local), nil
68+
return nil
5769
}

0 commit comments

Comments
 (0)