diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 8e8d050dc1..bcc9ab4088 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -118,7 +118,7 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CI=true + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests - name: Run functional graceful-recovery tests @@ -132,5 +132,5 @@ jobs: run: | ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CI=true + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true working-directory: ./tests diff --git a/internal/mode/static/nginx/config/generator.go b/internal/mode/static/nginx/config/generator.go index be2877bf29..6e2e88194f 100644 --- a/internal/mode/static/nginx/config/generator.go +++ b/internal/mode/static/nginx/config/generator.go @@ -49,6 +49,7 @@ const ( ) // ConfigFolders is a list of folders where NGINX configuration files are stored. +// Volumes here also need to be added to our crossplane ephemeral test container. var ConfigFolders = []string{httpFolder, secretsFolder, includesFolder, modulesIncludesFolder, streamFolder} // Generator generates NGINX configuration files. diff --git a/tests/Makefile b/tests/Makefile index abc4aa3ec3..21618725fb 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,7 @@ GW_SERVICE_TYPE = NodePort## Service type to use for the gateway GW_SVC_GKE_INTERNAL = false NGF_VERSION ?= edge## NGF version to be tested PULL_POLICY = Never## Pull policy for the images +NGINX_CONF_DIR = internal/mode/static/nginx/conf PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC @@ -38,6 +39,10 @@ update-go-modules: ## Update the gateway-api go modules to latest main version build-test-runner-image: ## Build conformance test runner image docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile . +.PHONY: build-crossplane-image +build-crossplane-image: ## Build the crossplane image + docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile .. + .PHONY: run-conformance-tests run-conformance-tests: ## Run conformance tests kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME) @@ -80,9 +85,6 @@ ifeq ($(PLUS_ENABLED),true) NGINX_PREFIX := $(NGINX_PLUS_PREFIX) endif -.PHONY: setup-gcp-and-run-tests -setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests - .PHONY: setup-gcp-and-run-nfr-tests setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests @@ -102,13 +104,9 @@ create-gke-router: ## Create a GKE router to allow egress traffic from private n sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM ./scripts/sync-files-to-vm.sh -.PHONY: run-tests-on-vm -run-tests-on-vm: ## Run the functional tests on a GCP VM - ./scripts/run-tests-gcp-vm.sh - .PHONY: nfr-test nfr-test: ## Run the NFR tests on a GCP VM - NFR=true CI=$(CI) ./scripts/run-tests-gcp-vm.sh + CI=$(CI) ./scripts/run-tests-gcp-vm.sh .PHONY: start-longevity-test start-longevity-test: export START_LONGEVITY=true @@ -130,7 +128,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL) .PHONY: test -test: ## Runs the functional tests on your default k8s cluster +test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster + kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME) go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \ --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \ --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ diff --git a/tests/README.md b/tests/README.md index 62531de3e2..297bf8aece 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,10 +28,8 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide - [System Testing](#system-testing) - [Logging in tests](#logging-in-tests) - [Step 1 - Run the tests](#step-1---run-the-tests) - - [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally) - - [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm) - - [Functional Tests](#functional-tests) - - [NFR tests](#nfr-tests) + - [Run the functional tests locally](#run-the-functional-tests-locally) + - [Run the NFR tests on a GKE cluster from a GCP VM](#run-the-nfr-tests-on-a-gke-cluster-from-a-gcp-vm) - [Longevity testing](#longevity-testing) - [Common test amendments](#common-test-amendments) - [Step 2 - Cleanup](#step-2---cleanup) @@ -47,7 +45,7 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide - [yq](https://github.com/mikefarah/yq/#install) - Make. -If running NFR tests, or running functional tests in GKE: +If running NFR tests: - The [gcloud CLI](https://cloud.google.com/sdk/docs/install) - A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file) @@ -59,9 +57,7 @@ All the commands below are executed from the `tests` directory. You can see all ### Step 1 - Create a Kubernetes cluster -This can be done in a cloud provider of choice, or locally using `kind`. - -**Important**: NFR tests can only be run on a GKE cluster. +**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster. To create a local `kind` cluster: @@ -237,7 +233,7 @@ When running locally, the tests create a port-forward from your NGF Pod to local test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an internal LoadBalancer service which will receive the test traffic. -**Important**: NFR tests can only be run on a GKE cluster. +**Important**: Functional tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster. Directory structure is as follows: @@ -252,7 +248,7 @@ To log in the tests, use the `GinkgoWriter` interface described here: https://on ### Step 1 - Run the tests -#### 1a - Run the functional tests locally +#### Run the functional tests locally ```makefile make test TAG=$(whoami) @@ -273,9 +269,7 @@ To run the telemetry test: make test TAG=$(whoami) GINKGO_LABEL=telemetry ``` -#### 1b - Run the tests on a GKE cluster from a GCP VM - -This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM. +#### Run the NFR tests on a GKE cluster from a GCP VM Before running the below `make` commands, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions. @@ -292,7 +286,7 @@ To just set up the VM with no router (this will not run the tests): make create-and-setup-vm ``` -Otherwise, you can set up the VM, router, and run the tests with a single command. See the options in the sections below. +Otherwise, you can set up the VM, router, and run the tests with a single command. See the options below. By default, the tests run using the version of NGF that was `git cloned` during the setup. If you want to make incremental changes and copy your local changes to the VM to test, you can run @@ -301,22 +295,6 @@ incremental changes and copy your local changes to the VM to test, you can run make sync-files-to-vm ``` -#### Functional Tests - -To set up the GCP environment with the router and VM and then run the tests, run the following command: - -```makefile -make setup-gcp-and-run-tests -``` - -To use an existing VM to run the tests, run the following - -```makefile -make run-tests-on-vm -``` - -#### NFR tests - To set up the GCP environment with the router and VM and then run the tests, run the following command: ```makefile @@ -374,7 +352,7 @@ or to pass a specific flag, e.g. run a specific test, use the GINKGO_FLAGS varia make test TAG=$(whoami) GINKGO_FLAGS='-ginkgo.focus "writes the system info to a results file"' ``` -> Note: if filtering on NFR tests (or functional tests on GKE), set the filter in the appropriate field in your `vars.env` file. +> Note: if filtering on NFR tests, set the filter in the appropriate field in your `vars.env` file. If you are running the tests in GCP, add your required label/ flags to `scripts/var.env`. diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go new file mode 100644 index 0000000000..d186e47606 --- /dev/null +++ b/tests/framework/crossplane.go @@ -0,0 +1,225 @@ +package framework + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// ExpectedNginxField contains an nginx directive key and value, +// and the expected file, server, and location block that it should exist in. +type ExpectedNginxField struct { + // Directive is the directive name. + Directive string + // Value is the value for the directive. Can be the full value or a substring. If it's a substring, + // then ValueSubstringAllowed should be true. + Value string + // File is the file name that should contain the directive. Can be a full filename or a substring. + File string + // Location is the location name that the directive should exist in. + Location string + // Servers are the server names that the directive should exist in. + Servers []string + // ValueSubstringAllowed allows the expected value to be a substring of the real value. + // This makes it easier for cases when real values are complex file names or contain things we + // don't care about, and we just want to check if a substring exists. + ValueSubstringAllowed bool +} + +// ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, +// and returns whether or not that field exists where it should. +func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error { + for _, config := range conf.Config { + if !strings.Contains(config.File, expFieldCfg.File) { + continue + } + + for _, directive := range config.Parsed { + if len(expFieldCfg.Servers) == 0 { + if expFieldCfg.fieldFound(directive) { + return nil + } + continue + } + + for _, serverName := range expFieldCfg.Servers { + if directive.Directive == "server" && getServerName(directive.Block) == serverName { + for _, serverDirective := range directive.Block { + if expFieldCfg.Location == "" && expFieldCfg.fieldFound(serverDirective) { + return nil + } else if serverDirective.Directive == "location" && + fieldExistsInLocation(serverDirective, expFieldCfg) { + return nil + } + } + } + } + } + } + + b, err := json.Marshal(conf) + if err != nil { + return fmt.Errorf("error marshaling nginx config: %w", err) + } + + return fmt.Errorf("field not found; expected: %+v\nNGINX conf: %s", expFieldCfg, string(b)) +} + +func getServerName(serverBlock Directives) string { + for _, directive := range serverBlock { + if directive.Directive == "server_name" { + return directive.Args[0] + } + } + + return "" +} + +func (e ExpectedNginxField) fieldFound(directive *Directive) bool { + arg := strings.Join(directive.Args, " ") + + valueMatch := arg == e.Value + if e.ValueSubstringAllowed { + valueMatch = strings.Contains(arg, e.Value) + } + + return directive.Directive == e.Directive && valueMatch +} + +func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool { + // location could start with '=', so get the last element which is the path + loc := locationDirective.Args[len(locationDirective.Args)-1] + if loc == expFieldCfg.Location { + for _, directive := range locationDirective.Block { + if expFieldCfg.fieldFound(directive) { + return true + } + } + } + + return false +} + +// injectCrossplaneContainer adds an ephemeral container that contains crossplane for parsing +// nginx config. It attaches to the nginx container and shares volumes with it. +func injectCrossplaneContainer( + k8sClient kubernetes.Interface, + timeout time.Duration, + ngfPodName, + namespace string, +) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + pod := &core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: ngfPodName, + Namespace: namespace, + }, + Spec: core.PodSpec{ + EphemeralContainers: []core.EphemeralContainer{ + { + TargetContainerName: "nginx", + EphemeralContainerCommon: core.EphemeralContainerCommon{ + Name: "crossplane", + Image: "nginx-crossplane:latest", + ImagePullPolicy: "Never", + Stdin: true, + VolumeMounts: []core.VolumeMount{ + { + MountPath: "/etc/nginx/conf.d", + Name: "nginx-conf", + }, + { + MountPath: "/etc/nginx/stream-conf.d", + Name: "nginx-stream-conf", + }, + { + MountPath: "/etc/nginx/module-includes", + Name: "module-includes", + }, + { + MountPath: "/etc/nginx/secrets", + Name: "nginx-secrets", + }, + { + MountPath: "/etc/nginx/includes", + Name: "nginx-includes", + }, + }, + }, + }, + }, + }, + } + + podClient := k8sClient.CoreV1().Pods(namespace) + if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("error adding ephemeral container: %w", err) + } + + return nil +} + +// createCrossplaneExecutor creates the executor for the crossplane command. +func createCrossplaneExecutor( + k8sClient kubernetes.Interface, + k8sConfig *rest.Config, + ngfPodName, + namespace string, +) (remotecommand.Executor, error) { + cmd := []string{"./crossplane", "/etc/nginx/nginx.conf"} + opts := &core.PodExecOptions{ + Command: cmd, + Container: "crossplane", + Stdout: true, + Stderr: true, + } + + req := k8sClient.CoreV1().RESTClient().Post(). + Resource("pods"). + SubResource("exec"). + Name(ngfPodName). + Namespace(namespace). + VersionedParams(opts, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) + if err != nil { + return nil, fmt.Errorf("error creating executor: %w", err) + } + + return exec, nil +} + +// The following types are copied from https://github.com/nginxinc/nginx-go-crossplane, +// with unnecessary fields stripped out. +type Payload struct { + Config []Config `json:"config"` +} + +type Config struct { + File string `json:"file"` + Parsed Directives `json:"parsed"` +} + +type Directive struct { + Comment *string `json:"comment,omitempty"` + Directive string `json:"directive"` + File string `json:"file,omitempty"` + Args []string `json:"args"` + Includes []int `json:"includes,omitempty"` + Block Directives `json:"block,omitempty"` + Line int `json:"line"` +} + +type Directives []*Directive diff --git a/tests/framework/crossplane/Dockerfile b/tests/framework/crossplane/Dockerfile new file mode 100644 index 0000000000..07b069f505 --- /dev/null +++ b/tests/framework/crossplane/Dockerfile @@ -0,0 +1,23 @@ +# syntax=docker/dockerfile:1.10 +FROM golang:1.23-alpine AS builder + +WORKDIR / +ARG DIR=tests/framework/crossplane + +COPY ${DIR}/go.mod ${DIR}/go.sum . +RUN go mod download + +COPY ${DIR}/cmd/crossplane . +RUN go build -o crossplane + +FROM alpine + +WORKDIR /crossplane +ARG NGINX_CONF_DIR + +COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf +COPY --from=builder /crossplane . + +USER 101:1001 + +ENTRYPOINT ["sh"] diff --git a/tests/framework/crossplane/cmd/crossplane/main.go b/tests/framework/crossplane/cmd/crossplane/main.go new file mode 100644 index 0000000000..e7ba749e24 --- /dev/null +++ b/tests/framework/crossplane/cmd/crossplane/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + + crossplane "github.com/nginxinc/nginx-go-crossplane" +) + +// This binary accepts a single argument, the path of the base nginx config, and prints out the JSON representation +// of the full nginx config, in crossplane format. +// See https://github.com/nginxinc/nginx-go-crossplane for more info. +func main() { + if len(os.Args) != 2 { + panic(errors.New("must have exactly one argument, the path of the base nginx config")) + } + + path := os.Args[1] + + payload, err := crossplane.Parse(path, &crossplane.ParseOptions{}) + if err != nil { + panic(err) + } + + b, err := json.Marshal(payload) + if err != nil { + panic(err) + } + + fmt.Println(string(b)) +} diff --git a/tests/framework/crossplane/go.mod b/tests/framework/crossplane/go.mod new file mode 100644 index 0000000000..6b988bae84 --- /dev/null +++ b/tests/framework/crossplane/go.mod @@ -0,0 +1,14 @@ +module github.com/nginxinc/nginx-gateway-fabric/tests/framework/crossplane + +go 1.23.1 + +require github.com/nginxinc/nginx-go-crossplane v0.4.63 + +require ( + github.com/jstemmer/go-junit-report v1.0.0 // indirect + github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.23.0 // indirect +) diff --git a/tests/framework/crossplane/go.sum b/tests/framework/crossplane/go.sum new file mode 100644 index 0000000000..89581ecd59 --- /dev/null +++ b/tests/framework/crossplane/go.sum @@ -0,0 +1,30 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds= +github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= +github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= +github.com/nginxinc/nginx-go-crossplane v0.4.63 h1:nx5e+EXzPepWVM3YsTEhcs8kp8XDTK1BCzPTTmdgK1E= +github.com/nginxinc/nginx-go-crossplane v0.4.63/go.mod h1:b7L/JSru3rvbbxVJxBgkePkNvC+LXo/IWE4iJJJvUUw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= +github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index 1cb0a5065b..434a5ecaed 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -23,6 +23,7 @@ import ( "bytes" "context" "embed" + "encoding/json" "errors" "fmt" "io" @@ -31,8 +32,6 @@ import ( "strings" "time" - "k8s.io/client-go/util/retry" - apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,6 +41,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" v1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -50,6 +52,7 @@ import ( type ResourceManager struct { K8sClient client.Client ClientGoClient kubernetes.Interface // used when k8sClient is not enough + K8sConfig *rest.Config FS embed.FS TimeoutConfig TimeoutConfig } @@ -812,3 +815,55 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( }, ) } + +// GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. +func (rm *ResourceManager) GetNginxConfig(ngfPodName, namespace string) (*Payload, error) { + if err := injectCrossplaneContainer( + rm.ClientGoClient, + rm.TimeoutConfig.UpdateTimeout, + ngfPodName, + namespace, + ); err != nil { + return nil, err + } + + exec, err := createCrossplaneExecutor(rm.ClientGoClient, rm.K8sConfig, ngfPodName, namespace) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.RequestTimeout) + defer cancel() + + buf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + if err := wait.PollUntilContextCancel( + ctx, + 500*time.Millisecond, + true, /* poll immediately */ + func(ctx context.Context) (bool, error) { + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: buf, + Stderr: errBuf, + }); err != nil { + return false, nil //nolint:nilerr // we want to retry if there's an error + } + + if errBuf.String() != "" { + return false, nil + } + + return true, nil + }, + ); err != nil { + return nil, fmt.Errorf("could not connect to ephemeral container: %w", err) + } + + conf := &Payload{} + if err := json.Unmarshal(buf.Bytes(), conf); err != nil { + return nil, fmt.Errorf("error unmarshaling nginx config: %w", err) + } + + return conf, nil +} diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh deleted file mode 100755 index a15ea5cbc2..0000000000 --- a/tests/scripts/remote-scripts/run-tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source "${HOME}"/vars.env - -cd nginx-gateway-fabric/tests && make test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 77dc6f4761..42a6d85c45 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -6,18 +6,13 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source scripts/vars.env -SCRIPT=run-tests.sh -if [ "${NFR}" = "true" ]; then - SCRIPT=run-nfr-tests.sh -fi - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="export START_LONGEVITY=${START_LONGEVITY} &&\ export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ export CI=${CI} &&\ - bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT} + bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh retcode=$? if [ ${retcode} -ne 0 ]; then @@ -25,14 +20,12 @@ if [ ${retcode} -ne 0 ]; then exit 1 fi -if [ "${NFR}" = "true" ]; then - ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config - if [ "${CI}" = "false" ]; then - gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null - rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . - else - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . - fi +## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config +if [ "${CI}" = "false" ]; then + gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null + rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . +else + gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . fi ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index baffadb0ef..b5e5a8ec48 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -91,9 +91,122 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" } }) + Context("nginx config", func() { + var conf *framework.Payload + filePrefix := fmt.Sprintf("/etc/nginx/includes/ClientSettingsPolicy_%s", namespace) + + BeforeAll(func() { + podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + Expect(err).ToNot(HaveOccurred()) + Expect(podNames).To(HaveLen(1)) + + ngfPodName := podNames[0] + + conf, err = resourceManager.GetNginxConfig(ngfPodName, ngfNamespace) + Expect(err).ToNot(HaveOccurred()) + }) + + DescribeTable("is set properly for", + func(expCfgs []framework.ExpectedNginxField) { + for _, expCfg := range expCfgs { + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + } + }, + Entry("gateway policy", []framework.ExpectedNginxField{ + { + Directive: "include", + Value: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + File: "http.conf", + Servers: []string{"*.example.com", "cafe.example.com"}, + }, + { + Directive: "client_max_body_size", + Value: "1000", + File: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + }, + { + Directive: "client_body_timeout", + Value: "30s", + File: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + }, + { + Directive: "keepalive_requests", + Value: "100", + File: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + }, + { + Directive: "keepalive_time", + Value: "5s", + File: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + }, + { + Directive: "keepalive_timeout", + Value: "2s 1s", + File: fmt.Sprintf("%s_gw-csp.conf", filePrefix), + }, + }), + Entry("coffee route policy", []framework.ExpectedNginxField{ + { + Directive: "include", + Value: fmt.Sprintf("%s_coffee-route-csp.conf", filePrefix), + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/coffee", + }, + { + Directive: "client_max_body_size", + Value: "2000", + File: fmt.Sprintf("%s_coffee-route-csp.conf", filePrefix), + }, + }), + Entry("tea route policy", []framework.ExpectedNginxField{ + { + Directive: "include", + Value: fmt.Sprintf("%s_tea-route-csp.conf", filePrefix), + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/tea", + }, + { + Directive: "keepalive_requests", + Value: "200", + File: fmt.Sprintf("%s_tea-route-csp.conf", filePrefix), + }, + }), + Entry("soda route policy", []framework.ExpectedNginxField{ + { + Directive: "include", + Value: fmt.Sprintf("%s_soda-route-csp.conf", filePrefix), + File: "http.conf", + Servers: []string{"cafe.example.com"}, + Location: "/soda", + }, + { + Directive: "client_max_body_size", + Value: "3000", + File: fmt.Sprintf("%s_soda-route-csp.conf", filePrefix), + }, + }), + Entry("grpc route policy", []framework.ExpectedNginxField{ + { + Directive: "include", + Value: fmt.Sprintf("%s_grpc-route-csp.conf", filePrefix), + File: "http.conf", + Servers: []string{"*.example.com"}, + Location: "/helloworld.Greeter/SayHello", + }, + { + Directive: "client_max_body_size", + Value: "0", + File: fmt.Sprintf("%s_grpc-route-csp.conf", filePrefix), + }, + }), + ) + }) + // We only test that the client_max_body_size directive in this test is propagated correctly. // This is because we can easily verify this directive by sending requests with different sized payloads. - DescribeTable("the settings are propagated to the nginx config", + DescribeTable("client_max_body_size requests work as expected", func(uri string, byteLengthOfRequestBody, expStatus int) { url := baseURL + uri diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 5bd0747a7a..525933fe2c 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -128,6 +128,7 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { resourceManager = framework.ResourceManager{ K8sClient: k8sClient, ClientGoClient: clientGoClient, + K8sConfig: k8sConfig, FS: manifests, TimeoutConfig: timeoutConfig, }