Skip to content

Commit 7a51fd2

Browse files
VietND96VietND96
authored
K8s: Update template for service configs (#2593)
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent 7cbd637 commit 7a51fd2

21 files changed

+163
-15
lines changed

‎charts/selenium-grid/CONFIGURATION.md

+23-1
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
194194
| components.router.externalName | string | `""` | Set specific externalName when serviceType is ExternalName (see https://kubernetes.io/docs/concepts/services-networking/service/#type-externalname) |
195195
| components.router.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
196196
| components.router.serviceAnnotations | object | `{}` | Custom annotations for router service |
197+
| components.router.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
198+
| components.router.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
197199
| components.router.tolerations | list | `[]` | Tolerations for router pods |
198200
| components.router.nodeSelector | object | `{}` | Node selector for router pods |
199201
| components.router.priorityClassName | string | `""` | Priority class name for router pods |
@@ -217,6 +219,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
217219
| components.distributor.securityContext | object | `{}` | SecurityContext for Distributor container |
218220
| components.distributor.serviceType | string | `"ClusterIP"` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
219221
| components.distributor.serviceAnnotations | object | `{}` | Custom annotations for Distributor service |
222+
| components.distributor.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
223+
| components.distributor.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
220224
| components.distributor.tolerations | list | `[]` | Tolerations for Distributor pods |
221225
| components.distributor.nodeSelector | object | `{}` | Node selector for Distributor pods |
222226
| components.distributor.priorityClassName | string | `""` | Priority class name for Distributor pods |
@@ -243,6 +247,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
243247
| components.eventBus.externalName | string | `""` | Set specific externalName when serviceType is ExternalName (see https://kubernetes.io/docs/concepts/services-networking/service/#type-externalname) |
244248
| components.eventBus.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
245249
| components.eventBus.serviceAnnotations | object | `{}` | Custom annotations for Event Bus service |
250+
| components.eventBus.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
251+
| components.eventBus.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
246252
| components.eventBus.tolerations | list | `[]` | Tolerations for Event Bus pods |
247253
| components.eventBus.nodeSelector | object | `{}` | Node selector for Event Bus pods |
248254
| components.eventBus.priorityClassName | string | `""` | Priority class name for Event Bus pods |
@@ -261,6 +267,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
261267
| components.sessionMap.securityContext | object | `{}` | SecurityContext for Session Map container |
262268
| components.sessionMap.serviceType | string | `"ClusterIP"` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
263269
| components.sessionMap.serviceAnnotations | object | `{}` | Custom annotations for Session Map service |
270+
| components.sessionMap.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
271+
| components.sessionMap.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
264272
| components.sessionMap.tolerations | list | `[]` | Tolerations for Session Map pods |
265273
| components.sessionMap.nodeSelector | object | `{}` | Node selector for Session Map pods |
266274
| components.sessionMap.priorityClassName | string | `""` | Priority class name for Session Map pods |
@@ -284,6 +292,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
284292
| components.sessionQueue.securityContext | object | `{}` | SecurityContext for Session Queue container |
285293
| components.sessionQueue.serviceType | string | `"ClusterIP"` | Kubernetes service type (see https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) |
286294
| components.sessionQueue.serviceAnnotations | object | `{}` | Custom annotations for Session Queue service |
295+
| components.sessionQueue.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
296+
| components.sessionQueue.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
287297
| components.sessionQueue.tolerations | list | `[]` | Tolerations for Session Queue pods |
288298
| components.sessionQueue.nodeSelector | object | `{}` | Node selector for Session Queue pods |
289299
| components.sessionQueue.priorityClassName | string | `""` | Priority class name for Session Queue pods |
@@ -323,13 +333,15 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
323333
| hub.externalName | string | `""` | Set specific externalName when serviceType is ExternalName (see https://kubernetes.io/docs/concepts/services-networking/service/#type-externalname) |
324334
| hub.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
325335
| hub.serviceAnnotations | object | `{}` | Custom annotations for Selenium Hub service |
336+
| hub.serviceExternalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
337+
| hub.serviceSessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
326338
| hub.tolerations | list | `[]` | Tolerations for selenium-hub pods |
327339
| hub.nodeSelector | object | `{}` | Node selector for selenium-hub pods |
328340
| hub.priorityClassName | string | `""` | Priority class name for selenium-hub pods |
329341
| tracing.enabled | bool | `false` | Enable tracing. Implies installing Jaeger |
330342
| tracing.enabledWithExistingEndpoint | bool | `false` | Enable tracing without automatically installing Jaeger |
331343
| tracing.exporter | string | `"otlp"` | Exporter type for tracing. Recommended `otlp` for wide compatibility with observability backends (e.g. Jaeger, Elastic, etc.) |
332-
| tracing.exporterEndpoint | string | `"http://{{ .Release.Name }}-jaeger-collector.{{ .Release.Namespace }}:4317"` | Exporter endpoint for pushing trace data |
344+
| tracing.exporterEndpoint | string | `"http://{{ .Release.Name }}-jaeger-collector:4317"` | Exporter endpoint for pushing trace data |
333345
| tracing.globalAutoConfigure | bool | `true` | Enable global auto-configuration for tracing |
334346
| tracing.ingress.enabled | bool | `true` | Enable ingress resource to access the Jaeger |
335347
| tracing.ingress.annotations | string | `nil` | Annotations for Jaeger ingress resource |
@@ -350,6 +362,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
350362
| monitoring.exporter.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
351363
| monitoring.exporter.service.nodePort | int | `30199` | Node port for service |
352364
| monitoring.exporter.service.annotations | object | `{}` | Annotations for exporter service |
365+
| monitoring.exporter.service.externalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
366+
| monitoring.exporter.service.sessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
353367
| monitoring.exporter.replicas | int | `1` | |
354368
| monitoring.additionalScrapeConfigs.key | string | `""` | |
355369
| monitoring.additionalScrapeConfigs.value | string | `""` | |
@@ -413,6 +427,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
413427
| chromeNode.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
414428
| chromeNode.service.ports | string | `nil` | Extra ports exposed in node service |
415429
| chromeNode.service.annotations | object | `{}` | Custom annotations for service |
430+
| chromeNode.service.externalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
431+
| chromeNode.service.sessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
416432
| chromeNode.dshmVolumeSizeLimit | string | `""` | Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") |
417433
| chromeNode.priorityClassName | string | `""` | Priority class name for chrome-node pods |
418434
| chromeNode.startupProbe | object | `{"enabled":true,"failureThreshold":12,"initialDelaySeconds":0,"path":"/status","periodSeconds":5,"successThreshold":1,"timeoutSeconds":60}` | Startup probe settings |
@@ -465,6 +481,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
465481
| firefoxNode.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
466482
| firefoxNode.service.ports | string | `nil` | Extra ports exposed in node service |
467483
| firefoxNode.service.annotations | object | `{}` | Custom annotations for service |
484+
| firefoxNode.service.externalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
485+
| firefoxNode.service.sessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
468486
| firefoxNode.dshmVolumeSizeLimit | string | `"2Gi"` | Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") |
469487
| firefoxNode.priorityClassName | string | `""` | Priority class name for firefox-node pods |
470488
| firefoxNode.startupProbe | object | `{"enabled":true,"failureThreshold":12,"initialDelaySeconds":0,"path":"/status","periodSeconds":5,"successThreshold":1,"timeoutSeconds":60}` | Startup probe settings |
@@ -517,6 +535,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
517535
| edgeNode.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
518536
| edgeNode.service.ports | string | `nil` | Extra ports exposed in node service |
519537
| edgeNode.service.annotations | object | `{}` | Custom annotations for service |
538+
| edgeNode.service.externalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
539+
| edgeNode.service.sessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
520540
| edgeNode.dshmVolumeSizeLimit | string | `""` | Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") |
521541
| edgeNode.priorityClassName | string | `""` | Priority class name for edge-node pods |
522542
| edgeNode.startupProbe | object | `{"enabled":true,"failureThreshold":12,"initialDelaySeconds":0,"path":"/status","periodSeconds":5,"successThreshold":1,"timeoutSeconds":60}` | Startup probe settings |
@@ -569,6 +589,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
569589
| relayNode.service.loadBalancerIP | string | `""` | Set specific loadBalancerIP when serviceType is LoadBalancer (see https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) |
570590
| relayNode.service.ports | string | `nil` | Extra ports exposed in node service |
571591
| relayNode.service.annotations | object | `{}` | Custom annotations for service |
592+
| relayNode.service.externalTrafficPolicy | string | `""` | Set externalTrafficPolicy to Local or Cluster (see https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) |
593+
| relayNode.service.sessionAffinity | string | `""` | Set session affinity to None, ClientIP or ClientIPString |
572594
| relayNode.dshmVolumeSizeLimit | string | `""` | Size limit for DSH volume mounted in container (if not set, default is disabled, e.g "1Gi") |
573595
| relayNode.priorityClassName | string | `""` | Priority class name for relay-node pods |
574596
| relayNode.startupProbe | object | `{"enabled":true,"failureThreshold":12,"initialDelaySeconds":0,"path":"/status","periodSeconds":5,"successThreshold":1,"timeoutSeconds":60}` | Startup probe settings |

‎charts/selenium-grid/README.md

+16
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
1616
* [Settings when scalingType with `deployment`](#settings-when-scalingtype-with-deployment-)
1717
* [Settings when scalingType with `job`](#settings-when-scalingtype-with-job)
1818
* [Scaler trigger configuration](#scaler-trigger-configuration)
19+
* [Define multiple scalers with different trigger parameters.](#define-multiple-scalers-with-different-trigger-parameters)
1920
* [Settings fixed-sized thread pool for the Distributor to create new sessions](#settings-fixed-sized-thread-pool-for-the-distributor-to-create-new-sessions)
21+
* [Troubleshooting](#troubleshooting)
2022
* [Updating Selenium-Grid release](#updating-selenium-grid-release)
2123
* [Uninstalling Selenium Grid release](#uninstalling-selenium-grid-release)
2224
* [Ingress Configuration](#ingress-configuration)
@@ -356,6 +358,20 @@ In autoscaling, by default, it will calculate based on `no. of node types * maxR
356358

357359
You can override the default calculation by another value via `components.distributor.newSessionThreadPoolSize` (in full distributed mode) or `hub.newSessionThreadPoolSize` (in basic mode).
358360

361+
### Troubleshooting
362+
363+
This is a list of common issues that you might encounter when enabling autoscaling in Selenium Grid.
364+
365+
- Autoscaling isn't working when deploying KEDA core and Selenium Grid in different namespaces.
366+
367+
For example, you deployed KEDA core in `keda` namespace, and Selenium Grid in `selenium` namespace with config `autoscaling.enableWithExistingKEDA=true`. You observe that the ScaledObject/ScaledJob is created but the scaler isn't triggered.
368+
Use `kubectl logs` to see `keda-operator` pod logs, if you see the error message looks like
369+
370+
> ERROR scale_handler Error getting scaler metrics and activity, but continue {"scaledJob.Name": "selenium-node-firefox-latest", "Scaler": "*scalers.seleniumGridScaler:", "error": "error requesting selenium grid endpoint: Post \"http://selenium-router:4444/graphql\": dial tcp: lookup selenium-router on 10.96.0.10:53: no such host"}
371+
372+
It probably is the DNS issue. You need to check the GraphQL endpoint provided in the trigger metadata is accessible from the KEDA core namespace. In case different namespaces, and using svc name, you might need to use <service-name>.<namespace-name> as the domain name.
373+
At cluster level, you might need to configure network policies to allow traffic between namespaces properly.
374+
359375
## Updating Selenium-Grid release
360376

361377
Once you have a new chart version, you can update your selenium-grid running:

‎charts/selenium-grid/configs/scrape/selenium-grid.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@
1212
insecure_skip_verify: true
1313
static_configs:
1414
- targets:
15-
- '{{ template "seleniumGrid.monitoring.exporter.fullname" $ }}.{{ .Release.Namespace }}:{{ $.Values.monitoring.exporter.port }}'
15+
- '{{ template "seleniumGrid.monitoring.exporter.fullname" $ }}:{{ $.Values.monitoring.exporter.port }}'

‎charts/selenium-grid/templates/chrome-node-service.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ spec:
4242
{{- end }}
4343
{{- end }}
4444
{{- end }}
45+
{{- with $nodeConfig.service.externalTrafficPolicy }}
46+
externalTrafficPolicy: {{ . }}
47+
{{- end }}
48+
{{- with $nodeConfig.service.sessionAffinity }}
49+
sessionAffinity: {{ . }}
50+
{{- end }}
4551
---
4652
{{- end }}
4753
{{- end }}

‎charts/selenium-grid/templates/distributor-deployment.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ spec:
5252
- name: SE_DISTRIBUTOR_PORT
5353
value: {{ .Values.components.distributor.port | quote }}
5454
- name: SE_ROUTER_HOST
55-
value: '{{ template "seleniumGrid.router.fullname" . }}.{{ .Release.Namespace }}'
55+
value: '{{ template "seleniumGrid.router.fullname" . }}'
5656
- name: SE_ROUTER_PORT
5757
value: {{ .Values.components.router.port | quote }}
5858
- name: SE_SESSIONS_MAP_HOST
59-
value: '{{ template "seleniumGrid.sessionMap.fullname" . }}.{{ .Release.Namespace }}'
59+
value: '{{ template "seleniumGrid.sessionMap.fullname" . }}'
6060
- name: SE_SESSIONS_MAP_PORT
6161
value: {{ .Values.components.sessionMap.port | quote }}
6262
- name: SE_SESSION_QUEUE_HOST
63-
value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}.{{ .Release.Namespace }}'
63+
value: '{{ template "seleniumGrid.sessionQueue.fullname" . }}'
6464
- name: SE_SESSION_QUEUE_PORT
6565
value: {{ .Values.components.sessionQueue.port | quote }}
6666
{{- if .Values.components.distributor.newSessionThreadPoolSize }}

‎charts/selenium-grid/templates/distributor-service.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,10 @@ spec:
2626
{{- if and (eq .Values.components.distributor.serviceType "NodePort") .Values.components.distributor.nodePort }}
2727
nodePort: {{ .Values.components.distributor.nodePort }}
2828
{{- end }}
29+
{{- with .Values.components.distributor.serviceExternalTrafficPolicy }}
30+
externalTrafficPolicy: {{ . }}
31+
{{- end }}
32+
{{- with .Values.components.distributor.serviceSessionAffinity }}
33+
sessionAffinity: {{ . }}
34+
{{- end }}
2935
{{- end }}

‎charts/selenium-grid/templates/edge-node-service.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ spec:
4242
{{- end }}
4343
{{- end }}
4444
{{- end }}
45+
{{- with $nodeConfig.service.externalTrafficPolicy }}
46+
externalTrafficPolicy: {{ . }}
47+
{{- end }}
48+
{{- with $nodeConfig.service.sessionAffinity }}
49+
sessionAffinity: {{ . }}
50+
{{- end }}
4551
---
4652
{{- end }}
4753
{{- end }}

‎charts/selenium-grid/templates/event-bus-configmap.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{{- $eventBusHost := printf "%s.%s" (ternary (include "seleniumGrid.eventBus.fullname" .) (include "seleniumGrid.hub.fullname" .) .Values.isolateComponents) (.Release.Namespace) -}}
1+
{{- $eventBusHost := printf "%s" (ternary (include "seleniumGrid.eventBus.fullname" .) (include "seleniumGrid.hub.fullname" .) .Values.isolateComponents) -}}
22
{{- $eventBusPublishPort := ternary .Values.components.eventBus.publishPort .Values.hub.publishPort .Values.isolateComponents -}}
33
{{- $eventBusSubscribePort := ternary .Values.components.eventBus.subscribePort .Values.hub.subscribePort .Values.isolateComponents -}}
44
apiVersion: v1

‎charts/selenium-grid/templates/event-bus-service.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,10 @@ spec:
4949
{{- if and (eq .Values.components.eventBus.serviceType "NodePort") .Values.components.eventBus.subscribeNodePort }}
5050
nodePort: {{ .Values.components.eventBus.subscribeNodePort }}
5151
{{- end }}
52+
{{- with .Values.components.eventBus.serviceExternalTrafficPolicy }}
53+
externalTrafficPolicy: {{ . }}
54+
{{- end }}
55+
{{- with .Values.components.eventBus.serviceSessionAffinity }}
56+
sessionAffinity: {{ . }}
57+
{{- end }}
5258
{{- end }}

‎charts/selenium-grid/templates/firefox-node-service.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,12 @@ spec:
4242
{{- end }}
4343
{{- end }}
4444
{{- end }}
45+
{{- with $nodeConfig.service.externalTrafficPolicy }}
46+
externalTrafficPolicy: {{ . }}
47+
{{- end }}
48+
{{- with $nodeConfig.service.sessionAffinity }}
49+
sessionAffinity: {{ . }}
50+
{{- end }}
4551
---
4652
{{- end }}
4753
{{- end }}

0 commit comments

Comments
 (0)