Skip to content

Feat/add otel support #7642

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
+115 −23
Draft

Feat/add otel support #7642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b2e7fc
accept otel configmap keys
haywoodsh Apr 11, 2025
7530e71
add otel-module to NIC images
pdabelf5 Apr 11, 2025
3f111b2
remove unused config map keys
haywoodsh Apr 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions build/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ ARG WAF_VERSION=v4
ARG PACKAGE_REPO=pkgs.nginx.com


############################################# Base images containing libs for Opentracing and FIPS #############################################
FROM ghcr.io/nginx/dependencies/nginx-ot:nginx-1.27.4@sha256:5c2b2b4a7732985295153ec4ebc8764385e1215548d3715f7b679004798b9a18 AS opentracing-lib
FROM ghcr.io/nginx/dependencies/nginx-ot:nginx-1.27.4-alpine@sha256:540c7aa2ccfb26a39304324dfb9972c3eeb9734074397e4582b062e1db3b867c AS alpine-opentracing-lib
############################################# Base images containing libs for FIPS #############################################
FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le
FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19
FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.21@sha256:5221dec2e33436f2586c743c7aa3ef4626c0ec54184dc3364d101036d4f4a060 AS alpine-fips-3.21
Expand All @@ -23,23 +21,25 @@ FROM golang:1.24-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754
############################################# Base image for Alpine #############################################
FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine

RUN --mount=type=bind,from=alpine-opentracing-lib,target=/tmp/ot/ \
apk add --no-cache libcap libstdc++ \
&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
&& cp -av /tmp/ot/usr/lib/nginx/modules/ngx_http_opentracing_module.so /usr/lib/nginx/modules/ \
&& ldconfig /usr/local/lib/
RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \
&& apk add --no-cache libcap libstdc++ nginx-module-otel \
&& sed -i -e '/nginx.org/d' /etc/apk/repositories


############################################# Base image for Debian #############################################
FROM nginx:1.27.4@sha256:124b44bfc9ccd1f3cedf4b592d4d1e8bddb78b51ec2ed5056c52d3692baebc19 AS debian

RUN --mount=type=bind,from=opentracing-lib,target=/tmp/ot/ \
apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y libcap2-bin \
&& cp -av /tmp/ot/usr/local/lib/libopentracing.so* /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
&& cp -av /tmp/ot/usr/lib/nginx/modules/ngx_http_opentracing_module.so /usr/lib/nginx/modules/ \
&& ldconfig

RUN apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y \
libcap2-bin curl gnupg2 ca-certificates lsb-release debian-archive-keyring \
&& curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \
&& echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \
http://nginx.org/packages/mainline/debian `lsb_release -cs` nginx" > /etc/apt/sources.list.d/nginx.list \
&& printf "%s" "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" > /etc/apt/preferences.d/99nginx \
&& apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y nginx-module-otel \
&& apt-get purge --auto-remove -y gnupg2 lsb-release curl \
&& rm -rf /var/lib/apt/lists/* /etc/apt/preferences.d/99nginx /etc/apt/sources.list.d/nginx.list

############################################# NGINX files #############################################
FROM scratch AS nginx-files
Expand Down Expand Up @@ -119,7 +119,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
--mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \
export $(cat /tmp/user_agent) \
&& printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check libcap libcurl \
&& apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check libcap libcurl \
&& mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \
&& ldconfig /usr/local/lib/ \
&& sed -i -e '/nginx.com/d' /etc/apk/repositories
Expand Down Expand Up @@ -161,7 +161,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \
&& printf "%s\n" "https://${PACKAGE_REPO}/app-protect/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& printf "%s\n" "https://pkgs.nginx.com/app-protect-security-updates/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \
&& mkdir -p /usr/ssl \
&& cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \
Expand Down Expand Up @@ -197,7 +197,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \
printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& printf "%s\n" "https://${PACKAGE_REPO}/app-protect-x-plus/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
&& apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \
&& mkdir -p /usr/ssl \
&& cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \
Expand Down Expand Up @@ -236,7 +236,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
&& gpg --dearmor -o /usr/share/keyrings/app-protect-archive-keyring.gpg /tmp/app-protect-security-updates.key \
&& cp /tmp/nginx-plus.sources /etc/apt/sources.list.d/nginx-plus.sources \
&& apt-get update \
&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& apt-get purge --auto-remove -y gpg \
&& mkdir -p /etc/nginx/reporting/ \
&& cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \
Expand Down Expand Up @@ -356,7 +356,7 @@ RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_s
printf "%s\n" "[nginx]" "name=nginx repo" \
"baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \
"gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \
&& microdnf --nodocs install -y nginx nginx-module-njs nginx-module-image-filter nginx-module-xslt \
&& microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \
&& rm /etc/yum.repos.d/nginx.repo; \
fi \
&& ubi-clean.sh
Expand All @@ -378,7 +378,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
--mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \
mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \
&& ubi-setup.sh \
&& microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& ubi-clean.sh


Expand Down Expand Up @@ -483,7 +483,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
&& groupadd --system --gid 101 nginx \
&& useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \
&& rpm --import /tmp/nginx_signing.key \
&& dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \
&& sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \
&& subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \
Expand Down Expand Up @@ -530,7 +530,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
&& groupadd --system --gid 101 nginx \
&& useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \
&& rpm --import /tmp/nginx_signing.key \
&& dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \
&& dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \
&& if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \
## end of duplicated code
&& sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \
Expand Down
7 changes: 7 additions & 0 deletions internal/configs/config_params.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ type ConfigParams struct {
MainOpenTracingLoadModule bool
MainOpenTracingTracer string
MainOpenTracingTracerConfig string
MainOtelLoadModule bool
MainOtelGlobalTraceEnabled bool
MainOtelExporterEndpoint string
MainOtelExporterTrustedCA string
MainOtelExporterHeaderName string
MainOtelExporterHeaderValue string
MainOtelServiceName string
MainServerNamesHashBucketSize string
MainServerNamesHashMaxSize string
MainStreamLogFormat []string
Expand Down
55 changes: 55 additions & 0 deletions internal/configs/configmaps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,54 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has
}
}

if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists {
otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint)
if otelExporterEndpoint != "" {
cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint
}
}

if otelExporterTrustedCA, exists := cfgm.Data["otel-exporter-trusted-ca"]; exists {
otelExporterTrustedCA = strings.TrimSpace(otelExporterTrustedCA)
if otelExporterTrustedCA != "" {
cfgParams.MainOtelExporterTrustedCA = otelExporterTrustedCA
}
}

if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists {
otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName)
if otelExporterHeaderName != "" {
cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName
}
}

if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists {
otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue)
if otelExporterHeaderValue != "" {
cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue
}
}

if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists {
otelServiceName = strings.TrimSpace(otelServiceName)
if otelServiceName != "" {
cfgParams.MainOtelServiceName = otelServiceName
}
}

if otelGlobalTraceEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-global-trace-enabled", cfgm); exists {
if err != nil {
nl.Error(l, err)
eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error())
configOk = false
}
cfgParams.MainOtelGlobalTraceEnabled = otelGlobalTraceEnabled
}

if cfgParams.MainOtelExporterEndpoint != "" {
cfgParams.MainOtelLoadModule = true
}

if hasAppProtect {
if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists {
if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" {
Expand Down Expand Up @@ -961,6 +1009,13 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config
OpenTracingLoadModule: config.MainOpenTracingLoadModule,
OpenTracingTracer: config.MainOpenTracingTracer,
OpenTracingTracerConfig: config.MainOpenTracingTracerConfig,
MainOtelLoadModule: config.MainOtelLoadModule,
MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled,
MainOtelExporterEndpoint: config.MainOtelExporterEndpoint,
MainOtelExporterTrustedCA: config.MainOtelExporterTrustedCA,
MainOtelExporterHeaderName: config.MainOtelExporterHeaderName,
MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue,
MainOtelServiceName: config.MainOtelServiceName,
ProxyProtocol: config.ProxyProtocol,
ResolverAddresses: config.ResolverAddresses,
ResolverIPV6: config.ResolverIPV6,
Expand Down
8 changes: 8 additions & 0 deletions internal/configs/version1/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,14 @@ type MainConfig struct {
OpenTracingLoadModule bool
OpenTracingTracer string
OpenTracingTracerConfig string
MainOtelEnabled bool
MainOtelLoadModule bool
MainOtelGlobalTraceEnabled bool
MainOtelExporterEndpoint string
MainOtelExporterTrustedCA string
MainOtelExporterHeaderName string
MainOtelExporterHeaderValue string
MainOtelServiceName string
ProxyProtocol bool
ResolverAddresses []string
ResolverIPV6 bool
Expand Down
22 changes: 22 additions & 0 deletions internal/configs/version1/nginx-plus.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ daemon off;
error_log stderr {{.ErrorLogLevel}};
pid /var/lib/nginx/nginx.pid;

{{- if .MainOtelLoadModule}}
load_module modules/ngx_otel_module.so;
{{- end}}
{{- if .AppProtectLoadModule}}
load_module modules/ngx_http_app_protect_module.so;
{{- end}}
Expand Down Expand Up @@ -143,6 +146,25 @@ http {
ssl_dhparam {{.SSLDHParam}};
{{- end}}

{{- if .MainOtelLoadModule}}
otel_exporter {
endpoint {{ .MainOtelExporterEndpoint}};
{{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }}
header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}";
{{- end }}
{{ if .MainOtelExporterTrustedCA}}
# trusted_certificate <path>;
{{- end }}
}

{{ if .MainOtelServiceName}}
otel_service_name {{ .MainOtelServiceName }};
{{- end }}
{{ if .MainOtelGlobalTraceEnabled }}
otel_trace on;
{{- end}}
{{- end}}

{{ $resolverIPV6HTTPBool := boolToPointerBool .ResolverIPV6 -}}
{{ makeResolver .ResolverAddresses .ResolverValid $resolverIPV6HTTPBool }}
{{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}}
Expand Down