From 497cf8109da2337f771a1110a66f4f5d249967a5 Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 14:31:45 +0200 Subject: [PATCH 1/7] PMM-10512 Capture username and application_name for pg_stat_activity --- cmd/postgres_exporter/postgres_exporter.go | 10 ++++++---- cmd/postgres_exporter/queries.go | 10 ++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 11fece772..f206b9e02 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -297,10 +297,12 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, "pg_stat_activity": { map[string]ColumnMapping{ - "datname": {LABEL, "Name of this database", nil, nil}, - "state": {LABEL, "connection state", nil, semver.MustParseRange(">=9.2.0")}, - "count": {GAUGE, "number of connections in this state", nil, nil}, - "max_tx_duration": {GAUGE, "max duration in seconds any active transaction has been running", nil, nil}, + "datname": {LABEL, "Name of this database", nil, nil}, + "state": {LABEL, "connection state", nil, semver.MustParseRange(">=9.2.0")}, + "usename": {LABEL, "connection usename", nil, nil}, + "application_name": {LABEL, "connection application_name", nil, nil}, + "count": {GAUGE, "number of connections in this state", nil, nil}, + "max_tx_duration": {GAUGE, "max duration in seconds any active transaction has been running", nil, nil}, }, true, 0, diff --git a/cmd/postgres_exporter/queries.go b/cmd/postgres_exporter/queries.go index e27480f4f..97642d991 100644 --- a/cmd/postgres_exporter/queries.go +++ b/cmd/postgres_exporter/queries.go @@ -137,6 +137,8 @@ var queryOverrides = map[string][]OverrideQuery{ SELECT pg_database.datname, tmp.state, + tmp2.usename, + tmp2.application_name, COALESCE(count,0) as count, COALESCE(max_tx_duration,0) as max_tx_duration FROM @@ -153,9 +155,11 @@ var queryOverrides = map[string][]OverrideQuery{ SELECT datname, state, + usename, + application_name, count(*) AS count, MAX(EXTRACT(EPOCH FROM now() - xact_start))::float AS max_tx_duration - FROM pg_stat_activity GROUP BY datname,state) AS tmp2 + FROM pg_stat_activity GROUP BY datname,state,usename,application_name) AS tmp2 ON tmp.state = tmp2.state AND pg_database.datname = tmp2.datname `, }, @@ -165,9 +169,11 @@ var queryOverrides = map[string][]OverrideQuery{ SELECT datname, 'unknown' AS state, + usename, + application_name, COALESCE(count(*),0) AS count, COALESCE(MAX(EXTRACT(EPOCH FROM now() - xact_start))::float,0) AS max_tx_duration - FROM pg_stat_activity GROUP BY datname + FROM pg_stat_activity GROUP BY datname,usename,application_name `, }, }, From 2d70b095a3ec3ecdba238bce209720bdb75f8baf Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 14:45:37 +0200 Subject: [PATCH 2/7] PMM-10512 Dump metrics update --- percona_tests/Makefile | 3 ++- percona_tests/metrics_test.go | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/percona_tests/Makefile b/percona_tests/Makefile index 74a4ac33a..4ec82bc35 100644 --- a/percona_tests/Makefile +++ b/percona_tests/Makefile @@ -8,6 +8,7 @@ test-performance: extraMetrics = false multipleLabels = false dumpMetrics = false +endpoint = '' test-metrics: go test -v -run '^TestMissingMetrics$$' -args -doRun=true @@ -22,7 +23,7 @@ test-resolutions: go test -v -run '^TestResolutions$$' -args -doRun=true dump-metrics: - go test -v -run '^TestDumpMetrics$$' -args -doRun=true -extraMetrics=$(extraMetrics) -multipleLabels=$(multipleLabels) -dumpMetrics=$(dumpMetrics) + go test -v -run '^TestDumpMetrics$$' -args -doRun=true -extraMetrics=$(extraMetrics) -multipleLabels=$(multipleLabels) -endpoint=$(endpoint) -dumpMetrics=true test-consistency: test-metrics test-resolutions test-resolutions-duplicates diff --git a/percona_tests/metrics_test.go b/percona_tests/metrics_test.go index cc156338c..9b622a0bc 100644 --- a/percona_tests/metrics_test.go +++ b/percona_tests/metrics_test.go @@ -14,6 +14,7 @@ import ( var dumpMetricsFlag = flag.Bool("dumpMetrics", false, "") var printExtraMetrics = flag.Bool("extraMetrics", false, "") var printMultipleLabels = flag.Bool("multipleLabels", false, "") +var endpointFlag = flag.String("endpoint", "", "") type Metric struct { name string @@ -86,13 +87,26 @@ func TestDumpMetrics(t *testing.T) { return } - newMetrics, err := getMetrics(updatedExporterFileName) + var ep string + if endpointFlag == nil || *endpointFlag == "" { + ep = "metrics" + } else if *endpointFlag == "hr" { + ep = highResolutionEndpoint + } else if *endpointFlag == "mr" { + ep = medResolutionEndpoint + } else if *endpointFlag == "lr" { + ep = lowResolutionEndpoint + } else { + ep = "metrics" + } + + newMetrics, err := getMetricsFrom(updatedExporterFileName, ep) if err != nil { t.Error(err) return } - oldMetrics, err := getMetrics(oldExporterFileName) + oldMetrics, err := getMetricsFrom(oldExporterFileName, ep) if err != nil { t.Error(err) return From c2d9b00d36918b9cfe2de14faa03e186ab3eecd9 Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 15:42:36 +0200 Subject: [PATCH 3/7] PMM-10512 Labels test fix --- percona_tests/metrics_test.go | 66 +++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/percona_tests/metrics_test.go b/percona_tests/metrics_test.go index 9b622a0bc..f0a3252b3 100644 --- a/percona_tests/metrics_test.go +++ b/percona_tests/metrics_test.go @@ -17,8 +17,9 @@ var printMultipleLabels = flag.Bool("multipleLabels", false, "") var endpointFlag = flag.String("endpoint", "", "") type Metric struct { - name string - labels string + name string + labelsRawStr string + labelsWithValues []string } type MetricsCollection struct { @@ -216,20 +217,50 @@ func testResolution(t *testing.T, resolutionEp, resolutionName string) { missingCount := 0 missingMetrics := "" - for _, metric := range oldMetricsCollection.MetricNamesWithLabels { - if metric == "" || strings.HasPrefix(metric, "# ") { + missingLabelsCount := 0 + missingLabels := "" + for _, oldMetric := range oldMetricsCollection.MetricsData { + if oldMetric.name == "" || strings.HasPrefix(oldMetric.name, "# ") { continue } - if !contains(newMetricsCollection.MetricNamesWithLabels, metric) { + metricFound := false + labelsMatch := false + for _, newMetric := range newMetricsCollection.MetricsData { + if newMetric.name != oldMetric.name { + continue + } + + metricFound = true + + if newMetric.labelsRawStr == oldMetric.labelsRawStr { + labelsMatch = true + break + } + + if arrIsSubsetOf(oldMetric.labelsWithValues, newMetric.labelsWithValues) { + labelsMatch = true + break + } + } + + if !metricFound { missingCount++ - missingMetrics += fmt.Sprintf("%s\n", metric) + missingMetrics += fmt.Sprintf("%s\n", oldMetric.name) + } else if !labelsMatch { + missingLabelsCount++ + missingLabels += fmt.Sprintf("%s\n", oldMetric.name) } } + if missingCount > 0 { t.Errorf("%d metrics are missing in new exporter for %s resolution:\n%s", missingCount, resolutionName, missingMetrics) } + if missingLabelsCount > 0 { + t.Errorf("%d metrics's labels missing in new exporter for %s resolution:\n%s", missingCount, resolutionName, missingLabels) + } + extraCount := 0 extraMetrics := "" for _, metric := range newMetricsCollection.MetricNamesWithLabels { @@ -376,13 +407,13 @@ func parseMetricsCollection(metricRaw string) MetricsCollection { } } -func arrIsSubsetOf(a, b []string) bool { - if len(a) == 0 { - return len(b) == 0 +func arrIsSubsetOf(smaller, larger []string) bool { + if len(smaller) == 0 { + return len(larger) == 0 } - for _, x := range a { - if !contains(b, x) { + for _, x := range smaller { + if !contains(larger, x) { return false } } @@ -408,10 +439,10 @@ func groupByMetrics(metrics []Metric) map[string][]string { metric := metrics[i] if _, ok := mtr[metric.name]; ok { labels := mtr[metric.name] - labels = append(labels, metric.labels) + labels = append(labels, metric.labelsRawStr) mtr[metric.name] = labels } else { - mtr[metric.name] = []string{metric.labels} + mtr[metric.name] = []string{metric.labelsRawStr} } } @@ -428,16 +459,21 @@ func parseMetrics(metrics []string) []Metric { } var mName, mLabels string + var labelsArr []string if strings.Contains(metricRawStr, "{") { mName = metricRawStr[:strings.Index(metricRawStr, "{")] mLabels = metricRawStr[strings.Index(metricRawStr, "{")+1 : len(metricRawStr)-1] + if mLabels != "" { + labelsArr = strings.Split(mLabels, ",") + } } else { mName = metricRawStr } metric := Metric{ - name: mName, - labels: mLabels, + name: mName, + labelsRawStr: mLabels, + labelsWithValues: labelsArr, } metricsData = append(metricsData, metric) From 3ed9044456eefb5569f074d5ed98856f2cb8a151 Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 16:10:58 +0200 Subject: [PATCH 4/7] build trigger From 9dd91a8d3f1f2eeaf56ad65364d35d27367ec37c Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 16:20:50 +0200 Subject: [PATCH 5/7] PMM-10512 switch-case --- percona_tests/metrics_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/percona_tests/metrics_test.go b/percona_tests/metrics_test.go index f0a3252b3..2c4885ee9 100644 --- a/percona_tests/metrics_test.go +++ b/percona_tests/metrics_test.go @@ -89,16 +89,19 @@ func TestDumpMetrics(t *testing.T) { } var ep string - if endpointFlag == nil || *endpointFlag == "" { - ep = "metrics" - } else if *endpointFlag == "hr" { + switch *endpointFlag { + case "hr": ep = highResolutionEndpoint - } else if *endpointFlag == "mr" { + break + case "mr": ep = medResolutionEndpoint - } else if *endpointFlag == "lr" { + break + case "lr": ep = lowResolutionEndpoint - } else { + break + default: ep = "metrics" + break } newMetrics, err := getMetricsFrom(updatedExporterFileName, ep) From e558fa77dec9bb7ff8699cfe7c66d171cb24aa7a Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 16:24:04 +0200 Subject: [PATCH 6/7] PMM-10512 column description --- cmd/postgres_exporter/postgres_exporter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index f206b9e02..1396d36b3 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -299,8 +299,8 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ map[string]ColumnMapping{ "datname": {LABEL, "Name of this database", nil, nil}, "state": {LABEL, "connection state", nil, semver.MustParseRange(">=9.2.0")}, - "usename": {LABEL, "connection usename", nil, nil}, - "application_name": {LABEL, "connection application_name", nil, nil}, + "usename": {LABEL, "Name of the user logged into this backend", nil, nil}, + "application_name": {LABEL, "Name of the application that is connected to this backend", nil, nil}, "count": {GAUGE, "number of connections in this state", nil, nil}, "max_tx_duration": {GAUGE, "max duration in seconds any active transaction has been running", nil, nil}, }, From 96f18878d727481d7972e8535a26296d37dac76b Mon Sep 17 00:00:00 2001 From: "taras.shcherban" Date: Tue, 4 Oct 2022 16:26:29 +0200 Subject: [PATCH 7/7] PMM-10512 S1023 fix --- percona_tests/metrics_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/percona_tests/metrics_test.go b/percona_tests/metrics_test.go index 2c4885ee9..fe6c7f61d 100644 --- a/percona_tests/metrics_test.go +++ b/percona_tests/metrics_test.go @@ -92,16 +92,12 @@ func TestDumpMetrics(t *testing.T) { switch *endpointFlag { case "hr": ep = highResolutionEndpoint - break case "mr": ep = medResolutionEndpoint - break case "lr": ep = lowResolutionEndpoint - break default: ep = "metrics" - break } newMetrics, err := getMetricsFrom(updatedExporterFileName, ep)