Merge remote-tracking branch 'origin/main' into 667-fleet-data-tags

* origin/main:
  fix package policy secrets (#821)
  chore(deps): update codecov/codecov-action digest to b9fd7d1 (#815)
  Bump release memory
  Switch to golang 1.23.2 in release
  Bump release memory
  Prepare release v0.11.8 (#810)
  Add key_id as an explicit attribute (#789)
  standalone-output resource (#811)
  Add URL support to data_view field_formats (#812)
  migrate fleet integration_policy to framework, fix secret churn (#797)
  Allow mappings to be unknown to support mappings defined in index templates (#803)

Author: tobio

.buildkite/release.yml

@@ -1,8 +1,8 @@
 steps:
   - label: Release
     agents:
-      image: "golang:1.23.1@sha256:4f063a24d429510e512cc730c3330292ff49f3ade3ae79bda8f84a24fa25ecb0"
+      image: "golang:1.23.2@sha256:adee809c2d0009a4199a11a1b2618990b244c6515149fe609e2788ddf164bd10"
       cpu: "8"
-      memory: "8G"
+      memory: "16G"
     command:
       - ".buildkite/scripts/release.sh"

.github/workflows/test.yml

@@ -63,7 +63,6 @@ jobs:
           xpack.security.enabled: true
           xpack.security.authc.api_key.enabled: true
           xpack.security.authc.token.enabled: true
-          xpack.security.http.ssl.enabled: false
           xpack.watcher.enabled: true
           xpack.license.self_generated.type: trial
           repositories.url.allowed_urls: https://example.com/*
@@ -80,10 +79,28 @@ jobs:
           ELASTICSEARCH_USERNAME: ${{ env.KIBANA_SYSTEM_USERNAME }}
           ELASTICSEARCH_PASSWORD: ${{ env.KIBANA_SYSTEM_PASSWORD }}
           XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: a7a6311933d3503b89bc2dbc36572c33a6c10925682e591bffcab6911c06786d
-#          LOGGING_ROOT_LEVEL: debug
+          # LOGGING_ROOT_LEVEL: debug
         ports:
           - 5601:5601
         options: --health-cmd="curl http://localhost:5601/api/status" --health-interval=10s --health-timeout=5s --health-retries=10
+      fleet:
+        image: docker.elastic.co/beats/elastic-agent:${{ matrix.version }}
+        env:
+          SERVER_NAME: fleet
+          FLEET_ENROLL: "1"
+          FLEET_URL: https://fleet:8220
+          FLEET_INSECURE: "true"
+          FLEET_SERVER_ENABLE: "1"
+          FLEET_SERVER_POLICY_ID: fleet-server
+          FLEET_SERVER_ELASTICSEARCH_HOST: http://elasticsearch:9200
+          FLEET_SERVER_ELASTICSEARCH_INSECURE: "true"
+          FLEET_SERVER_INSECURE_HTTP: "true"
+          KIBANA_HOST: http://kibana:5601
+          KIBANA_FLEET_SETUP: "1"
+          KIBANA_FLEET_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
+        ports:
+          - 8220:8220
+        options: --restart="unless-stopped"
 
     timeout-minutes: 15
     strategy:
@@ -123,8 +140,6 @@ jobs:
       - name: Setup Kibana user
         run: make set-kibana-password
         env:
-          ELASTICSEARCH_ENDPOINTS: "http://localhost:9200"
-          ELASTICSEARCH_USERNAME: "elastic"
           ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
           KIBANA_SYSTEM_USERNAME: ${{ env.KIBANA_SYSTEM_USERNAME }}
           KIBANA_SYSTEM_PASSWORD: ${{ env.KIBANA_SYSTEM_PASSWORD }}
@@ -134,10 +149,17 @@ jobs:
         run: |-
           echo "apikey=$(make create-es-api-key | jq -r .encoded)" >> "$GITHUB_OUTPUT"
         env:
-          ELASTICSEARCH_ENDPOINTS: "http://localhost:9200"
-          ELASTICSEARCH_USERNAME: "elastic"
           ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
 
+      - id: setup-fleet
+        name: Setup Fleet
+        if: matrix.version == '8.10.3' || matrix.version == '8.11.4' || matrix.version == '8.12.2' || matrix.version == '8.13.4' || matrix.version == '8.14.3' || matrix.version == '8.15.0'
+        run: |-
+          make setup-kibana-fleet
+        env:
+          ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
+          FLEET_NAME: "fleet"
+
       - id: force-install-synthetics
         name: Force install synthetics
         if: matrix.version == '8.14.3' || matrix.version == '8.15.0'

CHANGELOG.md

@@ -1,8 +1,15 @@
 ## [Unreleased]
 
+- Fix secret handling `elasticstack_fleet_integration_policy` resource. ([#821](https://github.com./elastic/terraform-provider-elasticstack/pull/821))
+
+## [0.11.8] - 2024-10-02
+
+- Add key_id to the `elasticstack_elasticsearch_api_key` resource. ([#789](https://github.com./elastic/terraform-provider-elasticstack/pull/789))
 - Fix handling of `sys_monitoring` in `elasticstack_fleet_agent_policy` ([#792](https://github.com./elastic/terraform-provider-elasticstack/pull/792))
 - Migrate `elasticstack_fleet_agent_policy`, `elasticstack_fleet_integration` (both), and `elasticstack_fleet_server_host` to terraform-plugin-framework ([#785](https://github.com./elastic/terraform-provider-elasticstack/pull/785))
 - Fix for synthetics http/tcp monitor produces inconsistent result after apply ([#801](https://github.com./elastic/terraform-provider-elasticstack/pull/801))
+- Migrate `elasticstack_fleet_integration_policy` to terraform-plugin-framework. Fix drift in integration policy secrets. ([#797](https://github.com./elastic/terraform-provider-elasticstack/pull/797))
+- Migrate `elasticstack_fleet_output` to terraform-plugin-framework. ([#811](https://github.com./elastic/terraform-provider-elasticstack/pull/811))
 
 ## [0.11.7] - 2024-09-20
 
@@ -345,8 +352,9 @@
 - Initial set of docs
 - CI integration
 
-[Unreleased]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.7...HEAD
-[0.11.6]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.6...v0.11.7
+[Unreleased]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.8...HEAD
+[0.11.8]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.7...v0.11.8
+[0.11.7]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.6...v0.11.7
 [0.11.6]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.5...v0.11.6
 [0.11.5]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.4...v0.11.5
 [0.11.4]: https://github.com./elastic/terraform-provider-elasticstack/compare/v0.11.3...v0.11.4

Makefile

@@ -2,7 +2,7 @@
 SHELL := /bin/bash
 
 
-VERSION ?= 0.11.7
+VERSION ?= 0.11.8
 
 NAME = elasticstack
 BINARY = terraform-provider-${NAME}
@@ -31,7 +31,11 @@ KIBANA_SYSTEM_USERNAME ?= kibana_system
 KIBANA_SYSTEM_PASSWORD ?= password
 KIBANA_API_KEY_NAME ?= kibana-api-key
 
+FLEET_NAME ?= terraform-elasticstack-fleet
+FLEET_ENDPOINT ?= https://$(FLEET_NAME):8220
+
 SOURCE_LOCATION ?= $(shell pwd)
+, := ,
 
 export GOBIN = $(shell pwd)/bin
 
@@ -72,7 +76,7 @@ retry = until [ $$(if [ -z "$$attempt" ]; then echo -n "0"; else echo -n "$$atte
 # To run specific test (e.g. TestAccResourceActionConnector) execute `make docker-testacc TESTARGS='-run ^TestAccResourceActionConnector$$'`
 # To enable tracing (or debugging), execute `make docker-testacc TF_LOG=TRACE`
 .PHONY: docker-testacc
-docker-testacc: docker-elasticsearch docker-kibana ## Run acceptance tests in the docker container
+docker-testacc: docker-elasticsearch docker-kibana docker-fleet ## Run acceptance tests in the docker container
 	@ docker run --rm \
 		-e ELASTICSEARCH_ENDPOINTS="$(ELASTICSEARCH_ENDPOINTS)" \
 		-e KIBANA_ENDPOINT="$(KIBANA_ENDPOINT)" \
@@ -163,6 +167,30 @@ docker-kibana-with-tls: docker-network docker-elasticsearch set-kibana-password
 		docker.elastic.co/kibana/kibana:$(STACK_VERSION); \
 		fi)
 
+.PHONY: docker-fleet
+docker-fleet: docker-network docker-elasticsearch docker-kibana setup-kibana-fleet ## Start Fleet node in docker container
+	@ docker rm -f $(FLEET_NAME)  &> /dev/null || true
+	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(FLEET_NAME) > /dev/null 2>&1 ; then \
+		docker run -d \
+		-p 8220:8220 \
+		-e SERVER_NAME=fleet \
+      	-e FLEET_ENROLL=1 \
+      	-e FLEET_URL=$(FLEET_ENDPOINT) \
+      	-e FLEET_INSECURE=true \
+      	-e FLEET_SERVER_ENABLE=1 \
+      	-e FLEET_SERVER_POLICY_ID=fleet-server \
+      	-e FLEET_SERVER_ELASTICSEARCH_HOST=$(ELASTICSEARCH_ENDPOINTS) \
+      	-e FLEET_SERVER_ELASTICSEARCH_INSECURE=true \
+      	-e FLEET_SERVER_INSECURE_HTTP=true \
+      	-e KIBANA_HOST=$(KIBANA_ENDPOINT) \
+      	-e KIBANA_FLEET_SETUP=1 \
+      	-e KIBANA_FLEET_USERNAME=$(ELASTICSEARCH_USERNAME) \
+      	-e KIBANA_FLEET_PASSWORD=$(ELASTICSEARCH_PASSWORD) \
+		--name $(FLEET_NAME) \
+		--network $(ELASTICSEARCH_NETWORK) \
+		docker.elastic.co/beats/elastic-agent:$(STACK_VERSION); \
+		fi)
+
 
 .PHONY: docker-network
 docker-network: ## Create a dedicated network for ES and test runs
@@ -172,19 +200,25 @@ docker-network: ## Create a dedicated network for ES and test runs
 
 .PHONY: set-kibana-password
 set-kibana-password: ## Sets the ES KIBANA_SYSTEM_USERNAME's password to KIBANA_SYSTEM_PASSWORD. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d "{\"password\":\"$(KIBANA_SYSTEM_PASSWORD)\"}" | grep -q "^{}")
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d '{"password":"$(KIBANA_SYSTEM_PASSWORD)"}' | grep -q "^{}")
 
 .PHONY: create-es-api-key
 create-es-api-key: ## Creates and outputs a new API Key. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/api_key -d "{\"name\":\"$(KIBANA_API_KEY_NAME)\"}")
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/api_key -d '{"name":"$(KIBANA_API_KEY_NAME)"}')
 
 .PHONY: create-es-bearer-token
-create-es-bearer-token:
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/oauth2/token -d "{\"grant_type\": \"client_credentials\"}")
+create-es-bearer-token: ## Creates and outputs a new OAuth bearer token. This expects Elasticsearch to be available at localhost:9200
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/oauth2/token -d '{"grant_type":"client_credentials"}')
+
+.PHONY: setup-kibana-fleet
+setup-kibana-fleet: ## Creates the agent and integration policies required to run Fleet. This expects Kibana to be available at localhost:5601
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/fleet_server_hosts -d '{"name":"default"$(,)"host_urls":["$(FLEET_ENDPOINT)"]$(,)"is_default":true}')
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/agent_policies -d '{"id":"fleet-server"$(,)"name":"Fleet Server"$(,)"namespace":"default"$(,)"monitoring_enabled":["logs"$(,)"metrics"]}')
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/package_policies -d '{"name":"fleet-server"$(,)"namespace":"default"$(,)"policy_id":"fleet-server"$(,)"enabled":true$(,)"inputs":[{"type":"fleet-server"$(,)"enabled":true$(,)"streams":[]$(,)"vars":{}}]$(,)"package":{"name":"fleet_server"$(,)"version":"1.5.0"}}')
 
 .PHONY: docker-clean
 docker-clean: ## Try to remove provisioned nodes and assigned network
-	@ docker rm -f $(ELASTICSEARCH_NAME) $(KIBANA_NAME) || true
+	@ docker rm -f $(ELASTICSEARCH_NAME) $(KIBANA_NAME) $(FLEET_NAME) || true
 	@ docker network rm $(ELASTICSEARCH_NETWORK) || true
 
 

docs/resources/elasticsearch_security_api_key.md

@@ -98,6 +98,7 @@ output "api_key" {
 - `encoded` (String, Sensitive) API key credentials which is the Base64-encoding of the UTF-8 representation of the id and api_key joined by a colon (:).
 - `expiration_timestamp` (Number) Expiration time in milliseconds for the API key. By default, API keys never expire.
 - `id` (String) Internal identifier of the resource.
+- `key_id` (String) Unique id for this API key.
 
 <a id="nestedblock--elasticsearch_connection"></a>
 ### Nested Schema for `elasticsearch_connection`

docs/resources/fleet_integration_policy.md

@@ -93,7 +93,7 @@ resource "elasticstack_fleet_integration_policy" "sample" {
 - `description` (String) The description of the integration policy.
 - `enabled` (Boolean) Enable the integration policy.
 - `force` (Boolean) Force operations, such as creation and deletion, to occur.
-- `input` (Block List) (see [below for nested schema](#nestedblock--input))
+- `input` (Block List) Integration inputs. (see [below for nested schema](#nestedblock--input))
 - `policy_id` (String) Unique identifier of the integration policy.
 - `vars_json` (String, Sensitive) Integration-level variables as JSON.
 

docs/resources/fleet_output.md

@@ -48,7 +48,7 @@ resource "elasticstack_fleet_output" "test_output" {
 - `default_monitoring` (Boolean) Make this output the default for agent monitoring.
 - `hosts` (List of String) A list of hosts.
 - `output_id` (String) Unique identifier of the output.
-- `ssl` (Block List, Max: 1) SSL configuration. (see [below for nested schema](#nestedblock--ssl))
+- `ssl` (Block List) SSL configuration. (see [below for nested schema](#nestedblock--ssl))
 
 ### Read-Only
 

docs/resources/kibana_data_view.md

@@ -88,7 +88,9 @@ Optional:
 
 Optional:
 
+- `labeltemplate` (String)
 - `pattern` (String)
+- `urltemplate` (String)
 
 
 

generated/fleet/fleet.gen.go

@@ -71,6 +71,7 @@ var transformers = []TransformFunc{
 	transformSchemasInputsType,
 	transformInlinePackageDefinitions,
 	transformAddPackagePolicyVars,
+	transformAddPackagePolicySecretReferences,
 	transformFixPackageSearchResult,
 }
 
@@ -334,6 +335,30 @@ func transformAddPackagePolicyVars(schema *Schema) {
 	}
 }
 
+// transformAddPackagePolicySecretReferences adds the missing 'secretReferences'
+// field to the PackagePolicy schema struct.
+func transformAddPackagePolicySecretReferences(schema *Schema) {
+	inputs, ok := schema.Components.GetFields("schemas.new_package_policy.properties")
+	if !ok {
+		panic("properties not found")
+	}
+
+	// Only add it if it doesn't exist.
+	if _, ok = inputs.Get("secret_references"); !ok {
+		inputs.Set("secret_references", map[string]any{
+			"type": "array",
+			"items": map[string]any{
+				"type": "object",
+				"properties": map[string]any{
+					"id": map[string]any{
+						"type": "string",
+					},
+				},
+			},
+		})
+	}
+}
+
 // transformFixPackageSearchResult removes unneeded fields from the
 // SearchResult struct. These fields are also causing parsing errors.
 func transformFixPackageSearchResult(schema *Schema) {