[Backport 8.0] Use external elastic-transport-go (#381)

* [Transport] Use elastic-transport-go as transport (#375)

* Initial commit to use estransport as a external dependency
* Adapt import name to transport package change
* Rename estransport to elastictransport
* Update transport reference in makefile
* Update reference to elastic-transport-go
* Apply external transport to examples
* Rename transport in readme files
* Adding go.sum for elastic-transport

* Bump github.com/tidwall/gjson from 1.2.1 to 1.9.3 in /_examples/encoding (#376)

Bumps [github.com/tidwall/gjson](https://github.com/tidwall/gjson) from 1.2.1 to 1.9.3.
- [Release notes](https://github.com/tidwall/gjson/releases)
- [Commits](tidwall/gjson@v1.2.1...v1.9.3)

---
updated-dependencies:
- dependency-name: github.com/tidwall/gjson
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Client: Reeintegrate compatibility header with external transport (#379)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: Anaethelion

Makefile

@@ -34,11 +34,11 @@ endif
   		export ELASTICSEARCH_URL='http://elastic:elastic@localhost:9200'; \
 		echo "gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml --" $(testintegargs); \
 		gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "."; \
-		gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./estransport" "./esapi" "./esutil"; \
+		gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./esapi" "./esutil"; \
 	else \
 	  	export ELASTICSEARCH_URL='http://elastic:elastic@localhost:9200'; \
 		echo "go test -v" $(testintegargs) "."; \
-		go test -v $(testintegargs) "./estransport" "./esapi" "./esutil"; \
+		go test -v $(testintegargs) "./esapi" "./esutil"; \
 	fi;
 
 test-api:  ## Run generated API integration tests
@@ -103,15 +103,17 @@ test-examples: ## Execute the _examples
 			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\n"; \
 			printf "\033[1mUpdating dependencies for $$d\033[0m\n"; \
 			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
-			(cd $$d && go mod download && make setup) || \
+			(cd $$d && go mod download all && make setup test) || \
 			( \
 				printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
 				printf "\033[31;1m⨯ ERROR\033[0m\n"; \
 				false; \
 			); \
 	    done; \
 	    \
-		for f in _examples/*.go; do \
+	    ( \
+	    cd _examples; \
+		for f in *.go; do \
 			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\n"; \
 			printf "\033[1m$$f\033[0m\n"; \
 			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
@@ -122,18 +124,19 @@ test-examples: ## Execute the _examples
 				false; \
 			); \
 		done; \
-		\
-		for f in _examples/*/; do \
-			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
-			printf "\033[1m$$f\033[0m\n"; \
-			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
-			(cd $$f && make test && true) || \
-			( \
-				printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
-				printf "\033[31;1m⨯ ERROR\033[0m\n"; \
-				false; \
-			); \
-		done; \
+		);\
+#		\
+#		for f in _examples/*/; do \
+#			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
+#			printf "\033[1m$$f\033[0m\n"; \
+#			printf "\033[2m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
+#			(cd $$f && make test && true) || \
+#			( \
+#				printf "\033[31m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
+#				printf "\033[31;1m⨯ ERROR\033[0m\n"; \
+#				false; \
+#			); \
+#		done; \
 		printf "\033[32m────────────────────────────────────────────────────────────────────────────────\033[0m\n"; \
 		\
 		printf "\033[32;1mSUCCESS\033[0m\n"; \
@@ -263,7 +266,6 @@ godoc: ## Display documentation for the package
 	@printf "\033[2m→ Generating documentation...\033[0m\n"
 	@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8"
 	@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/esapi"
-	@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/estransport"
 	@echo "* http://localhost:6060/pkg/github.com/elastic/go-elasticsearch/v8/esutil"
 	@printf "\n"
 	godoc --http=localhost:6060 --play

README.md

@@ -84,7 +84,7 @@ go run main.go
 
 ## Usage
 
-The `elasticsearch` package ties together two separate packages for calling the Elasticsearch APIs and transferring data over HTTP: `esapi` and `estransport`, respectively.
+The `elasticsearch` package ties together two separate packages for calling the Elasticsearch APIs and transferring data over HTTP: `esapi` and `elastictransport`, respectively.
 
 Use the `elasticsearch.NewDefaultClient()` function to create the client with the default settings.
 
@@ -351,7 +351,7 @@ func main() {
 As you see in the example above, the `esapi` package allows to call the Elasticsearch APIs in two distinct ways: either by creating a struct, such as `IndexRequest`, and calling its `Do()` method by passing it a context and the client, or by calling the `Search()` function on the client directly, using the option functions such as `WithIndex()`. See more information and examples in the
 [package documentation](https://godoc.org/github.com/elastic/go-elasticsearch/esapi).
 
-The `estransport` package handles the transfer of data to and from Elasticsearch, including retrying failed requests, keeping a connection pool, discovering cluster nodes and logging.
+The `elastictransport` package handles the transfer of data to and from Elasticsearch, including retrying failed requests, keeping a connection pool, discovering cluster nodes and logging.
 
 Read more about the client internals and usage in the following blog posts:
 

_benchmarks/benchmarks/actions/006_bulk_helper.go

@@ -23,9 +23,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/elastic/elastic-transport-go/v8/elastictransport"
+
 	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/elastic/go-elasticsearch/v8/esapi"
-	"github.com/elastic/go-elasticsearch/v8/estransport"
 	"github.com/elastic/go-elasticsearch/v8/esutil"
 
 	"github.com/elastic/go-elasticsearch/v8/benchmarks"
@@ -75,7 +76,7 @@ func init() {
 				)
 
 				var addresses []string
-				for _, u := range c.RunnerClient.Transport.(*estransport.Client).URLs() {
+				for _, u := range c.RunnerClient.Transport.(*elastictransport.Client).URLs() {
 					addresses = append(addresses, u.String())
 				}
 

_benchmarks/benchmarks/cmd/main.go

@@ -32,7 +32,7 @@ import (
 	"github.com/montanaflynn/stats"
 
 	"github.com/elastic/go-elasticsearch/v8"
-	"github.com/elastic/go-elasticsearch/v8/estransport"
+	"github.com/elastic/elastic-transport-go/v8/elastictransport"
 
 	"github.com/elastic/go-elasticsearch/v8/benchmarks"
 	"github.com/elastic/go-elasticsearch/v8/benchmarks/runner"
@@ -125,8 +125,8 @@ func main() {
 		EnableRetryOnTimeout: true,
 	}
 	if os.Getenv("DEBUG") != "" {
-		runnerClientConfig.Logger = &estransport.ColorLogger{Output: os.Stdout}
-		reportClientConfig.Logger = &estransport.ColorLogger{Output: os.Stdout}
+		runnerClientConfig.Logger = &elastictransport.ColorLogger{Output: os.Stdout}
+		reportClientConfig.Logger = &elastictransport.ColorLogger{Output: os.Stdout}
 	}
 
 	runnerClient, _ := elasticsearch.NewClient(runnerClientConfig)

_examples/bulk/Makefile

@@ -13,12 +13,7 @@ test-benchmarks: clean setup
 	cd benchmarks && go run benchmarks.go
 
 setup:
-	@go get -u github.com/mailru/easyjson/...
-	@go mod download github.com/dustin/go-humanize
-	@go mod download github.com/cenkalti/backoff/v4
-	@cd benchmarks && \
-	go mod download && \
-	go mod download github.com/mailru/easyjson && \
+	go get -u github.com/mailru/easyjson/...
 	go generate ./model
 
 clean:

_examples/bulk/README.md

@@ -5,8 +5,8 @@
 The [`default.go`](default.go) example demonstrates how to properly operate the Elasticsearch's
 [Bulk API]([https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html]).
 
-The example intentionally doesn't use any abstractions or helper functions, to
-demonstrate the low-level mechanics of working with the Bulk API:
+The example intentionally doesn't use any abstractions or helper functions, to demonstrate the low-level mechanics of
+working with the Bulk API:
 
 * iterating over a slice of data and preparing the `meta`/`data` pairs,
 * filling a buffer with the payload until the configured threshold for a single batch is reached,
@@ -29,7 +29,8 @@ go run default.go -count=100000 -batch=25000
 
 ## `indexer.go`
 
-The [`indexer.go`](indexer.go) example demonstrates how to use the [`esutil.BulkIndexer`](../esutil/bulk_indexer.go) helper for efficient indexing in parallel.
+The [`indexer.go`](indexer.go) example demonstrates how to use the [`esutil.BulkIndexer`](../esutil/bulk_indexer.go)
+helper for efficient indexing in parallel.
 
 ```bash
 go run indexer.go -count=100000 -flush=1000000

_examples/bulk/benchmarks/README.md

@@ -1,6 +1,8 @@
 # Bulk Indexer Benchmarks
 
-The [`benchmarks.go`](benchmarks.go) file executes end-to-end benchmarks for `esutil.NewBulkIndexer`. It allows to configure indexer parameters, index settings, number of runs. See `go run benchmarks.go --help` for an overview of configuration options:
+The [`benchmarks.go`](benchmarks.go) file executes end-to-end benchmarks for `esutil.NewBulkIndexer`. It allows to
+configure indexer parameters, index settings, number of runs. See `go run benchmarks.go --help` for an overview of
+configuration options:
 
 ```
 go run benchmarks.go --help
@@ -67,7 +69,8 @@ docs/sec: min [279,173] max [289,351] mean [286,987]
 
 ## HTTP Log Event
 
-The [`httplog`](data/httplog/document.json) dataset uses a bigger document (2.5K), corresponding to a log event gathered by [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-nginx.html) from Nginx.
+The [`httplog`](data/httplog/document.json) dataset uses a bigger document (2.5K), corresponding to a log event gathered
+by [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-nginx.html) from Nginx.
 
 ```
 ELASTICSEARCH_URL=http://server:9200 go run benchmarks.go --dataset=httplog --count=1_000_000 --flush=3MB --shards=5 --replicas=0 --fasthttp=true --easyjson=true
@@ -89,7 +92,8 @@ docs/sec: min [50,165] max [53,072] mean [52,011]
 
 ## Mock Server
 
-The `--mockserver` flag allows to run the benchmark against a "mock server", in this case Nginx, to understand a theoretical performance of the client, without the overhead of a real Elasticsearch cluster.
+The `--mockserver` flag allows to run the benchmark against a "mock server", in this case Nginx, to understand a
+theoretical performance of the client, without the overhead of a real Elasticsearch cluster.
 
 ```
 ELASTICSEARCH_URL=http://server:8000 go run benchmarks.go --dataset=small --count=1_000_000 --flush=2MB --warmup=0 --mockserver
@@ -117,8 +121,12 @@ the size and structure of your data, the index settings and mappings, the cluste
 The benchmarks have been run in the following environment:
 
 * OS: Ubuntu 18.04.4 LTS (5.0.0-1031-gcp)
-* Client: A `n2-standard-8` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (8 vCPUs/32GB RAM)
-* Server: A `n2-standard-16` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (16 vCPUs/64GB RAM)
-* Disk: A [local SSD](https://cloud.google.com/compute/docs/disks#localssds) formatted as `ext4` on NVMe interface for Elasticsearch data
-* A single-node Elasticsearch cluster, `7.6.0`, [default distribution](https://www.elastic.co/downloads/elasticsearch), installed from a TAR, with 4GB locked for heap
+* Client: A `n2-standard-8` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (8
+  vCPUs/32GB RAM)
+* Server: A `n2-standard-16` [GCP instance](https://cloud.google.com/compute/docs/machine-types#n2_machine_types) (16
+  vCPUs/64GB RAM)
+* Disk: A [local SSD](https://cloud.google.com/compute/docs/disks#localssds) formatted as `ext4` on NVMe interface for
+  Elasticsearch data
+* A single-node Elasticsearch cluster, `7.6.0`, [default distribution](https://www.elastic.co/downloads/elasticsearch),
+  installed from a TAR, with 4GB locked for heap
 * Nginx 1.17.8 with [`nginx.conf`](etc/nginx.conf)

_examples/bulk/benchmarks/benchmarks.go

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//go:build ignore
 // +build ignore
 
 // This example demonstrates indexing documents using the esutil.BulkIndexer helper.
@@ -41,8 +42,8 @@ import (
 	"github.com/mailru/easyjson"
 	"github.com/valyala/fasthttp"
 
+	"github.com/elastic/elastic-transport-go/v8/elastictransport"
 	"github.com/elastic/go-elasticsearch/v8"
-	"github.com/elastic/go-elasticsearch/v8/estransport"
 	"github.com/elastic/go-elasticsearch/v8/esutil"
 
 	"github.com/elastic/go-elasticsearch/v8/_examples/bulk/benchmarks/model"
@@ -108,7 +109,7 @@ func main() {
 	}
 
 	if debug {
-		clientCfg.Logger = &estransport.ColorLogger{Output: os.Stdout, EnableRequestBody: true, EnableResponseBody: true}
+		clientCfg.Logger = &elastictransport.ColorLogger{Output: os.Stdout, EnableRequestBody: true, EnableResponseBody: true}
 	}
 
 	es, _ := elasticsearch.NewClient(clientCfg)

_examples/bulk/benchmarks/data/httplog/mapping.json

@@ -3,7 +3,6 @@
     "_meta": {
       "version": "8-SNAPSHOT"
     },
-
     "dynamic_templates": [
       {
         "labels": {
@@ -60,9 +59,7 @@
         }
       }
     ],
-
     "date_detection": false,
-
     "properties": {
       "@timestamp": {
         "type": "date"

_examples/bulk/benchmarks/data/small/mapping.json

@@ -3,13 +3,20 @@
     "_meta": {
       "version": "8-SNAPSHOT"
     },
-
     "properties": {
-      "count": { "type": "long" },
-      "tags":  { "type": "keyword" },
-      "time":  { "type": "date" },
-      "title": { "type": "text" }
+      "count": {
+        "type": "long"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "time": {
+        "type": "date"
+      },
+      "title": {
+        "type": "text"
       }
     }
   }
 }
+}