From commits-return-716-archive-asf-public=cust-asf.ponee.io@zipkin.apache.org Fri May 3 11:09:01 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 7711918064D for ; Fri, 3 May 2019 13:09:00 +0200 (CEST) Received: (qmail 18123 invoked by uid 500); 3 May 2019 11:08:59 -0000 Mailing-List: contact commits-help@zipkin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zipkin.apache.org Delivered-To: mailing list commits@zipkin.apache.org Received: (qmail 18114 invoked by uid 99); 3 May 2019 11:08:59 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 May 2019 11:08:59 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id B2B44871F3; Fri, 3 May 2019 11:08:54 +0000 (UTC) Date: Fri, 03 May 2019 11:08:54 +0000 To: "commits@zipkin.apache.org" Subject: [incubator-zipkin] branch master updated: Supports Elasticsearch 7.x (#2398) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155688173456.16359.813529442423935527@gitbox.apache.org> From: adriancole@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: incubator-zipkin X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: f2d9cc8c7e874bae7531cdc94c11eb5cc23f987d X-Git-Newrev: 762a79539ac3ed945038ee759cfec3a28971105c X-Git-Rev: 762a79539ac3ed945038ee759cfec3a28971105c X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. adriancole pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-zipkin.git The following commit(s) were added to refs/heads/master by this push: new 762a795 Supports Elasticsearch 7.x (#2398) 762a795 is described below commit 762a79539ac3ed945038ee759cfec3a28971105c Author: Adrian Cole AuthorDate: Fri May 3 19:08:49 2019 +0800 Supports Elasticsearch 7.x (#2398) Fixes #2219 --- ...nElasticsearchStorageAutoConfigurationTest.java | 39 +----- zipkin-storage/elasticsearch/README.md | 15 ++- .../elasticsearch/ElasticsearchSpanConsumer.java | 13 +- .../elasticsearch/ElasticsearchStorage.java | 72 +++++----- .../zipkin2/elasticsearch/EnsureIndexTemplate.java | 18 ++- .../java/zipkin2/elasticsearch/IndexTemplates.java | 9 ++ .../elasticsearch/VersionSpecificTemplates.java | 65 +++++---- .../elasticsearch/internal/HttpBulkIndexer.java | 56 ++------ .../elasticsearch/internal/IndexNameFormatter.java | 49 ++++++- .../ElasticsearchSpanConsumerTest.java | 1 - .../elasticsearch/ElasticsearchStorageTest.java | 4 +- .../zipkin2/elasticsearch/InternalForTests.java | 4 +- .../VersionSpecificTemplatesTest.java | 49 ++++++- .../integration/ElasticsearchStorageRule.java | 2 +- .../integration/ITElasticsearchStorageV7.java | 145 +++++++++++++++++++++ .../internal/IndexNameFormatterTest.java | 88 ++++++------- 16 files changed, 411 insertions(+), 218 deletions(-) diff --git a/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java b/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java index 3f1d7ec..d9cfa8e 100644 --- a/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java +++ b/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java @@ -38,18 +38,15 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Rule public ExpectedException thrown = ExpectedException.none(); - AnnotationConfigApplicationContext context; + final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); @After public void close() { - if (context != null) { - context.close(); - } + context.close(); } @Test public void doesntProvideStorageComponent_whenStorageTypeNotElasticsearch() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of("zipkin.storage.type:cassandra").applyTo(context); Access.registerElasticsearchHttp(context); context.refresh(); @@ -60,7 +57,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void providesStorageComponent_whenStorageTypeElasticsearchAndHostsAreUrls() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200") @@ -73,7 +69,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void canOverridesProperty_hostsWithList() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200,http://host2:9200") @@ -87,7 +82,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void configuresPipeline() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -101,7 +95,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void configuresMaxRequests() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -116,7 +109,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { /** This helps ensure old setups don't break (provided they have http port 9200 open) */ @Test public void coersesPort9300To9200() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:host1:9300") @@ -129,7 +121,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void httpPrefixOptional() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:host1:9200") @@ -142,7 +133,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void defaultsToPort9200() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:host1") @@ -175,7 +165,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { /** Ensures we can wire up network interceptors, such as for logging or authentication */ @Test public void usesInterceptorsQualifiedWith_zipkinElasticsearchHttp() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:host1:9200") @@ -190,7 +179,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void timeout_defaultsTo10Seconds() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:host1:9200") @@ -206,7 +194,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void timeout_override() { - context = new AnnotationConfigApplicationContext(); int timeout = 30_000; TestPropertyValues.of( "zipkin.storage.type:elasticsearch", @@ -224,7 +211,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void strictTraceId_defaultsToTrue() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200") @@ -236,7 +222,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void strictTraceId_canSetToFalse() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -250,7 +235,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void dailyIndexFormat() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200") @@ -259,12 +243,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { context.refresh(); assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0)) - .isEqualTo("zipkin:span-1970-01-01"); + .isEqualTo("zipkin*span-1970-01-01"); } @Test public void dailyIndexFormat_overridingPrefix() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -274,12 +257,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { context.refresh(); assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0)) - .isEqualTo("zipkin_prod:span-1970-01-01"); + .isEqualTo("zipkin_prod*span-1970-01-01"); } @Test public void dailyIndexFormat_overridingDateSeparator() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -289,12 +271,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { context.refresh(); assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0)) - .isEqualTo("zipkin:span-1970.01.01"); + .isEqualTo("zipkin*span-1970.01.01"); } @Test public void dailyIndexFormat_overridingDateSeparator_empty() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -304,12 +285,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { context.refresh(); assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0)) - .isEqualTo("zipkin:span-19700101"); + .isEqualTo("zipkin*span-19700101"); } @Test public void dailyIndexFormat_overridingDateSeparator_invalidToBeMultiChar() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -323,7 +303,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void namesLookbackAssignedFromQueryLookback() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -337,7 +316,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void doesntProvideBasicAuthInterceptor_whenBasicAuthUserNameandPasswordNotConfigured() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200") @@ -351,7 +329,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void providesBasicAuthInterceptor_whenBasicAuthUserNameAndPasswordConfigured() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.elasticsearch.hosts:http://host1:9200", @@ -368,7 +345,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void searchEnabled_false() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.search-enabled:false") @@ -381,7 +357,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void autocompleteKeys_list() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.autocomplete-keys:environment") @@ -395,7 +370,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void autocompleteTtl() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.autocomplete-ttl:60000") @@ -409,7 +383,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest { @Test public void autocompleteCardinality() { - context = new AnnotationConfigApplicationContext(); TestPropertyValues.of( "zipkin.storage.type:elasticsearch", "zipkin.storage.autocomplete-cardinality:5000") diff --git a/zipkin-storage/elasticsearch/README.md b/zipkin-storage/elasticsearch/README.md index f3bcc0a..a3ee202 100644 --- a/zipkin-storage/elasticsearch/README.md +++ b/zipkin-storage/elasticsearch/README.md @@ -3,7 +3,7 @@ This is is a plugin to the Elasticsearch storage component, which uses HTTP by way of [OkHttp 3](https://github.com/square/okttp) and [Moshi](https://github.com/square/moshi). This currently supports 2.x, -5.x and 6.x version families. +5.x, 6.x and 7.x version families. ## Multiple hosts Most users will supply a DNS name that's mapped to multiple A or AAAA @@ -33,7 +33,8 @@ spans. This is mapped to the Elasticsearch date type, so can be used to any date ## Indexes Spans are stored into daily indices, for example spans with a timestamp -falling on 2016/03/19 will be stored in the index named 'zipkin:span-2016-03-19'. +falling on 2016/03/19 will be stored in the index named 'zipkin:span-2016-03-19' +or 'zipkin-span-2016-03-19' if using Elasticsearch version 7 or higher. There is no support for TTL through this SpanStore. It is recommended instead to use [Elastic Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html) to remove indices older than the point you are interested in. @@ -45,9 +46,9 @@ the date separator from '-' to something else. `ElasticsearchStorage.Builder.index` and `ElasticsearchStorage.Builder.dateSeparator` control the daily index format. -For example, spans with a timestamp falling on 2016/03/19 end up in the -index 'zipkin:span-2016-03-19'. When the date separator is '.', the index -would be 'zipkin:span-2016.03.19'. +For example, using Elasticsearch 7+, spans with a timestamp falling on +2016/03/19 end up in the index 'zipkin-span-2016-03-19'. When the date +separator is '.', the index would be 'zipkin-span-2016.03.19'. ### String Mapping The Zipkin api implies aggregation and exact match (keyword) on string @@ -63,7 +64,7 @@ The values in `q` are limited to 256 characters and searched as keywords. You can check these manually like so: ```bash -$ curl -s localhost:9200/zipkin:span-2017-08-11/_search?q=_q:error=500 +$ curl -s 'localhost:9200/zipkin*span-2017-08-11/_search?q=_q:error=500' ``` The reason for special casing is around dotted name constraints. Tags @@ -103,7 +104,7 @@ your indexes: ```bash # the output below shows which tokens will match on the trace id supplied. -$ curl -s localhost:9200/zipkin:span-2017-08-22/_analyze -d '{ +$ curl -s 'localhost:9200/zipkin*span-2017-08-22/_analyze' -d '{ "text": "48485a3953bb61246b221d5bc9e6496c", "analyzer": "traceId_analyzer" }'|jq '.tokens|.[]|.token' diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java index 84993db..e793394 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java @@ -35,6 +35,7 @@ import zipkin2.codec.SpanBytesEncoder; import zipkin2.elasticsearch.internal.HttpBulkIndexer; import zipkin2.elasticsearch.internal.IndexNameFormatter; import zipkin2.internal.DelayLimiter; +import zipkin2.internal.Nullable; import zipkin2.storage.SpanConsumer; import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; @@ -50,6 +51,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi final ElasticsearchStorage es; final Set autocompleteKeys; final IndexNameFormatter indexNameFormatter; + final char indexTypeDelimiter; final boolean searchEnabled; final DelayLimiter delayLimiter; @@ -57,12 +59,18 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi this.es = es; this.autocompleteKeys = new LinkedHashSet<>(es.autocompleteKeys()); this.indexNameFormatter = es.indexNameFormatter(); + this.indexTypeDelimiter = es.indexTypeDelimiter(); this.searchEnabled = es.searchEnabled(); this.delayLimiter = DelayLimiter.newBuilder() .ttl(es.autocompleteTtl()) .cardinality(es.autocompleteCardinality()).build(); } + String formatTypeAndTimestampForInsert(String type, long timestampMillis) { + return indexNameFormatter.formatTypeAndTimestampForInsert(type, indexTypeDelimiter, + timestampMillis); + } + @Override public Call accept(List spans) { if (spans.isEmpty()) return Call.create(null); BulkSpanIndexer indexer = new BulkSpanIndexer(this); @@ -104,8 +112,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi } void add(long indexTimestamp, Span span, long timestampMillis) { - String index = consumer.indexNameFormatter - .formatTypeAndTimestamp(SPAN, indexTimestamp); + String index = consumer.formatTypeAndTimestampForInsert(SPAN, indexTimestamp); byte[] document = consumer.searchEnabled ? prefixWithTimestampMillisAndQuery(span, timestampMillis) : SpanBytesEncoder.JSON_V2.encode(span); @@ -113,7 +120,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi } void addAutocompleteValues(long indexTimestamp, Span span) { - String idx = consumer.indexNameFormatter.formatTypeAndTimestamp(AUTOCOMPLETE, indexTimestamp); + String idx = consumer.formatTypeAndTimestampForInsert(AUTOCOMPLETE, indexTimestamp); for (Map.Entry tag : span.tags().entrySet()) { int length = tag.getKey().length() + tag.getValue().length() + 1; if (length > INDEX_CHARS_LIMIT) continue; diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java index 93f5cb6..a19541f 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java @@ -29,7 +29,6 @@ import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.RequestBody; import okio.Buffer; import okio.BufferedSource; import zipkin2.CheckResult; @@ -46,6 +45,7 @@ import zipkin2.storage.StorageComponent; import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY; import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN; +import static zipkin2.elasticsearch.EnsureIndexTemplate.ensureIndexTemplate; import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; @AutoValue @@ -145,7 +145,11 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon */ public abstract Builder namesLookback(int namesLookback); - /** Visible for testing */ + /** + * Internal and visible only for testing. + * + *

See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-refresh.html + */ public abstract Builder flushOnWrites(boolean flushOnWrites); /** The index prefix to use when generating daily index names. Defaults to zipkin. */ @@ -158,9 +162,9 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon * The date separator to use when generating daily index names. Defaults to '-'. * *

By default, spans with a timestamp falling on 2016/03/19 end up in the index - * 'zipkin:span-2016-03-19'. When the date separator is '.', the index would be - * 'zipkin:span-2016.03.19'. If the date separator is 0, there is no delimiter. Ex the index - * would be 'zipkin:span-20160319' + * 'zipkin-span-2016-03-19'. When the date separator is '.', the index would be + * 'zipkin-span-2016.03.19'. If the date separator is 0, there is no delimiter. Ex the index + * would be 'zipkin-span-20160319' */ public final Builder dateSeparator(char dateSeparator) { indexNameFormatterBuilder().dateSeparator(dateSeparator); @@ -275,7 +279,11 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon return ensureIndexTemplates().version(); } - /** This is a blocking call, only used in tests. */ + char indexTypeDelimiter() { + return ensureIndexTemplates().indexTypeDelimiter(); + } + + /** This is an internal blocking call, only used in tests. */ public void clear() throws IOException { Set toClear = new LinkedHashSet<>(); toClear.add(indexNameFormatter().formatType(SPAN)); @@ -284,29 +292,10 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon } void clear(String index) throws IOException { - Request deleteRequest = - new Request.Builder() - .url(http().baseUrl.newBuilder().addPathSegment(index).build()) - .delete() - .tag("delete-index") - .build(); - - http().newCall(deleteRequest, BodyConverters.NULL).execute(); - - flush(http(), index); - } - - /** This is a blocking call, only used in tests. */ - public static void flush(HttpCall.Factory factory, String index) throws IOException { - Request flushRequest = - new Request.Builder() - .url( - factory.baseUrl.newBuilder().addPathSegment(index).addPathSegment("_flush").build()) - .post(RequestBody.create(APPLICATION_JSON, "")) - .tag("flush-index") - .build(); - - factory.newCall(flushRequest, BodyConverters.NULL).execute(); + HttpUrl.Builder url = http().baseUrl.newBuilder().addPathSegment(index); + //if (version() >= 6.0 ) url.addQueryParameter("refresh", "wait_for"); + Request delete = new Request.Builder().url(url.build()).delete().tag("delete-index").build(); + http().newCall(delete, BodyConverters.NULL).execute(); } /** This is blocking so that we can determine if the cluster is healthy or not */ @@ -334,7 +323,7 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon @Override public CheckResult convert(BufferedSource b) throws IOException { b.request(Long.MAX_VALUE); // Buffer the entire body. - Buffer body = b.buffer(); + Buffer body = b.getBuffer(); JsonReader status = enterPath(JsonReader.of(body.clone()), "status"); if (status == null) { throw new IllegalStateException("Health status couldn't be read " + body.readUtf8()); @@ -353,23 +342,28 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon @Memoized // since we don't want overlapping calls to apply the index templates IndexTemplates ensureIndexTemplates() { - String index = indexNameFormatter().index(); try { IndexTemplates templates = new VersionSpecificTemplates(this).get(http()); - EnsureIndexTemplate.apply(http(), index + ":" + SPAN + "_template", templates.span()); - EnsureIndexTemplate.apply( - http(), index + ":" + DEPENDENCY + "_template", templates.dependency()); - EnsureIndexTemplate.apply( - http(), index + ":" + AUTOCOMPLETE + "_template", templates.autocomplete()); + HttpCall.Factory http = http(); + ensureIndexTemplate(http, buildUrl(http, templates, SPAN), templates.span()); + ensureIndexTemplate(http, buildUrl(http, templates, DEPENDENCY), templates.dependency()); + ensureIndexTemplate(http, buildUrl(http, templates, AUTOCOMPLETE), templates.autocomplete()); return templates; } catch (IOException e) { throw Platform.get().uncheckedIOException(e); } } - @Memoized - public // hosts resolution might imply a network call, and we might make a new okhttp instance - HttpCall.Factory http() { + HttpUrl buildUrl(HttpCall.Factory http, IndexTemplates templates, String type) { + HttpUrl.Builder builder = http.baseUrl.newBuilder("_template"); + // ES 7.x defaults include_type_name to false https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_literal_include_type_name_literal_now_defaults_to_literal_false_literal + if (templates.version() >= 7) builder.addQueryParameter("include_type_name", "true"); + String indexPrefix = indexNameFormatter().index() + templates.indexTypeDelimiter(); + return builder.addPathSegment(indexPrefix + type + "_template").build(); + } + + @Memoized // hosts resolution might imply a network call, and we might make a new okhttp instance + public HttpCall.Factory http() { List hosts = hostsSupplier().get(); if (hosts.isEmpty()) throw new IllegalArgumentException("no hosts configured"); OkHttpClient ok = diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java index 68f12fe..302688b 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java @@ -20,7 +20,7 @@ import java.io.IOException; import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.RequestBody; -import zipkin2.elasticsearch.internal.client.HttpCall; +import zipkin2.elasticsearch.internal.client.HttpCall.Factory; /** Ensures the index template exists and saves off the version */ final class EnsureIndexTemplate { @@ -29,19 +29,17 @@ final class EnsureIndexTemplate { * This is a blocking call, used inside a lazy. That's because no writes should occur until the * template is available. */ - static void apply(HttpCall.Factory callFactory, String name, String indexTemplate) - throws IOException { - HttpUrl templateUrl = callFactory.baseUrl.newBuilder("_template").addPathSegment(name).build(); + static void ensureIndexTemplate(Factory callFactory, HttpUrl templateUrl, String indexTemplate) + throws IOException { Request getTemplate = new Request.Builder().url(templateUrl).tag("get-template").build(); try { callFactory.newCall(getTemplate, BodyConverters.NULL).execute(); } catch (IllegalStateException e) { // TODO: handle 404 slightly more nicely - Request updateTemplate = - new Request.Builder() - .url(templateUrl) - .put(RequestBody.create(ElasticsearchStorage.APPLICATION_JSON, indexTemplate)) - .tag("update-template") - .build(); + Request updateTemplate = new Request.Builder() + .url(templateUrl) + .put(RequestBody.create(ElasticsearchStorage.APPLICATION_JSON, indexTemplate)) + .tag("update-template") + .build(); callFactory.newCall(updateTemplate, BodyConverters.NULL).execute(); } } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java index 5b52235..41e6f8c 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java @@ -32,6 +32,15 @@ abstract class IndexTemplates { abstract String autocomplete(); + /** + * This returns a delimiter based on what's supported by the Elasticsearch version. + * + *

See https://github.com/openzipkin/zipkin/issues/2219 + */ + char indexTypeDelimiter() { + return version() < 7 ? ':' : '-'; + } + @AutoValue.Builder interface Builder { Builder version(float version); diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java index c49f170..b48a9a1 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java @@ -23,9 +23,9 @@ import okhttp3.Request; import okio.BufferedSource; import zipkin2.elasticsearch.internal.client.HttpCall; +import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY; import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN; -import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE; import static zipkin2.elasticsearch.internal.JsonReaders.enterPath; /** Returns a version-specific span and dependency index template */ @@ -91,9 +91,11 @@ final class VersionSpecificTemplates { + " },\n"; if (searchEnabled) { return result - + (" \"mappings\": {\n" - + " \"_default_\": {\n" - + " DISABLE_ALL" // don't concat all fields into big string + + (" \"mappings\": {\nDISABLE_ALL" + + " \"" + + SPAN + + "\": {\n" + + " \"_source\": {\"excludes\": [\"_q\"] },\n" + " \"dynamic_templates\": [\n" + " {\n" + " \"strings\": {\n" @@ -105,12 +107,7 @@ final class VersionSpecificTemplates { + " \"match\": \"*\"\n" + " }\n" + " }\n" - + " ]\n" - + " },\n" - + " \"" - + SPAN - + "\": {\n" - + " \"_source\": {\"excludes\": [\"_q\"] },\n" + + " ],\n" + " \"properties\": {\n" + " \"traceId\": ${__TRACE_ID_MAPPING__},\n" + " \"name\": { KEYWORD },\n" @@ -138,8 +135,7 @@ final class VersionSpecificTemplates { + "}"); } return result - + (" \"mappings\": {\n" - + " \"_default_\": { DISABLE_ALL },\n" + + (" \"mappings\": {\nDISABLE_ALL" + " \"" + SPAN + "\": {\n" @@ -181,12 +177,12 @@ final class VersionSpecificTemplates { + " \"index.number_of_shards\": ${__NUMBER_OF_SHARDS__},\n" + " \"index.number_of_replicas\": ${__NUMBER_OF_REPLICAS__},\n" + " \"index.requests.cache.enable\": true,\n" - + " \"index.mapper.dynamic\": true\n" + + " \"index.mapper.dynamic\": false\n" + " },\n" + " \"mappings\": {\"" + AUTOCOMPLETE + "\": { \"enabled\": true,\n" - + " \t\"properties\": {\n" + + " \"properties\": {\n" + " \"tagKey\": { KEYWORD },\n" + " \"tagValue\": { KEYWORD }\n" + " }}}\n" @@ -228,47 +224,62 @@ final class VersionSpecificTemplates { } private String versionSpecificSpanIndexTemplate(float version) { + String result; if (version >= 2 && version < 3) { - return spanIndexTemplate + result = spanIndexTemplate .replace("TEMPLATE", "template") .replace("STRING", "string") - .replace("DISABLE_ALL", "\"_all\": {\"enabled\": false}" + (searchEnabled ? ",\n" : "")) + .replace("DISABLE_ALL", "\"_default_\": { \"_all\": {\"enabled\": false} },\n") .replace( "KEYWORD", "\"type\": \"string\", \"norms\": {\"enabled\": false }, \"index\": \"not_analyzed\""); } else if (version >= 5) { - return spanIndexTemplate + result = spanIndexTemplate .replace("TEMPLATE", version >= 6 ? "index_patterns" : "template") .replace("STRING", "text") - .replace("DISABLE_ALL", "") // _all isn't supported in 6.x anyway + // 6.x _all disabled https://www.elastic.co/guide/en/elasticsearch/reference/6.7/breaking-changes-6.0.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default + // 7.x _default disallowed https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_the_literal__default__literal_mapping_is_no_longer_allowed + .replace("DISABLE_ALL", "") .replace("KEYWORD", "\"type\": \"keyword\", \"norms\": false") .replace( "\"analyzer\": \"traceId_analyzer\" }", "\"fielddata\": \"true\", \"analyzer\": \"traceId_analyzer\" }"); } else { - throw new IllegalStateException( - "Elasticsearch 2.x, 5.x and 6.x are supported, was: " + version); + throw new IllegalStateException("Elasticsearch 2-7.x are supported, was: " + version); } + return maybeReviseFor7x(SPAN, version, result); } private String versionSpecificDependencyLinkIndexTemplate(float version) { - return dependencyIndexTemplate.replace( - "TEMPLATE", version >= 6 ? "index_patterns" : "template"); + String result = dependencyIndexTemplate.replace( + "TEMPLATE", version >= 6 ? "index_patterns" : "template"); + return maybeReviseFor7x(DEPENDENCY, version, result); } + private String versionSpecificAutocompleteIndexTemplate(float version) { + String result; if (version >= 2 && version < 3) { - return autocompleteIndexTemplate + result = autocompleteIndexTemplate .replace("TEMPLATE", "template") .replace("KEYWORD", "\"type\": \"string\", \"norms\": {\"enabled\": false }, \"index\": " + "\"not_analyzed\""); } else if (version >= 5) { - return autocompleteIndexTemplate + result = autocompleteIndexTemplate .replace("TEMPLATE", version >= 6 ? "index_patterns" : "template") .replace("KEYWORD", "\"type\": \"keyword\",\"norms\": false\n"); - }else { - throw new IllegalStateException( - "Elasticsearch 2.x, 5.x and 6.x are supported, was: " + version); + } else { + throw new IllegalStateException("Elasticsearch 2-7.x are supported, was: " + version); } + return maybeReviseFor7x(AUTOCOMPLETE, version, result); + } + + private String maybeReviseFor7x(String type, float version, String result) { + if (version < 7) return result; + // Colons are no longer allowed in index names. Make sure the pattern in our index template + // doesn't use them either. + result = result.replaceAll(":" + type, "-" + type); + result = result.replaceAll(",\n +\"index\\.mapper\\.dynamic\": false", ""); + return result; } } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java index 08d21f5..c52cda1 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java @@ -17,10 +17,6 @@ package zipkin2.elasticsearch.internal; import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Set; import java.util.concurrent.RejectedExecutionException; import okhttp3.HttpUrl; import okhttp3.MediaType; @@ -42,34 +38,16 @@ public final class HttpBulkIndexer { final String tag; final HttpCall.Factory http; final String pipeline; - final boolean flushOnWrites; + final boolean waitForRefresh; // Mutated for each call to add final Buffer body = new Buffer(); - final Set indices; - final HttpCall.BodyConverter maybeFlush; public HttpBulkIndexer(String tag, ElasticsearchStorage es) { this.tag = tag; http = es.http(); pipeline = es.pipeline(); - flushOnWrites = es.flushOnWrites(); - if (flushOnWrites) { - indices = new LinkedHashSet<>(); - maybeFlush = - new HttpCall.BodyConverter() { - @Override - public Void convert(BufferedSource b) throws IOException { - CheckForErrors.INSTANCE.convert(b); - if (indices.isEmpty()) return null; - ElasticsearchStorage.flush(http, join(indices)); - return null; - } - }; - } else { - indices = null; - maybeFlush = CheckForErrors.INSTANCE; - } + waitForRefresh = es.flushOnWrites(); } enum CheckForErrors implements HttpCall.BodyConverter { @@ -95,7 +73,6 @@ public final class HttpBulkIndexer { } void writeIndexMetadata(String index, String typeName, @Nullable String id) { - if (flushOnWrites) indices.add(index); body.writeUtf8("{\"index\":{\"_index\":\"").writeUtf8(index).writeByte('"'); // the _type parameter is needed for Elasticsearch <6.x body.writeUtf8(",\"_type\":\"").writeUtf8(typeName).writeByte('"'); @@ -112,27 +89,16 @@ public final class HttpBulkIndexer { /** Creates a bulk request when there is more than one object to store */ public HttpCall newCall() { - HttpUrl url = - pipeline != null - ? http.baseUrl.newBuilder("_bulk").addQueryParameter("pipeline", pipeline).build() - : http.baseUrl.resolve("_bulk"); - - Request request = - new Request.Builder() - .url(url) - .tag(tag) - .post(RequestBody.create(APPLICATION_JSON, body.readByteString())) - .build(); + HttpUrl.Builder urlBuilder = http.baseUrl.newBuilder("_bulk"); + if (pipeline != null) urlBuilder.addQueryParameter("pipeline", pipeline); + if (waitForRefresh) urlBuilder.addQueryParameter("refresh", "wait_for"); - return http.newCall(request, maybeFlush); - } + Request request = new Request.Builder() + .url(urlBuilder.build()) + .tag(tag) + .post(RequestBody.create(APPLICATION_JSON, body.readByteString())) + .build(); - static String join(Collection parts) { - Iterator iterator = parts.iterator(); - StringBuilder result = new StringBuilder(iterator.next()); - while (iterator.hasNext()) { - result.append(',').append(iterator.next()); - } - return result.toString(); + return http.newCall(request, CheckForErrors.INSTANCE); } } diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java index 5f21779..4b70737 100644 --- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java +++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java @@ -31,6 +31,44 @@ import zipkin2.internal.Nullable; import static java.lang.String.format; +/** + *

Index-Prefix/type delimiter

+ * When Elasticsearch dropped support for multiple type indexes, we introduced a delimited naming + * convention to distinguish between span, dependency and autocomplete documents. Originally, this + * was a colon prefix pattern. In version 7, Elasticsearch dropped support for colons in indexes. To + * keep existing writes consistent, we still use colon in versions prior to ES 7, eventhough + * starting at version 7, we change to hyphens. {@code zipkin2.elasticsearch.IndexTemplates} is + * responsible for this decision. + * + *

Creating indexes

+ * Using the default index prefix of "zipkin", when indexes are created, they look like the + * following, based on the version. + * + *
    + *
  • ES up to v6: zipkin:span-2019-05-03 zipkin:dependency-2019-05-03 zipkin:autocomplete-2019-05-03
    • + *
  • ES v7: zipkin-span-2019-05-03 zipkin-dependency-2019-05-03 zipkin-autocomplete-2019-05-03
    • + *
+ * + *

We can allow an index prefix of up to 231 UTF-8 encoded bytes, subject to the index naming + * constraints. This is the normal 255 limit minus the longest suffix (ex. -autocomplete-2019-05-03). + * + *

Reading indexes

+ * While ES 7 cannot write new indexes with a colons, it can read them. Upon upgrade, some sites + * will have a mixed read state where some indexes delimit types with a colon and others a hyphen. + * Accordingly, we use * in read patterns in place of a type delimiter. We use * because there is no + * support for single character wildcards in ES. + * + *

Elasticsearch 7 naming constraints

+ * According to a recent + * reference, the following index naming constraints apply to index names as of ES 7: + * + *
    + *
  • No more than 255 UTF-8 encoded bytes
    • + *
  • Cannot be . or ..
    • + *
  • Cannot contain : or #
    • + *
  • Cannot start with _ - or +
    • + *
+ */ @AutoValue public abstract class IndexNameFormatter { public static Builder newBuilder() { @@ -157,12 +195,21 @@ public abstract class IndexNameFormatter { return result; } + /** On insert, require a version-specific index-type delimiter as ES 7+ dropped colons */ + public String formatTypeAndTimestampForInsert(String type, char indexTypeDelimiter, + long timestampMillis) { + return index() + indexTypeDelimiter + type + '-' + dateFormat().get() + .format(new Date(timestampMillis)); + } + public String formatTypeAndTimestamp(@Nullable String type, long timestampMillis) { return prefix(type) + "-" + dateFormat().get().format(new Date(timestampMillis)); } private String prefix(@Nullable String type) { - return type != null ? index() + ":" + type : index(); + // We use single-character wildcard here in order to read both : and - as starting in ES 7, : + // is no longer permitted. + return type != null ? index() + "*" + type : index(); } // for testing diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java index 7cdc812..f74fae8 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java @@ -335,7 +335,6 @@ public class ElasticsearchSpanConsumerTest { .contains( "" + " \"mappings\": {\n" - + " \"_default_\": { },\n" + " \"span\": {\n" + " \"properties\": {\n" + " \"traceId\": { \"type\": \"keyword\", \"norms\": false },\n" diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java index 4edbcb6..0609fe3 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java @@ -60,9 +60,9 @@ public class ElasticsearchStorageTest { es.takeRequest(); // get tags template assertThat(es.takeRequest().getPath()) - .startsWith("/zipkin:dependency-2016-10-01,zipkin:dependency-2016-10-02/_search"); + .startsWith("/zipkin*dependency-2016-10-01,zipkin*dependency-2016-10-02/_search"); assertThat(es.takeRequest().getPath()) - .startsWith("/zipkin:dependency-2016-10-01,zipkin:dependency-2016-10-02/_search"); + .startsWith("/zipkin*dependency-2016-10-01,zipkin*dependency-2016-10-02/_search"); } String healthResponse = diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java index 2b67613..c38b8eb 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java @@ -27,8 +27,8 @@ import zipkin2.elasticsearch.internal.HttpBulkIndexer; public class InternalForTests { public static void writeDependencyLinks(ElasticsearchStorage es, List links, long midnightUTC) { - String index = - es.indexNameFormatter().formatTypeAndTimestamp("dependency", midnightUTC); + String index = ((ElasticsearchSpanConsumer) es.spanConsumer()) + .formatTypeAndTimestampForInsert("dependency", midnightUTC); HttpBulkIndexer indexer = new HttpBulkIndexer("indexlinks", es); for (DependencyLink link : links) { byte[] document = DependencyLinkBytesEncoder.JSON_V1.encode(link); diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java index 1161011..d747ffa 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java @@ -16,7 +16,6 @@ */ package zipkin2.elasticsearch; -import java.io.IOException; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.junit.After; @@ -34,8 +33,6 @@ public class VersionSpecificTemplatesTest { ElasticsearchStorage storage = ElasticsearchStorage.newBuilder().hosts(asList(es.url("").toString())).build(); - VersionSpecificTemplates client = new VersionSpecificTemplates(storage); - @After public void close() { storage.close(); @@ -128,4 +125,50 @@ public class VersionSpecificTemplatesTest { assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(6.0f); } + + @Test public void getVersion_6_7() throws Exception { + es.enqueue(new MockResponse().setBody( + "{\n" + + " \"name\" : \"PV-NhJd\",\n" + + " \"cluster_name\" : \"CollectorDBCluster\",\n" + + " \"cluster_uuid\" : \"UjZaM0fQRC6tkHINCg9y8w\",\n" + + " \"version\" : {\n" + + " \"number\" : \"6.7.0\",\n" + + " \"build_flavor\" : \"oss\",\n" + + " \"build_type\" : \"tar\",\n" + + " \"build_hash\" : \"8453f77\",\n" + + " \"build_date\" : \"2019-03-21T15:32:29.844721Z\",\n" + + " \"build_snapshot\" : false,\n" + + " \"lucene_version\" : \"7.7.0\",\n" + + " \"minimum_wire_compatibility_version\" : \"5.6.0\",\n" + + " \"minimum_index_compatibility_version\" : \"5.0.0\"\n" + + " },\n" + + " \"tagline\" : \"You Know, for Search\"\n" + + "}")); + + assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(6.7f); + } + + @Test public void getVersion_7() throws Exception { + es.enqueue(new MockResponse().setBody( + "{\n" + + " \"name\" : \"zipkin-elasticsearch\",\n" + + " \"cluster_name\" : \"docker-cluster\",\n" + + " \"cluster_uuid\" : \"wByRPgSgTryYl0TZXW4MsA\",\n" + + " \"version\" : {\n" + + " \"number\" : \"7.0.1\",\n" + + " \"build_flavor\" : \"default\",\n" + + " \"build_type\" : \"tar\",\n" + + " \"build_hash\" : \"e4efcb5\",\n" + + " \"build_date\" : \"2019-04-29T12:56:03.145736Z\",\n" + + " \"build_snapshot\" : false,\n" + + " \"lucene_version\" : \"8.0.0\",\n" + + " \"minimum_wire_compatibility_version\" : \"6.7.0\",\n" + + " \"minimum_index_compatibility_version\" : \"6.0.0-beta1\"\n" + + " },\n" + + " \"tagline\" : \"You Know, for Search\"\n" + + "}")); + + assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(7.0f); + } } diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java index b397c94..d42bb34 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java @@ -27,7 +27,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import zipkin2.CheckResult; import zipkin2.elasticsearch.ElasticsearchStorage; diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java new file mode 100644 index 0000000..e2c37f3 --- /dev/null +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package zipkin2.elasticsearch.integration; + +import java.io.IOException; +import java.util.List; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import zipkin2.Span; +import zipkin2.elasticsearch.ElasticsearchStorage; +import zipkin2.elasticsearch.InternalForTests; +import zipkin2.storage.StorageComponent; + +import static zipkin2.elasticsearch.integration.ElasticsearchStorageRule.index; + +@RunWith(Enclosed.class) +public class ITElasticsearchStorageV7 { + + static ElasticsearchStorageRule classRule() { + return new ElasticsearchStorageRule("openzipkin/zipkin-elasticsearch7:2.12.9", + "test_elasticsearch3"); + } + + public static class ITSpanStore extends zipkin2.storage.ITSpanStore { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + ElasticsearchStorage storage; + + @Before public void connect() { + storage = backend.computeStorageBuilder().index(index(testName)).build(); + } + + @Override protected StorageComponent storage() { + return storage; + } + + @Override @Test @Ignore("No consumer-side span deduplication") public void deduplicates() { + } + + @Before @Override public void clear() throws IOException { + storage.clear(); + } + } + + public static class ITSearchEnabledFalse extends zipkin2.storage.ITSearchEnabledFalse { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + ElasticsearchStorage storage; + + @Before public void connect() { + storage = backend.computeStorageBuilder().index(index(testName)) + .searchEnabled(false).build(); + } + + @Override protected StorageComponent storage() { + return storage; + } + + @Before @Override public void clear() throws IOException { + storage.clear(); + } + } + + public static class ITAutocompleteTags extends zipkin2.storage.ITAutocompleteTags { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + @Override protected StorageComponent.Builder storageBuilder() { + return backend.computeStorageBuilder().index(index(testName)); + } + + @Before @Override public void clear() throws IOException { + ((ElasticsearchStorage) storage).clear(); + } + } + + public static class ITStrictTraceIdFalse extends zipkin2.storage.ITStrictTraceIdFalse { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + ElasticsearchStorage storage; + + @Before public void connect() { + storage = backend.computeStorageBuilder().index(index(testName)).strictTraceId(false).build(); + } + + @Override protected StorageComponent storage() { + return storage; + } + + @Before @Override public void clear() throws IOException { + storage.clear(); + } + } + + public static class ITDependencies extends zipkin2.storage.ITDependencies { + @ClassRule public static ElasticsearchStorageRule backend = classRule(); + @Rule public TestName testName = new TestName(); + + ElasticsearchStorage storage; + + @Before public void connect() { + storage = backend.computeStorageBuilder().index(index(testName)).build(); + } + + @Override protected StorageComponent storage() { + return storage; + } + + /** + * The current implementation does not include dependency aggregation. It includes retrieval of + * pre-aggregated links, usually made via zipkin-dependencies + */ + @Override protected void processDependencies(List spans) throws Exception { + aggregateLinks(spans).forEach( + (midnight, links) -> InternalForTests.writeDependencyLinks(storage, links, midnight)); + } + + @Before @Override public void clear() throws IOException { + storage.clear(); + } + } +} diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java index fb1ccf1..c867f63 100644 --- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java +++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java @@ -39,7 +39,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-01T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016-11-01"); + .containsExactly("zipkin*span-2016-11-01"); } @Test @@ -48,7 +48,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-16T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016-11-15", "zipkin:span-2016-11-16"); + .containsExactly("zipkin*span-2016-11-15", "zipkin*span-2016-11-16"); } @Test @@ -58,7 +58,7 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016-11-01", "zipkin:span-2016-11-02", "zipkin:span-2016-11-03"); + "zipkin*span-2016-11-01", "zipkin*span-2016-11-02", "zipkin*span-2016-11-03"); } @Test @@ -67,7 +67,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-01T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016-10-31", "zipkin:span-2016-11-01"); + .containsExactly("zipkin*span-2016-10-31", "zipkin*span-2016-11-01"); } @Test @@ -76,7 +76,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-10-31T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016-10-*"); + .containsExactly("zipkin*span-2016-10-*"); } @Test @@ -86,7 +86,7 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016-10-31", "zipkin:span-2016-11-*", "zipkin:span-2016-12-01"); + "zipkin*span-2016-10-31", "zipkin*span-2016-11-*", "zipkin*span-2016-12-01"); } @Test @@ -96,10 +96,10 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016-02-28", - "zipkin:span-2016-02-29", - "zipkin:span-2016-03-*", - "zipkin:span-2016-04-01"); + "zipkin*span-2016-02-28", + "zipkin*span-2016-02-29", + "zipkin*span-2016-03-*", + "zipkin*span-2016-04-01"); } @Test @@ -108,7 +108,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-12-31T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016-*"); + .containsExactly("zipkin*span-2016-*"); } @Test @@ -118,11 +118,11 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016-10-31", - "zipkin:span-2016-11-*", - "zipkin:span-2016-12-*", - "zipkin:span-2017-*", - "zipkin:span-2018-01-01"); + "zipkin*span-2016-10-31", + "zipkin*span-2016-11-*", + "zipkin*span-2016-12-*", + "zipkin*span-2017-*", + "zipkin*span-2018-01-01"); } @Test @@ -132,7 +132,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-01T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.11.01"); + .containsExactly("zipkin*span-2016.11.01"); } @Test @@ -142,7 +142,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-16T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.11.15", "zipkin:span-2016.11.16"); + .containsExactly("zipkin*span-2016.11.15", "zipkin*span-2016.11.16"); } @Test @@ -153,7 +153,7 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.11.01", "zipkin:span-2016.11.02", "zipkin:span-2016.11.03"); + "zipkin*span-2016.11.01", "zipkin*span-2016.11.02", "zipkin*span-2016.11.03"); } @Test @@ -163,7 +163,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-11-01T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.10.31", "zipkin:span-2016.11.01"); + .containsExactly("zipkin*span-2016.10.31", "zipkin*span-2016.11.01"); } @Test @@ -173,7 +173,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-10-31T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.10.*"); + .containsExactly("zipkin*span-2016.10.*"); } @Test @@ -184,7 +184,7 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.10.31", "zipkin:span-2016.11.*", "zipkin:span-2016.12.01"); + "zipkin*span-2016.10.31", "zipkin*span-2016.11.*", "zipkin*span-2016.12.01"); } @Test @@ -195,10 +195,10 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.02.28", - "zipkin:span-2016.02.29", - "zipkin:span-2016.03.*", - "zipkin:span-2016.04.01"); + "zipkin*span-2016.02.28", + "zipkin*span-2016.02.29", + "zipkin*span-2016.03.*", + "zipkin*span-2016.04.01"); } @Test @@ -208,7 +208,7 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-12-31T23:59:59Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.*"); + .containsExactly("zipkin*span-2016.*"); } @Test @@ -219,11 +219,11 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.10.31", - "zipkin:span-2016.11.*", - "zipkin:span-2016.12.*", - "zipkin:span-2017.*", - "zipkin:span-2018.01.01"); + "zipkin*span-2016.10.31", + "zipkin*span-2016.11.*", + "zipkin*span-2016.12.*", + "zipkin*span-2017.*", + "zipkin*span-2018.01.01"); } @Test @@ -234,10 +234,10 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.10.0*", - "zipkin:span-2016.10.1*", - "zipkin:span-2016.10.2*", - "zipkin:span-2016.10.30"); + "zipkin*span-2016.10.0*", + "zipkin*span-2016.10.1*", + "zipkin*span-2016.10.2*", + "zipkin*span-2016.10.30"); } @Test @@ -248,10 +248,10 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.10.09", - "zipkin:span-2016.10.1*", - "zipkin:span-2016.10.2*", - "zipkin:span-2016.10.30"); + "zipkin*span-2016.10.09", + "zipkin*span-2016.10.1*", + "zipkin*span-2016.10.2*", + "zipkin*span-2016.10.30"); } @Test @@ -262,9 +262,9 @@ public class IndexNameFormatterTest { assertThat(formatter.formatTypeAndRange("span", start, end)) .containsExactly( - "zipkin:span-2016.10.19", - "zipkin:span-2016.10.2*", - "zipkin:span-2016.10.30"); + "zipkin*span-2016.10.19", + "zipkin*span-2016.10.2*", + "zipkin*span-2016.10.30"); } @Test @@ -274,6 +274,6 @@ public class IndexNameFormatterTest { long end = iso8601.parse("2016-06-30T01:01:01Z").getTime(); assertThat(formatter.formatTypeAndRange("span", start, end)) - .containsExactly("zipkin:span-2016.06.*"); + .containsExactly("zipkin*span-2016.06.*"); } }