Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 04B6D200D51 for ; Fri, 8 Dec 2017 00:17:31 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 0388D160C0C; Thu, 7 Dec 2017 23:17:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2D08D160C1E for ; Fri, 8 Dec 2017 00:17:30 +0100 (CET) Received: (qmail 35593 invoked by uid 500); 7 Dec 2017 23:17:25 -0000 Mailing-List: contact commits-help@beam.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@beam.apache.org Delivered-To: mailing list commits@beam.apache.org Received: (qmail 35458 invoked by uid 99); 7 Dec 2017 23:17:25 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Dec 2017 23:17:25 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 9987A85695; Thu, 7 Dec 2017 23:17:24 +0000 (UTC) Date: Thu, 07 Dec 2017 23:17:25 +0000 To: "commits@beam.apache.org" Subject: [beam] 02/02: [BEAM-2957] Change assert in testSplit to allow up to 50% of empty splits MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: lcwik@apache.org In-Reply-To: <151268864371.31410.9707669677361305078@gitbox.apache.org> References: <151268864371.31410.9707669677361305078@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: beam X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 16a506b582d7a583bf771ce3fe1f260572d6ebc9 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20171207231724.9987A85695@gitbox.apache.org> archived-at: Thu, 07 Dec 2017 23:17:31 -0000 This is an automated email from the ASF dual-hosted git repository. lcwik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git commit 16a506b582d7a583bf771ce3fe1f260572d6ebc9 Author: Etienne Chauchot AuthorDate: Wed Dec 6 09:56:40 2017 +0100 [BEAM-2957] Change assert in testSplit to allow up to 50% of empty splits --- .../beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 18 ++++++++++++------ .../beam/sdk/io/elasticsearch/ElasticsearchIOTest.java | 15 ++++++++++----- .../io/elasticsearch/ElasticsearchIOTestCommon.java | 1 + 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java index 06298cd..c1e7662 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-2/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java @@ -20,11 +20,14 @@ package org.apache.beam.sdk.io.elasticsearch; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read; +import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ACCEPTABLE_EMPTY_SPLITS_PERCENTAGE; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_INDEX; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_TYPE; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.NUM_DOCS_UTESTS; import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource; +import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import java.io.IOException; import java.io.Serializable; @@ -172,14 +175,17 @@ public class ElasticsearchIOTest implements Serializable { SourceTestUtils.assertSourcesEqualReferenceSource(initialSource, splits, options); //this is the number of ES shards // (By default, each index in Elasticsearch is allocated 5 primary shards) - int expectedNumSplits = 5; - assertEquals(expectedNumSplits, splits.size()); - int nonEmptySplits = 0; + int expectedNumSources = 5; + assertEquals("Wrong number of splits", expectedNumSources, splits.size()); + int emptySplits = 0; for (BoundedSource subSource : splits) { - if (readFromSource(subSource, options).size() > 0) { - nonEmptySplits += 1; + if (readFromSource(subSource, options).isEmpty()) { + emptySplits += 1; } } - assertEquals("Wrong number of non empty splits", expectedNumSplits, nonEmptySplits); + assertThat( + "There are too many empty splits, parallelism is sub-optimal", + emptySplits, + lessThan((int) (ACCEPTABLE_EMPTY_SPLITS_PERCENTAGE * splits.size()))); } } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java index c1d844b..ec81074 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-5/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTest.java @@ -20,10 +20,12 @@ package org.apache.beam.sdk.io.elasticsearch; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.BoundedElasticsearchSource; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.ConnectionConfiguration; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.Read; +import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ACCEPTABLE_EMPTY_SPLITS_PERCENTAGE; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_INDEX; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.ES_TYPE; import static org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOTestCommon.NUM_DOCS_UTESTS; import static org.apache.beam.sdk.testing.SourceTestUtils.readFromSource; +import static org.hamcrest.Matchers.lessThan; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import java.io.Serializable; @@ -173,13 +175,16 @@ public class ElasticsearchIOTest extends ESIntegTestCase implements Serializable long indexSize = BoundedElasticsearchSource.estimateIndexSize(connectionConfiguration); float expectedNumSourcesFloat = (float) indexSize / desiredBundleSizeBytes; int expectedNumSources = (int) Math.ceil(expectedNumSourcesFloat); - assertEquals(expectedNumSources, splits.size()); - int nonEmptySplits = 0; + assertEquals("Wrong number of splits", expectedNumSources, splits.size()); + int emptySplits = 0; for (BoundedSource subSource : splits) { - if (readFromSource(subSource, options).size() > 0) { - nonEmptySplits += 1; + if (readFromSource(subSource, options).isEmpty()) { + emptySplits += 1; } } - assertEquals("Wrong number of non empty splits", expectedNumSources, nonEmptySplits); + assertThat( + "There are too many empty splits, parallelism is sub-optimal", + emptySplits, + lessThan((int) (ACCEPTABLE_EMPTY_SPLITS_PERCENTAGE * splits.size()))); } } diff --git a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java index 03eaf00..55d1fe7 100644 --- a/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java +++ b/sdks/java/io/elasticsearch-tests/elasticsearch-tests-common/src/test/java/org/apache/beam/sdk/io/elasticsearch/ElasticsearchIOTestCommon.java @@ -59,6 +59,7 @@ class ElasticsearchIOTestCommon implements Serializable { static final String ES_TYPE = "test"; static final long NUM_DOCS_UTESTS = 400L; static final long NUM_DOCS_ITESTS = 50000L; + static final float ACCEPTABLE_EMPTY_SPLITS_PERCENTAGE = 0.5f; private static final long AVERAGE_DOC_SIZE = 25L; -- To stop receiving notification emails like this one, please contact "commits@beam.apache.org" .