pinot-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mcvsu...@apache.org
Subject [incubator-pinot] branch master updated: Add a new table config field for peer segment download. (#5478)
Date Tue, 02 Jun 2020 21:01:00 GMT
This is an automated email from the ASF dual-hosted git repository.

mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 8a3eb43  Add a new table config field for peer segment download. (#5478)
8a3eb43 is described below

commit 8a3eb4315df59d6d73a4ec11308d30eab1c51042
Author: Ting Chen <tingchen@uber.com>
AuthorDate: Tue Jun 2 14:00:47 2020 -0700

    Add a new table config field for peer segment download. (#5478)
    
    * Add a new table config field for peer segment download.
    
    * Address feedbacks.
    
    * Update pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
    
    Co-authored-by: Subbu Subramaniam <mcvsubbu@users.noreply.github.com>
    
    * Update pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
    
    Co-authored-by: Subbu Subramaniam <mcvsubbu@users.noreply.github.com>
    
    Co-authored-by: Subbu Subramaniam <mcvsubbu@users.noreply.github.com>
---
 .../pinot/common/utils/config/TableConfigUtils.java    | 18 ++++++++++++++++++
 .../common/utils/config/TableConfigSerDeTest.java      | 11 +++++++++++
 .../table/SegmentsValidationAndRetentionConfig.java    | 11 +++++++++++
 .../pinot/spi/utils/builder/TableConfigBuilder.java    |  7 +++++++
 4 files changed, 47 insertions(+)

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
index d93b24b..b7b7d52 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TableConfigUtils.java
@@ -181,9 +181,15 @@ public class TableConfigUtils {
    * Validates the table config with the following rules:
    * <ul>
    *   <li>Text index column must be raw</li>
+   *   <li>peerSegmentDownloadScheme in ValidationConfig must be http or https</li>
    * </ul>
    */
   public static void validate(TableConfig tableConfig) {
+    validateFieldConfigList(tableConfig);
+    validateValidationConfig(tableConfig);
+  }
+
+  private static void validateFieldConfigList(TableConfig tableConfig) {
     List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList();
     if (fieldConfigList != null) {
       List<String> noDictionaryColumns = tableConfig.getIndexingConfig().getNoDictionaryColumns();
@@ -201,4 +207,16 @@ public class TableConfigUtils {
       }
     }
   }
+
+  private static void validateValidationConfig(TableConfig tableConfig) {
+    SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig();
+    if (validationConfig != null) {
+      String peerSegmentDownloadScheme = validationConfig.getPeerSegmentDownloadScheme();
+      if (peerSegmentDownloadScheme != null) {
+        if (!"http".equalsIgnoreCase(peerSegmentDownloadScheme) && !"https".equalsIgnoreCase(peerSegmentDownloadScheme))
{
+          throw new IllegalStateException("Invalid value '" + peerSegmentDownloadScheme +
"' for peerSegmentDownloadScheme. Must be one of http nor https" );
+        }
+      }
+    }
+  }
 }
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index 4d34012..d2d2ebf 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -243,6 +243,17 @@ public class TableConfigSerDeTest {
       checkTableConfigWithUpsertConfig(JsonUtils.stringToObject(tableConfig.toJsonString(),
TableConfig.class));
       checkTableConfigWithUpsertConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig)));
     }
+    {
+      // with SegmentsValidationAndRetentionConfig
+      TableConfig tableConfig = tableConfigBuilder.setPeerSegmentDownloadScheme("http").build();
+      checkSegmentsValidationAndRetentionConfig(JsonUtils.stringToObject(tableConfig.toJsonString(),
TableConfig.class));
+      checkSegmentsValidationAndRetentionConfig(TableConfigUtils.fromZNRecord(TableConfigUtils.toZNRecord(tableConfig)));
+    }
+  }
+
+  private void checkSegmentsValidationAndRetentionConfig(TableConfig tableConfig) {
+    // TODO validate other fields of SegmentsValidationAndRetentionConfig.
+    assertEquals(tableConfig.getValidationConfig().getPeerSegmentDownloadScheme(), "http");
   }
 
   private void checkDefaultTableConfig(TableConfig tableConfig) {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
index 4e0f3e2..30451ab 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/SegmentsValidationAndRetentionConfig.java
@@ -39,6 +39,11 @@ public class SegmentsValidationAndRetentionConfig extends BaseJsonConfig
{
   private String _segmentAssignmentStrategy;
   private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig;
   private CompletionConfig _completionConfig;
+  // Possible values can be http or https. If this field is set, a Pinot server can download
segments from peer servers
+  // using the specified download scheme. Both realtime tables and offline tables can set
this field.
+  // For more usage of this field, please refer to this design doc:
+  // https://cwiki.apache.org/confluence/display/PINOT/By-passing+deep-store+requirement+for+Realtime+segment+completion#By-passingdeep-storerequirementforRealtimesegmentcompletion-EnablebesteffortsegmentuploadinSplitSegmentCommiteranddownloadsegmentfrompeerservers.
+  private String _peerSegmentDownloadScheme;
 
   // Number of replicas per partition of low-level consumers. This config is used for realtime
tables only.
   private String _replicasPerPartition;
@@ -152,4 +157,10 @@ public class SegmentsValidationAndRetentionConfig extends BaseJsonConfig
{
   public int getReplicasPerPartitionNumber() {
     return Integer.parseInt(_replicasPerPartition);
   }
+
+  public String getPeerSegmentDownloadScheme() { return _peerSegmentDownloadScheme; }
+
+  public void setPeerSegmentDownloadScheme(String peerSegmentDownloadScheme) {
+    _peerSegmentDownloadScheme = peerSegmentDownloadScheme;
+  }
 }
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
index 63cc43c..dbd0c03 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
@@ -64,6 +64,7 @@ public class TableConfigBuilder {
   private String _segmentPushFrequency;
   private String _segmentPushType = DEFAULT_SEGMENT_PUSH_TYPE;
   private String _segmentAssignmentStrategy = DEFAULT_SEGMENT_ASSIGNMENT_STRATEGY;
+  private String _peerSegmentDownloadScheme;
   private ReplicaGroupStrategyConfig _replicaGroupStrategyConfig;
   private CompletionConfig _completionConfig;
 
@@ -287,6 +288,11 @@ public class TableConfigBuilder {
     return this;
   }
 
+  public TableConfigBuilder setPeerSegmentDownloadScheme(String peerSegmentDownloadScheme)
{
+    _peerSegmentDownloadScheme = peerSegmentDownloadScheme;
+    return this;
+  }
+
   public TableConfig build() {
     // Validation config
     SegmentsValidationAndRetentionConfig validationConfig = new SegmentsValidationAndRetentionConfig();
@@ -301,6 +307,7 @@ public class TableConfigBuilder {
     validationConfig.setCompletionConfig(_completionConfig);
     validationConfig.setSchemaName(_schemaName);
     validationConfig.setReplication(_numReplicas);
+    validationConfig.setPeerSegmentDownloadScheme(_peerSegmentDownloadScheme);
     if (_isLLC) {
       validationConfig.setReplicasPerPartition(_numReplicas);
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org


Mime
View raw message