asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From AsterixDB Code Review <do-not-re...@asterix-gerrit.ics.uci.edu>
Subject Change in asterixdb[cheshire-cat]: [ASTERIXDB-2919] Azure Blob external datasets: add support to anonymo...
Date Thu, 17 Jun 2021 15:42:25 GMT
>From Hussain Towaileb <hussainht@gmail.com>:

Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11983
)


Change subject: [ASTERIXDB-2919] Azure Blob external datasets: add support to anonymous authentication
......................................................................

[ASTERIXDB-2919] Azure Blob external datasets: add support to anonymous authentication

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Added support to creating external datasets with no
  credentials (anonymous).

Change-Id: I19a161a3613371572fa26c786bab690e332fdd08
---
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
C asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.001.query.sqlpp
R asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.099.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/result.001.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
10 files changed, 81 insertions(+), 24 deletions(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/83/11983/1

diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
new file mode 100644
index 0000000..bd3e340
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: no auth method is provided
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.001.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.099.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/result.001.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 1e302da..5c6c1c0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -68,9 +68,14 @@
       </compilation-unit>
     </test-case>
     <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
-      <compilation-unit name="invalid-no-auth">
-        <output-dir compare="Text">invalid-no-auth</output-dir>
-        <expected-error>ASX1134: No authentication parameters provided</expected-error>
+      <compilation-unit name="anonymous-no-auth">
+        <output-dir compare="Text">anonymous-no-auth</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+      <compilation-unit name="anonymous-no-auth-no-endpoint">
+        <output-dir compare="Text">anonymous-no-auth-no-endpoint</output-dir>
+        <expected-error>ASX1151: No authentication credentials provided, 'blobEndpoint'
field is required for anonymous access</expected-error>
       </compilation-unit>
     </test-case>
   </test-group>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 4078ea2..e80dbfb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -235,6 +235,7 @@
     CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148),
     ILLEGAL_FUNCTION_RECURSION(1149),
     ILLEGAL_FUNCTION_USE(1150),
+    NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS(1151),
 
     // Feed errors
     DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 18898bb..cd4a289 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -237,6 +237,7 @@
 1148 = Cannot drop %1$s %2$s being used by %3$s %4$s
 1149 = Illegal function recursion
 1150 = Illegal use of function %1$s
+1151 = No authentication credentials provided, '%1$s' field is required for anonymous access
 
 # Feed Errors
 3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8e94263..6fb0271 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -22,17 +22,7 @@
 import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
 import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ERROR_METHOD_NOT_IMPLEMENTED;
 import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_KEY;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_BLOB_ENDPOINT;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ENDPOINT_SUFFIX;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_SHARED_ACCESS_SIGNATURE;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.*;
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_EXTERNAL_SCAN_BUFFER_SIZE;
@@ -899,7 +889,7 @@
             String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME);
             String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME);
 
-            // Constructor the connection string
+            // Construct the connection string
             // Connection string format: name1=value1;name2=value2;....
             StringBuilder connectionStringBuilder = new StringBuilder();
             BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
@@ -926,34 +916,38 @@
                 if (authMethodFound) {
                     throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
                 }
-                authMethodFound = true;
                 // account name + shared access token
                 connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
                         .append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=")
                         .append(sharedAccessSignature).append(";");
             }
 
-            if (!authMethodFound) {
-                throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED);
-            }
-
             // Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed
             if (blobEndpoint != null) {
                 connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint)
                         .append(";");
+
                 if (endpointSuffix != null) {
-                    String endpointSuffixUpdated;
+                    String endpointSuffixSlashCorrected;
+
                     if (blobEndpoint.endsWith("/")) {
-                        endpointSuffixUpdated =
+                        endpointSuffixSlashCorrected =
                                 endpointSuffix.startsWith("/") ? endpointSuffix.substring(1)
: endpointSuffix;
                     } else {
-                        endpointSuffixUpdated = endpointSuffix.startsWith("/") ? endpointSuffix
: "/" + endpointSuffix;
+                        endpointSuffixSlashCorrected =
+                                endpointSuffix.startsWith("/") ? endpointSuffix : "/" + endpointSuffix;
                     }
                     connectionStringBuilder.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=")
-                            .append(endpointSuffixUpdated).append(";");
+                            .append(endpointSuffixSlashCorrected).append(";");
                 }
             }
 
+            // No credentials or endpoint provided
+            if (connectionStringBuilder.length() == 0) {
+                throw new CompilationException(ErrorCode.NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS,
+                        BLOB_ENDPOINT_FIELD_NAME);
+            }
+
             try {
                 return builder.connectionString(connectionStringBuilder.toString()).buildClient();
             } catch (Exception ex) {

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11983
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: cheshire-cat
Gerrit-Change-Id: I19a161a3613371572fa26c786bab690e332fdd08
Gerrit-Change-Number: 11983
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hussainht@gmail.com>
Gerrit-MessageType: newchange

Mime
View raw message