reef-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (REEF-2031) Azure Batch credentials exposed in job-submission-params.json in REEF.NET
Date Thu, 14 Jun 2018 20:34:00 GMT

    [ https://issues.apache.org/jira/browse/REEF-2031?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16512951#comment-16512951
] 

ASF GitHub Bot commented on REEF-2031:
--------------------------------------

motus closed pull request #1469: [REEF-2031]Azure Batch credentials exposed in job-submission-params.json
in REEF.NET
URL: https://github.com/apache/reef/pull/1469
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
index 78bfe4a323..6409dc87fe 100644
--- a/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
+++ b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
@@ -30,7 +30,7 @@ namespace Org.Apache.REEF.Client.Avro.AzureBatch
     [DataContract(Namespace = "org.apache.reef.reef.bridge.client.avro")]
     public sealed class AvroAzureBatchJobSubmissionParameters
     {
-        private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters"",""doc"":""Job
submission parameters used by the Azure Batch runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General
cross-language job submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""AzureBatchAccountKey"",""type"":""string""},{""name"":""AzureBatchAccountName"",""type"":""string""},{""name"":""AzureBatchAccountUri"",""type"":""string""},{""name"":""AzureBatchPoolId"",""type"":""string""},{""name"":""AzureStorageAccountKey"",""type"":""string""},{""name"":""AzureStorageAccountName"",""type"":""string""},{""name"":""AzureStorageContainerName"",""type"":""string""},{""name"":""AzureBatchIsWindows"",""type"":""boolean""}]}";
+        private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters"",""doc"":""Job
submission parameters used by the Azure Batch runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General
cross-language job submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""AzureBatchAccountName"",""type"":""string""},{""name"":""AzureBatchAccountUri"",""type"":""string""},{""name"":""AzureBatchPoolId"",""type"":""string""},{""name"":""AzureStorageAccountName"",""type"":""string""},{""name"":""AzureStorageContainerName"",""type"":""string""}]}";
 
         /// <summary>
         /// Gets the schema.
@@ -49,12 +49,6 @@ public static string Schema
         [DataMember]
         public AvroJobSubmissionParameters sharedJobSubmissionParameters { get; set; }
 
-        /// <summary>
-        /// Gets or sets the AzureBatchAccountKey field.
-        /// </summary>
-        [DataMember]
-        public string AzureBatchAccountKey { get; set; }
-
         /// <summary>
         /// Gets or sets the AzureBatchAccountName field.
         /// </summary>
@@ -73,12 +67,6 @@ public static string Schema
         [DataMember]
         public string AzureBatchPoolId { get; set; }
 
-        /// <summary>
-        /// Gets or sets the AzureStorageAccountKey field.
-        /// </summary>
-        [DataMember]
-        public string AzureStorageAccountKey { get; set; }
-
         /// <summary>
         /// Gets or sets the AzureStorageAccountName field.
         /// </summary>
@@ -91,12 +79,6 @@ public static string Schema
         [DataMember]
         public string AzureStorageContainerName { get; set; }
 
-        /// <summary>
-        /// Gets or sets the AzureBatchIsWindows field.
-        /// </summary>
-        [DataMember]
-        public bool AzureBatchIsWindows { get; set; }
-
         /// <summary>
         /// Initializes a new instance of the <see cref="AvroAzureBatchJobSubmissionParameters"/>
class.
         /// </summary>
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
index 48e62e53d9..4d434d764a 100644
--- a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
@@ -40,11 +40,9 @@ internal sealed class JobJarMaker
             IResourceArchiveFileGenerator resourceArchiveFileGenerator,
             DriverFolderPreparationHelper driverFolderPreparationHelper,
             REEFFileNames fileNames,
-            [Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey,
             [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
             [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
             [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
-            [Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey,
             [Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
             [Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName)
         {
@@ -53,14 +51,11 @@ internal sealed class JobJarMaker
             _fileNames = fileNames;
             _avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
             {
-                AzureBatchAccountKey = azureBatchAccountKey,
                 AzureBatchAccountName = azureBatchAccountName,
                 AzureBatchAccountUri = azureBatchAccountUri,
                 AzureBatchPoolId = azureBatchPoolId,
-                AzureStorageAccountKey = azureStorageAccountKey,
                 AzureStorageAccountName = azureStorageAccountName,
                 AzureStorageContainerName = azureStorageContainerName,
-                AzureBatchIsWindows = true
             };
         }
 
diff --git a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
index e2a30f39b3..d3bda9f256 100644
--- a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
+++ b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
@@ -73,14 +73,11 @@
       "doc": "Cross-language submission parameters to the Azure Batch runtime",
       "fields": [
         { "name": "sharedJobSubmissionParameters", "type": "AvroJobSubmissionParameters"
},
-        { "name": "AzureBatchAccountKey", "type": "string" },
         { "name": "AzureBatchAccountName", "type": "string" },
         { "name": "AzureBatchAccountUri", "type": "string" },
         { "name": "AzureBatchPoolId", "type": "string" },
-        { "name": "AzureStorageAccountKey", "type": "string" },
         { "name": "AzureStorageAccountName", "type": "string" },
-        { "name": "AzureStorageContainerName", "type": "string" },
-        { "name": "AzureBatchIsWindows", "type": "boolean" }
+        { "name": "AzureStorageContainerName", "type": "string" }
       ]
   }
 ]
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
index c054722a81..5f09abd02a 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
@@ -18,9 +18,6 @@
  */
 package org.apache.reef.bridge.client;
 
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.JsonDecoder;
-import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters;
 import org.apache.reef.runtime.common.client.DriverConfigurationProvider;
 import org.apache.reef.runtime.common.driver.parameters.ClientRemoteIdentifier;
@@ -28,9 +25,6 @@
 
 import javax.inject.Inject;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -49,15 +43,10 @@ private AzureBatchBootstrapDriverConfigGenerator(final DriverConfigurationProvid
     this.driverConfigurationProvider = driverConfigurationProvider;
   }
 
-  Configuration getDriverConfigurationFromParams(final String bootstrapJobArgsLocation) throws
IOException {
-
-    final File bootstrapJobArgsFile = new File(bootstrapJobArgsLocation).getCanonicalFile();
-
-    final AvroAzureBatchJobSubmissionParameters azureBatchBootstrapJobArgs =
-        readAzureBatchJobSubmissionParametersFromFile(bootstrapJobArgsFile);
-
-    final String jobId = azureBatchBootstrapJobArgs.getSharedJobSubmissionParameters().getJobId().toString();
-    final File jobFolder = new File(azureBatchBootstrapJobArgs
+  Configuration getDriverConfigurationFromParams(
+      final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters)
{
+    final String jobId = avroAzureBatchJobSubmissionParameters.getSharedJobSubmissionParameters().getJobId().toString();
+    final File jobFolder = new File(avroAzureBatchJobSubmissionParameters
         .getSharedJobSubmissionParameters().getJobSubmissionFolder().toString());
 
     LOG.log(Level.INFO, "jobFolder {0} jobId {1}.", new Object[]{jobFolder.toURI(), jobId});
@@ -66,20 +55,4 @@ Configuration getDriverConfigurationFromParams(final String bootstrapJobArgsLoca
         jobFolder.toURI(), ClientRemoteIdentifier.NONE, jobId,
         Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER);
   }
-
-  private AvroAzureBatchJobSubmissionParameters readAzureBatchJobSubmissionParametersFromFile(final
File file)
-      throws IOException {
-    try (final FileInputStream fileInputStream = new FileInputStream(file)) {
-      return readAzureBatchSubmissionParametersFromInputStream(fileInputStream);
-    }
-  }
-
-  private static AvroAzureBatchJobSubmissionParameters readAzureBatchSubmissionParametersFromInputStream(
-      final InputStream inputStream) throws IOException {
-    final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
-        AvroAzureBatchJobSubmissionParameters.getClassSchema(), inputStream);
-    final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = new SpecificDatumReader<>(
-        AvroAzureBatchJobSubmissionParameters.class);
-    return reader.read(null, decoder);
-  }
 }
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
index 35a1e5942d..4f2a3f66f3 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
@@ -23,10 +23,21 @@
 import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.reef.annotations.audience.Interop;
 import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters;
-import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration;
-import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationCreator;
+import org.apache.reef.runtime.azbatch.AzureBatchClasspathProvider;
+import org.apache.reef.runtime.azbatch.AzureBatchJVMPathProvider;
+import org.apache.reef.runtime.azbatch.client.AzureBatchDriverConfigurationProviderImpl;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountName;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri;
+import org.apache.reef.runtime.azbatch.parameters.AzureBatchPoolId;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName;
+import org.apache.reef.runtime.azbatch.parameters.AzureStorageContainerName;
+import org.apache.reef.runtime.azbatch.util.command.CommandBuilder;
+import org.apache.reef.runtime.azbatch.util.command.WindowsCommandBuilder;
 import org.apache.reef.runtime.common.REEFEnvironment;
+import org.apache.reef.runtime.common.client.DriverConfigurationProvider;
 import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler;
+import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
+import org.apache.reef.runtime.common.files.RuntimePathProvider;
 import org.apache.reef.runtime.common.launch.REEFErrorHandler;
 import org.apache.reef.runtime.common.launch.REEFMessageCodec;
 import org.apache.reef.tang.Configuration;
@@ -73,9 +84,10 @@ public static void main(final String[] args) throws IOException, InjectionExcept
       throw fatal(message, new IllegalArgumentException(message));
     }
 
-    final File partialConfigFile = new File(args[0]);
+    final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters =
+        readAvroJobSubmissionParameters(new File(args[0]));
     final AzureBatchBootstrapDriverConfigGenerator azureBatchBootstrapDriverConfigGenerator
=
-        TANG.newInjector(generateConfigurationFromJobSubmissionParameters(partialConfigFile))
+        TANG.newInjector(generateConfiguration(avroAzureBatchJobSubmissionParameters))
             .getInstance(AzureBatchBootstrapDriverConfigGenerator.class);
 
     final Configuration launcherConfig =
@@ -87,7 +99,8 @@ public static void main(final String[] args) throws IOException, InjectionExcept
             .build();
 
     try (final REEFEnvironment reef = REEFEnvironment.fromConfiguration(
-        azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams(args[0]),
launcherConfig)) {
+        azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams(
+            avroAzureBatchJobSubmissionParameters), launcherConfig)) {
       reef.run();
     } catch (final InjectionException ex) {
       throw fatal("Unable to configure and start REEFEnvironment.", ex);
@@ -98,33 +111,35 @@ public static void main(final String[] args) throws IOException, InjectionExcept
     System.exit(0); // TODO[REEF-1715]: Should be able to exit cleanly at the end of main()
   }
 
-  private static Configuration generateConfigurationFromJobSubmissionParameters(final File
params) throws IOException {
-
+  private static AvroAzureBatchJobSubmissionParameters readAvroJobSubmissionParameters(
+      final File paramsFile) throws IOException {
     final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters;
-
-    try (final FileInputStream fileInputStream = new FileInputStream(params)) {
+    try (final FileInputStream fileInputStream = new FileInputStream(paramsFile)) {
       final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
           AvroAzureBatchJobSubmissionParameters.getClassSchema(), fileInputStream);
       final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader =
           new SpecificDatumReader<>(AvroAzureBatchJobSubmissionParameters.class);
       avroAzureBatchJobSubmissionParameters = reader.read(null, decoder);
     }
+    return avroAzureBatchJobSubmissionParameters;
+  }
 
-    return AzureBatchRuntimeConfigurationCreator
-        .getOrCreateAzureBatchRuntimeConfiguration(avroAzureBatchJobSubmissionParameters.getAzureBatchIsWindows())
-        .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_NAME,
+  private static Configuration generateConfiguration(
+      final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters)
{
+    return TANG.newConfigurationBuilder()
+        .bindImplementation(DriverConfigurationProvider.class, AzureBatchDriverConfigurationProviderImpl.class)
+        .bindImplementation(RuntimeClasspathProvider.class, AzureBatchClasspathProvider.class)
+        .bindImplementation(RuntimePathProvider.class, AzureBatchJVMPathProvider.class)
+        .bindImplementation(CommandBuilder.class, WindowsCommandBuilder.class)
+        .bindNamedParameter(AzureBatchAccountName.class,
             avroAzureBatchJobSubmissionParameters.getAzureBatchAccountName().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_KEY,
-            avroAzureBatchJobSubmissionParameters.getAzureBatchAccountKey().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_URI,
+        .bindNamedParameter(AzureBatchAccountUri.class,
             avroAzureBatchJobSubmissionParameters.getAzureBatchAccountUri().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_POOL_ID,
+        .bindNamedParameter(AzureBatchPoolId.class,
             avroAzureBatchJobSubmissionParameters.getAzureBatchPoolId().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_NAME,
+        .bindNamedParameter(AzureStorageAccountName.class,
             avroAzureBatchJobSubmissionParameters.getAzureStorageAccountName().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_KEY,
-            avroAzureBatchJobSubmissionParameters.getAzureStorageAccountKey().toString())
-        .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_CONTAINER_NAME,
+        .bindNamedParameter(AzureStorageContainerName.class,
             avroAzureBatchJobSubmissionParameters.getAzureStorageContainerName().toString())
         .build();
   }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Azure Batch credentials exposed in job-submission-params.json in REEF.NET
> -------------------------------------------------------------------------
>
>                 Key: REEF-2031
>                 URL: https://issues.apache.org/jira/browse/REEF-2031
>             Project: REEF
>          Issue Type: Improvement
>          Components: REEF.NET Driver
>            Reporter: Chenxi Zhao
>            Assignee: Chenxi Zhao
>            Priority: Major
>
> When running REEF .NET to submit a job to AzureBatch, it generates job-submission-params.json
file which carries configuration data from .NET client to Java bootstrap launcher.
>  
> Azure Batch REEF .Java is using "AZ_BATCH_AUTH_TOKEN_ENV" for Azure Batch credential
and shared access token for BlobStorage.
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message