ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject ambari git commit: AMBARI-22082: Create missing parent directory when expanding mpack tarball (jluniya)
Date Thu, 28 Sep 2017 17:50:19 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-14714 530df165d -> 4a278c1ce


AMBARI-22082: Create missing parent directory when expanding mpack tarball (jluniya)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a278c1c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a278c1c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a278c1c

Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 4a278c1ce49043e80a081ec9dcc3d8c4dd50fb02
Parents: 530df16
Author: Jayush Luniya <jluniya@hortonworks.com>
Authored: Thu Sep 28 10:50:13 2017 -0700
Committer: Jayush Luniya <jluniya@hortonworks.com>
Committed: Thu Sep 28 10:50:13 2017 -0700

----------------------------------------------------------------------
 .../internal/MpackResourceProvider.java         |  46 ++++-----
 .../ambari/server/mpack/MpackManager.java       | 102 ++++++++++++-------
 2 files changed, 90 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4a278c1c/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
index 94b5fdf..2642d10 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
@@ -57,12 +57,10 @@ import org.apache.ambari.server.registry.RegistryMpack;
 import org.apache.ambari.server.registry.RegistryMpackVersion;
 import org.apache.ambari.server.state.Packlet;
 import org.apache.ambari.server.state.StackId;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.Validate;
 
 import com.google.inject.Inject;
 
-
 /**
  * ResourceProvider for Mpack instances
  */
@@ -78,10 +76,11 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
   public static final String MPACK_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP
+ "mpack_uri";
   public static final String PACKLETS = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP +
"packlets";
   public static final String STACK_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP
+ "stack_name";
-  public static final String STACK_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP
+ "stack_version";
+  public static final String STACK_VERSION_PROPERTY_ID =
+    RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "stack_version";
 
   private static Set<String> pkPropertyIds = new HashSet<>(
-          Arrays.asList(MPACK_ID, STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID));
+    Arrays.asList(MPACK_ID, STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID));
 
   /**
    * The property ids for an mpack resource.
@@ -129,8 +128,6 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
     return pkPropertyIds;
   }
 
-
-
   @Override
   public RequestStatus createResourcesAuthorized(final Request request)
     throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException,
@@ -169,6 +166,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
 
   /***
    * Validates the request body for the required properties in order to create an Mpack resource.
+   *
    * @param mpackRequest
    */
   private void validateCreateRequest(MpackRequest mpackRequest) {
@@ -177,24 +175,24 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
     final Long registryId = mpackRequest.getRegistryId();
     final String mpackVersion = mpackRequest.getMpackVersion();
 
-    if(registryId == null) {
+    if (registryId == null) {
       Validate.isTrue(mpackUrl != null);
       LOG.info("Received a createMpack request"
-              + ", mpackUrl=" + mpackUrl);
+        + ", mpackUrl=" + mpackUrl);
     } else {
       Validate.notNull(mpackName, "MpackName should not be null");
       Validate.notNull(mpackVersion, "MpackVersion should not be null");
       LOG.info("Received a createMpack request"
-              + ", mpackName=" + mpackName
-              + ", mpackVersion=" + mpackVersion
-              + ", registryId=" + registryId);
+        + ", mpackName=" + mpackName
+        + ", mpackVersion=" + mpackVersion
+        + ", registryId=" + registryId);
     }
     try {
       URI uri = new URI(mpackUrl);
       URL url = uri.toURL();
-      String jsonString = IOUtils.toString(url);
-    }catch(Exception e){
-      Validate.isTrue(e == null, e.getMessage() + " is an invalid mpack uri. Please check
the download link for the mpack again.");
+    } catch (Exception e) {
+      Validate.isTrue(e == null,
+        e.getMessage() + " is an invalid mpack uri. Please check the download link for the
mpack again.");
     }
   }
 
@@ -207,7 +205,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
         return null;
         //Fetch Mpack Download Url using the given registry id
       else if (!propertyMap.containsKey(MPACK_URI)) {
-        mpackRequest.setRegistryId(Long.valueOf ((String) propertyMap.get(REGISTRY_ID)));
+        mpackRequest.setRegistryId(Long.valueOf((String) propertyMap.get(REGISTRY_ID)));
         mpackRequest.setMpackName((String) propertyMap.get(MPACK_NAME));
         mpackRequest.setMpackVersion((String) propertyMap.get(MPACK_VERSION));
         mpackRequest.setMpackUri(getMpackUri(mpackRequest));
@@ -222,6 +220,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
 
   /***
    * Uses the Registries functions to get the mpack uri.
+   *
    * @param mpackRequest
    * @return
    * @throws AmbariException
@@ -235,8 +234,8 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
 
   @Override
   public Set<Resource> getResources(Request request, Predicate predicate)
-          throws SystemException, UnsupportedPropertyException,
-          NoSuchResourceException, NoSuchParentResourceException {
+    throws SystemException, UnsupportedPropertyException,
+    NoSuchResourceException, NoSuchParentResourceException {
 
     Set<Resource> results = new LinkedHashSet<>();
     Long mpackId = null;
@@ -299,7 +298,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
       }
       if (results.isEmpty()) {
         throw new NoSuchResourceException(
-                "The requested resource doesn't exist: " + predicate);
+          "The requested resource doesn't exist: " + predicate);
       }
     }
     return results;
@@ -307,15 +306,16 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
 
   @Override
   protected RequestStatus deleteResourcesAuthorized(final Request request, Predicate predicate)
-          throws SystemException, UnsupportedPropertyException, NoSuchResourceException,
NoSuchParentResourceException {
+    throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException
{
 
     final Long mpackId;
     Map<String, Object> propertyMap = new HashMap<>(PredicateHelper.getProperties(predicate));
     DeleteStatusMetaData deleteStatusMetaData = null;
 
-    //Allow deleting mpack only if there are no cluster services deploying using this mpack.
Support deleting mpacks only if no cluster has been deployed
+    // Allow deleting mpack only if there are no cluster services deploying using this mpack.
+    // Support deleting mpacks only if no cluster has been deployed
     // (i.e. you should be able to delete an mpack during install wizard only).
-    //Todo : Relax the rule
+    // TODO : Relax the rule
     if (getManagementController().getClusters().getClusters().size() > 0) {
       throw new SystemException("Delete request cannot be completed since there is a cluster
deployed");
     } else {
@@ -335,7 +335,8 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
                 @Override
                 public DeleteStatusMetaData invoke() throws AmbariException {
                   if (stackEntity != null) {
-                    repositoryVersionDAO.removeByStack(new StackId(stackEntity.getStackName()
+ "-" + stackEntity.getStackVersion()));
+                    repositoryVersionDAO
+                      .removeByStack(new StackId(stackEntity.getStackName() + "-" + stackEntity.getStackVersion()));
                     stackDAO.removeByMpack(mpackId);
                     notifyDelete(Resource.Type.Stack, predicate);
                   }
@@ -360,6 +361,5 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider
{
       return getRequestStatus(null, null, deleteStatusMetaData);
     }
   }
-
 }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a278c1c/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
index 86c12ce..0fcc285 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
@@ -70,7 +70,11 @@ public class MpackManager {
   private File stackRoot;
 
   @AssistedInject
-  public MpackManager(@Assisted("mpacksv2Staging") File mpacksStagingLocation, @Assisted("stackRoot")
File stackRootDir, MpackDAO mpackDAOObj, StackDAO stackDAOObj) {
+  public MpackManager(
+    @Assisted("mpacksv2Staging") File mpacksStagingLocation,
+    @Assisted("stackRoot") File stackRootDir,
+    MpackDAO mpackDAOObj,
+    StackDAO stackDAOObj) {
     mpacksStaging = mpacksStagingLocation;
     mpackDAO = mpackDAOObj;
     stackRoot = stackRootDir;
@@ -101,8 +105,9 @@ public class MpackManager {
                 if (resultSet.size() > 0) {
                   MpackEntity mpackEntity = (MpackEntity) resultSet.get(0);
 
-                  //Read the mpack.json file into Mpack Object for further use.
-                  String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file
+ "/" + MPACK_METADATA))), "UTF-8");
+                  // Read the mpack.json file into Mpack Object for further use.
+                  String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file
+ "/" + MPACK_METADATA))),
+                    "UTF-8");
                   Gson gson = new Gson();
                   Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class);
                   mpackMap.put(mpackEntity.getMpackId(), existingMpack);
@@ -117,7 +122,6 @@ public class MpackManager {
     }
   }
 
-
   /**
    * Parses mpack.json to fetch mpack and associated packlet information and
    * stores the mpack to the database and mpackMap
@@ -128,7 +132,8 @@ public class MpackManager {
    * @throws IllegalArgumentException
    * @throws ResourceAlreadyExistsException
    */
-  public MpackResponse registerMpack(MpackRequest mpackRequest) throws IOException, IllegalArgumentException,
ResourceAlreadyExistsException {
+  public MpackResponse registerMpack(MpackRequest mpackRequest)
+    throws IOException, IllegalArgumentException, ResourceAlreadyExistsException {
 
     Long mpackId;
     String mpackName = "";
@@ -151,7 +156,9 @@ public class MpackManager {
         if (isValidMetadata) {
           mpackDirectory = mpacksStaging + File.separator + mpack.getName() + File.separator
+ mpack.getVersion();
         } else {
-          String message = "Incorrect information : Mismatch in - (" + mpackName + "," +
mpack.getName() + ") or (" + mpackVersion + "," + mpack.getVersion() + ")";
+          String message =
+            "Incorrect information : Mismatch in - (" + mpackName + "," + mpack.getName()
+ ") or (" + mpackVersion
+              + "," + mpack.getVersion() + ")";
           throw new IllegalArgumentException(message); //Mismatch in information
         }
 
@@ -173,45 +180,54 @@ public class MpackManager {
       populateStackDB(mpack);
       return new MpackResponse(mpack);
     } else {
-      String message = "Mpack :" + mpackRequest.getMpackName() + " version: " + mpackRequest.getMpackVersion()
+ " already exists in server";
+      String message = "Mpack :" + mpackRequest.getMpackName() + " version: " + mpackRequest.getMpackVersion()
+        + " already exists in server";
       throw new ResourceAlreadyExistsException(message);
     }
   }
 
   /***
    * A generic method to extract tar files.
+   *
    * @param tarPath
    * @throws IOException
    */
-  private void extractTar(Path tarPath, File tempOutputDirectory) throws IOException {
-    TarArchiveInputStream tarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new
BufferedInputStream(new FileInputStream(new File(String.valueOf(tarPath))))));
+  private void extractTar(Path tarPath, File untarDirectory) throws IOException {
+    TarArchiveInputStream tarFile = new TarArchiveInputStream(
+      new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(tarPath))))));
     TarArchiveEntry entry = null;
     File outputFile = null;
 
     //Create a loop to read every single entry in TAR file
     while ((entry = tarFile.getNextTarEntry()) != null) {
-      outputFile = new File(tempOutputDirectory, entry.getName());
+      outputFile = new File(untarDirectory, entry.getName());
       if (entry.isDirectory()) {
-        LOG.debug("Attempting to write output directory" + outputFile.getAbsolutePath());
         if (!outputFile.exists()) {
-          LOG.debug("Attempting to create output directory " + outputFile.getAbsolutePath());
+          LOG.debug("Creating output directory " + outputFile.getAbsolutePath());
           if (!outputFile.mkdirs()) {
-            throw new IllegalStateException(String.format("Couldn't create directory %s.",
outputFile.getAbsolutePath()));
+            throw new IllegalStateException("Couldn't create directory " + outputFile.getAbsolutePath());
           }
         }
       } else {
-        LOG.debug("Creating output file %s." + outputFile.getAbsolutePath());
+        File parentDir = outputFile.getParentFile();
+        if (!parentDir.exists()) {
+          LOG.debug("Attempting to create output directory " + parentDir.getAbsolutePath());
+          if (!parentDir.mkdirs()) {
+            throw new IllegalStateException("Couldn't create directory " + parentDir.getAbsolutePath());
+          }
+        }
+        LOG.debug("Creating output file " + outputFile.getAbsolutePath());
         final OutputStream outputFileStream = new FileOutputStream(outputFile);
         IOUtils.copy(tarFile, outputFileStream);
         outputFileStream.close();
       }
     }
-
     tarFile.close();
   }
 
   /**
-   * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/
directory
+   * Mpack is downloaded as a tar.gz file.
+   * It is extracted into /var/lib/ambari-server/resources/mpack-v2/{mpack-name}/{mpack-version}
directory
    *
    * @param mpack          Mpack to process
    * @param mpackTarPath   Path to mpack tarball
@@ -224,8 +240,9 @@ public class MpackManager {
 
     String mpackTarDirectory = mpackTarPath.toString();
     Path extractedMpackDirectory = Files.move
-            (Paths.get(mpacksStaging + File.separator + mpackTarDirectory.substring(mpackTarDirectory.lastIndexOf('/')
+ 1, mpackTarDirectory.indexOf(".tar")) + File.separator),
-                    Paths.get(mpackDirectory), StandardCopyOption.REPLACE_EXISTING);
+      (Paths.get(mpacksStaging + File.separator + mpackTarDirectory
+          .substring(mpackTarDirectory.lastIndexOf('/') + 1, mpackTarDirectory.indexOf(".tar"))
+ File.separator),
+        Paths.get(mpackDirectory), StandardCopyOption.REPLACE_EXISTING);
 
     createServicesDirectory(extractedMpackDirectory, mpack);
     createSymLinks(mpack);
@@ -233,6 +250,7 @@ public class MpackManager {
 
   /***
    * Create a services directory and extract all the services tar file inside it. This readies
it for cluster deployment
+   *
    * @param extractedMpackDirectory
    * @param mpack
    * @throws IOException
@@ -248,11 +266,13 @@ public class MpackManager {
       if (packlet.getType() == Packlet.PackletType.SERVICE_PACKLET) {
         String packletSourceLocation = packlet.getSourceLocation();
         File serviceTargetDir = new File(servicesDir + File.separator + packlet.getName());
-        extractTar(Paths.get(extractedMpackDirectory + File.separator + packlet.getSourceLocation())
, servicesDir );
-        Path extractedServiceDirectory = Files.move(Paths.get(servicesDir + File.separator
+ packletSourceLocation.substring(packletSourceLocation.indexOf("/")+1,packletSourceLocation.indexOf(".tar.gz"))),serviceTargetDir.toPath(),
StandardCopyOption.REPLACE_EXISTING);
-        }
+        extractTar(Paths.get(extractedMpackDirectory + File.separator + packlet.getSourceLocation()),
servicesDir);
+        Path extractedServiceDirectory = Files.move(Paths.get(servicesDir + File.separator
+ packletSourceLocation
+            .substring(packletSourceLocation.indexOf("/") + 1, packletSourceLocation.indexOf(".tar.gz"))),
+          serviceTargetDir.toPath(), StandardCopyOption.REPLACE_EXISTING);
       }
     }
+  }
 
   /**
    * Reads the mpack.json file within the {mpack-name}.tar.gz file and populates Mpack object.
@@ -263,9 +283,11 @@ public class MpackManager {
    * @return boolean
    * @throws IOException
    */
-  private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath) throws IOException,
ResourceAlreadyExistsException {
+  private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath)
+    throws IOException, ResourceAlreadyExistsException {
 
-    TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new
BufferedInputStream(new FileInputStream(new File(mpackTarPath.toString())))));
+    TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(
+      new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(mpackTarPath.toString())))));
     TarArchiveEntry entry = null;
     String individualFiles;
     int offset;
@@ -302,7 +324,8 @@ public class MpackManager {
           }
           return true;
         } else {
-          String message = "Mpack: " + mpack.getName() + " version: " + mpack.getVersion()
+ " already exists in server";
+          String message =
+            "Mpack: " + mpack.getName() + " version: " + mpack.getVersion() + " already exists
in server";
           throw new ResourceAlreadyExistsException(message);
         }
       }
@@ -337,9 +360,9 @@ public class MpackManager {
     Files.createSymbolicLink(stackPath, mpackPath);
   }
 
-
   /***
    * Download the mpack from the given uri
+   *
    * @param mpackURI
    * @return
    */
@@ -358,7 +381,6 @@ public class MpackManager {
     return targetPath;
   }
 
-
   /**
    * Compares if the user's mpack information matches the downloaded mpack information.
    *
@@ -368,14 +390,19 @@ public class MpackManager {
    * @param actualMpackVersion
    * @return boolean
    */
-  protected boolean validateMpackInfo(String expectedMpackName, String expectedMpackVersion,
String actualMpackName, String actualMpackVersion) {
-
-    String strippedActualMpackVersion = actualMpackVersion.substring(0,actualMpackVersion.lastIndexOf('.'));
-    if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion.equalsIgnoreCase(strippedActualMpackVersion))
{
+  protected boolean validateMpackInfo(
+    String expectedMpackName,
+    String expectedMpackVersion,
+    String actualMpackName,
+    String actualMpackVersion) {
+
+    String strippedActualMpackVersion = actualMpackVersion.substring(0, actualMpackVersion.lastIndexOf('.'));
+    if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion
+      .equalsIgnoreCase(strippedActualMpackVersion)) {
       return true;
-    }
-    else {
-      LOG.info("Incorrect information : Mismatch in - (" + expectedMpackName + "," + actualMpackName
+ ") or (" + expectedMpackVersion + "," + actualMpackVersion + ")");
+    } else {
+      LOG.info("Incorrect information : Mismatch in - (" + expectedMpackName + "," + actualMpackName
+ ") or ("
+        + expectedMpackVersion + "," + actualMpackVersion + ")");
       return false;
     }
   }
@@ -409,7 +436,9 @@ public class MpackManager {
   }
 
   /***
-   * Makes an entry or updates the entry in the stack table to establish a link between the
mpack and the associated stack
+   * Makes an entry or updates the entry in the stack table to establish a link between the
mpack and the
+   * associated stack
+   *
    * @param mpack
    * @throws IOException
    */
@@ -436,6 +465,7 @@ public class MpackManager {
       stackDAO.merge(stackEntity);
     }
   }
+
   /**
    * Fetches the packlet info stored in the memory for mpacks/{mpack_id} call.
    *
@@ -453,13 +483,15 @@ public class MpackManager {
 
   /***
    * Remove the mpack and stack directories when a request comes in to delete a particular
mpack.
+   *
    * @param mpackEntity
    * @throws IOException
    */
   public boolean removeMpack(MpackEntity mpackEntity, StackEntity stackEntity) throws IOException
{
 
     boolean stackDelete = false;
-    File mpackDirToDelete = new File(mpacksStaging + File.separator + mpackEntity.getMpackName()
+ File.separator + mpackEntity.getMpackVersion());
+    File mpackDirToDelete = new File(
+      mpacksStaging + File.separator + mpackEntity.getMpackName() + File.separator + mpackEntity.getMpackVersion());
     File mpackDirectory = new File(mpacksStaging + "/" + mpackEntity.getMpackName());
     String mpackName = mpackEntity.getMpackName() + "-" + mpackEntity.getMpackVersion() +
".tar.gz";
     Path mpackTarFile = Paths.get(mpacksStaging + File.separator + MPACK_TAR_LOCATION + File.separator
+ mpackName);


Mime
View raw message