zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject zeppelin git commit: [ZEPPELIN-2088] Bundle helium packages one by one
Date Sat, 08 Apr 2017 22:48:36 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 495be1ede -> 241fd0344


[ZEPPELIN-2088] Bundle helium packages one by one

### What is this PR for?

Bundle helium packages one by one because

#### Summary

Let's say you have one helium package called `ultimate-line-chart`.

**1. Package Path**

- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/node_modules/ultimate-line-chart`
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/bundles/ultimate-line-chart`

**2. Zeppelin Local Module Path**

- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/node_modules/zeppelin-tabledata`
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/local_modules/zeppelin-tabledata`

**3. Bundle Cache Path**

- **(before)** `$ZEPPELIN_HOME/local-repo/helium-bundles/helium.bundle.cache.js` (mixed one)
- **(after)** `$ZEPPELIN_HOME/local-repo/helium-bundles/bundles/ultimate-line-chart/helium.bundle.cache.js` (for each helium package)

#### Details

- Bundling them in one file is not good idea because the whole bundling is broken when only 1 bundle fails
- Some node packages might not installed correctly. For example, `amchart/amcharts3` will be installed in `helium-bundles/node_modules/my-helium-vis/node_modules` but `amchart-export` will be placed in `helium-bundles/node_modules` so `my-helium-vis` can't import `amchart-export`. (This is real case)
- Additionally, I used yarn to install required node_modules. separated bundling requires install same dependencies multiple times. This takes more time than before and npm is too slow to install them all.
- Install Zeppelin framework modules using [local path](https://docs.npmjs.com/files/package.json#local-paths). It's more safe and correct way to install local modules.

### What type of PR is it?
[Improvement]

### Todos

NONE

### What is the Jira issue?

[ZEPPELIN-2088](https://issues.apache.org/jira/browse/ZEPPELIN-2088)

### How should this be tested?

1. Install any local helium package
2. Install any online helium package
3. Use them
4. Disable
5. Test `npm run dev:helium`

### Screenshots (if appropriate)

NONE

### Questions:
* Does the licenses files need update? - NO
* Is there breaking changes for older versions? - NO
* Does this needs documentation? - NO

Author: 1ambda <1amb4a@gmail.com>

Closes #2210 from 1ambda/ZEPPELIN-2088/evaluate-helium-bundle-one-by-one and squashes the following commits:

bea6c09 [1ambda] fix: Bundling errors
53145fa [1ambda] fix: Add error handling for failed bundle
212c737 [1ambda] feat: Apply #2178
736acee [1ambda] fix: Add nodeInstallationDir
11ef0ae [1ambda] fix: HeliumBundleFactoryTest
5a2a938 [1ambda] fix: Log single bundle error to browser console
a99f981 [1ambda] test: Fix APIs
743aba4 [1ambda] feat: Loading bundles
697c5e6 [1ambda] feat: enable, disable
002e66f [1ambda] feat: Build online packages
512508d [1ambda] feat: Build each bundle using yarn


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/241fd034
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/241fd034
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/241fd034

Branch: refs/heads/master
Commit: 241fd0344da3f6fbda7ba8d0976a85a51e3c33e9
Parents: 495be1e
Author: 1ambda <1amb4a@gmail.com>
Authored: Tue Apr 4 17:59:14 2017 +0900
Committer: Lee moon soo <moon@apache.org>
Committed: Sun Apr 9 07:48:31 2017 +0900

----------------------------------------------------------------------
 .../org/apache/zeppelin/rest/HeliumRestApi.java |  50 +-
 .../apache/zeppelin/server/ZeppelinServer.java  |   4 +-
 .../src/components/helium/helium.service.js     | 100 ++--
 zeppelin-zengine/pom.xml                        |   5 +
 .../java/org/apache/zeppelin/helium/Helium.java |  36 +-
 .../zeppelin/helium/HeliumBundleFactory.java    | 479 +++++++++++++------
 .../zeppelin/helium/HeliumLocalRegistry.java    |   1 -
 .../src/main/resources/helium/package.json      |   8 +-
 .../src/main/resources/helium/webpack.config.js |  55 +--
 .../helium/HeliumBundleFactoryTest.java         |  45 +-
 10 files changed, 513 insertions(+), 270 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
index 9234cc5..44b583c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java
@@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.helium.Helium;
 import org.apache.zeppelin.helium.HeliumPackage;
+import org.apache.zeppelin.helium.HeliumPackageSearchResult;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.notebook.Notebook;
 import org.apache.zeppelin.notebook.Paragraph;
@@ -69,6 +70,16 @@ public class HeliumRestApi {
   }
 
   /**
+   * Get all enabled package infos
+   */
+  @GET
+  @Path("enabledPackage")
+  public Response getAllEnabledPackageInfo() {
+    return new JsonResponse(
+            Response.Status.OK, "", helium.getAllEnabledPackages()).build();
+  }
+
+  /**
    * Get single package info
    */
   @GET
@@ -130,26 +141,43 @@ public class HeliumRestApi {
   }
 
   @GET
-  @Path("bundle/load")
+  @Path("bundle/load/{packageName}")
   @Produces("text/javascript")
-  public Response bundleLoad(@QueryParam("refresh") String refresh) {
+  public Response bundleLoad(@QueryParam("refresh") String refresh,
+                             @PathParam("packageName") String packageName) {
+    if (StringUtils.isEmpty(packageName)) {
+      return new JsonResponse(
+          Response.Status.BAD_REQUEST,
+          "Can't get bundle due to empty package name").build();
+    }
+
+    HeliumPackageSearchResult psr = null;
+    List<HeliumPackageSearchResult> enabledPackages = helium.getAllEnabledPackages();
+    for (HeliumPackageSearchResult e : enabledPackages) {
+      if (e.getPkg().getName().equals(packageName)) {
+        psr = e;
+        break;
+      }
+    }
+
+    if (psr == null) {
+      // return empty to specify
+      return Response.ok().build();
+    }
+
     try {
       File bundle;
-      if (refresh != null && refresh.equals("true")) {
-        bundle = helium.recreateBundle();
-      } else {
-        bundle = helium.getBundleFactory().getCurrentCacheBundle();
-      }
+      boolean rebuild = (refresh != null && refresh.equals("true"));
+      bundle = helium.getBundle(psr.getPkg(), rebuild);
 
       if (bundle == null) {
         return Response.ok().build();
       } else {
-        String stringifiedBundle = FileUtils.readFileToString(bundle);
-        return Response.ok(stringifiedBundle).build();
+        String stringified = FileUtils.readFileToString(bundle);
+        return Response.ok(stringified).build();
       }
     } catch (Exception e) {
       logger.error(e.getMessage(), e);
-
       // returning error will prevent zeppelin front-end render any notebook.
       // visualization load fail doesn't need to block notebook rendering work.
       // so it's better return ok instead of any error.
@@ -172,7 +200,7 @@ public class HeliumRestApi {
 
   @POST
   @Path("disable/{packageName}")
-  public Response enablePackage(@PathParam("packageName") String packageName) {
+  public Response disablePackage(@PathParam("packageName") String packageName) {
     try {
       helium.disable(packageName);
       return new JsonResponse(Response.Status.OK).build();

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index ce38e46..fe2823c 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -115,6 +115,7 @@ public class ZeppelinServer extends Application {
        */
       heliumBundleFactory = new HeliumBundleFactory(
           conf,
+          null,
           new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
           new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
           new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
@@ -122,6 +123,7 @@ public class ZeppelinServer extends Application {
     } else {
       heliumBundleFactory = new HeliumBundleFactory(
           conf,
+          null,
           new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)),
           new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata")),
           new File(conf.getRelativeDir("zeppelin-web/src/app/visualization")),
@@ -138,7 +140,7 @@ public class ZeppelinServer extends Application {
 
     // create bundle
     try {
-      heliumBundleFactory.buildBundle(helium.getBundlePackagesToBundle());
+      heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
     } catch (Exception e) {
       LOG.error(e.getMessage(), e);
     }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-web/src/components/helium/helium.service.js
----------------------------------------------------------------------
diff --git a/zeppelin-web/src/components/helium/helium.service.js b/zeppelin-web/src/components/helium/helium.service.js
index 8b3e6a3..34520e1 100644
--- a/zeppelin-web/src/components/helium/helium.service.js
+++ b/zeppelin-web/src/components/helium/helium.service.js
@@ -27,41 +27,14 @@ angular.module('zeppelinWebApp').service('heliumService', heliumService);
 export default function heliumService($http, $sce, baseUrlSrv) {
   'ngInject';
 
-  var url = baseUrlSrv.getRestApiBase() + '/helium/bundle/load';
-  if (process.env.HELIUM_BUNDLE_DEV) {
-    url = url + '?refresh=true';
-  }
-
   let visualizationBundles = [];
-  // name `heliumBundles` should be same as `HelumBundleFactory.HELIUM_BUNDLES_VAR`
+  // name `heliumBundles` should be same as `HeliumBundleFactory.HELIUM_BUNDLES_VAR`
   let heliumBundles = [];
   // map for `{ magic: interpreter }`
   let spellPerMagic = {};
   // map for `{ magic: package-name }`
   let pkgNamePerMagic = {}
 
-  // load should be promise
-  this.load = $http.get(url).success(function(response) {
-    if (response.substring(0, 'ERROR:'.length) !== 'ERROR:') {
-      // evaluate bundles
-      eval(response);
-
-      // extract bundles by type
-      heliumBundles.map(b => {
-        if (b.type === HeliumType.SPELL) {
-          const spell = new b.class(); // eslint-disable-line new-cap
-          const pkgName = b.id;
-          spellPerMagic[spell.getMagic()] = spell;
-          pkgNamePerMagic[spell.getMagic()] = pkgName;
-        } else if (b.type === HeliumType.VISUALIZATION) {
-          visualizationBundles.push(b);
-        }
-      });
-    } else {
-      console.error(response);
-    }
-  });
-
   /**
    * @param magic {string} e.g `%flowchart`
    * @returns {SpellBase} undefined if magic is not registered
@@ -160,6 +133,37 @@ export default function heliumService($http, $sce, baseUrlSrv) {
       });
   };
 
+  this.getAllEnabledPackages = function() {
+    return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/enabledPackage`)
+      .then(function(response, status) {
+        return response.data.body;
+      })
+      .catch(function(error) {
+        console.error('Failed to get all enabled package infos', error);
+      });
+  };
+
+  this.getSingleBundle = function(pkgName) {
+    let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}`
+    if (process.env.HELIUM_BUNDLE_DEV) {
+      url = url + '?refresh=true';
+    }
+
+    return $http.get(url)
+      .then(function(response, status) {
+        const bundle = response.data
+        if (bundle.substring(0, 'ERROR:'.length) === 'ERROR:') {
+          console.error(`Failed to get bundle: ${pkgName}`, bundle);
+          return '' // empty bundle will be filtered later
+        }
+
+        return bundle
+      })
+      .catch(function(error) {
+        console.error(`Failed to get single bundle: ${pkgName}`, error);
+      });
+  }
+
   this.getDefaultPackages = function() {
     return this.getAllPackageInfo()
       .then(pkgSearchResults => {
@@ -241,4 +245,44 @@ export default function heliumService($http, $sce, baseUrlSrv) {
       return merged;
     });
   }
+
+  const p = this.getAllEnabledPackages()
+    .then(enabledPackageSearchResults => {
+      const promises = enabledPackageSearchResults.map(packageSearchResult => {
+        const pkgName = packageSearchResult.pkg.name
+        return this.getSingleBundle(pkgName)
+      })
+
+      return Promise.all(promises)
+    })
+    .then(bundles => {
+      return bundles.reduce((acc, b) => {
+        // filter out empty bundle
+        if (b === '') { return acc }
+        acc.push(b)
+        return acc;
+      }, [])
+    })
+
+  // load should be promise
+  this.load = p.then(availableBundles => {
+
+    // evaluate bundles
+    availableBundles.map(b => {
+      eval(b)
+    })
+
+    // extract bundles by type
+    heliumBundles.map(b => {
+      if (b.type === HeliumType.SPELL) {
+        const spell = new b.class() // eslint-disable-line new-cap
+        const pkgName = b.id
+        spellPerMagic[spell.getMagic()] = spell
+        pkgNamePerMagic[spell.getMagic()] = pkgName
+      } else if (b.type === HeliumType.VISUALIZATION) {
+        visualizationBundles.push(b)
+      }
+    })
+  })
+
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index d5fef16..6589e3b 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -290,6 +290,11 @@
       <artifactId>mongo-java-driver</artifactId>
       <version>3.4.1</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.5</version>
+    </dependency>
 
   </dependencies>
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index b4d5a79..f5f6695 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -242,6 +242,22 @@ public class Helium {
     }
   }
 
+  public List<HeliumPackageSearchResult> getAllEnabledPackages() {
+    Map<String, List<HeliumPackageSearchResult>> allPackages = getAllPackageInfoWithoutRefresh();
+    List<HeliumPackageSearchResult> enabledPackages = new ArrayList<>();
+
+    for (List<HeliumPackageSearchResult> versionedPackages : allPackages.values()) {
+      for (HeliumPackageSearchResult psr : versionedPackages) {
+        if (psr.isEnabled()) {
+          enabledPackages.add(psr);
+          break;
+        }
+      }
+    }
+
+    return enabledPackages;
+  }
+
   public List<HeliumPackageSearchResult> getSinglePackageInfo(String packageName) {
     Map<String, List<HeliumPackageSearchResult>> result = getAllPackageInfo(false, packageName);
 
@@ -281,8 +297,8 @@ public class Helium {
     return null;
   }
 
-  public File recreateBundle() throws IOException {
-    return bundleFactory.buildBundle(getBundlePackagesToBundle(), true);
+  public File getBundle(HeliumPackage pkg, boolean rebuild) throws IOException {
+    return bundleFactory.buildPackage(pkg, rebuild, true);
   }
 
   public void enable(String name, String artifact) throws IOException {
@@ -293,14 +309,13 @@ public class Helium {
       return;
     }
 
-    // enable package
-    heliumConf.enablePackage(name, artifact);
-
     // if package is visualization, rebuild bundle
     if (HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
-      bundleFactory.buildBundle(getBundlePackagesToBundle());
+      bundleFactory.buildPackage(pkgInfo.getPkg(), true, true);
     }
 
+    // update conf and save
+    heliumConf.enablePackage(name, artifact);
     save();
   }
 
@@ -311,13 +326,8 @@ public class Helium {
       return;
     }
 
+    // update conf and save
     heliumConf.disablePackage(name);
-
-    HeliumPackageSearchResult pkgInfo = getPackageInfo(name, artifact);
-    if (pkgInfo == null || HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
-      bundleFactory.buildBundle(getBundlePackagesToBundle());
-    }
-
     save();
   }
 
@@ -445,7 +455,7 @@ public class Helium {
     heliumConf.setBundleDisplayOrder(orderedPackageList);
 
     // if package is visualization, rebuild buildBundle
-    bundleFactory.buildBundle(getBundlePackagesToBundle());
+    bundleFactory.buildAllPackages(getBundlePackagesToBundle());
 
     save();
   }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
index 2dbefe0..d0e9b00 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
@@ -17,10 +17,18 @@
 package org.apache.zeppelin.helium;
 
 import com.github.eirslett.maven.plugins.frontend.lib.*;
+
 import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Appender;
 import org.apache.log4j.PatternLayout;
 import org.apache.log4j.WriterAppender;
@@ -30,7 +38,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.net.URL;
 import java.util.*;
 
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -42,17 +49,25 @@ public class HeliumBundleFactory {
   Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
   private final String NODE_VERSION = "v6.9.1";
   private final String NPM_VERSION = "3.10.8";
+  private final String YARN_VERSION = "v0.21.3";
   public static final String HELIUM_LOCAL_REPO = "helium-bundle";
+  public static final String HELIUM_BUNDLES_DIR = "bundles";
+  public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
+  public static final String HELIUM_BUNDLES_SRC_DIR = "src";
+  public static final String HELIUM_BUNDLES_SRC = "load.js";
+  public static final String PACKAGE_JSON = "package.json";
   public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
   public static final String HELIUM_BUNDLE = "helium.bundle.js";
   public static final String HELIUM_BUNDLES_VAR = "heliumBundles";
   private final int FETCH_RETRY_COUNT = 2;
   private final int FETCH_RETRY_FACTOR_COUNT = 1;
-  // Milliseconds
-  private final int FETCH_RETRY_MIN_TIMEOUT = 5000;
+  private final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds
 
   private final FrontendPluginFactory frontEndPluginFactory;
-  private final File workingDirectory;
+  private final File nodeInstallationDirectory;
+  private final File heliumLocalRepoDirectory;
+  private final File heliumBundleDirectory;
+  private final File heliumLocalModuleDirectory;
   private ZeppelinConfiguration conf;
   private File tabledataModulePath;
   private File visualizationModulePath;
@@ -61,18 +76,16 @@ public class HeliumBundleFactory {
   private Gson gson;
   private boolean nodeAndNpmInstalled = false;
 
-  String bundleCacheKey = "";
-  File currentCacheBundle;
-
   ByteArrayOutputStream out  = new ByteArrayOutputStream();
 
   public HeliumBundleFactory(
       ZeppelinConfiguration conf,
+      File nodeInstallationDir,
       File moduleDownloadPath,
       File tabledataModulePath,
       File visualizationModulePath,
       File spellModulePath) throws TaskRunnerException {
-    this(conf, moduleDownloadPath);
+    this(conf, nodeInstallationDir, moduleDownloadPath);
     this.tabledataModulePath = tabledataModulePath;
     this.visualizationModulePath = visualizationModulePath;
     this.spellModulePath = spellModulePath;
@@ -80,16 +93,20 @@ public class HeliumBundleFactory {
 
   public HeliumBundleFactory(
       ZeppelinConfiguration conf,
+      File nodeInstallationDir,
       File moduleDownloadPath) throws TaskRunnerException {
-    this.workingDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+    this.heliumLocalRepoDirectory = new File(moduleDownloadPath, HELIUM_LOCAL_REPO);
+    this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
+    this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
     this.conf = conf;
     this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
-    File installDirectory = workingDirectory;
+
+    nodeInstallationDirectory = (nodeInstallationDir == null) ?
+        heliumLocalRepoDirectory : nodeInstallationDir;
 
     frontEndPluginFactory = new FrontendPluginFactory(
-        workingDirectory, installDirectory);
+            heliumLocalRepoDirectory, nodeInstallationDirectory);
 
-    currentCacheBundle = new File(workingDirectory, HELIUM_BUNDLE_CACHE);
     gson = new Gson();
   }
 
@@ -98,13 +115,18 @@ public class HeliumBundleFactory {
       return;
     }
     try {
+      NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
+      nodeInstaller.setNodeVersion(NODE_VERSION);
+      nodeInstaller.install();
+
       NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
       npmInstaller.setNpmVersion(NPM_VERSION);
       npmInstaller.install();
 
-      NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
-      nodeInstaller.setNodeVersion(NODE_VERSION);
-      nodeInstaller.install();
+      YarnInstaller yarnInstaller = frontEndPluginFactory.getYarnInstaller(getProxyConfig());
+      yarnInstaller.setYarnVersion(YARN_VERSION);
+      yarnInstaller.install();
+
       configureLogger();
       nodeAndNpmInstalled = true;
     } catch (InstallationException e) {
@@ -117,33 +139,60 @@ public class HeliumBundleFactory {
     return new ProxyConfig(proxy);
   }
 
-  public File buildBundle(List<HeliumPackage> pkgs) throws IOException {
-    return buildBundle(pkgs, false);
+  public void buildAllPackages(List<HeliumPackage> pkgs) throws IOException {
+    buildAllPackages(pkgs, false);
   }
 
-  public synchronized File buildBundle(List<HeliumPackage> pkgs, boolean forceRefresh)
-      throws IOException {
+  public File getHeliumPackageDirectory(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName);
+  }
 
-    if (pkgs == null || pkgs.size() == 0) {
-      // when no package is selected, simply return an empty file instead of try bundle package
-      synchronized (this) {
-        currentCacheBundle.getParentFile().mkdirs();
-        currentCacheBundle.delete();
-        currentCacheBundle.createNewFile();
-        bundleCacheKey = "";
-        return currentCacheBundle;
+  public File getHeliumPackageSourceDirectory(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLES_SRC_DIR);
+  }
+
+  public File getHeliumPackageBundleCache(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLE_CACHE);
+  }
+
+  public static List<String> unTgz(File tarFile, File directory) throws IOException {
+    List<String> result = new ArrayList<String>();
+    InputStream is = new FileInputStream(tarFile);
+    GzipCompressorInputStream gcis = new GzipCompressorInputStream(is);
+    TarArchiveInputStream in = new TarArchiveInputStream(gcis);
+    TarArchiveEntry entry = in.getNextTarEntry();
+    while (entry != null) {
+      if (entry.isDirectory()) {
+        entry = in.getNextTarEntry();
+        continue;
       }
+      File curfile = new File(directory, entry.getName());
+      File parent = curfile.getParentFile();
+      if (!parent.exists()) {
+        parent.mkdirs();
+      }
+      OutputStream out = new FileOutputStream(curfile);
+      IOUtils.copy(in, out);
+      out.close();
+      result.add(entry.getName());
+      entry = in.getNextTarEntry();
     }
+    in.close();
+    return result;
+  }
 
-    installNodeAndNpm();
-
-    // package.json
-    URL pkgUrl = Resources.getResource("helium/package.json");
-    String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8);
-    StringBuilder dependencies = new StringBuilder();
-    StringBuilder cacheKeyBuilder = new StringBuilder();
+  /**
+   * @return main file name of this helium package (relative path)
+   */
+  public String downloadPackage(HeliumPackage pkg, String[] nameAndVersion, File bundleDir,
+                                String templateWebpackConfig, String templatePackageJson,
+                                FrontendPluginFactory fpf) throws IOException, TaskRunnerException {
+    if (bundleDir.exists()) {
+      FileUtils.deleteQuietly(bundleDir);
+    }
+    FileUtils.forceMkdir(bundleDir);
 
-    FileFilter npmPackageCopyFilter = new FileFilter() {
+    FileFilter copyFilter = new FileFilter() {
       @Override
       public boolean accept(File pathname) {
         String fileName = pathname.getName();
@@ -155,73 +204,97 @@ public class HeliumBundleFactory {
       }
     };
 
-    for (HeliumPackage pkg : pkgs) {
-      String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
-      if (moduleNameVersion == null) {
-        logger.error("Can't get module name and version of package " + pkg.getName());
-        continue;
-      }
-      if (dependencies.length() > 0) {
-        dependencies.append(",\n");
-      }
-      dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\"");
-      cacheKeyBuilder.append(pkg.getName() + pkg.getArtifact());
+    if (isLocalPackage(pkg)) {
+      FileUtils.copyDirectory(
+              new File(pkg.getArtifact()),
+              bundleDir,
+              copyFilter);
+    } else {
+      // if online package
+      String version = nameAndVersion[1];
+      File tgz = new File(heliumLocalRepoDirectory, pkg.getName() + "-" + version + ".tgz");
+      tgz.delete();
+
+      // wget, extract and move dir to `bundles/${pkg.getName()}`, and remove tgz
+      npmCommand(fpf, "pack " + pkg.getArtifact());
+      File extracted = new File(heliumBundleDirectory, "package");
+      FileUtils.deleteDirectory(extracted);
+      unTgz(tgz, heliumBundleDirectory);
+      tgz.delete();
+      FileUtils.copyDirectory(extracted, bundleDir);
+      FileUtils.deleteDirectory(extracted);
+    }
+
+    // 1. setup package.json
+    File existingPackageJson = new File(bundleDir, "package.json");
+    JsonReader reader = new JsonReader(new FileReader(existingPackageJson));
+    Map<String, Object> packageJson = gson.fromJson(reader,
+            new TypeToken<Map<String, Object>>(){}.getType());
+    Map<String, String> existingDeps = (Map<String, String>) packageJson.get("dependencies");
+    String mainFileName = (String) packageJson.get("main");
 
-      File pkgInstallDir = new File(workingDirectory, "node_modules/" + pkg.getName());
-      if (pkgInstallDir.exists()) {
-        FileUtils.deleteDirectory(pkgInstallDir);
+    StringBuilder dependencies = new StringBuilder();
+    int index = 0;
+    for (Map.Entry<String, String> e: existingDeps.entrySet()) {
+      dependencies.append("    \"").append(e.getKey()).append("\": ");
+      if (e.getKey().equals("zeppelin-vis") ||
+          e.getKey().equals("zeppelin-tabledata") ||
+          e.getKey().equals("zeppelin-spell")) {
+        dependencies.append("\"file:../../" + HELIUM_LOCAL_MODULE_DIR + "/")
+                .append(e.getKey()).append("\"");
+      } else {
+        dependencies.append("\"").append(e.getValue()).append("\"");
       }
 
-      if (isLocalPackage(pkg)) {
-        FileUtils.copyDirectory(
-            new File(pkg.getArtifact()),
-            pkgInstallDir,
-            npmPackageCopyFilter);
+      if (index < existingDeps.size() - 1) {
+        dependencies.append(",\n");
       }
+      index = index + 1;
     }
-    pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString());
 
-    // check if we can use previous buildBundle or not
-    if (cacheKeyBuilder.toString().equals(bundleCacheKey) &&
-        currentCacheBundle.isFile() && !forceRefresh) {
-      return currentCacheBundle;
-    }
+    FileUtils.deleteQuietly(new File(bundleDir, PACKAGE_JSON));
+    templatePackageJson = templatePackageJson.replaceFirst("PACKAGE_NAME", pkg.getName());
+    templatePackageJson = templatePackageJson.replaceFirst("MAIN_FILE", mainFileName);
+    templatePackageJson = templatePackageJson.replaceFirst("DEPENDENCIES", dependencies.toString());
+    FileUtils.write(new File(bundleDir, PACKAGE_JSON), templatePackageJson);
 
-    // webpack.config.js
-    URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js");
-    String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8);
+    // 2. setup webpack.config
+    FileUtils.write(new File(bundleDir, "webpack.config.js"), templateWebpackConfig);
 
-    // generate load.js
+    return mainFileName;
+  }
+
+  public void prepareSource(HeliumPackage pkg, String[] moduleNameVersion,
+                            String mainFileName) throws IOException {
     StringBuilder loadJsImport = new StringBuilder();
     StringBuilder loadJsRegister = new StringBuilder();
+    String className = "bundles" + pkg.getName().replaceAll("[-_]", "");
 
-    long idx = 0;
-    for (HeliumPackage pkg : pkgs) {
-      String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
-      if (moduleNameVersion == null) {
-        continue;
-      }
-
-      String className = "bundles" + idx++;
-      loadJsImport.append(
-          "import " + className + " from \"" + moduleNameVersion[0] + "\"\n");
-
-      loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
-      loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
-      loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
-      loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
-      loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
-      loadJsRegister.append("class: " + className + "\n");
-      loadJsRegister.append("})\n");
+    // remove postfix `.js` for ES6 import
+    if (mainFileName.endsWith(".js")) {
+      mainFileName = mainFileName.substring(0, mainFileName.length() - 3);
     }
 
-    FileUtils.write(new File(workingDirectory, "package.json"), pkgJson);
-    FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig);
-    FileUtils.write(new File(workingDirectory, "load.js"),
-        loadJsImport.append(loadJsRegister).toString());
-
-    copyFrameworkModuleToInstallPath(npmPackageCopyFilter);
+    loadJsImport
+        .append("import ")
+        .append(className)
+        .append(" from \"../" + mainFileName + "\"\n");
+
+    loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
+    loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
+    loadJsRegister.append("name: \"" + pkg.getName() + "\",\n");
+    loadJsRegister.append("icon: " + gson.toJson(pkg.getIcon()) + ",\n");
+    loadJsRegister.append("type: \"" + pkg.getType() + "\",\n");
+    loadJsRegister.append("class: " + className + "\n");
+    loadJsRegister.append("})\n");
+
+    File srcDir = getHeliumPackageSourceDirectory(pkg.getName());
+    FileUtils.forceMkdir(srcDir);
+    FileUtils.write(new File(srcDir, HELIUM_BUNDLES_SRC),
+            loadJsImport.append(loadJsRegister).toString());
+  }
 
+  public synchronized void installNodeModules(FrontendPluginFactory fpf) throws IOException {
     try {
       out.reset();
       String commandForNpmInstall =
@@ -229,112 +302,213 @@ public class HeliumBundleFactory {
                               "--fetch-retry-mintimeout=%d",
                       FETCH_RETRY_COUNT, FETCH_RETRY_FACTOR_COUNT, FETCH_RETRY_MIN_TIMEOUT);
       logger.info("Installing required node modules");
-      npmCommand(commandForNpmInstall);
+      yarnCommand(fpf, commandForNpmInstall);
       logger.info("Installed required node modules");
     } catch (TaskRunnerException e) {
-      // ignore `(empty)` warning
-      String cause = new String(out.toByteArray());
-      if (!cause.contains("(empty)")) {
-        throw new IOException(cause);
-      }
+      throw new IOException(e);
     }
+  }
 
+  public synchronized File bundleHeliumPackage(FrontendPluginFactory fpf,
+                                               File bundleDir) throws IOException {
     try {
       out.reset();
       logger.info("Bundling helium packages");
-      npmCommand("run bundle");
+      yarnCommand(fpf, "run bundle");
       logger.info("Bundled helium packages");
     } catch (TaskRunnerException e) {
       throw new IOException(new String(out.toByteArray()));
     }
 
     String bundleStdoutResult = new String(out.toByteArray());
-
-    File heliumBundle = new File(workingDirectory, HELIUM_BUNDLE);
+    File heliumBundle = new File(bundleDir, HELIUM_BUNDLE);
     if (!heliumBundle.isFile()) {
       throw new IOException(
-          "Can't create bundle: \n" + bundleStdoutResult);
+              "Can't create bundle: \n" + bundleStdoutResult);
     }
 
     WebpackResult result = getWebpackResultFromOutput(bundleStdoutResult);
     if (result.errors.length > 0) {
-      heliumBundle.delete();
+      FileUtils.deleteQuietly(heliumBundle);
       throw new IOException(result.errors[0]);
     }
 
-    synchronized (this) {
-      currentCacheBundle.delete();
-      FileUtils.moveFile(heliumBundle, currentCacheBundle);
-      bundleCacheKey = cacheKeyBuilder.toString();
+    return heliumBundle;
+  }
+
+  public synchronized File buildPackage(HeliumPackage pkg,
+                                        boolean rebuild,
+                                        boolean recopyLocalModule) throws IOException {
+    if (pkg == null) {
+      return null;
+    }
+
+    String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
+    if (moduleNameVersion == null) {
+      return null;
+    }
+
+    if (moduleNameVersion == null) {
+      logger.error("Can't get module name and version of package " + pkg.getName());
+      return null;
+    }
+
+    String pkgName = pkg.getName();
+    File bundleDir = getHeliumPackageDirectory(pkgName);
+    File bundleCache = getHeliumPackageBundleCache(pkgName);
+
+    if (!rebuild && bundleCache.exists() && !bundleCache.isDirectory()) {
+      return bundleCache;
+    }
+
+    // 0. install node, npm (should be called before `downloadPackage`
+    installNodeAndNpm();
+
+    // 1. prepare directories
+    if (!heliumLocalRepoDirectory.exists() || !heliumLocalRepoDirectory.isDirectory()) {
+      FileUtils.deleteQuietly(heliumLocalRepoDirectory);
+      FileUtils.forceMkdir(heliumLocalRepoDirectory);
+    }
+    FrontendPluginFactory fpf = new FrontendPluginFactory(
+            bundleDir, nodeInstallationDirectory);
+
+    // resources: webpack.js, package.json
+    String templateWebpackConfig = Resources.toString(
+        Resources.getResource("helium/webpack.config.js"), Charsets.UTF_8);
+    String templatePackageJson = Resources.toString(
+        Resources.getResource("helium/" + PACKAGE_JSON), Charsets.UTF_8);
+
+    // 2. download helium package using `npm pack`
+    String mainFileName = null;
+    try {
+      mainFileName = downloadPackage(pkg, moduleNameVersion, bundleDir,
+              templateWebpackConfig, templatePackageJson, fpf);
+    } catch (TaskRunnerException e) {
+      throw new IOException(e);
+    }
+
+    // 3. prepare bundle source
+    prepareSource(pkg, moduleNameVersion, mainFileName);
+
+    // 4. install node and local modules for a bundle
+    copyFrameworkModuleToInstallPath(recopyLocalModule); // should copy local modules first
+    installNodeModules(fpf);
+
+    // 5. let's bundle and update cache
+    File heliumBundle = bundleHeliumPackage(fpf, bundleDir);
+    bundleCache.delete();
+    FileUtils.moveFile(heliumBundle, bundleCache);
+
+    return bundleCache;
+  }
+
+  public synchronized void buildAllPackages(List<HeliumPackage> pkgs, boolean rebuild)
+      throws IOException {
+
+    if (pkgs == null || pkgs.size() == 0) {
+      return;
+    }
+
+    // DON't recopy local modules when build all packages to avoid duplicated copies.
+    boolean recopyLocalModules = false;
+
+    for (HeliumPackage pkg : pkgs) {
+      try {
+        buildPackage(pkg, rebuild, recopyLocalModules);
+      } catch (IOException e) {
+        logger.error("Failed to build helium package: " + pkg.getArtifact(), e);
+      }
     }
-    return currentCacheBundle;
   }
 
-  private void copyFrameworkModuleToInstallPath(FileFilter npmPackageCopyFilter)
+  void copyFrameworkModuleToInstallPath(boolean recopy)
       throws IOException {
+
+    FileFilter npmPackageCopyFilter = new FileFilter() {
+      @Override
+      public boolean accept(File pathname) {
+        String fileName = pathname.getName();
+        if (fileName.startsWith(".") || fileName.startsWith("#") || fileName.startsWith("~")) {
+          return false;
+        } else {
+          return true;
+        }
+      }
+    };
+
     // install tabledata module
-    File tabledataModuleInstallPath = new File(workingDirectory,
-        "node_modules/zeppelin-tabledata");
+    FileUtils.forceMkdir(heliumLocalModuleDirectory);
+
+    File tabledataModuleInstallPath = new File(heliumLocalModuleDirectory,
+        "zeppelin-tabledata");
     if (tabledataModulePath != null) {
-      if (tabledataModuleInstallPath.exists()) {
+      if (recopy && tabledataModuleInstallPath.exists()) {
         FileUtils.deleteDirectory(tabledataModuleInstallPath);
+
+      }
+
+      if (!tabledataModuleInstallPath.exists()) {
+        FileUtils.copyDirectory(
+            tabledataModulePath,
+            tabledataModuleInstallPath,
+            npmPackageCopyFilter);
       }
-      FileUtils.copyDirectory(
-          tabledataModulePath,
-          tabledataModuleInstallPath,
-          npmPackageCopyFilter);
     }
 
     // install visualization module
-    File visModuleInstallPath = new File(workingDirectory,
-        "node_modules/zeppelin-vis");
+    File visModuleInstallPath = new File(heliumLocalModuleDirectory,
+        "zeppelin-vis");
     if (visualizationModulePath != null) {
-      if (visModuleInstallPath.exists()) {
-        // when zeppelin-vis and zeppelin-table package is published to npm repository
-        // we don't need to remove module because npm install command will take care
-        // dependency version change. However, when two dependencies are copied manually
-        // into node_modules directory, changing vis package version results inconsistent npm
-        // install behavior.
-        //
-        // Remote vis package everytime and let npm download every time bundle as a workaround
+      if (recopy && visModuleInstallPath.exists()) {
         FileUtils.deleteDirectory(visModuleInstallPath);
       }
-      FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath, npmPackageCopyFilter);
+
+      if (!visModuleInstallPath.exists()) {
+        FileUtils.copyDirectory(
+            visualizationModulePath,
+            visModuleInstallPath,
+            npmPackageCopyFilter);
+      }
     }
 
     // install spell module
-    File spellModuleInstallPath = new File(workingDirectory,
-        "node_modules/zeppelin-spell");
+    File spellModuleInstallPath = new File(heliumLocalModuleDirectory,
+        "zeppelin-spell");
     if (spellModulePath != null) {
-      if (spellModuleInstallPath.exists()) {
+      if (recopy && spellModuleInstallPath.exists()) {
         FileUtils.deleteDirectory(spellModuleInstallPath);
       }
 
-      FileUtils.copyDirectory(
-          spellModulePath,
-          spellModuleInstallPath,
-          npmPackageCopyFilter);
+      if (!spellModuleInstallPath.exists()) {
+        FileUtils.copyDirectory(
+            spellModulePath,
+            spellModuleInstallPath,
+            npmPackageCopyFilter);
+      }
     }
   }
 
   private WebpackResult getWebpackResultFromOutput(String output) {
     BufferedReader reader = new BufferedReader(new StringReader(output));
 
-    String line;
     boolean webpackRunDetected = false;
     boolean resultJsonDetected = false;
     StringBuffer sb = new StringBuffer();
     try {
-      while ((line = reader.readLine()) != null) {
+      String next, line = reader.readLine();
+      for (boolean last = (line == null); !last; line = next) {
+        last = ((next = reader.readLine()) == null);
+
         if (!webpackRunDetected) {
-          if (line.contains("webpack.js") && line.endsWith("--json")) {
+          String trimed = line.trim();
+          if (trimed.contains("webpack") && trimed.endsWith("--json")) {
             webpackRunDetected = true;
           }
           continue;
         }
 
         if (!resultJsonDetected) {
-          if (line.equals("{")) {
+          if (line.trim().equals("{")) {
             sb.append(line);
             resultJsonDetected = true;
           }
@@ -342,10 +516,12 @@ public class HeliumBundleFactory {
         }
 
         if (resultJsonDetected && webpackRunDetected) {
-          sb.append(line);
+          // yarn command always ends with `Done in ... seconds `
+          if (!last) {
+            sb.append(line);
+          }
         }
       }
-
       Gson gson = new Gson();
       return gson.fromJson(sb.toString(), WebpackResult.class);
     } catch (IOException e) {
@@ -354,16 +530,6 @@ public class HeliumBundleFactory {
     }
   }
 
-  public File getCurrentCacheBundle() {
-    synchronized (this) {
-      if (currentCacheBundle.isFile()) {
-        return currentCacheBundle;
-      } else {
-        return null;
-      }
-    }
-  }
-
   private boolean isLocalPackage(HeliumPackage pkg) {
     return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/"));
   }
@@ -423,14 +589,27 @@ public class HeliumBundleFactory {
   }
 
   private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
-    installNodeAndNpm();
     NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), defaultNpmRegistryUrl);
     npm.execute(args, env);
   }
 
-  private void configureLogger() {
+  private void npmCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
+    npmCommand(args, new HashMap<String, String>());
+  }
+
+  private void yarnCommand(FrontendPluginFactory fpf, String args) throws TaskRunnerException {
+    yarnCommand(fpf, args, new HashMap<String, String>());
+  }
+
+  private void yarnCommand(FrontendPluginFactory fpf,
+                           String args, Map<String, String> env) throws TaskRunnerException {
+    YarnRunner yarn = fpf.getYarnRunner(getProxyConfig(), defaultNpmRegistryUrl);
+    yarn.execute(args, env);
+  }
+
+  private synchronized void configureLogger() {
     org.apache.log4j.Logger npmLogger = org.apache.log4j.Logger.getLogger(
-        "com.github.eirslett.maven.plugins.frontend.lib.DefaultNpmRunner");
+        "com.github.eirslett.maven.plugins.frontend.lib.DefaultYarnRunner");
     Enumeration appenders = org.apache.log4j.Logger.getRootLogger().getAllAppenders();
 
     if (appenders != null) {
@@ -440,7 +619,7 @@ public class HeliumBundleFactory {
 
           @Override
           public int decide(LoggingEvent loggingEvent) {
-            if (loggingEvent.getLoggerName().contains("DefaultNpmRunner")) {
+            if (loggingEvent.getLoggerName().contains("DefaultYarnRunner")) {
               return DENY;
             } else {
               return NEUTRAL;

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
index e54349b..dc4de44 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java
@@ -43,7 +43,6 @@ public class HeliumLocalRegistry extends HeliumRegistry {
 
   }
 
-
   @Override
   public synchronized List<HeliumPackage> getAll() throws IOException {
     List<HeliumPackage> result = new LinkedList<>();

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/resources/helium/package.json
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/resources/helium/package.json b/zeppelin-zengine/src/main/resources/helium/package.json
index 197ef65..6e0fe92 100644
--- a/zeppelin-zengine/src/main/resources/helium/package.json
+++ b/zeppelin-zengine/src/main/resources/helium/package.json
@@ -1,11 +1,11 @@
 {
-  "name": "zeppelin-helium-bundle",
-  "main": "load",
+  "name": "PACKAGE_NAME",
+  "main": "MAIN_FILE",
   "scripts": {
-    "bundle": "node/node node_modules/webpack/bin/webpack.js --display-error-details --json"
+    "bundle": "webpack --display-error-details --json"
   },
   "dependencies": {
-    DEPENDENCIES
+DEPENDENCIES
   },
   "devDependencies": {
     "webpack": "^1.12.2",

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/main/resources/helium/webpack.config.js
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/resources/helium/webpack.config.js b/zeppelin-zengine/src/main/resources/helium/webpack.config.js
index 0163ca1..c7d3384 100644
--- a/zeppelin-zengine/src/main/resources/helium/webpack.config.js
+++ b/zeppelin-zengine/src/main/resources/helium/webpack.config.js
@@ -16,43 +16,22 @@
  */
 
 module.exports = {
-    entry: './load.js',
+    entry: './src/load.js',
     output: { path: './', filename: 'helium.bundle.js', },
-    module: {
-        loaders: [
-          {
-            test: /\.js$/,
-            // DON'T exclude. since zeppelin will bundle all necessary packages: `exclude: /node_modules/,`
-            loader: 'babel-loader',
-            query: { presets: ['es2015', 'stage-0'] },
-          },
-          {
-            test: /(\.css)$/,
-            loaders: ['style', 'css?sourceMap&importLoaders=1'],
-          },
-          {
-            test: /\.woff(\?\S*)?$/,
-            loader: 'url-loader?limit=10000&minetype=application/font-woff',
-          },
-          {
-            test: /\.woff2(\?\S*)?$/,
-            loader: 'url-loader?limit=10000&minetype=application/font-woff',
-          },
-          {
-            test: /\.eot(\?\S*)?$/,
-            loader: 'url-loader',
-          }, {
-            test: /\.ttf(\?\S*)?$/,
-            loader: 'url-loader',
-          },
-          {
-            test: /\.svg(\?\S*)?$/,
-            loader: 'url-loader',
-          },
-          {
-            test: /\.json$/,
-            loader: 'json-loader'
-          },
-        ],
-    }
+  module: {
+    loaders: [
+      {
+        test: /\.js$/,
+        loader: 'babel-loader',
+        exclude: /node_modules\/(?!(zeppelin-spell|zeppelin-vis|zeppelin-tabledata)\/).*/,
+        query: { presets: ['es2015', 'stage-0'] },
+      },
+      { test: /(\.css)$/, loaders: ['style', 'css?sourceMap&importLoaders=1'], },
+      { test: /\.woff(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
+      { test: /\.woff2(\?\S*)?$/, loader: 'url-loader?limit=10000&minetype=application/font-woff', },
+      { test: /\.eot(\?\S*)?$/, loader: 'url-loader', }, {
+        test: /\.ttf(\?\S*)?$/, loader: 'url-loader', }, {
+        test: /\.svg(\?\S*)?$/, loader: 'url-loader', }, {
+        test: /\.json$/, loader: 'json-loader' }, ],
+  }
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/241fd034/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
index 7b816e6..d55358f 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
@@ -22,6 +22,7 @@ import com.google.common.io.Resources;
 import org.apache.commons.io.FileUtils;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.File;
@@ -38,9 +39,16 @@ public class HeliumBundleFactoryTest {
   private File tmpDir;
   private ZeppelinConfiguration conf;
   private HeliumBundleFactory hbf;
+  static File nodeInstallationDir = new File(
+      System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_nodeCache");
+
+  @BeforeClass
+  static public void beforeAll() throws IOException {
+    FileUtils.deleteDirectory(nodeInstallationDir);
+  }
 
   @Before
-  public void setUp() throws InstallationException, TaskRunnerException {
+  public void setUp() throws InstallationException, TaskRunnerException, IOException {
     tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
     tmpDir.mkdirs();
 
@@ -52,10 +60,13 @@ public class HeliumBundleFactoryTest {
     conf = new ZeppelinConfiguration();
 
     hbf = new HeliumBundleFactory(conf,
+        nodeInstallationDir,
         tmpDir,
         new File(moduleDir, "tabledata"),
         new File(moduleDir, "visualization"),
         new File(moduleDir, "spell"));
+    hbf.installNodeAndNpm();
+    hbf.copyFrameworkModuleToInstallPath(true);
   }
 
   @After
@@ -65,17 +76,9 @@ public class HeliumBundleFactoryTest {
 
   @Test
   public void testInstallNpm() throws InstallationException {
-    assertFalse(new File(tmpDir,
-        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
-    assertFalse(new File(tmpDir,
-        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
-
-    hbf.installNodeAndNpm();
-
-    assertTrue(new File(tmpDir,
-        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/npm").isFile());
-    assertTrue(new File(tmpDir,
-        HeliumBundleFactory.HELIUM_LOCAL_REPO + "/node/node").isFile());
+    assertTrue(new File(nodeInstallationDir, "/node/npm").isFile());
+    assertTrue(new File(nodeInstallationDir, "/node/node").isFile());
+    assertTrue(new File(nodeInstallationDir, "/node/yarn/dist/bin/yarn").isFile());
   }
 
   @Test
@@ -107,14 +110,12 @@ public class HeliumBundleFactoryTest {
         "license",
         "icon"
     );
-    List<HeliumPackage> pkgs = new LinkedList<>();
-    pkgs.add(pkg);
-    File bundle = hbf.buildBundle(pkgs);
+    File bundle = hbf.buildPackage(pkg, true, true);
     assertTrue(bundle.isFile());
     long lastModified = bundle.lastModified();
 
     // buildBundle again and check if it served from cache
-    bundle = hbf.buildBundle(pkgs);
+    bundle = hbf.buildPackage(pkg, false, true);
     assertEquals(lastModified, bundle.lastModified());
   }
 
@@ -135,9 +136,7 @@ public class HeliumBundleFactoryTest {
         "license",
         "fa fa-coffee"
     );
-    List<HeliumPackage> pkgs = new LinkedList<>();
-    pkgs.add(pkg);
-    File bundle = hbf.buildBundle(pkgs);
+    File bundle = hbf.buildPackage(pkg, true, true);
     assertTrue(bundle.isFile());
   }
 
@@ -157,11 +156,9 @@ public class HeliumBundleFactoryTest {
         "license",
         "fa fa-coffee"
     );
-    List<HeliumPackage> pkgs = new LinkedList<>();
-    pkgs.add(pkg);
     File bundle = null;
     try {
-      bundle = hbf.buildBundle(pkgs);
+      bundle = hbf.buildPackage(pkg, true, true);
       // should throw exception
       assertTrue(false);
     } catch (IOException e) {
@@ -202,8 +199,8 @@ public class HeliumBundleFactoryTest {
     List<HeliumPackage> pkgsV2 = new LinkedList<>();
     pkgsV2.add(pkgV2);
 
-    File bundle1 = hbf.buildBundle(pkgsV1);
-    File bundle2 = hbf.buildBundle(pkgsV2);
+    File bundle1 = hbf.buildPackage(pkgV1, true, true);
+    File bundle2 = hbf.buildPackage(pkgV2, true, true);
 
     assertNotSame(bundle1.lastModified(), bundle2.lastModified());
   }


Mime
View raw message