zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject zeppelin git commit: [ZEPPELIN-3182] Support saving notebooks to Google Cloud Storage
Date Wed, 14 Feb 2018 04:39:55 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/branch-0.8 d6fc7b604 -> ec4f3e08a


[ZEPPELIN-3182] Support saving notebooks to Google Cloud Storage

Same as #2738 , but targeted toward `branch-0.8`

Author: Karthik Palaniappan <karthikpal@google.com>

Closes #2751 from karth295/master and squashes the following commits:

c4a45b7 [Karthik Palaniappan] [ZEPPELIN-3182] Support saving notebooks to Google Cloud Storage
8dc819e [Karthik Palaniappan] Unify logic to clear notebook runtime state on load from storage


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/ec4f3e08
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/ec4f3e08
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/ec4f3e08

Branch: refs/heads/branch-0.8
Commit: ec4f3e08a25eb05d5a7703555344ebb21d270490
Parents: d6fc7b6
Author: Karthik Palaniappan <karthikpal@google.com>
Authored: Sun Jan 28 19:21:34 2018 -0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Wed Feb 14 12:39:42 2018 +0800

----------------------------------------------------------------------
 LICENSE                                         |   5 +-
 conf/zeppelin-env.sh.template                   |   6 +
 conf/zeppelin-site.xml.template                 |  17 ++
 docs/index.md                                   |   1 +
 docs/interpreter/bigquery.md                    |  20 +-
 docs/setup/storage/storage.md                   |  92 ++++++++
 .../zeppelin/conf/ZeppelinConfiguration.java    |  11 +-
 zeppelin-zengine/pom.xml                        | 103 +++++++-
 .../java/org/apache/zeppelin/notebook/Note.java |  13 +
 .../notebook/repo/AzureNotebookRepo.java        |  28 +--
 .../zeppelin/notebook/repo/GCSNotebookRepo.java | 234 ++++++++++++++++++
 .../notebook/repo/MongoNotebookRepo.java        |  42 +---
 .../zeppelin/notebook/repo/S3NotebookRepo.java  |  17 +-
 .../zeppelin/notebook/repo/VFSNotebookRepo.java |  27 +--
 .../notebook/repo/GCSNotebookRepoTest.java      | 235 +++++++++++++++++++
 15 files changed, 744 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 2252d65..3b34053 100644
--- a/LICENSE
+++ b/LICENSE
@@ -259,6 +259,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
     (Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras)
     (Apache 2.0) Nimbus JOSE+JWT (https://bitbucket.org/connect2id/nimbus-jose-jwt/wiki/Home)
     (Apache 2.0) jarchivelib (https://github.com/thrau/jarchivelib)
+    (Apache 2.0) Google Cloud Client Library for Java (https://github.com/GoogleCloudPlatform/google-cloud-java)
 
 ========================================================================
 BSD 3-Clause licenses
@@ -274,6 +275,8 @@ The following components are provided under the BSD 3-Clause license.  See file
 
   (BSD 3 Clause) diff.js (https://github.com/kpdecker/jsdiff)
 
+  (BSD 3-Clause) Google Auth Library for Java (https://github.com/google/google-auth-library-java)
+
 ========================================================================
 BSD 2-Clause licenses
 ========================================================================
@@ -287,4 +290,4 @@ Jython Software License
 ========================================================================
 The following components are provided under the Jython Software License.  See file headers and project links for details.
 
-  (Jython Software License) jython-standalone - http://www.jython.org/
\ No newline at end of file
+  (Jython Software License) jython-standalone - http://www.jython.org/

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/conf/zeppelin-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template
index 7bc38d6..c7204bd 100644
--- a/conf/zeppelin-env.sh.template
+++ b/conf/zeppelin-env.sh.template
@@ -30,16 +30,22 @@
 # export ZEPPELIN_NOTEBOOK_DIR   		# Where notebook saved
 # export ZEPPELIN_NOTEBOOK_HOMESCREEN		# Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
 # export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE	# hide homescreen notebook from list when this value set to "true". default "false"
+
 # export ZEPPELIN_NOTEBOOK_S3_BUCKET        # Bucket where notebook saved
 # export ZEPPELIN_NOTEBOOK_S3_ENDPOINT      # Endpoint of the bucket
 # export ZEPPELIN_NOTEBOOK_S3_USER          # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
 # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID    # AWS KMS key ID
 # export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION      # AWS KMS key region
 # export ZEPPELIN_NOTEBOOK_S3_SSE      # Server-side encryption enabled for notebooks
+
+# export ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR      # GCS "directory" (prefix) under which notebooks are saved. E.g. gs://example-bucket/path/to/dir
+# export GOOGLE_APPLICATION_CREDENTIALS         # Provide a service account key file for GCS and BigQuery API calls (overrides application default credentials)
+
 # export ZEPPELIN_NOTEBOOK_MONGO_URI				# MongoDB connection URI used to connect to a MongoDB database server. Default "mongodb://localhost"
 # export ZEPPELIN_NOTEBOOK_MONGO_DATABASE		# Database name to store notebook. Default "zeppelin"
 # export ZEPPELIN_NOTEBOOK_MONGO_COLLECTION # Collection name to store notebook. Default "notes"
 # export ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT	# If "true" import local notes under ZEPPELIN_NOTEBOOK_DIR on startup. Default "false"
+
 # export ZEPPELIN_IDENT_STRING   		# A string representing this instance of zeppelin. $USER by default.
 # export ZEPPELIN_NICENESS       		# The scheduling priority for daemons. Defaults to 0.
 # export ZEPPELIN_INTERPRETER_LOCALREPO         # Local repository for interpreter's additional dependency loading

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index 9e9898b..9774f0d 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -67,6 +67,23 @@
   <description>hide homescreen notebook from list when this value set to true</description>
 </property>
 
+<!-- Google Cloud Storage notebook storage -->
+<!--
+<property>
+  <name>zeppelin.notebook.gcs.dir</name>
+  <value></value>
+  <description>
+    A GCS path in the form gs://bucketname/path/to/dir.
+    Notes are stored at {zeppelin.notebook.gcs.dir}/{notebook-id}/note.json
+ </description>
+</property>
+
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
+  <description>notebook persistence layer implementation</description>
+</property>
+-->
 
 <!-- Amazon S3 notebook storage -->
 <!-- Creates the following directory structure: s3://{bucket}/{username}/{notebook-id}/note.json -->

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index 587ae93..3d42735 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -104,6 +104,7 @@ limitations under the License.
   * [Git Storage](./setup/storage/storage.html#notebook-storage-in-local-git-repository)
   * [S3 Storage](./setup/storage/storage.html#notebook-storage-in-s3)
   * [Azure Storage](./setup/storage/storage.html#notebook-storage-in-azure)
+  * [Google Cloud Storage](./setup/storage/storage.html#notebook-storage-in-gcs)
   * [ZeppelinHub Storage](./setup/storage/storage.html#notebook-storage-in-zeppelinhub)
   * [MongoDB Storage](./setup/storage/storage.html#notebook-storage-in-mongodb)
 * Operation 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/docs/interpreter/bigquery.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/bigquery.md b/docs/interpreter/bigquery.md
index 7ebe2e2..1b90f99 100644
--- a/docs/interpreter/bigquery.md
+++ b/docs/interpreter/bigquery.md
@@ -58,20 +58,12 @@ Zeppelin is built against BigQuery API version v2-rev265-1.21.0 - [API Javadocs]
 
 In a notebook, to enable the **BigQuery** interpreter, click the **Gear** icon and select **bigquery**.
 
-### Setup service account credentials
-
-In order to run BigQuery interpreter outside of Google Cloud Engine you need to provide authentication credentials,
-by [following this instructions](https://developers.google.com/identity/protocols/application-default-credentials):
-
- - Go to the [API Console Credentials page](https://console.developers.google.com/project/_/apis/credentials)
- - From the project drop-down, select your project.
- - On the `Credentials` page, select the `Create credentials` drop-down, then select `Service account key`.
- - From the Service account drop-down, select an existing service account or create a new one.
- - For `Key type`, select the `JSON` key option, then select `Create`. The file automatically downloads to your computer.
- - Put the `*.json` file you just downloaded in a directory of your choosing. This directory must be private (you can't let anyone get access to this), but accessible to your Zeppelin instance.
- - Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the path of the JSON file downloaded.
-    * either though GUI: in interpreter configuration page property names in CAPITAL_CASE set up env vars
-    * or though `zeppelin-env.sh`: just add it to the end of the file.
+### Provide Application Default Credentials
+
+Within Google Cloud Platform (e.g. Google App Engine, Google Compute Engine),
+built-in credentials are used by default.
+
+Outside of GCP, follow the Google API authentication instructions for [Zeppelin Google Cloud Storage](https://zeppelin.apache.org/docs/latest/storage/storage.html#notebook-storage-in-gcs)
 
 ## Using the BigQuery Interpreter
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/docs/setup/storage/storage.md
----------------------------------------------------------------------
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index f34fc2c..6f2ace4 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -33,6 +33,7 @@ There are few notebook storage systems available for a use out of the box:
   * all notes are saved in the notebook folder in hadoop compatible file system - `FileSystemNotebookRepo`
   * storage using Amazon S3 service - `S3NotebookRepo`
   * storage using Azure service - `AzureNotebookRepo`
+  * storage using Google Cloud Storage - `GCSNotebookRepo`
   * storage using MongoDB - `MongoNotebookRepo`
   * storage using GitHub - `GitHubNotebookRepo`
 
@@ -264,6 +265,97 @@ Optionally, you can specify Azure folder structure name in the file **zeppelin-s
 ```
 
 </br>
+## Notebook Storage in Google Cloud Storage<a name="GCS"></a>
+
+Using `GCSNotebookRepo` you can connect Zeppelin with Google Cloud Storage using [Application Default Credentials](https://cloud.google.com/docs/authentication/production).
+
+First, choose a GCS path under which to store notebooks.
+
+```
+<property>
+  <name>zeppelin.notebook.gcs.dir</name>
+  <value></value>
+  <description>
+    A GCS path in the form gs://bucketname/path/to/dir.
+    Notes are stored at {zeppelin.notebook.gcs.dir}/{notebook-id}/note.json
+ </description>
+</property>
+```
+
+Then, initialize the `GCSNotebookRepo` class in the file **zeppelin-site.xml** by commenting the next property:
+
+```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.GitNotebookRepo</value>
+  <description>versioned notebook persistence layer implementation</description>
+</property>
+```
+
+and commenting out:
+
+```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
+  <description>notebook persistence layer implementation</description>
+</property>
+```
+
+Or, if you want to simultaneously use your local git storage with GCS, use the following property instead:
+
+ ```
+<property>
+  <name>zeppelin.notebook.storage</name>
+  <value>org.apache.zeppelin.notebook.repo.GitNotebookRepo,org.apache.zeppelin.notebook.repo.GCSNotebookRepo</value>
+  <description>notebook persistence layer implementation</description>
+</property>
+```
+
+### Google Cloud API Authentication
+
+Note: On Google App Engine, Google Cloud Shell, and Google Compute Engine, these
+steps are not necessary, as build-in credentials are used by default.
+
+For more information, see [Application Default Credentials](https://cloud.google.com/docs/authentication/production)
+
+#### Using gcloud auth application-default login
+
+See the [gcloud docs](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login)
+
+As the user running the zeppelin daemon, run:
+
+```bash
+gcloud auth application-default login
+```
+
+You can also use `--scopes` to restrict access to specific Google APIs, such as
+Cloud Storage and BigQuery.
+
+#### Using service account key files
+
+Alternatively, to use a [service account](https://cloud.google.com/compute/docs/access/service-accounts)
+for authentication with GCS, you will need a JSON service account key file.
+
+1. Navigate to the [service accounts page](https://console.cloud.google.com/iam-admin/serviceaccounts/project)
+2. Click `CREATE SERVICE ACCOUNT`
+3. Select at least `Storage -> Storage Object Admin`. Note that this is
+   **different** than `Storage Admin`.
+4. If you are also using the BigQuery Interpreter, add the appropriate
+   permissions (e.g. `Bigquery -> Bigquery Data Viewer and BigQuery User`)
+5. Name your service account, and select "Furnish a new private key" to download
+   a `.json` file. Click "Create".
+6. Move the downloaded file to a location of your choice (e.g.
+   `/path/to/my/key.json`), and give it appropriate permissions. Ensure at
+   least the user running the zeppelin daemon can read it.
+
+Then, point `GOOGLE_APPLICATION_CREDENTIALS` at your new key file in **zeppelin-env.sh**. For example:
+
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/my/key.json
+```
+
+</br>
 ## Notebook Storage in ZeppelinHub  <a name="ZeppelinHub"></a>
 
 ZeppelinHub storage layer allows out of the box connection of Zeppelin instance with your ZeppelinHub account. First of all, you need to either comment out the following  property in **zeppelin-site.xml**:

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 030d781..3a2b5a2 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -360,15 +360,19 @@ public class ZeppelinConfiguration extends XMLConfiguration {
         "org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage");
   }
 
-  public String getUser() {
+  public String getGCSStorageDir() {
+    return getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR);
+  }
+
+  public String getS3User() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER);
   }
 
-  public String getBucketName() {
+  public String getS3BucketName() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
   }
 
-  public String getEndpoint() {
+  public String getS3Endpoint() {
     return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
   }
 
@@ -697,6 +701,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),
     // whether homescreen notebook will be hidden from notebook list or not
     ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
+    ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR("zeppelin.notebook.gcs.dir", ""),
     ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
     ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
     ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index 9ba9ac1..886664c 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -39,6 +39,7 @@
     <hadoop.version>2.7.3</hadoop.version>
     <commons.lang3.version>3.4</commons.lang3.version>
     <commons.vfs2.version>2.0</commons.vfs2.version>
+    <gcs.storage.version>1.14.0</gcs.storage.version>
     <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version>
     <adl.sdk.version>2.1.4</adl.sdk.version>
     <jackrabbit.webdav.version>1.5.2</jackrabbit.webdav.version>
@@ -51,6 +52,7 @@
 
     <!--test library versions-->
     <google.truth.version>0.27</google.truth.version>
+    <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version>
   </properties>
 
   <dependencies>
@@ -115,9 +117,83 @@
     </dependency>
 
     <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <version>${gcs.storage.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.protobuf</groupId>
+          <artifactId>protobuf-java</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.api</groupId>
+          <artifactId>api-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.http-client</groupId>
+          <artifactId>google-http-client-jackson2</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.http-client</groupId>
+          <artifactId>google-http-client</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-core-asl</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.api</groupId>
+      <artifactId>api-common</artifactId>
+      <version>1.2.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.http-client</groupId>
+      <artifactId>google-http-client-jackson2</artifactId>
+      <version>1.23.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
       <version>${aws.sdk.s3.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>joda-time</groupId>
+          <artifactId>joda-time</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -146,7 +222,7 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>15.0</version>
+      <version>20.0</version>
     </dependency>
 
     <dependency>
@@ -228,6 +304,23 @@
     </dependency>
 
     <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-nio</artifactId>
+      <version>${google.testing.nio.version}</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
       <groupId>com.google.truth</groupId>
       <artifactId>truth</artifactId>
       <version>${google.truth.version}</version>
@@ -612,6 +705,10 @@
               <artifactId>protobuf-java</artifactId>
             </exclusion>
             <exclusion>
+              <groupId>com.google.protobuf</groupId>
+              <artifactId>protobuf-java-util</artifactId>
+            </exclusion>
+            <exclusion>
               <groupId>com.google.guava</groupId>
               <artifactId>guava</artifactId>
             </exclusion>
@@ -623,6 +720,10 @@
               <groupId>io.grpc</groupId>
               <artifactId>grpc-context</artifactId>
             </exclusion>
+            <exclusion>
+              <groupId>com.google.api.grpc</groupId>
+              <artifactId>proto-google-common-protos</artifactId>
+            </exclusion>
           </exclusions>
         </dependency>
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 281c4de..0a8fb12 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -963,6 +963,19 @@ public class Note implements ParagraphJobListener, JsonSerializable {
     for (Paragraph p : paragraphs) {
       p.clearRuntimeInfos();
       p.parseText();
+
+      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
+        p.setStatus(Status.ABORT);
+      }
+
+      List<ApplicationState> appStates = p.getAllApplicationStates();
+      if (appStates != null) {
+        for (ApplicationState app : appStates) {
+          if (app.getStatus() != ApplicationState.Status.ERROR) {
+            app.setStatus(ApplicationState.Status.UNLOADED);
+          }
+        }
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
index de337fa..731a3e8 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
@@ -17,6 +17,13 @@
 
 package org.apache.zeppelin.notebook.repo;
 
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.file.CloudFile;
+import com.microsoft.azure.storage.file.CloudFileClient;
+import com.microsoft.azure.storage.file.CloudFileDirectory;
+import com.microsoft.azure.storage.file.CloudFileShare;
+import com.microsoft.azure.storage.file.ListFileItem;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,26 +35,15 @@ import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.file.CloudFile;
-import com.microsoft.azure.storage.file.CloudFileClient;
-import com.microsoft.azure.storage.file.CloudFileDirectory;
-import com.microsoft.azure.storage.file.CloudFileShare;
-import com.microsoft.azure.storage.file.ListFileItem;
-
 /**
  * Azure storage backend for notebooks
  */
@@ -128,15 +124,7 @@ public class AzureNotebookRepo implements NotebookRepo {
     String json = IOUtils.toString(ins,
         conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING));
     ins.close();
-    Note note = Note.fromJson(json);
-
-    for (Paragraph p : note.getParagraphs()) {
-      if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) {
-        p.setStatus(Job.Status.ABORT);
-      }
-    }
-
-    return note;
+    return Note.fromJson(json);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
new file mode 100644
index 0000000..591c532
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobListOption;
+import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.JsonParseException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A NotebookRepo implementation for storing notebooks in Google Cloud Storage.
+ *
+ * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path
+ * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an
+ * object store, so this "directory" should not itself be an object. Instead, it represents the base
+ * path for the note.json files.
+ *
+ * Authentication is provided by google-auth-library-java.
+ * @see <a href="https://github.com/google/google-auth-library-java">
+ *   google-auth-library-java</a>.
+ */
+public class GCSNotebookRepo implements NotebookRepo {
+
+  private static final Logger LOG = LoggerFactory.getLogger(GCSNotebookRepo.class);
+  private String encoding;
+  private String bucketName;
+  private Optional<String> basePath;
+  private Pattern noteNamePattern;
+  private Storage storage;
+
+  public GCSNotebookRepo(ZeppelinConfiguration conf) throws IOException {
+    this(conf, StorageOptions.getDefaultInstance().getService());
+  }
+
+  // For tests to use an in-memory storage implementation
+  GCSNotebookRepo(ZeppelinConfiguration conf, Storage storage) throws IOException {
+    this.encoding =  conf.getString(ConfVars.ZEPPELIN_ENCODING);
+
+    String gcsStorageDir = conf.getGCSStorageDir();
+    if (gcsStorageDir.isEmpty()) {
+      throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'");
+    }
+    if (!gcsStorageDir.startsWith("gs://")) {
+      throw new IOException(String.format(
+          "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir));
+    }
+    String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length());
+
+    // pathComponents excludes empty string if trailing slash is present
+    List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/"));
+    if (pathComponents.size() < 1) {
+      throw new IOException(String.format(
+          "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir",
+          gcsStorageDir));
+    }
+    this.bucketName = pathComponents.get(0);
+    if (pathComponents.size() > 1) {
+      this.basePath = Optional.of(StringUtils.join(
+          pathComponents.subList(1, pathComponents.size()), "/"));
+    } else {
+      this.basePath = Optional.absent();
+    }
+
+    // Notes are stored at gs://bucketName/basePath/<note-id>/note.json
+    if (basePath.isPresent()) {
+      this.noteNamePattern = Pattern.compile(
+          "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$");
+    } else {
+      this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
+    }
+
+    this.storage = storage;
+  }
+
+  private BlobId makeBlobId(String noteId) {
+    if (basePath.isPresent()) {
+      return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json");
+    } else {
+      return BlobId.of(bucketName, noteId + "/note.json");
+    }
+  }
+
+  @Override
+  public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+    try {
+      List<NoteInfo> infos = new ArrayList<>();
+      Iterable<Blob> blobsUnderDir;
+      if (basePath.isPresent()) {
+        blobsUnderDir = storage
+          .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
+          .iterateAll();
+      } else {
+        blobsUnderDir = storage
+          .list(bucketName)
+          .iterateAll();
+      }
+      for (Blob b : blobsUnderDir) {
+        Matcher matcher = noteNamePattern.matcher(b.getName());
+        if (matcher.matches()) {
+          // Callers only use the id field, so do not fetch each note
+          // This matches the implementation in FileSystemNoteRepo#list
+          infos.add(new NoteInfo(matcher.group(1), "", null));
+        }
+      }
+      return infos;
+    } catch (StorageException se) {
+      throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
+    }
+  }
+
+  @Override
+  public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+    BlobId blobId = makeBlobId(noteId);
+    byte[] contents;
+    try {
+      contents = storage.readAllBytes(blobId);
+    } catch (StorageException se) {
+      throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se);
+    }
+
+    try {
+      return Note.fromJson(new String(contents, encoding));
+    } catch (JsonParseException jpe) {
+      throw new IOException(
+          "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe);
+    }
+  }
+
+  @Override
+  public void save(Note note, AuthenticationInfo subject) throws IOException {
+    BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId()))
+        .setContentType("application/json")
+        .build();
+    try {
+      storage.create(info, note.toJson().getBytes("UTF-8"));
+    } catch (StorageException se) {
+      throw new IOException("Could not write " + info.toString() + ": " + se.getMessage(), se);
+    }
+  }
+
+  @Override
+  public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+    Preconditions.checkArgument(!Strings.isNullOrEmpty(noteId));
+    BlobId blobId = makeBlobId(noteId);
+    try {
+      boolean deleted = storage.delete(blobId);
+      if (!deleted) {
+        throw new IOException("Tried to remove nonexistent blob " + blobId.toString());
+      }
+    } catch (StorageException se) {
+      throw new IOException("Could not remove " + blobId.toString() + ": " + se.getMessage(), se);
+    }
+  }
+
+  @Override
+  public void close() {
+    //no-op
+  }
+
+  @Override
+  public Revision checkpoint(String noteId, String checkpointMsg, AuthenticationInfo subject)
+      throws IOException {
+    LOG.warn("checkpoint is not implemented for GCSNotebookRepo");
+    return null;
+  }
+
+  @Override
+  public Note get(String noteId, String revId, AuthenticationInfo subject) throws IOException {
+    LOG.warn("get revId is not implemented for GCSNotebookRepo");
+    return null;
+  }
+
+  @Override
+  public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
+    LOG.warn("revisionHistory is not implemented for GCSNotebookRepo");
+    return Collections.emptyList();
+  }
+
+  @Override
+  public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject)
+      throws IOException {
+    LOG.warn("setNoteRevision is not implemented for GCSNotebookRepo");
+    return null;
+  }
+
+  @Override
+  public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+    LOG.warn("getSettings is not implemented for GCSNotebookRepo");
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+    LOG.warn("updateSettings is not implemented for GCSNotebookRepo");
+  }
+}

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
index 273d75d..376a986 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
@@ -1,5 +1,9 @@
 package org.apache.zeppelin.notebook.repo;
 
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.in;
+import static com.mongodb.client.model.Filters.type;
+
 import com.mongodb.MongoBulkWriteException;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
@@ -7,18 +11,18 @@ import com.mongodb.bulk.BulkWriteError;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
-import static com.mongodb.client.model.Filters.eq;
-import static com.mongodb.client.model.Filters.type;
-import static com.mongodb.client.model.Filters.in;
-
 import com.mongodb.client.model.InsertManyOptions;
 import com.mongodb.client.model.UpdateOptions;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.notebook.ApplicationState;
-import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.bson.BsonType;
 import org.bson.Document;
@@ -26,11 +30,6 @@ import org.bson.types.ObjectId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 /**
  * Backend for storing Notebook on MongoDB
  */
@@ -161,24 +160,7 @@ public class MongoNotebookRepo implements NotebookRepo {
     // document to JSON
     String json = doc.toJson();
     // JSON to note
-    Note note = Note.fromJson(json);
-
-    for (Paragraph p : note.getParagraphs()) {
-      if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) {
-        p.setStatus(Job.Status.ABORT);
-      }
-
-      List<ApplicationState> appStates = p.getAllApplicationStates();
-      if (appStates != null) {
-        for (ApplicationState app : appStates) {
-          if (app.getStatus() != ApplicationState.Status.ERROR) {
-            app.setStatus(ApplicationState.Status.UNLOADED);
-          }
-        }
-      }
-    }
-
-    return note;
+    return Note.fromJson(json);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 8828985..7d64702 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -90,8 +90,8 @@ public class S3NotebookRepo implements NotebookRepo {
 
   public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
     this.conf = conf;
-    bucketName = conf.getBucketName();
-    user = conf.getUser();
+    bucketName = conf.getS3BucketName();
+    user = conf.getS3User();
     useServerSideEncryption = conf.isS3ServerSideEncryption();
 
     // always use the default provider chain
@@ -123,7 +123,7 @@ public class S3NotebookRepo implements NotebookRepo {
     }
 
     // set S3 endpoint to use
-    s3client.setEndpoint(conf.getEndpoint());
+    s3client.setEndpoint(conf.getS3Endpoint());
   }
 
   /**
@@ -205,19 +205,10 @@ public class S3NotebookRepo implements NotebookRepo {
       throw new IOException("Unable to retrieve object from S3: " + ace, ace);
     }
 
-    Note note;
     try (InputStream ins = s3object.getObjectContent()) {
       String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
-      note = Note.fromJson(json);
+      return Note.fromJson(json);
     }
-
-    for (Paragraph p : note.getParagraphs()) {
-      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
-        p.setStatus(Status.ABORT);
-      }
-    }
-
-    return note;
   }
 
   private NoteInfo getNoteInfo(String key) throws IOException {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
index 63395f9..481ea3d 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
@@ -17,6 +17,7 @@
 
 package org.apache.zeppelin.notebook.repo;
 
+import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -24,11 +25,9 @@ import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Collections;
-import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.vfs2.FileContent;
@@ -40,17 +39,12 @@ import org.apache.commons.vfs2.Selectors;
 import org.apache.commons.vfs2.VFS;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.notebook.ApplicationState;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.Paragraph;
-import org.apache.zeppelin.scheduler.Job.Status;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
-
 /**
 *
 */
@@ -167,24 +161,7 @@ public class VFSNotebookRepo implements NotebookRepo {
     String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
     ins.close();
 
-    Note note = Note.fromJson(json);
-
-    for (Paragraph p : note.getParagraphs()) {
-      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
-        p.setStatus(Status.ABORT);
-      }
-
-      List<ApplicationState> appStates = p.getAllApplicationStates();
-      if (appStates != null) {
-        for (ApplicationState app : appStates) {
-          if (app.getStatus() != ApplicationState.Status.ERROR) {
-            app.setStatus(ApplicationState.Status.UNLOADED);
-          }
-        }
-      }
-    }
-
-    return note;
+    return Note.fromJson(json);
   }
 
   private NoteInfo getNoteInfo(FileObject noteDir) throws IOException {

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ec4f3e08/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
new file mode 100644
index 0000000..c1fae67
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import static com.google.common.truth.Truth.assertThat;
+import static junit.framework.TestCase.fail;
+
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class GCSNotebookRepoTest {
+  private static final AuthenticationInfo AUTH_INFO = AuthenticationInfo.ANONYMOUS;
+
+  private GCSNotebookRepo notebookRepo;
+  private Storage storage;
+
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {
+        { "bucketname", Optional.absent(), "gs://bucketname" },
+        { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" },
+        { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" },
+        { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" }
+    });
+  }
+
+  @Parameter(0)
+  public String bucketName;
+
+  @Parameter(1)
+  public Optional<String> basePath;
+
+  @Parameter(2)
+  public String uriPath;
+
+  private Note runningNote;
+
+  @Before
+  public void setUp() throws Exception {
+    this.runningNote = makeRunningNote();
+
+    this.storage = LocalStorageHelper.getOptions().getService();
+
+    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), uriPath);
+    this.notebookRepo = new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+  }
+
+  private static Note makeRunningNote() {
+    Note note = new Note();
+    note.setConfig(ImmutableMap.<String, Object>of("key", "value"));
+
+    Paragraph p = new Paragraph(note, null, null);
+    p.setText("text");
+    p.setStatus(Status.RUNNING);
+    note.addParagraph(p);
+    return note;
+  }
+
+  @Test
+  public void testList_nonexistent() throws Exception {
+    assertThat(notebookRepo.list(AUTH_INFO)).isEmpty();
+  }
+
+  @Test
+  public void testList() throws Exception {
+    createAt(runningNote, "note.json");
+    createAt(runningNote, "/note.json");
+    createAt(runningNote, "validid/note.json");
+    createAt(runningNote, "validid-2/note.json");
+    createAt(runningNote, "cannot-be-dir/note.json/foo");
+    createAt(runningNote, "cannot/be/nested/note.json");
+
+    List<NoteInfo> infos = notebookRepo.list(AUTH_INFO);
+    List<String> noteIds = new ArrayList<>();
+    for (NoteInfo info : infos) {
+      noteIds.add(info.getId());
+    }
+    // Only valid paths are gs://bucketname/path/<noteid>/note.json
+    assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("validid", "validid-2"));
+  }
+
+  @Test
+  public void testGet_nonexistent() throws Exception {
+    try {
+      notebookRepo.get("id", AUTH_INFO);
+      fail();
+    } catch (IOException e) {}
+  }
+
+  @Test
+  public void testGet() throws Exception {
+    create(runningNote);
+
+    // Status of saved running note is removed in get()
+    Note got = notebookRepo.get(runningNote.getId(), AUTH_INFO);
+    assertThat(got.getLastParagraph().getStatus()).isEqualTo(Status.ABORT);
+
+    // But otherwise equal
+    got.getLastParagraph().setStatus(Status.RUNNING);
+    assertThat(got).isEqualTo(runningNote);
+  }
+
+  @Test
+  public void testGet_malformed() throws Exception {
+    createMalformed("id");
+    try {
+      notebookRepo.get("id", AUTH_INFO);
+      fail();
+    } catch (IOException e) {}
+  }
+
+  @Test
+  public void testSave_create() throws Exception {
+    notebookRepo.save(runningNote, AUTH_INFO);
+    // Output is saved
+    assertThat(storage.readAllBytes(makeBlobId(runningNote.getId())))
+        .isEqualTo(runningNote.toJson().getBytes("UTF-8"));
+  }
+
+  @Test
+  public void testSave_update() throws Exception {
+    notebookRepo.save(runningNote, AUTH_INFO);
+    // Change name of runningNote
+    runningNote.setName("new-name");
+    notebookRepo.save(runningNote, AUTH_INFO);
+    assertThat(storage.readAllBytes(makeBlobId(runningNote.getId())))
+        .isEqualTo(runningNote.toJson().getBytes("UTF-8"));
+  }
+
+  @Test
+  public void testRemove_nonexistent() throws Exception {
+    try {
+      notebookRepo.remove("id", AUTH_INFO);
+      fail();
+    } catch (IOException e) {}
+  }
+
+  @Test
+  public void testRemove() throws Exception {
+    create(runningNote);
+    notebookRepo.remove(runningNote.getId(), AUTH_INFO);
+    assertThat(storage.get(makeBlobId(runningNote.getId()))).isNull();
+  }
+
+  private String makeName(String relativePath) {
+    if (basePath.isPresent()) {
+      return basePath.get() + "/" + relativePath;
+    } else {
+      return relativePath;
+    }
+  }
+
+  private BlobId makeBlobId(String noteId) {
+    return BlobId.of(bucketName, makeName(noteId + "/note.json"));
+  }
+
+  private void createAt(Note note, String relativePath) throws IOException {
+    BlobId id = BlobId.of(bucketName, makeName(relativePath));
+    BlobInfo info = BlobInfo.newBuilder(id).setContentType("application/json").build();
+    storage.create(info, note.toJson().getBytes("UTF-8"));
+  }
+
+  private void create(Note note) throws IOException {
+    BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId()))
+        .setContentType("application/json")
+        .build();
+    storage.create(info, note.toJson().getBytes("UTF-8"));
+  }
+
+  private void createMalformed(String noteId) throws IOException {
+    BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId))
+        .setContentType("application/json")
+        .build();
+    storage.create(info, "{ invalid-json }".getBytes("UTF-8"));
+  }
+
+  /* These tests test path parsing for illegal paths, and do not use the parameterized vars */
+
+  @Test
+  public void testInitialization_pathNotSet() throws Exception {
+    try {
+      System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "");
+      new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+      fail();
+    } catch (IOException e) {}
+  }
+
+  @Test
+  public void testInitialization_malformedPath() throws Exception {
+    try {
+      System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo");
+      new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+      fail();
+    } catch (IOException e) {}
+  }
+}


Mime
View raw message