zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject incubator-zeppelin git commit: Sync functionality with another storage system
Date Wed, 22 Jul 2015 03:34:34 GMT
Repository: incubator-zeppelin
Updated Branches:
  refs/heads/master 6a011b993 -> b71f04eb9


Sync functionality with another storage system

This PR is to handle the synchronization of notebooks between local Zeppelin fs and remote
storage system.

Corresponding issue: https://issues.apache.org/jira/browse/ZEPPELIN-133

TODO:
- [x] sync non existing files at start
- [x] sync modified files at start
- [x] save to both storages on save()
- [x] delete from both storages on remove()
- [x] pull new/updated notes from secondary storage
- [x] add comma separated storage class definitions
- [x] corresponding tests
- [x] handle failure cases of secondary storage
- [x] handle updated notes on Paragraph level instead of Note class itself

Author: Khalid Huseynov <khalidhnv@nflabs.com>

Closes #123 from khalidhuseynov/sync-with-remote and squashes the following commits:

14c87ff [Khalid Huseynov] Merge branch 'master' into sync-with-remote Latest updates
cbe167c [Khalid Huseynov] minor test change + log removal
d33ffc7 [Khalid Huseynov] check latest modified from Paragraph fields
c2827ff [Khalid Huseynov] delete modTime field from Note and NoteInfo
4293723 [Khalid Huseynov] add initial tests
a441f9d [Khalid Huseynov] catch write to secondary storage failed exception, write to main
storage
5a967f1 [Khalid Huseynov] logic ramification; add comma-separated storage class names; use
list of repos
3314971 [Khalid Huseynov] remove more comments
b6a3590 [Khalid Huseynov] minor refactoring of getting storage class
7c4366d [Khalid Huseynov] remove commented code section and redundant type conversion
d001ebf [Khalid Huseynov] function name/description change
3899f8e [Khalid Huseynov] minor changes to log comments
359a41e [Khalid Huseynov] minor fix of default classname
51e6271 [Khalid Huseynov] change modTime of Note object on save()
5f7ff18 [Khalid Huseynov] use modTime field for uploading modified notes
5b8d994 [Khalid Huseynov] modify containsID() function to return object instead of boolean
032e59b [Khalid Huseynov] add modification time field for Note and NoteInfo
e778c4e [Khalid Huseynov] initial sync implementation


Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/b71f04eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/b71f04eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/b71f04eb

Branch: refs/heads/master
Commit: b71f04eb97d4c924f754b0ba7cc666dc09cefc55
Parents: 6a011b9
Author: Khalid Huseynov <khalidhnv@nflabs.com>
Authored: Tue Jul 21 11:17:35 2015 +0900
Committer: Lee moon soo <moon@apache.org>
Committed: Wed Jul 22 12:34:25 2015 +0900

----------------------------------------------------------------------
 .../apache/zeppelin/server/ZeppelinServer.java  |   7 +-
 zeppelin-zengine/pom.xml                        |  19 +-
 .../org/apache/zeppelin/notebook/NoteInfo.java  |   1 +
 .../notebook/repo/NotebookRepoSync.java         | 297 +++++++++++++++++++
 .../notebook/repo/NotebookRepoSyncTest.java     | 214 +++++++++++++
 .../notebook/repo/mock/VFSNotebookRepoMock.java |  26 ++
 6 files changed, 553 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 403d30a..2bd23bb 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -34,6 +34,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
 import org.apache.zeppelin.interpreter.InterpreterFactory;
 import org.apache.zeppelin.notebook.Notebook;
 import org.apache.zeppelin.notebook.repo.NotebookRepo;
+import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
 import org.apache.zeppelin.rest.InterpreterRestApi;
 import org.apache.zeppelin.rest.NotebookRestApi;
 import org.apache.zeppelin.rest.ZeppelinRestApi;
@@ -307,11 +308,7 @@ public class ZeppelinServer extends Application {
     this.schedulerFactory = new SchedulerFactory();
 
     this.replFactory = new InterpreterFactory(conf, notebookServer);
-    Class<?> notebookStorageClass = getClass().forName(
-        conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE));
-    Constructor<?> constructor = notebookStorageClass.getConstructor(
-        ZeppelinConfiguration.class);
-    this.notebookRepo = (NotebookRepo) constructor.newInstance(conf);
+    this.notebookRepo = new NotebookRepoSync(conf);
     notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, notebookServer);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index d4207a5..e6de0b8 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -89,7 +89,7 @@
        <exclusions>
          <exclusion>
            <groupId>commons-httpclient</groupId>
-           <artifactId>commons-httpclient</artifactId>           
+           <artifactId>commons-httpclient</artifactId>
          </exclusion>
        </exclusions>
      </dependency>
@@ -105,17 +105,17 @@
       <artifactId>quartz</artifactId>
       <version>2.2.1</version>
     </dependency>
-    
+
     <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
     </dependency>
-    
+
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
-            
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -134,14 +134,21 @@
         <exclusion>
           <groupId>xml-apis</groupId>
           <artifactId>xml-apis</artifactId>
-        </exclusion>        
-      </exclusions>      
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency> <!-- because there are two of them above  -->
       <groupId>xml-apis</groupId>
       <artifactId>xml-apis</artifactId>
       <version>1.4.01</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.9.0</version>
+      <scope>test</scope>
     </dependency>    
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
index db07e50..f75a107 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInfo.java
@@ -64,4 +64,5 @@ public class NoteInfo {
   public void setConfig(Map<String, Object> config) {
     this.config = config;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
new file mode 100644
index 0000000..288cff9
--- /dev/null
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Notebook repository sync with remote storage
+ */
+public class NotebookRepoSync implements NotebookRepo{
+  private List<NotebookRepo> repos = new ArrayList<NotebookRepo>();
+  private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSync.class);
+  private static final int maxRepoNum = 2;
+  private static final String pushKey = "pushNoteIDs";
+  private static final String pullKey = "pullNoteIDs";
+
+  /**
+   * @param (conf)
+   * @throws - Exception
+   */
+  public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception {
+    
+    String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim();
+    if (allStorageClassNames.isEmpty()) {
+      throw new IOException("Empty ZEPPELIN_NOTEBOOK_STORAGE conf parameter");
+    }
+    String[] storageClassNames = allStorageClassNames.split(",");
+    if (storageClassNames.length > getMaxRepoNum()) {
+      throw new IOException("Unsupported number of storage classes (" + 
+        storageClassNames.length + ") in ZEPPELIN_NOTEBOOK_STORAGE");
+    }
+
+    for (int i = 0; i < storageClassNames.length; i++) {
+      Class<?> notebookStorageClass = getClass().forName(storageClassNames[i].trim());
+      Constructor<?> constructor = notebookStorageClass.getConstructor(
+                ZeppelinConfiguration.class);
+      repos.add((NotebookRepo) constructor.newInstance(conf));
+    }
+    if (getRepoCount() > 1) {
+      sync(0, 1);
+    }
+  }
+
+  /* by default lists from first repository */
+  public List<NoteInfo> list() throws IOException {
+    return getRepo(0).list();
+  }
+  
+  /* list from specific repo (for tests) */
+  List<NoteInfo> list(int repoIndex) throws IOException {
+    return getRepo(repoIndex).list();
+  }
+
+  /* by default returns from first repository */ 
+  public Note get(String noteId) throws IOException {
+    return getRepo(0).get(noteId);
+  }
+
+  /* get note from specific repo (for tests) */
+  Note get(int repoIndex, String noteId) throws IOException {
+    return getRepo(repoIndex).get(noteId);
+  }
+  
+  /* by default saves to all repos */
+  public void save(Note note) throws IOException {
+    getRepo(0).save(note);
+    if (getRepoCount() > 1) {
+      try {
+        getRepo(1).save(note);
+      }
+      catch (IOException e) {
+        LOG.info(e.getMessage() + ": Failed to write to secondary storage");
+      }
+    }
+  }
+
+  /* save note to specific repo (for tests) */
+  void save(int repoIndex, Note note) throws IOException {
+    getRepo(repoIndex).save(note);
+  }
+
+  public void remove(String noteId) throws IOException {
+    for (NotebookRepo repo : repos) {
+      repo.remove(noteId);
+    }
+    /* TODO(khalid): handle case when removing from secondary storage fails */
+  }
+
+  /**
+   * copy new/updated notes from source to destination storage 
+   * @throws IOException
+   */
+  public void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
+    LOG.info("Sync started");
+    NotebookRepo sourceRepo = getRepo(sourceRepoIndex);
+    NotebookRepo destRepo = getRepo(destRepoIndex);
+    List <NoteInfo> sourceNotes = sourceRepo.list();
+    List <NoteInfo> destNotes = destRepo.list();
+    
+    Map<String, List<String>> noteIDs = notesCheckDiff(sourceNotes,
+                                                       sourceRepo,
+                                                       destNotes,
+                                                       destRepo);
+    List<String> pushNoteIDs = noteIDs.get(pushKey);
+    List<String> pullNoteIDs = noteIDs.get(pullKey);
+    if (!pushNoteIDs.isEmpty()) {
+      LOG.info("Notes with the following IDs will be pushed");
+      for (String id : pushNoteIDs) {
+        LOG.info("ID : " + id);
+      }
+      pushNotes(pushNoteIDs, sourceRepo, destRepo);
+    } else {
+      LOG.info("Nothing to push");
+    }
+    
+    if (!pullNoteIDs.isEmpty()) {
+      LOG.info("Notes with the following IDs will be pulled");
+      for (String id : pullNoteIDs) {
+        LOG.info("ID : " + id);
+      }
+      pushNotes(pullNoteIDs, destRepo, sourceRepo);
+    } else {
+      LOG.info("Nothing to pull");
+    }
+    
+    LOG.info("Sync ended");
+  }
+
+  public void sync() throws IOException {
+    sync(0, 1);
+  }
+  
+  private void pushNotes(List<String> ids, NotebookRepo localRepo,
+                            NotebookRepo remoteRepo) throws IOException {
+    for (String id : ids) {
+      remoteRepo.save(localRepo.get(id));
+    }
+  }
+
+  int getRepoCount() {
+    return repos.size();
+  }
+  
+  int getMaxRepoNum() {
+    return maxRepoNum;
+  }
+
+  private NotebookRepo getRepo(int repoIndex) throws IOException {
+    if (repoIndex < 0 || repoIndex >= getRepoCount()) {
+      throw new IOException("Storage repo index is out of range");
+    }
+    return repos.get(repoIndex);
+  }
+  
+  private Map<String, List<String>> notesCheckDiff(List <NoteInfo> sourceNotes,
+                                                   NotebookRepo sourceRepo,
+                                                   List <NoteInfo> destNotes,
+                                                   NotebookRepo destRepo) throws IOException
{
+    List <String> pushIDs = new ArrayList<String>();
+    List <String> pullIDs = new ArrayList<String>();
+    
+    NoteInfo dnote;
+    Date sdate, ddate;
+    for (NoteInfo snote : sourceNotes) {
+      dnote = containsID(destNotes, snote.getId());
+      if (dnote != null) {
+        /* note exists in source and destination storage systems */
+        sdate = lastModificationDate(sourceRepo.get(snote.getId()));
+        ddate = lastModificationDate(destRepo.get(dnote.getId()));
+        if (sdate.after(ddate)) {
+          /* source contains more up to date note - push */
+          pushIDs.add(snote.getId());
+          LOG.info("Modified note is added to push list : " + sdate);
+        } else if (sdate.compareTo(ddate) != 0) {
+          /* destination contains more up to date note - pull */
+          LOG.info("Modified note is added to pull list : " + ddate);
+          pullIDs.add(snote.getId());
+        }
+      } else {
+        /* note exists in source storage, and absent in destination
+         * view source as up to date - push
+         * (another scenario : note was deleted from destination - not considered)*/
+        pushIDs.add(snote.getId());
+      }
+    }
+    
+    for (NoteInfo note : destNotes) {
+      dnote = containsID(sourceNotes, note.getId());
+      if (dnote == null) {
+        /* note exists in destination storage, and absent in source - pull*/
+        pullIDs.add(note.getId());
+      }
+    }
+    
+    Map<String, List<String>> map = new HashMap<String, List<String>>();
+    map.put(pushKey, pushIDs);
+    map.put(pullKey, pullIDs);
+    return map;
+  }
+
+  private NoteInfo containsID(List <NoteInfo> notes, String id) { 
+    for (NoteInfo note : notes) {
+      if (note.getId().equals(id)) {
+        return note;
+      }
+    }
+    return null;
+  }
+  /**
+   * checks latest modification date based on Paragraph fields
+   * @return -Date
+   */
+  private Date lastModificationDate(Note note) {
+    Date latest = new Date(0L);
+    Date tempCreated, tempStarted, tempFinished;
+    
+    for (Paragraph paragraph : note.getParagraphs()) {
+      tempCreated = paragraph.getDateCreated();
+      tempStarted = paragraph.getDateStarted();
+      tempFinished = paragraph.getDateFinished();
+
+      if (tempCreated != null && tempCreated.after(latest)) {
+        latest = tempCreated;
+      }
+      if (tempStarted != null && tempStarted.after(latest)) {
+        latest = tempStarted;
+      }
+      if (tempFinished != null && tempFinished.after(latest)) {
+        latest = tempFinished;
+      }
+    }
+    return latest;
+  }
+  
+  private void printParagraphs(Note note) {
+    LOG.info("Note name :  " + note.getName());
+    LOG.info("Note ID :  " + note.id());
+    for (Paragraph p : note.getParagraphs()) {
+      printParagraph(p);
+    }
+  }
+  
+  private void printParagraph(Paragraph paragraph) {
+    LOG.info("Date created :  " + paragraph.getDateCreated());
+    LOG.info("Date started :  " + paragraph.getDateStarted());
+    LOG.info("Date finished :  " + paragraph.getDateFinished());
+    LOG.info("Paragraph ID : " + paragraph.getId());
+    LOG.info("Paragraph title : " + paragraph.getTitle());
+  }
+  
+  private void printNoteInfos(List <NoteInfo> notes) {
+    LOG.info("The following is a list of note infos");
+    for (NoteInfo note : notes) {
+      printNoteInfo(note);
+    }
+  }
+
+  private void printNoteInfo(NoteInfo note) {
+    LOG.info("Note info of notebook with name : " + note.getName());
+    LOG.info("ID : " + note.getId());
+    Map<String, Object> configs = note.getConfig();
+    for (Map.Entry<String, Object> entry : configs.entrySet()) {
+      LOG.info("Config Key = " + entry.getKey() + "  , Value = " + 
+        entry.getValue().toString() + "of class " + entry.getClass());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
new file mode 100644
index 0000000..981ed6e
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterOption;
+import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
+import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
+import org.apache.zeppelin.notebook.JobListenerFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Notebook;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.scheduler.JobListener;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class NotebookRepoSyncTest implements JobListenerFactory{
+
+  private File mainZepDir;
+  private ZeppelinConfiguration conf;
+  private SchedulerFactory schedulerFactory;
+  private File mainNotebookDir;
+  private File secNotebookDir;
+  private Notebook notebookSync;
+  private NotebookRepoSync notebookRepoSync;
+  private InterpreterFactory factory;
+  private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
+  
+  @Before
+  public void setUp() throws Exception {
+    String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
+    mainZepDir = new File(zpath);
+    mainZepDir.mkdirs();
+    new File(mainZepDir, "conf").mkdirs();
+    String mainNotePath = zpath+"/notebook";
+    String secNotePath = mainNotePath + "_secondary";
+    mainNotebookDir = new File(mainNotePath);
+    secNotebookDir = new File(secNotePath);
+    mainNotebookDir.mkdirs();
+    secNotebookDir.mkdirs();
+
+    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
+    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+    System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2");
+    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock");
+    LOG.info("main Note dir : " + mainNotePath);
+    LOG.info("secondary note dir : " + secNotePath);
+    conf = ZeppelinConfiguration.create();
+
+    this.schedulerFactory = new SchedulerFactory();
+
+    MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
+    MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
+
+    factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
+    
+    notebookRepoSync = new NotebookRepoSync(conf);
+    notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    delete(mainZepDir);
+  }
+  
+  @Test
+  public void testRepoCount() throws IOException {
+    assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
+  }
+  
+  @Test
+  public void testSyncOnCreate() throws IOException {
+    /* check that both storage systems are empty */
+    assertTrue(notebookRepoSync.getRepoCount() > 1);
+    assertEquals(0, notebookRepoSync.list(0).size());
+    assertEquals(0, notebookRepoSync.list(1).size());
+    
+    /* create note */
+    Note note = notebookSync.createNote();
+
+    // check that automatically saved on both storages
+    assertEquals(1, notebookRepoSync.list(0).size());
+    assertEquals(1, notebookRepoSync.list(1).size());
+    assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
+    
+  }
+
+  @Test
+  public void testSyncOnDelete() throws IOException {
+    /* create note */
+    assertTrue(notebookRepoSync.getRepoCount() > 1);
+    assertEquals(0, notebookRepoSync.list(0).size());
+    assertEquals(0, notebookRepoSync.list(1).size());
+    
+    Note note = notebookSync.createNote();
+
+    /* check that created in both storage systems */
+    assertEquals(1, notebookRepoSync.list(0).size());
+    assertEquals(1, notebookRepoSync.list(1).size());
+    assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
+    
+    /* remove Note */
+    notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId());
+    
+    /* check that deleted in both storages */
+    assertEquals(0, notebookRepoSync.list(0).size());
+    assertEquals(0, notebookRepoSync.list(1).size());
+    
+  }
+  
+  @Test
+  public void testSyncUpdateMain() throws IOException {
+    
+    /* create note */
+    Note note = notebookSync.createNote();
+    Paragraph p1 = note.addParagraph();
+    p1.setText("hello world");
+    
+    /* new paragraph exists in note instance */
+    assertEquals(1, note.getParagraphs().size());
+    
+    /* new paragraph not yet saved into storages */
+    assertEquals(0, notebookRepoSync.get(0,
+        notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
+    assertEquals(0, notebookRepoSync.get(1,
+        notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
+    
+    /* save to storage under index 0 (first storage) */ 
+    notebookRepoSync.save(0, note);
+    
+    /* check paragraph saved to first storage */
+    assertEquals(1, notebookRepoSync.get(0,
+        notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
+    /* check paragraph isn't saved to second storage */
+    assertEquals(0, notebookRepoSync.get(1,
+        notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
+    /* apply sync */
+    notebookRepoSync.sync();
+    /* check whether added to second storage */
+    assertEquals(1, notebookRepoSync.get(1,
+    notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
+    /* check whether same paragraph id */
+    assertEquals(p1.getId(), notebookRepoSync.get(0,
+        notebookRepoSync.list(0).get(0).getId()).getLastParagraph().getId());
+    assertEquals(p1.getId(), notebookRepoSync.get(1,
+        notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId());
+  }
+  
+  private void delete(File file){
+    if(file.isFile()) file.delete();
+      else if(file.isDirectory()){
+        File [] files = file.listFiles();
+        if(files!=null && files.length>0){
+          for(File f : files){
+            delete(f);
+          }
+        }
+        file.delete();
+      }
+    
+  }
+  @Override
+  public JobListener getParagraphJobListener(Note note) {
+    return new JobListener(){
+
+      @Override
+      public void onProgressUpdate(Job job, int progress) {
+      }
+
+      @Override
+      public void beforeStatusChange(Job job, Status before, Status after) {
+      }
+
+      @Override
+      public void afterStatusChange(Job job, Status before, Status after) {
+      }
+    };
+  }
+	
+}

http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/b71f04eb/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
new file mode 100644
index 0000000..dd4e513
--- /dev/null
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
@@ -0,0 +1,26 @@
+package org.apache.zeppelin.notebook.repo.mock;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.commons.vfs2.VFS;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
+
+public class VFSNotebookRepoMock extends VFSNotebookRepo {
+  
+  private static ZeppelinConfiguration modifyNotebookDir(ZeppelinConfiguration conf) {
+    String secNotebookDir = conf.getNotebookDir() + "_secondary";
+    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir);
+    ZeppelinConfiguration secConf = ZeppelinConfiguration.create();
+    return secConf;
+  }
+
+  public VFSNotebookRepoMock(ZeppelinConfiguration conf) throws IOException {
+    super(modifyNotebookDir(conf));
+  }
+  
+}


Mime
View raw message