hdt-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rsha...@apache.org
Subject [07/27] git commit: HDT-50: Adding HDFS site if found - Minor tweaks to the hadoop perspective - Modified create/open of HDFS project in WorkspaceJob - Modified RunOnHadoop to add HDFS site. Show error in dialog if found. - Adding timeoutcheck to MR clus
Date Thu, 26 Jun 2014 08:36:30 GMT
HDT-50: Adding HDFS site if found - Minor tweaks to the hadoop perspective - Modified create/open
of HDFS project in WorkspaceJob - Modified RunOnHadoop to add HDFS site. Show error in dialog
if found. - Adding timeoutcheck to MR cluster


Project: http://git-wip-us.apache.org/repos/asf/incubator-hdt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hdt/commit/00646ae0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hdt/tree/00646ae0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hdt/diff/00646ae0

Branch: refs/heads/hadoop-eclipse-merge
Commit: 00646ae0b3e502dda9ec3a4ddb5484412cf709ef
Parents: 0e9e729
Author: Rahul Sharma <rsharma@apache.org>
Authored: Tue May 13 17:00:53 2014 +0530
Committer: Rahul Sharma <rsharma@apache.org>
Committed: Fri May 23 08:44:02 2014 +0530

----------------------------------------------------------------------
 .../hdt/core/internal/hdfs/HDFSManager.java     | 61 +++++++++++++++++---
 .../hdt/core/launch/AbstractHadoopCluster.java  |  4 ++
 .../hdt/hadoop/release/HadoopCluster.java       | 33 +++++++++++
 org.apache.hdt.ui/plugin.xml                    | 14 ++---
 .../hdt/ui/internal/hdfs/NewHDFSWizard.java     | 21 ++-----
 .../internal/launch/HadoopLocationWizard.java   | 43 +++++++++++++-
 .../hdt/ui/internal/launch/ServerRegistry.java  | 53 ++++++++++++-----
 7 files changed, 184 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java b/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
index 93f0696..5897cea 100644
--- a/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
+++ b/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
@@ -18,6 +18,7 @@
 
 package org.apache.hdt.core.internal.hdfs;
 
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -37,13 +38,19 @@ import org.eclipse.core.resources.IResource;
 import org.eclipse.core.resources.IWorkspace;
 import org.eclipse.core.resources.IWorkspaceRoot;
 import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.resources.WorkspaceJob;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.SubProgressMonitor;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.emf.common.util.EList;
+import org.eclipse.jface.dialogs.MessageDialog;
+import org.eclipse.swt.widgets.Display;
 import org.eclipse.team.core.RepositoryProvider;
 
 /**
@@ -176,14 +183,39 @@ public class HDFSManager {
 	 * @return
 	 * @throws CoreException
 	 */
-	private IProject createIProject(String name, java.net.URI hdfsURI) throws CoreException
{
+	private IProject createIProject(String name, final java.net.URI hdfsURI) {
 		final IWorkspace workspace = ResourcesPlugin.getWorkspace();
-		IProject project = workspace.getRoot().getProject(name);
-		IProjectDescription pd = workspace.newProjectDescription(name);
-		pd.setLocationURI(hdfsURI);
-		project.create(pd, new NullProgressMonitor());
-		project.open(new NullProgressMonitor());
-		RepositoryProvider.map(project, HDFSTeamRepositoryProvider.ID);
+		final IProject project = workspace.getRoot().getProject(name);
+		final IProjectDescription pd = workspace.newProjectDescription(name);
+	    WorkspaceJob operation = new WorkspaceJob("Adding HDFS Location") {
+
+			@Override
+			public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+				monitor.beginTask("Creating Project", 100);
+				try {
+					pd.setLocationURI(hdfsURI);
+					project.create(pd, new SubProgressMonitor(monitor, 70));
+					project.open(IResource.BACKGROUND_REFRESH, new SubProgressMonitor(monitor, 30));
+					RepositoryProvider.map(project, HDFSTeamRepositoryProvider.ID);
+					return Status.OK_STATUS;
+				} catch (final CoreException e) {
+					logger.error("error found in creating HDFS site", e);
+					Display.getDefault().syncExec(new Runnable(){
+						public void run(){
+							MessageDialog.openError(Display.getDefault().getActiveShell(), 
+									"HDFS Error", "Unable to create HDFS site :"+e.getMessage());
+						}
+					});
+					return e.getStatus();
+				} finally {
+					monitor.done();
+				}
+			}
+		   };
+		operation.setPriority(Job.LONG);
+		operation.setUser(true);
+		operation.setRule(project);
+		operation.schedule();
 		return project;
 	}
 
@@ -204,6 +236,8 @@ public class HDFSManager {
 		}
 		return uriToServerCacheMap.get(uri);
 	}
+	
+	
 
 	public String getProjectName(HDFSServer server) {
 		return serverToProjectMap.get(server);
@@ -282,4 +316,17 @@ public class HDFSManager {
 			return hdfsClientsMap.get(serverURI);
 		}
 	}
+	
+	public static org.eclipse.core.runtime.IStatus addServer(String serverName, String location,
String userId, List<String> groupId) {
+		try {
+			HDFSManager.INSTANCE.createServer(serverName, new URI(location), userId, groupId);
+		} catch (CoreException e) {
+			logger.warn(e.getMessage(), e);
+			return e.getStatus();
+		} catch (URISyntaxException e) {
+			logger.warn(e.getMessage(), e);
+			return new Status(Status.ERROR,"unknown",e.getMessage(),e);
+		}
+		return Status.OK_STATUS;
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
b/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
index 782a89c..cd06f0e 100644
--- a/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
+++ b/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
@@ -64,6 +64,8 @@ public abstract class AbstractHadoopCluster {
 
 	abstract public boolean loadFromXML(File file) throws IOException;
 	
+	abstract public boolean isAvailable() throws CoreException;
+	
 	public static AbstractHadoopCluster createCluster(File file) throws CoreException, IOException
{
 		AbstractHadoopCluster hadoopCluster = createCluster();
 		hadoopCluster.loadFromXML(file);
@@ -81,4 +83,6 @@ public abstract class AbstractHadoopCluster {
 		return hadoopCluster;
 	}
 
+	
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
b/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
index 466739b..67fcb75 100644
--- a/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
+++ b/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
@@ -31,6 +31,13 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.logging.Logger;
 
 import javax.xml.parsers.DocumentBuilder;
@@ -52,6 +59,8 @@ import org.apache.hdt.core.launch.IHadoopJob;
 import org.apache.hdt.core.launch.IJarModule;
 import org.apache.hdt.core.launch.IJobListener;
 import org.eclipse.core.internal.utils.FileUtil;
+import org.eclipse.core.resources.WorkspaceJob;
+import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.IStatus;
 import org.eclipse.core.runtime.Status;
@@ -81,6 +90,7 @@ import org.xml.sax.SAXException;
  */
 
 public class HadoopCluster extends AbstractHadoopCluster {
+	private ExecutorService service= Executors.newSingleThreadExecutor();
 
 	/**
 	 * Frequency of location status observations expressed as the delay in ms
@@ -566,4 +576,27 @@ public class HadoopCluster extends AbstractHadoopCluster {
 		}
 
 	}
+
+	/* (non-Javadoc)
+	 * @see org.apache.hdt.core.launch.AbstractHadoopCluster#isAvailable()
+	 */
+	@Override
+	public boolean isAvailable() throws CoreException {
+		Callable<JobClient> task= new Callable<JobClient>() {
+			
+			@Override
+			public JobClient call() throws Exception {
+				return getJobClient();
+			}
+		}; 
+		Future<JobClient> jobClientFuture = service.submit(task);
+		try{
+			JobClient jobClient = jobClientFuture.get(5, TimeUnit.SECONDS);
+			return jobClient!=null;
+		}catch(Exception e){
+			e.printStackTrace();
+			throw new CoreException(new Status(Status.ERROR, 
+					Activator.BUNDLE_ID, "unable to connect to server", e));
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/plugin.xml
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/plugin.xml b/org.apache.hdt.ui/plugin.xml
index 7bc5b47..af68ffa 100644
--- a/org.apache.hdt.ui/plugin.xml
+++ b/org.apache.hdt.ui/plugin.xml
@@ -39,31 +39,31 @@
       <perspectiveExtension
             targetID="org.apache.hdt.ui.perspective">
          <view
-               id="org.apache.hdt.ui.view.servers"
+               id="org.eclipse.ui.navigator.ProjectExplorer"
                minimized="false"
                ratio="0.25"
                relationship="left"
                relative="org.eclipse.ui.editorss">
          </view>
          <view
-               id="org.eclipse.ui.navigator.ProjectExplorer"
+               id="org.apache.hdt.ui.view.servers"
                minimized="false"
                ratio="0.5"
                relationship="bottom"
-               relative="org.apache.hdt.ui.view.servers">
+               relative="org.eclipse.ui.navigator.ProjectExplorer">
          </view>
          <view
-               id="org.eclipse.ui.views.PropertySheet"
+               id="org.apache.hdt.ui.ClusterView"
                minimized="false"
                ratio="0.66"
                relationship="bottom"
                relative="org.eclipse.ui.editorss">
          </view>
           <view
-                id=" org.apache.hdt.ui.ClusterView"
+                id="org.eclipse.ui.views.PropertySheet"
                 minimized="false"
                 relationship="stack"
-                relative="org.apache.hdt.ui.view.servers">
+                relative="org.apache.hdt.ui.ClusterView">
          </view>
          <newWizardShortcut
                id="org.apache.hdt.ui.wizard.newProjectWizard"/>
@@ -441,7 +441,7 @@
             class="org.eclipse.ui.navigator.CommonNavigator"
             icon="icons/hadoop-logo-16x16.png"
             id="org.apache.hdt.ui.view.servers"
-            name="Hadoop Servers"
+            name="ZooKeeper Servers"
             restorable="true">
       </view>
       <view

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
index 545ea3a..4cd40de 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
@@ -17,14 +17,9 @@
  */
 package org.apache.hdt.ui.internal.hdfs;
 
-import java.net.URI;
-import java.net.URISyntaxException;
-
 import org.apache.hdt.core.internal.hdfs.HDFSManager;
 import org.apache.hdt.ui.Activator;
 import org.apache.log4j.Logger;
-import org.eclipse.core.runtime.CoreException;
-import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.viewers.IStructuredSelection;
@@ -73,17 +68,9 @@ public class NewHDFSWizard extends Wizard implements INewWizard {
 
 				Job j = new Job("Creating HDFS project [" + serverLocationWizardPage.getHdfsServerName()
+ "]") {
 					protected org.eclipse.core.runtime.IStatus run(org.eclipse.core.runtime.IProgressMonitor
monitor) {
-						try {
-							HDFSManager.INSTANCE.createServer(serverLocationWizardPage.getHdfsServerName(), new
URI(serverLocationWizardPage
-									.getHdfsServerLocation()), serverLocationWizardPage.isOverrideDefaultSecurity()
? serverLocationWizardPage.getUserId()
-									: null, serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getGroupIds()
: null);
-						} catch (CoreException e) {
-							logger.warn(e.getMessage(), e);
-							return e.getStatus();
-						} catch (URISyntaxException e) {
-							logger.warn(e.getMessage(), e);
-						}
-						return Status.OK_STATUS;
+						return HDFSManager.addServer(serverLocationWizardPage.getHdfsServerName(),serverLocationWizardPage.getHdfsServerLocation(),
+								serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getUserId()
: null,
+								serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getGroupIds()
: null);
 					};
 				};
 				j.schedule();
@@ -93,4 +80,6 @@ public class NewHDFSWizard extends Wizard implements INewWizard {
 		return false;
 	}
 
+	
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
index 4f11128..3757c05 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
@@ -28,8 +28,12 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.apache.hdt.core.launch.ConfProp;
+import org.apache.hdt.core.internal.hdfs.HDFSManager;
 import org.apache.hdt.core.launch.AbstractHadoopCluster;
+import org.apache.hdt.core.launch.ConfProp;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
 import org.eclipse.core.runtime.CoreException;
 import org.eclipse.jface.dialogs.IMessageProvider;
 import org.eclipse.jface.wizard.WizardPage;
@@ -118,17 +122,49 @@ public class HadoopLocationWizard extends WizardPage {
 	public AbstractHadoopCluster performFinish() {
 		try {
 			if (this.original == null) {
+				Display.getDefault().syncExec(new Runnable() {
+					public void run() {
+						HDFSManager.addServer(location.getLocationName(),
+								location.getConfProp(ConfProp.FS_DEFAULT_URI), location
+								.getConfProp(ConfProp.PI_USER_NAME), null);
+					}
+				});
 				// New location
 				Display.getDefault().syncExec(new Runnable() {
 					public void run() {
-						ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location);
+							ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location);
 					}
 				});
 				return this.location;
 
 			} else {
+				
 				// Update location
 				final String originalName = this.original.getLocationName();
+				final String originalLoc = this.original.getConfProp(ConfProp.FS_DEFAULT_URI);
+				final String newName = this.location.getLocationName();
+				final String newLoc = this.location.getConfProp(ConfProp.FS_DEFAULT_URI);
+				
+				if (!originalName.equals(newName) || !originalLoc.equals(newLoc)){
+					IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+					final IProject project = root.getProject(originalName);
+					
+					Display.getDefault().syncExec(new Runnable() {
+						public void run() {
+							if(project.exists()){
+								try {
+									project.close(null);
+									project.delete(true, null);
+								} catch (CoreException e) {
+									e.printStackTrace();
+								}
+							}
+							HDFSManager.addServer(location.getLocationName(),
+									location.getConfProp(ConfProp.FS_DEFAULT_URI), location
+									.getConfProp(ConfProp.PI_USER_NAME), null);
+						}
+					});
+				}
 				this.original.load(this.location);
 
 				Display.getDefault().syncExec(new Runnable() {
@@ -139,6 +175,9 @@ public class HadoopLocationWizard extends WizardPage {
 				return this.original;
 
 			}
+			
+			
+			
 		} catch (Exception e) {
 			e.printStackTrace();
 			setMessage("Invalid server location values", IMessageProvider.ERROR);

http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
index 785286c..f03fb50 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
@@ -33,6 +33,12 @@ import javax.security.auth.login.Configuration;
 import org.apache.hdt.core.launch.AbstractHadoopCluster;
 import org.apache.hdt.core.launch.IHadoopClusterListener;
 import org.apache.hdt.ui.Activator;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.resources.WorkspaceJob;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.jface.dialogs.MessageDialog;
 
 /**
@@ -173,10 +179,22 @@ public class ServerRegistry {
 		fireListeners(server, SERVER_REMOVED);
 	}
 
-	public synchronized void addServer(AbstractHadoopCluster server) {
-		this.servers.put(server.getLocationName(), server);
-		store();
-		fireListeners(server, SERVER_ADDED);
+	public synchronized void addServer(final AbstractHadoopCluster server) {
+		WorkspaceJob job= new WorkspaceJob("Adding Hadoop Server") {
+			@Override
+			public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+				if(server.isAvailable()){
+					servers.put(server.getLocationName(), server);
+					store();
+					fireListeners(server, SERVER_ADDED);
+				}
+			return org.eclipse.core.runtime.Status.OK_STATUS;
+		}};
+		
+		job.setPriority(Job.LONG);
+		job.setRule(ResourcesPlugin.getWorkspace().getRoot());
+		job.setUser(true);
+		job.schedule();
 	}
 
 	/**
@@ -187,14 +205,23 @@ public class ServerRegistry {
 	 * @param server
 	 *            the location
 	 */
-	public synchronized void updateServer(String originalName, AbstractHadoopCluster server)
{
-
-		// Update the map if the location name has changed
-		if (!server.getLocationName().equals(originalName)) {
-			servers.remove(originalName);
-			servers.put(server.getLocationName(), server);
-		}
-		store();
-		fireListeners(server, SERVER_STATE_CHANGED);
+	public synchronized void updateServer(final String originalName, final AbstractHadoopCluster
server) {
+		WorkspaceJob job= new WorkspaceJob("Updating  Hadoop Server") {
+			@Override
+			public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+				// Update the map if the location name has changed
+				if (!server.getLocationName().equals(originalName) && server.isAvailable()) {
+					servers.remove(originalName);
+					servers.put(server.getLocationName(), server);
+					store();
+					fireListeners(server, SERVER_STATE_CHANGED);
+				}
+				
+				return org.eclipse.core.runtime.Status.OK_STATUS;
+			}};
+		
+		job.setPriority(Job.LONG);
+		job.setUser(true);
+		job.schedule();
 	}
 }


Mime
View raw message