cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "deng (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (CASSANDRA-3118) nodetool can not decommission a node
Date Mon, 05 Sep 2011 03:43:10 GMT

    [ https://issues.apache.org/jira/browse/CASSANDRA-3118?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13096995#comment-13096995
] 

deng commented on CASSANDRA-3118:
---------------------------------

I changed the code and added some debugg infomation,the code is:
	public static void calculatePendingRanges(
			AbstractReplicationStrategy strategy, String table) {
		logger_.debug("Calculating pending ranges for {} with {}", table, strategy);
		
		TokenMetadata tm = StorageService.instance.getTokenMetadata();
		Multimap<Range, InetAddress> pendingRanges = HashMultimap.create();
		Map<Token, InetAddress> bootstrapTokens = tm.getBootstrapTokens();
		Set<InetAddress> leavingEndpoints = tm.getLeavingEndpoints();
        for(InetAddress leave:leavingEndpoints){
        	System.out.println("leavingEndpoints===="+leave.getHostAddress());
        }
		
		if (bootstrapTokens.isEmpty() && leavingEndpoints.isEmpty()
				&& tm.getMovingEndpoints().isEmpty()) {
			if (logger_.isDebugEnabled())
				logger_.debug(
						"No bootstrapping, leaving or moving nodes -> empty pending ranges for {}",
						table);
			tm.setPendingRanges(table, pendingRanges);
			return;
		}

		Multimap<InetAddress, Range> addressRanges = strategy
				.getAddressRanges();

		// Copy of metadata reflecting the situation after all leave operations
		// are finished.
		TokenMetadata allLeftMetadata = tm.cloneAfterAllLeft();

		// get all ranges that will be affected by leaving nodes
		Set<Range> affectedRanges = new HashSet<Range>();
		for (InetAddress endpoint : leavingEndpoints)
			affectedRanges.addAll(addressRanges.get(endpoint));

		// for each of those ranges, find what new nodes will be responsible for
		// the range when
		// all leaving nodes are gone.
		for (Range range : affectedRanges) {
			Collection<InetAddress> currentEndpoints = strategy
					.calculateNaturalEndpoints(range.right, tm);
			Collection<InetAddress> newEndpoints = strategy
					.calculateNaturalEndpoints(range.right, allLeftMetadata);
			System.out.println("olddAddressSize===="+currentEndpoints.size());
			for (InetAddress olddAddress : currentEndpoints) {
				System.out.println("OLDAddress+++++"
						+ olddAddress.getHostAddress()
						+ olddAddress.getHostName());
			}
			
			System.out.println("newEndpointsSize===="+newEndpoints.size());
			for (InetAddress newAddress : newEndpoints) {
				System.out.println("NEWAddress+++++"
						+ newAddress.getHostAddress()
						+ newAddress.getHostName());
			}
			newEndpoints.removeAll(currentEndpoints);
			pendingRanges.putAll(range, newEndpoints);
		}

		// At this stage pendingRanges has been updated according to leave
		// operations. We can
		// now continue the calculation by checking bootstrapping nodes.

		// For each of the bootstrapping nodes, simply add and remove them one
		// by one to
		// allLeftMetadata and check in between what their ranges would be.
		for (Map.Entry<Token, InetAddress> entry : bootstrapTokens.entrySet()) {
			InetAddress endpoint = entry.getValue();

			allLeftMetadata.updateNormalToken(entry.getKey(), endpoint);
			for (Range range : strategy.getAddressRanges(allLeftMetadata).get(
					endpoint))
				pendingRanges.put(range, endpoint);
			allLeftMetadata.removeEndpoint(endpoint);
		}

		// At this stage pendingRanges has been updated according to leaving and
		// bootstrapping nodes.
		// We can now finish the calculation by checking moving nodes.

		// For each of the moving nodes, we do the same thing we did for
		// bootstrapping:
		// simply add and remove them one by one to allLeftMetadata and check in
		// between what their ranges would be.
		for (Pair<Token, InetAddress> moving : tm.getMovingEndpoints()) {
			InetAddress endpoint = moving.right; // address of the moving node

			// moving.left is a new token of the endpoint
			allLeftMetadata.updateNormalToken(moving.left, endpoint);

			for (Range range : strategy.getAddressRanges(allLeftMetadata).get(
					endpoint)) {
				pendingRanges.put(range, endpoint);
			}

			allLeftMetadata.removeEndpoint(endpoint);
		}

		tm.setPendingRanges(table, pendingRanges);

		if (logger_.isDebugEnabled())
			logger_.debug("Pending ranges:\n"
					+ (pendingRanges.isEmpty() ? "<empty>" : tm
							.printPendingRanges()));
	}

I run the commad decommission and the error is:
[root@devapp3 bin]# ./nodetool -h10.86.17.9 decommission
Exception in thread "main" java.lang.UnsupportedOperationException
        at java.util.AbstractList.remove(AbstractList.java:144)
        at java.util.AbstractList$Itr.remove(AbstractList.java:360)
        at java.util.AbstractCollection.removeAll(AbstractCollection.java:337)
        at org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1065)
        at org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1006)
        at org.apache.cassandra.service.StorageService.handleStateLeaving(StorageService.java:877)
        at org.apache.cassandra.service.StorageService.onChange(StorageService.java:732)
        at org.apache.cassandra.gms.Gossiper.doNotifications(Gossiper.java:839)
        at org.apache.cassandra.gms.Gossiper.addLocalApplicationState(Gossiper.java:986)
        at org.apache.cassandra.service.StorageService.startLeaving(StorageService.java:1866)
        at org.apache.cassandra.service.StorageService.decommission(StorageService.java:1885)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
        at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
        at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
        at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
        at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
        at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
        at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
        at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1426)
        at javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
        at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)
        at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1359)
        at javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
        at sun.rmi.transport.Transport$1.run(Transport.java:159)
        at java.security.AccessController.doPrivileged(Native Method)
        at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
        at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
        at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
        at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)

I saw the  log:
DEBUG 11:38:37,513 DECOMMISSIONING
DEBUG 11:38:37,528 Node /10.86.17.9 state leaving, token 14668421489936677549167
5769537457222604
DEBUG 11:38:37,528 Calculating pending ranges for edl with org.apache.cassandra.
locator.SimpleStrategy@7f8062
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
 INFO 11:38:38,872 Started hinted handoff for endpoint /10.86.12.224
 INFO 11:38:38,872 Finished hinted handoff of 0 rows to endpoint /10.86.12.224
DEBUG 11:38:38,872 Checking remote schema before delivering hints
DEBUG 11:38:38,872 schema for /172.28.16.127 matches local schema
DEBUG 11:38:38,872 Sleeping 8676ms to stagger hint delivery
NEWAddress+++++172.28.16.126172.28.16.126
DEBUG 11:38:42,044 Pending ranges:
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]

DEBUG 11:38:42,044 Calculating pending ranges for Staging with org.apache.cassan
dra.locator.SimpleStrategy@181b3d4
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
NEWAddress+++++172.28.16.126172.28.16.126
DEBUG 11:38:42,060 Pending ranges:
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]
172.28.16.126/172.28.16.126:(138871267011481555303305975400762244971,14668421489
9366775491675769537457222604]

DEBUG 11:38:42,075 Calculating pending ranges for test1 with org.apache.cassandr
a.locator.LocalStrategy@45378f
leavingEndpoints====10.86.17.9
olddAddressSize====1
OLDAddress+++++10.86.17.9szpc662G
newEndpointsSize====1
NEWAddress+++++10.86.17.9szpc662G



please check!!!!

> nodetool  can not  decommission a node
> --------------------------------------
>
>                 Key: CASSANDRA-3118
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-3118
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Tools
>    Affects Versions: 0.8.4
>         Environment: Cassandra0.84
>            Reporter: deng
>             Fix For: 0.8.5
>
>         Attachments: 3118-debug.txt
>
>
> when i use nodetool ring and get the result ,and than i want to decommission 100.86.17.90
 node ,but i get the error:
> [root@ip bin]# ./nodetool -h10.86.12.225 ring
> Address         DC          Rack        Status State   Load            Owns    Token
                                      
>                                                                                154562542458917734942660802527609328132
    
> 100.86.17.90      datacenter1 rack1       Up     Leaving 1.08 MB         11.21%  3493450320433654773610109291263389161
      
> 100.86.12.225    datacenter1 rack1       Up     Normal  558.25 MB       14.25%  27742979166206700793970535921354744095
     
> 100.86.12.224    datacenter1 rack1       Up     Normal  5.01 GB         6.58%   38945137636148605752956920077679425910
     
> ERROR:
> root@ip bin]# ./nodetool -h100.86.17.90 decommission
> Exception in thread "main" java.lang.UnsupportedOperationException
>         at java.util.AbstractList.remove(AbstractList.java:144)
>         at java.util.AbstractList$Itr.remove(AbstractList.java:360)
>         at java.util.AbstractCollection.removeAll(AbstractCollection.java:337)
>         at org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1041)
>         at org.apache.cassandra.service.StorageService.calculatePendingRanges(StorageService.java:1006)
>         at org.apache.cassandra.service.StorageService.handleStateLeaving(StorageService.java:877)
>         at org.apache.cassandra.service.StorageService.onChange(StorageService.java:732)
>         at org.apache.cassandra.gms.Gossiper.doNotifications(Gossiper.java:839)
>         at org.apache.cassandra.gms.Gossiper.addLocalApplicationState(Gossiper.java:986)
>         at org.apache.cassandra.service.StorageService.startLeaving(StorageService.java:1836)
>         at org.apache.cassandra.service.StorageService.decommission(StorageService.java:1855)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:597)
>         at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:93)
>         at com.sun.jmx.mbeanserver.StandardMBeanIntrospector.invokeM2(StandardMBeanIntrospector.java:27)
>         at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:208)
>         at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:120)
>         at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:262)
>         at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:836)
>         at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:761)
>         at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1426)
>         at javax.management.remote.rmi.RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)
>         at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)
>         at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1359)
>         at javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:788)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:597)
>         at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:305)
>         at sun.rmi.transport.Transport$1.run(Transport.java:159)
>         at java.security.AccessController.doPrivileged(Native Method)
>         at sun.rmi.transport.Transport.serviceCall(Transport.java:155)
>         at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:535)
>         at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:790)
>         at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:649)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>         at java.lang.Thread.run(Thread.java:619)

--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira

        

Mime
View raw message