Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6A078200C3E for ; Tue, 21 Mar 2017 17:36:00 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 6882F160B9C; Tue, 21 Mar 2017 16:36:00 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D7331160B6E for ; Tue, 21 Mar 2017 17:35:58 +0100 (CET) Received: (qmail 10100 invoked by uid 500); 21 Mar 2017 16:35:58 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 10087 invoked by uid 99); 21 Mar 2017 16:35:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Mar 2017 16:35:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DAC9DDFE8F; Tue, 21 Mar 2017 16:35:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: misty@apache.org To: commits@hbase.apache.org Date: Tue, 21 Mar 2017 16:35:57 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [01/52] [partial] hbase-site git commit: Published site at 1cfd22bf43c9b64afae35d9bf16f764d0da80cab. archived-at: Tue, 21 Mar 2017 16:36:00 -0000 Repository: hbase-site Updated Branches: refs/heads/asf-site c6ddb98fc -> fadf6d5a0 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/22cff34f/apidocs/src-html/org/apache/hadoop/hbase/util/RegionMover.html ---------------------------------------------------------------------- diff --git a/apidocs/src-html/org/apache/hadoop/hbase/util/RegionMover.html b/apidocs/src-html/org/apache/hadoop/hbase/util/RegionMover.html index 27b58df..c28cd5c 100644 --- a/apidocs/src-html/org/apache/hadoop/hbase/util/RegionMover.html +++ b/apidocs/src-html/org/apache/hadoop/hbase/util/RegionMover.html @@ -405,7 +405,7 @@ 397 LOG.info("Moving " + regionsToMove.size() + " regions to " + server + " using " 398 + this.maxthreads + " threads.Ack mode:" + this.ack); 399 ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); -400 List<Future<Boolean>> taskList = new ArrayList<Future<Boolean>>(); +400 List<Future<Boolean>> taskList = new ArrayList<>(); 401 int counter = 0; 402 while (counter < regionsToMove.size()) { 403 HRegionInfo region = regionsToMove.get(counter); @@ -469,7 +469,7 @@ 461 justification="FB is wrong; its size is read") 462 private void unloadRegions(Admin admin, String server, ArrayList<String> regionServers, 463 boolean ack, List<HRegionInfo> movedRegions) throws Exception { -464 List<HRegionInfo> regionsToMove = new ArrayList<HRegionInfo>();// FindBugs: DLS_DEAD_LOCAL_STORE +464 List<HRegionInfo> regionsToMove = new ArrayList<>();// FindBugs: DLS_DEAD_LOCAL_STORE 465 regionsToMove = getRegions(this.conf, server); 466 if (regionsToMove.isEmpty()) { 467 LOG.info("No Regions to move....Quitting now"); @@ -489,7 +489,7 @@ 481 + regionServers.size() + " servers using " + this.maxthreads + " threads .Ack Mode:" 482 + ack); 483 ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); -484 List<Future<Boolean>> taskList = new ArrayList<Future<Boolean>>(); +484 List<Future<Boolean>> taskList = new ArrayList<>(); 485 int serverIndex = 0; 486 while (counter < regionsToMove.size()) { 487 if (ack) { @@ -644,7 +644,7 @@ 636 } 637 638 private List<HRegionInfo> readRegionsFromFile(String filename) throws IOException { -639 List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); +639 List<HRegionInfo> regions = new ArrayList<>(); 640 File f = new File(filename); 641 if (!f.exists()) { 642 return regions; @@ -766,7 +766,7 @@ 758 * @return List of servers from the exclude file in format 'hostname:port'. 759 */ 760 private ArrayList<String> readExcludes(String excludeFile) throws IOException { -761 ArrayList<String> excludeServers = new ArrayList<String>(); +761 ArrayList<String> excludeServers = new ArrayList<>(); 762 if (excludeFile == null) { 763 return excludeServers; 764 } else { @@ -829,184 +829,183 @@ 821 * @throws IOException 822 */ 823 private ArrayList<String> getServers(Admin admin) throws IOException { -824 ArrayList<ServerName> serverInfo = -825 new ArrayList<ServerName>(admin.getClusterStatus().getServers()); -826 ArrayList<String> regionServers = new ArrayList<String>(serverInfo.size()); -827 for (ServerName server : serverInfo) { -828 regionServers.add(server.getServerName()); -829 } -830 return regionServers; -831 } -832 -833 private void deleteFile(String filename) { -834 File f = new File(filename); -835 if (f.exists()) { -836 f.delete(); -837 } -838 } -839 -840 /** -841 * Tries to scan a row from passed region -842 * @param admin -843 * @param region -844 * @throws IOException -845 */ -846 private void isSuccessfulScan(Admin admin, HRegionInfo region) throws IOException { -847 Scan scan = new Scan(region.getStartKey()); -848 scan.setBatch(1); -849 scan.setCaching(1); -850 scan.setFilter(new FirstKeyOnlyFilter()); -851 try { -852 Table table = admin.getConnection().getTable(region.getTable()); -853 try { -854 ResultScanner scanner = table.getScanner(scan); -855 try { -856 scanner.next(); -857 } finally { -858 scanner.close(); -859 } -860 } finally { -861 table.close(); -862 } -863 } catch (IOException e) { -864 LOG.error("Could not scan region:" + region.getEncodedName(), e); -865 throw e; -866 } -867 } -868 -869 /** -870 * Returns true if passed region is still on serverName when we look at hbase:meta. -871 * @param admin -872 * @param region -873 * @param serverName -874 * @return true if region is hosted on serverName otherwise false -875 * @throws IOException -876 */ -877 private boolean isSameServer(Admin admin, HRegionInfo region, String serverName) -878 throws IOException { -879 String serverForRegion = getServerNameForRegion(admin, region); -880 if (serverForRegion != null && serverForRegion.equals(serverName)) { -881 return true; -882 } -883 return false; -884 } -885 -886 /** -887 * Get servername that is up in hbase:meta hosting the given region. this is hostname + port + -888 * startcode comma-delimited. Can return null -889 * @param admin -890 * @param region -891 * @return regionServer hosting the given region -892 * @throws IOException -893 */ -894 private String getServerNameForRegion(Admin admin, HRegionInfo region) throws IOException { -895 String server = null; -896 if (!admin.isTableEnabled(region.getTable())) { -897 return null; -898 } -899 if (region.isMetaRegion()) { -900 ZooKeeperWatcher zkw = new ZooKeeperWatcher(admin.getConfiguration(), "region_mover", null); -901 MetaTableLocator locator = new MetaTableLocator(); -902 int maxWaitInSeconds = -903 admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); -904 try { -905 server = locator.waitMetaRegionLocation(zkw, maxWaitInSeconds * 1000).toString() + ","; -906 } catch (InterruptedException e) { -907 LOG.error("Interrupted while waiting for location of Meta", e); -908 } finally { -909 if (zkw != null) { -910 zkw.close(); -911 } -912 } -913 } else { -914 Table table = admin.getConnection().getTable(TableName.META_TABLE_NAME); -915 try { -916 Get get = new Get(region.getRegionName()); -917 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); -918 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); -919 Result result = table.get(get); -920 if (result != null) { -921 byte[] servername = -922 result.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); -923 byte[] startcode = -924 result.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); -925 if (servername != null) { -926 server = -927 Bytes.toString(servername).replaceFirst(":", ",") + "," + Bytes.toLong(startcode); -928 } -929 } -930 } catch (IOException e) { -931 LOG.error("Could not get Server Name for region:" + region.getEncodedName(), e); -932 throw e; -933 } finally { -934 table.close(); -935 } -936 } -937 return server; -938 } -939 -940 @Override -941 protected void addOptions() { -942 this.addRequiredOptWithArg("r", "regionserverhost", "region server <hostname>|<hostname:port>"); -943 this.addRequiredOptWithArg("o", "operation", "Expected: load/unload"); -944 this.addOptWithArg("m", "maxthreads", -945 "Define the maximum number of threads to use to unload and reload the regions"); -946 this.addOptWithArg("x", "excludefile", -947 "File with <hostname:port> per line to exclude as unload targets; default excludes only " -948 + "target host; useful for rack decommisioning."); -949 this.addOptWithArg("f", "filename", -950 "File to save regions list into unloading, or read from loading; " -951 + "default /tmp/<usernamehostname:port>"); -952 this.addOptNoArg("n", "noack", -953 "Turn on No-Ack mode(default: false) which won't check if region is online on target " -954 + "RegionServer, hence best effort. This is more performant in unloading and loading " -955 + "but might lead to region being unavailable for some time till master reassigns it " -956 + "in case the move failed"); -957 this.addOptWithArg("t", "timeout", "timeout in seconds after which the tool will exit " -958 + "irrespective of whether it finished or not;default Integer.MAX_VALUE"); -959 } -960 -961 @Override -962 protected void processOptions(CommandLine cmd) { -963 String hostname = cmd.getOptionValue("r"); -964 rmbuilder = new RegionMoverBuilder(hostname); -965 if (cmd.hasOption('m')) { -966 rmbuilder.maxthreads(Integer.parseInt(cmd.getOptionValue('m'))); -967 } -968 if (cmd.hasOption('n')) { -969 rmbuilder.ack(false); -970 } -971 if (cmd.hasOption('f')) { -972 rmbuilder.filename(cmd.getOptionValue('f')); -973 } -974 if (cmd.hasOption('x')) { -975 rmbuilder.excludeFile(cmd.getOptionValue('x')); -976 } -977 if (cmd.hasOption('t')) { -978 rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t'))); -979 } -980 this.loadUnload = cmd.getOptionValue("o").toLowerCase(Locale.ROOT); -981 } -982 -983 @Override -984 protected int doWork() throws Exception { -985 boolean success; -986 RegionMover rm = rmbuilder.build(); -987 if (loadUnload.equalsIgnoreCase("load")) { -988 success = rm.load(); -989 } else if (loadUnload.equalsIgnoreCase("unload")) { -990 success = rm.unload(); -991 } else { -992 printUsage(); -993 success = false; -994 } -995 return (success ? 0 : 1); -996 } -997 -998 public static void main(String[] args) { -999 new RegionMover().doStaticMain(args); -1000 } -1001} +824 ArrayList<ServerName> serverInfo = new ArrayList<>(admin.getClusterStatus().getServers()); +825 ArrayList<String> regionServers = new ArrayList<>(serverInfo.size()); +826 for (ServerName server : serverInfo) { +827 regionServers.add(server.getServerName()); +828 } +829 return regionServers; +830 } +831 +832 private void deleteFile(String filename) { +833 File f = new File(filename); +834 if (f.exists()) { +835 f.delete(); +836 } +837 } +838 +839 /** +840 * Tries to scan a row from passed region +841 * @param admin +842 * @param region +843 * @throws IOException +844 */ +845 private void isSuccessfulScan(Admin admin, HRegionInfo region) throws IOException { +846 Scan scan = new Scan(region.getStartKey()); +847 scan.setBatch(1); +848 scan.setCaching(1); +849 scan.setFilter(new FirstKeyOnlyFilter()); +850 try { +851 Table table = admin.getConnection().getTable(region.getTable()); +852 try { +853 ResultScanner scanner = table.getScanner(scan); +854 try { +855 scanner.next(); +856 } finally { +857 scanner.close(); +858 } +859 } finally { +860 table.close(); +861 } +862 } catch (IOException e) { +863 LOG.error("Could not scan region:" + region.getEncodedName(), e); +864 throw e; +865 } +866 } +867 +868 /** +869 * Returns true if passed region is still on serverName when we look at hbase:meta. +870 * @param admin +871 * @param region +872 * @param serverName +873 * @return true if region is hosted on serverName otherwise false +874 * @throws IOException +875 */ +876 private boolean isSameServer(Admin admin, HRegionInfo region, String serverName) +877 throws IOException { +878 String serverForRegion = getServerNameForRegion(admin, region); +879 if (serverForRegion != null && serverForRegion.equals(serverName)) { +880 return true; +881 } +882 return false; +883 } +884 +885 /** +886 * Get servername that is up in hbase:meta hosting the given region. this is hostname + port + +887 * startcode comma-delimited. Can return null +888 * @param admin +889 * @param region +890 * @return regionServer hosting the given region +891 * @throws IOException +892 */ +893 private String getServerNameForRegion(Admin admin, HRegionInfo region) throws IOException { +894 String server = null; +895 if (!admin.isTableEnabled(region.getTable())) { +896 return null; +897 } +898 if (region.isMetaRegion()) { +899 ZooKeeperWatcher zkw = new ZooKeeperWatcher(admin.getConfiguration(), "region_mover", null); +900 MetaTableLocator locator = new MetaTableLocator(); +901 int maxWaitInSeconds = +902 admin.getConfiguration().getInt(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); +903 try { +904 server = locator.waitMetaRegionLocation(zkw, maxWaitInSeconds * 1000).toString() + ","; +905 } catch (InterruptedException e) { +906 LOG.error("Interrupted while waiting for location of Meta", e); +907 } finally { +908 if (zkw != null) { +909 zkw.close(); +910 } +911 } +912 } else { +913 Table table = admin.getConnection().getTable(TableName.META_TABLE_NAME); +914 try { +915 Get get = new Get(region.getRegionName()); +916 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); +917 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); +918 Result result = table.get(get); +919 if (result != null) { +920 byte[] servername = +921 result.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); +922 byte[] startcode = +923 result.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER); +924 if (servername != null) { +925 server = +926 Bytes.toString(servername).replaceFirst(":", ",") + "," + Bytes.toLong(startcode); +927 } +928 } +929 } catch (IOException e) { +930 LOG.error("Could not get Server Name for region:" + region.getEncodedName(), e); +931 throw e; +932 } finally { +933 table.close(); +934 } +935 } +936 return server; +937 } +938 +939 @Override +940 protected void addOptions() { +941 this.addRequiredOptWithArg("r", "regionserverhost", "region server <hostname>|<hostname:port>"); +942 this.addRequiredOptWithArg("o", "operation", "Expected: load/unload"); +943 this.addOptWithArg("m", "maxthreads", +944 "Define the maximum number of threads to use to unload and reload the regions"); +945 this.addOptWithArg("x", "excludefile", +946 "File with <hostname:port> per line to exclude as unload targets; default excludes only " +947 + "target host; useful for rack decommisioning."); +948 this.addOptWithArg("f", "filename", +949 "File to save regions list into unloading, or read from loading; " +950 + "default /tmp/<usernamehostname:port>"); +951 this.addOptNoArg("n", "noack", +952 "Turn on No-Ack mode(default: false) which won't check if region is online on target " +953 + "RegionServer, hence best effort. This is more performant in unloading and loading " +954 + "but might lead to region being unavailable for some time till master reassigns it " +955 + "in case the move failed"); +956 this.addOptWithArg("t", "timeout", "timeout in seconds after which the tool will exit " +957 + "irrespective of whether it finished or not;default Integer.MAX_VALUE"); +958 } +959 +960 @Override +961 protected void processOptions(CommandLine cmd) { +962 String hostname = cmd.getOptionValue("r"); +963 rmbuilder = new RegionMoverBuilder(hostname); +964 if (cmd.hasOption('m')) { +965 rmbuilder.maxthreads(Integer.parseInt(cmd.getOptionValue('m'))); +966 } +967 if (cmd.hasOption('n')) { +968 rmbuilder.ack(false); +969 } +970 if (cmd.hasOption('f')) { +971 rmbuilder.filename(cmd.getOptionValue('f')); +972 } +973 if (cmd.hasOption('x')) { +974 rmbuilder.excludeFile(cmd.getOptionValue('x')); +975 } +976 if (cmd.hasOption('t')) { +977 rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t'))); +978 } +979 this.loadUnload = cmd.getOptionValue("o").toLowerCase(Locale.ROOT); +980 } +981 +982 @Override +983 protected int doWork() throws Exception { +984 boolean success; +985 RegionMover rm = rmbuilder.build(); +986 if (loadUnload.equalsIgnoreCase("load")) { +987 success = rm.load(); +988 } else if (loadUnload.equalsIgnoreCase("unload")) { +989 success = rm.unload(); +990 } else { +991 printUsage(); +992 success = false; +993 } +994 return (success ? 0 : 1); +995 } +996 +997 public static void main(String[] args) { +998 new RegionMover().doStaticMain(args); +999 } +1000}