Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 81EAD200CFA for ; Tue, 5 Sep 2017 17:13:50 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8087F161421; Tue, 5 Sep 2017 15:13:50 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AF40316149B for ; Tue, 5 Sep 2017 17:13:48 +0200 (CEST) Received: (qmail 38249 invoked by uid 500); 5 Sep 2017 15:13:45 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 37052 invoked by uid 99); 5 Sep 2017 15:13:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Sep 2017 15:13:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EA7E1F569B; Tue, 5 Sep 2017 15:13:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 05 Sep 2017 15:13:55 -0000 Message-Id: <3c94259955ff4ab1b4446ad2e752f10c@git.apache.org> In-Reply-To: <2b3c8569a4f449baba3f530e819900f9@git.apache.org> References: <2b3c8569a4f449baba3f530e819900f9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/17] hbase-site git commit: Published site at . archived-at: Tue, 05 Sep 2017 15:13:50 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/96bf4e08/devapidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit.html b/devapidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit.html index a73c866..dd2d739 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportSnapshotInputFormat.ExportSnapshotInputSplit.html @@ -834,289 +834,287 @@ 826 827 // Run the MR Job 828 if (!job.waitForCompletion(true)) { -829 // TODO: Replace the fixed string with job.getStatus().getFailureInfo() -830 // when it will be available on all the supported versions. -831 throw new ExportSnapshotException("Copy Files Map-Reduce Job failed"); -832 } -833 } -834 -835 private void verifySnapshot(final Configuration baseConf, -836 final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException { -837 // Update the conf with the current root dir, since may be a different cluster -838 Configuration conf = new Configuration(baseConf); -839 FSUtils.setRootDir(conf, rootDir); -840 FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf)); -841 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); -842 SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc); -843 } -844 -845 /** -846 * Set path ownership. -847 */ -848 private void setOwner(final FileSystem fs, final Path path, final String user, -849 final String group, final boolean recursive) throws IOException { -850 if (user != null || group != null) { -851 if (recursive && fs.isDirectory(path)) { -852 for (FileStatus child : fs.listStatus(path)) { -853 setOwner(fs, child.getPath(), user, group, recursive); -854 } -855 } -856 fs.setOwner(path, user, group); -857 } -858 } -859 -860 /** -861 * Set path permission. -862 */ -863 private void setPermission(final FileSystem fs, final Path path, final short filesMode, -864 final boolean recursive) throws IOException { -865 if (filesMode > 0) { -866 FsPermission perm = new FsPermission(filesMode); -867 if (recursive && fs.isDirectory(path)) { -868 for (FileStatus child : fs.listStatus(path)) { -869 setPermission(fs, child.getPath(), filesMode, recursive); -870 } -871 } -872 fs.setPermission(path, perm); -873 } -874 } -875 -876 private boolean verifyTarget = true; -877 private boolean verifyChecksum = true; -878 private String snapshotName = null; -879 private String targetName = null; -880 private boolean overwrite = false; -881 private String filesGroup = null; -882 private String filesUser = null; -883 private Path outputRoot = null; -884 private Path inputRoot = null; -885 private int bandwidthMB = Integer.MAX_VALUE; -886 private int filesMode = 0; -887 private int mappers = 0; -888 -889 @Override -890 protected void processOptions(CommandLine cmd) { -891 snapshotName = cmd.getOptionValue(Options.SNAPSHOT.getLongOpt(), snapshotName); -892 targetName = cmd.getOptionValue(Options.TARGET_NAME.getLongOpt(), targetName); -893 if (cmd.hasOption(Options.COPY_TO.getLongOpt())) { -894 outputRoot = new Path(cmd.getOptionValue(Options.COPY_TO.getLongOpt())); -895 } -896 if (cmd.hasOption(Options.COPY_FROM.getLongOpt())) { -897 inputRoot = new Path(cmd.getOptionValue(Options.COPY_FROM.getLongOpt())); -898 } -899 mappers = getOptionAsInt(cmd, Options.MAPPERS.getLongOpt(), mappers); -900 filesUser = cmd.getOptionValue(Options.CHUSER.getLongOpt(), filesUser); -901 filesGroup = cmd.getOptionValue(Options.CHGROUP.getLongOpt(), filesGroup); -902 filesMode = getOptionAsInt(cmd, Options.CHMOD.getLongOpt(), filesMode); -903 bandwidthMB = getOptionAsInt(cmd, Options.BANDWIDTH.getLongOpt(), bandwidthMB); -904 overwrite = cmd.hasOption(Options.OVERWRITE.getLongOpt()); -905 // And verifyChecksum and verifyTarget with values read from old args in processOldArgs(...). -906 verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt()); -907 verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt()); -908 } -909 -910 /** -911 * Execute the export snapshot by copying the snapshot metadata, hfiles and wals. -912 * @return 0 on success, and != 0 upon failure. -913 */ -914 @Override -915 public int doWork() throws IOException { -916 Configuration conf = getConf(); -917 -918 // Check user options -919 if (snapshotName == null) { -920 System.err.println("Snapshot name not provided."); -921 LOG.error("Use -h or --help for usage instructions."); -922 return 0; -923 } -924 -925 if (outputRoot == null) { -926 System.err.println("Destination file-system (--" + Options.COPY_TO.getLongOpt() -927 + ") not provided."); -928 LOG.error("Use -h or --help for usage instructions."); -929 return 0; -930 } -931 -932 if (targetName == null) { -933 targetName = snapshotName; -934 } -935 if (inputRoot == null) { -936 inputRoot = FSUtils.getRootDir(conf); -937 } else { -938 FSUtils.setRootDir(conf, inputRoot); -939 } -940 -941 Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); -942 srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); -943 FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf); -944 LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot); -945 Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); -946 destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); -947 FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf); -948 LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString()); +829 throw new ExportSnapshotException(job.getStatus().getFailureInfo()); +830 } +831 } +832 +833 private void verifySnapshot(final Configuration baseConf, +834 final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException { +835 // Update the conf with the current root dir, since may be a different cluster +836 Configuration conf = new Configuration(baseConf); +837 FSUtils.setRootDir(conf, rootDir); +838 FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf)); +839 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); +840 SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc); +841 } +842 +843 /** +844 * Set path ownership. +845 */ +846 private void setOwner(final FileSystem fs, final Path path, final String user, +847 final String group, final boolean recursive) throws IOException { +848 if (user != null || group != null) { +849 if (recursive && fs.isDirectory(path)) { +850 for (FileStatus child : fs.listStatus(path)) { +851 setOwner(fs, child.getPath(), user, group, recursive); +852 } +853 } +854 fs.setOwner(path, user, group); +855 } +856 } +857 +858 /** +859 * Set path permission. +860 */ +861 private void setPermission(final FileSystem fs, final Path path, final short filesMode, +862 final boolean recursive) throws IOException { +863 if (filesMode > 0) { +864 FsPermission perm = new FsPermission(filesMode); +865 if (recursive && fs.isDirectory(path)) { +866 for (FileStatus child : fs.listStatus(path)) { +867 setPermission(fs, child.getPath(), filesMode, recursive); +868 } +869 } +870 fs.setPermission(path, perm); +871 } +872 } +873 +874 private boolean verifyTarget = true; +875 private boolean verifyChecksum = true; +876 private String snapshotName = null; +877 private String targetName = null; +878 private boolean overwrite = false; +879 private String filesGroup = null; +880 private String filesUser = null; +881 private Path outputRoot = null; +882 private Path inputRoot = null; +883 private int bandwidthMB = Integer.MAX_VALUE; +884 private int filesMode = 0; +885 private int mappers = 0; +886 +887 @Override +888 protected void processOptions(CommandLine cmd) { +889 snapshotName = cmd.getOptionValue(Options.SNAPSHOT.getLongOpt(), snapshotName); +890 targetName = cmd.getOptionValue(Options.TARGET_NAME.getLongOpt(), targetName); +891 if (cmd.hasOption(Options.COPY_TO.getLongOpt())) { +892 outputRoot = new Path(cmd.getOptionValue(Options.COPY_TO.getLongOpt())); +893 } +894 if (cmd.hasOption(Options.COPY_FROM.getLongOpt())) { +895 inputRoot = new Path(cmd.getOptionValue(Options.COPY_FROM.getLongOpt())); +896 } +897 mappers = getOptionAsInt(cmd, Options.MAPPERS.getLongOpt(), mappers); +898 filesUser = cmd.getOptionValue(Options.CHUSER.getLongOpt(), filesUser); +899 filesGroup = cmd.getOptionValue(Options.CHGROUP.getLongOpt(), filesGroup); +900 filesMode = getOptionAsInt(cmd, Options.CHMOD.getLongOpt(), filesMode); +901 bandwidthMB = getOptionAsInt(cmd, Options.BANDWIDTH.getLongOpt(), bandwidthMB); +902 overwrite = cmd.hasOption(Options.OVERWRITE.getLongOpt()); +903 // And verifyChecksum and verifyTarget with values read from old args in processOldArgs(...). +904 verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt()); +905 verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt()); +906 } +907 +908 /** +909 * Execute the export snapshot by copying the snapshot metadata, hfiles and wals. +910 * @return 0 on success, and != 0 upon failure. +911 */ +912 @Override +913 public int doWork() throws IOException { +914 Configuration conf = getConf(); +915 +916 // Check user options +917 if (snapshotName == null) { +918 System.err.println("Snapshot name not provided."); +919 LOG.error("Use -h or --help for usage instructions."); +920 return 0; +921 } +922 +923 if (outputRoot == null) { +924 System.err.println("Destination file-system (--" + Options.COPY_TO.getLongOpt() +925 + ") not provided."); +926 LOG.error("Use -h or --help for usage instructions."); +927 return 0; +928 } +929 +930 if (targetName == null) { +931 targetName = snapshotName; +932 } +933 if (inputRoot == null) { +934 inputRoot = FSUtils.getRootDir(conf); +935 } else { +936 FSUtils.setRootDir(conf, inputRoot); +937 } +938 +939 Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); +940 srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true); +941 FileSystem inputFs = FileSystem.get(inputRoot.toUri(), srcConf); +942 LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot); +943 Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); +944 destConf.setBoolean("fs." + outputRoot.toUri().getScheme() + ".impl.disable.cache", true); +945 FileSystem outputFs = FileSystem.get(outputRoot.toUri(), destConf); +946 LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString()); +947 +948 boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false); 949 -950 boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false); -951 -952 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); -953 Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot); -954 Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot); -955 Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir; -956 -957 // Find the necessary directory which need to change owner and group -958 Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot); -959 if (outputFs.exists(needSetOwnerDir)) { -960 if (skipTmp) { -961 needSetOwnerDir = outputSnapshotDir; -962 } else { -963 needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot); -964 if (outputFs.exists(needSetOwnerDir)) { -965 needSetOwnerDir = snapshotTmpDir; -966 } -967 } -968 } -969 -970 // Check if the snapshot already exists -971 if (outputFs.exists(outputSnapshotDir)) { -972 if (overwrite) { -973 if (!outputFs.delete(outputSnapshotDir, true)) { -974 System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir); -975 return 1; -976 } -977 } else { -978 System.err.println("The snapshot '" + targetName + -979 "' already exists in the destination: " + outputSnapshotDir); -980 return 1; -981 } -982 } -983 -984 if (!skipTmp) { -985 // Check if the snapshot already in-progress -986 if (outputFs.exists(snapshotTmpDir)) { -987 if (overwrite) { -988 if (!outputFs.delete(snapshotTmpDir, true)) { -989 System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir); -990 return 1; -991 } -992 } else { -993 System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress"); -994 System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, "); -995 System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option"); -996 return 1; -997 } -998 } -999 } -1000 -1001 // Step 1 - Copy fs1:/.snapshot/<snapshot> to fs2:/.snapshot/.tmp/<snapshot> -1002 // The snapshot references must be copied before the hfiles otherwise the cleaner -1003 // will remove them because they are unreferenced. -1004 try { -1005 LOG.info("Copy Snapshot Manifest"); -1006 FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf); -1007 } catch (IOException e) { -1008 throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" + -1009 snapshotDir + " to=" + initialOutputSnapshotDir, e); -1010 } finally { -1011 if (filesUser != null || filesGroup != null) { -1012 LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to " -1013 + filesUser) -1014 + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to " -1015 + filesGroup)); -1016 setOwner(outputFs, needSetOwnerDir, filesUser, filesGroup, true); -1017 } -1018 if (filesMode > 0) { -1019 LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode); -1020 setPermission(outputFs, needSetOwnerDir, (short)filesMode, true); -1021 } -1022 } -1023 -1024 // Write a new .snapshotinfo if the target name is different from the source name -1025 if (!targetName.equals(snapshotName)) { -1026 SnapshotDescription snapshotDesc = -1027 SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir) -1028 .toBuilder() -1029 .setName(targetName) -1030 .build(); -1031 SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs); -1032 if (filesUser != null || filesGroup != null) { -1033 outputFs.setOwner(new Path(initialOutputSnapshotDir, -1034 SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup); -1035 } -1036 if (filesMode > 0) { -1037 outputFs.setPermission(new Path(initialOutputSnapshotDir, -1038 SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short)filesMode)); -1039 } -1040 } -1041 -1042 // Step 2 - Start MR Job to copy files -1043 // The snapshot references must be copied before the files otherwise the files gets removed -1044 // by the HFileArchiver, since they have no references. -1045 try { -1046 runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum, -1047 filesUser, filesGroup, filesMode, mappers, bandwidthMB); -1048 -1049 LOG.info("Finalize the Snapshot Export"); -1050 if (!skipTmp) { -1051 // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot> -1052 if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) { -1053 throw new ExportSnapshotException("Unable to rename snapshot directory from=" + -1054 snapshotTmpDir + " to=" + outputSnapshotDir); -1055 } -1056 } -1057 -1058 // Step 4 - Verify snapshot integrity -1059 if (verifyTarget) { -1060 LOG.info("Verify snapshot integrity"); -1061 verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir); -1062 } -1063 -1064 LOG.info("Export Completed: " + targetName); -1065 return 0; -1066 } catch (Exception e) { -1067 LOG.error("Snapshot export failed", e); -1068 if (!skipTmp) { -1069 outputFs.delete(snapshotTmpDir, true); -1070 } -1071 outputFs.delete(outputSnapshotDir, true); -1072 return 1; -1073 } finally { -1074 IOUtils.closeStream(inputFs); -1075 IOUtils.closeStream(outputFs); -1076 } -1077 } -1078 -1079 @Override -1080 protected void printUsage() { -1081 super.printUsage(); -1082 System.out.println("\n" -1083 + "Examples:\n" -1084 + " hbase snapshot export \\\n" -1085 + " --snapshot MySnapshot --copy-to hdfs://srv2:8082/hbase \\\n" -1086 + " --chuser MyUser --chgroup MyGroup --chmod 700 --mappers 16\n" -1087 + "\n" -1088 + " hbase snapshot export \\\n" -1089 + " --snapshot MySnapshot --copy-from hdfs://srv2:8082/hbase \\\n" -1090 + " --copy-to hdfs://srv1:50070/hbase"); -1091 } -1092 -1093 @Override protected void addOptions() { -1094 addRequiredOption(Options.SNAPSHOT); -1095 addOption(Options.COPY_TO); -1096 addOption(Options.COPY_FROM); -1097 addOption(Options.TARGET_NAME); -1098 addOption(Options.NO_CHECKSUM_VERIFY); -1099 addOption(Options.NO_TARGET_VERIFY); -1100 addOption(Options.OVERWRITE); -1101 addOption(Options.CHUSER); -1102 addOption(Options.CHGROUP); -1103 addOption(Options.CHMOD); -1104 addOption(Options.MAPPERS); -1105 addOption(Options.BANDWIDTH); -1106 } -1107 -1108 public static void main(String[] args) { -1109 new ExportSnapshot().doStaticMain(args); -1110 } -1111} +950 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); +951 Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot); +952 Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot); +953 Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir; +954 +955 // Find the necessary directory which need to change owner and group +956 Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot); +957 if (outputFs.exists(needSetOwnerDir)) { +958 if (skipTmp) { +959 needSetOwnerDir = outputSnapshotDir; +960 } else { +961 needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot); +962 if (outputFs.exists(needSetOwnerDir)) { +963 needSetOwnerDir = snapshotTmpDir; +964 } +965 } +966 } +967 +968 // Check if the snapshot already exists +969 if (outputFs.exists(outputSnapshotDir)) { +970 if (overwrite) { +971 if (!outputFs.delete(outputSnapshotDir, true)) { +972 System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir); +973 return 1; +974 } +975 } else { +976 System.err.println("The snapshot '" + targetName + +977 "' already exists in the destination: " + outputSnapshotDir); +978 return 1; +979 } +980 } +981 +982 if (!skipTmp) { +983 // Check if the snapshot already in-progress +984 if (outputFs.exists(snapshotTmpDir)) { +985 if (overwrite) { +986 if (!outputFs.delete(snapshotTmpDir, true)) { +987 System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir); +988 return 1; +989 } +990 } else { +991 System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress"); +992 System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, "); +993 System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option"); +994 return 1; +995 } +996 } +997 } +998 +999 // Step 1 - Copy fs1:/.snapshot/<snapshot> to fs2:/.snapshot/.tmp/<snapshot> +1000 // The snapshot references must be copied before the hfiles otherwise the cleaner +1001 // will remove them because they are unreferenced. +1002 try { +1003 LOG.info("Copy Snapshot Manifest"); +1004 FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf); +1005 } catch (IOException e) { +1006 throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" + +1007 snapshotDir + " to=" + initialOutputSnapshotDir, e); +1008 } finally { +1009 if (filesUser != null || filesGroup != null) { +1010 LOG.warn((filesUser == null ? "" : "Change the owner of " + needSetOwnerDir + " to " +1011 + filesUser) +1012 + (filesGroup == null ? "" : ", Change the group of " + needSetOwnerDir + " to " +1013 + filesGroup)); +1014 setOwner(outputFs, needSetOwnerDir, filesUser, filesGroup, true); +1015 } +1016 if (filesMode > 0) { +1017 LOG.warn("Change the permission of " + needSetOwnerDir + " to " + filesMode); +1018 setPermission(outputFs, needSetOwnerDir, (short)filesMode, true); +1019 } +1020 } +1021 +1022 // Write a new .snapshotinfo if the target name is different from the source name +1023 if (!targetName.equals(snapshotName)) { +1024 SnapshotDescription snapshotDesc = +1025 SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir) +1026 .toBuilder() +1027 .setName(targetName) +1028 .build(); +1029 SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs); +1030 if (filesUser != null || filesGroup != null) { +1031 outputFs.setOwner(new Path(initialOutputSnapshotDir, +1032 SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), filesUser, filesGroup); +1033 } +1034 if (filesMode > 0) { +1035 outputFs.setPermission(new Path(initialOutputSnapshotDir, +1036 SnapshotDescriptionUtils.SNAPSHOTINFO_FILE), new FsPermission((short)filesMode)); +1037 } +1038 } +1039 +1040 // Step 2 - Start MR Job to copy files +1041 // The snapshot references must be copied before the files otherwise the files gets removed +1042 // by the HFileArchiver, since they have no references. +1043 try { +1044 runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum, +1045 filesUser, filesGroup, filesMode, mappers, bandwidthMB); +1046 +1047 LOG.info("Finalize the Snapshot Export"); +1048 if (!skipTmp) { +1049 // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot> +1050 if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) { +1051 throw new ExportSnapshotException("Unable to rename snapshot directory from=" + +1052 snapshotTmpDir + " to=" + outputSnapshotDir); +1053 } +1054 } +1055 +1056 // Step 4 - Verify snapshot integrity +1057 if (verifyTarget) { +1058 LOG.info("Verify snapshot integrity"); +1059 verifySnapshot(destConf, outputFs, outputRoot, outputSnapshotDir); +1060 } +1061 +1062 LOG.info("Export Completed: " + targetName); +1063 return 0; +1064 } catch (Exception e) { +1065 LOG.error("Snapshot export failed", e); +1066 if (!skipTmp) { +1067 outputFs.delete(snapshotTmpDir, true); +1068 } +1069 outputFs.delete(outputSnapshotDir, true); +1070 return 1; +1071 } finally { +1072 IOUtils.closeStream(inputFs); +1073 IOUtils.closeStream(outputFs); +1074 } +1075 } +1076 +1077 @Override +1078 protected void printUsage() { +1079 super.printUsage(); +1080 System.out.println("\n" +1081 + "Examples:\n" +1082 + " hbase snapshot export \\\n" +1083 + " --snapshot MySnapshot --copy-to hdfs://srv2:8082/hbase \\\n" +1084 + " --chuser MyUser --chgroup MyGroup --chmod 700 --mappers 16\n" +1085 + "\n" +1086 + " hbase snapshot export \\\n" +1087 + " --snapshot MySnapshot --copy-from hdfs://srv2:8082/hbase \\\n" +1088 + " --copy-to hdfs://srv1:50070/hbase"); +1089 } +1090 +1091 @Override protected void addOptions() { +1092 addRequiredOption(Options.SNAPSHOT); +1093 addOption(Options.COPY_TO); +1094 addOption(Options.COPY_FROM); +1095 addOption(Options.TARGET_NAME); +1096 addOption(Options.NO_CHECKSUM_VERIFY); +1097 addOption(Options.NO_TARGET_VERIFY); +1098 addOption(Options.OVERWRITE); +1099 addOption(Options.CHUSER); +1100 addOption(Options.CHGROUP); +1101 addOption(Options.CHMOD); +1102 addOption(Options.MAPPERS); +1103 addOption(Options.BANDWIDTH); +1104 } +1105 +1106 public static void main(String[] args) { +1107 new ExportSnapshot().doStaticMain(args); +1108 } +1109}