hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject [4/4] incubator-hawq-docs git commit: HAWQ-1250 - removing legacy unreferenced files; adding version to subnav file; adding index.html for redirect
Date Fri, 06 Jan 2017 23:49:48 GMT
HAWQ-1250 - removing legacy unreferenced files; adding version to subnav file; adding index.html
for redirect


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/commit/06d7cd27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/tree/06d7cd27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/diff/06d7cd27

Branch: refs/heads/release/2.1.0.0-incubating
Commit: 06d7cd275683fbd7c0e2e4ab2288fc6cc5d20090
Parents: 92d7ddb
Author: David Yozie <yozie@apache.org>
Authored: Fri Jan 6 15:49:37 2017 -0800
Committer: David Yozie <yozie@apache.org>
Committed: Fri Jan 6 15:49:37 2017 -0800

----------------------------------------------------------------------
 book/config.yml                                 |   2 +-
 .../patch/dynamic_variable_interpretation.py    | 192 ----
 .../source/subnavs/apache-hawq-nav-210.erb      | 894 +++++++++++++++++++
 .../source/subnavs/apache-hawq-nav.erb          | 894 -------------------
 markdown/index.html.erb                         |   7 +
 5 files changed, 902 insertions(+), 1087 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/blob/06d7cd27/book/config.yml
----------------------------------------------------------------------
diff --git a/book/config.yml b/book/config.yml
index 22d2799..cc83bbf 100644
--- a/book/config.yml
+++ b/book/config.yml
@@ -6,7 +6,7 @@ sections:
  - repository:
      name: incubator-hawq-docs/markdown
    directory: docs/userguide/2.1.0.0-incubating
-   subnav_template: apache-hawq-nav
+   subnav_template: apache-hawq-nav-210
 
 template_variables:
   use_global_header: true

http://git-wip-us.apache.org/repos/asf/incubator-hawq-docs/blob/06d7cd27/book/master_middleman/source/patch/dynamic_variable_interpretation.py
----------------------------------------------------------------------
diff --git a/book/master_middleman/source/patch/dynamic_variable_interpretation.py b/book/master_middleman/source/patch/dynamic_variable_interpretation.py
deleted file mode 100644
index 66df9ff..0000000
--- a/book/master_middleman/source/patch/dynamic_variable_interpretation.py
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-__all__ = ["copy_tarballs_to_hdfs", ]
-import os
-import glob
-import re
-import tempfile
-from resource_management.libraries.functions.default import default
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.resources.copy_from_local import CopyFromLocal
-from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
-from resource_management.core.resources.system import Execute
-from resource_management.core.exceptions import Fail
-from resource_management.core.logger import Logger
-from resource_management.core import shell
-
-"""
-This file provides helper methods needed for the versioning of RPMs. Specifically, it does
dynamic variable
-interpretation to replace strings like {{ hdp_stack_version }}  where the value of the
-variables cannot be determined ahead of time, but rather, depends on what files are found.
-
-It assumes that {{ hdp_stack_version }} is constructed as ${major.minor.patch.rev}-${build_number}
-E.g., 998.2.2.1.0-998
-Please note that "-${build_number}" is optional.
-"""
-
-# These values must be the suffix of the properties in cluster-env.xml
-TAR_SOURCE_SUFFIX = "_tar_source"
-TAR_DESTINATION_FOLDER_SUFFIX = "_tar_destination_folder"
-
-
-def _get_tar_source_and_dest_folder(tarball_prefix):
-  """
-  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
-  :return: Returns a tuple of (x, y) after verifying the properties
-  """
-  component_tar_source_file = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(),
TAR_SOURCE_SUFFIX), None)
-  # E.g., /usr/hdp/current/hadoop-client/tez-{{ hdp_stack_version }}.tar.gz
-
-  component_tar_destination_folder = default("/configurations/cluster-env/%s%s" % (tarball_prefix.lower(),
TAR_DESTINATION_FOLDER_SUFFIX), None)
-  # E.g., hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/
-
-  if not component_tar_source_file or not component_tar_destination_folder:
-    Logger.warning("Did not find %s tar source file and destination folder properties in
cluster-env.xml" %
-                   tarball_prefix)
-    return None, None
-
-  if component_tar_source_file.find("/") == -1:
-    Logger.warning("The tar file path %s is not valid" % str(component_tar_source_file))
-    return None, None
-
-  if not component_tar_destination_folder.endswith("/"):
-    component_tar_destination_folder = component_tar_destination_folder + "/"
-
-  if not component_tar_destination_folder.startswith("hdfs://"):
-    return None, None
-
-  return component_tar_source_file, component_tar_destination_folder
-
-
-def _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed):
-  """
-  :param source_and_dest_pairs: List of tuples (x, y), where x is the source file in the
local file system,
-  and y is the destination file path in HDFS
-  :param file_owner: Owner to set for the file copied to HDFS (typically hdfs account)
-  :param group_owner: Owning group to set for the file copied to HDFS (typically hadoop group)
-  :param kinit_if_needed: kinit command if it is needed, otherwise an empty string
-  :return: Returns 0 if at least one file was copied and no exceptions occurred, and 1 otherwise.
-
-  Must kinit before calling this function.
-  """
-  import params
-
-  return_value = 1
-  if source_and_dest_pairs and len(source_and_dest_pairs) > 0:
-    return_value = 0
-    for (source, destination) in source_and_dest_pairs:
-      try:
-        destination_dir = os.path.dirname(destination)
-
-        params.HdfsDirectory(destination_dir,
-                             action="create",
-                             owner=file_owner,
-                             mode=0555
-        )
-
-        CopyFromLocal(source,
-                      mode=0444,
-                      owner=file_owner,
-                      group=group_owner,
-                      dest_dir=destination_dir,
-                      kinnit_if_needed=kinit_if_needed,
-                      hdfs_user=params.hdfs_user,
-                      hadoop_bin_dir=params.hadoop_bin_dir,
-                      hadoop_conf_dir=params.hadoop_conf_dir
-        )
-      except:
-        return_value = 1
-  return return_value
-
-
-def copy_tarballs_to_hdfs(tarball_prefix, component_user, file_owner, group_owner):
-  """
-  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
-  :param component_user: User that will execute the Hadoop commands
-  :param file_owner: Owner of the files copied to HDFS (typically hdfs account)
-  :param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
-  :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an
exception.
-
-  In order to call this function, params.py must have all of the following,
-  hdp_stack_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name,
hdfs_user_keytab,
-  hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
-  """
-  import params
-
-  if not hasattr(params, "hdp_stack_version") or params.hdp_stack_version is None:
-    Logger.warning("Could not find hdp_stack_version")
-    return 1
-
-  component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder(tarball_prefix)
-  if not component_tar_source_file or not component_tar_destination_folder:
-    Logger.warning("Could not retrieve properties for tarball with prefix: %s" % str(tarball_prefix))
-    return 1
-
-  if not os.path.exists(component_tar_source_file):
-    Logger.warning("Could not find file: %s" % str(component_tar_source_file))
-    return 1
-
-  # Ubuntu returns: "stdin: is not a tty", as subprocess output.
-  tmpfile = tempfile.NamedTemporaryFile()
-  with open(tmpfile.name, 'r+') as file:
-    get_hdp_version_cmd = '/usr/bin/hdp-select versions > %s' % tmpfile.name
-    code, stdoutdata = shell.call(get_hdp_version_cmd)
-    out = file.read()
-  pass
-  if code != 0 or out is None:
-    Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output:
%s." %
-                   (get_hdp_version_cmd, str(code), str(out)))
-    return 1
-
-  hdp_version = out.strip() # this should include the build number
-
-  file_name = os.path.basename(component_tar_source_file)
-  destination_file = os.path.join(component_tar_destination_folder, file_name)
-  destination_file = destination_file.replace("{{ hdp_stack_version }}", hdp_version)
-
-  does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
-
-  kinit_if_needed = ""
-  if params.security_enabled:
-    kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
-
-  if kinit_if_needed:
-    Execute(kinit_if_needed,
-            user=component_user,
-            path='/bin'
-    )
-
-  does_hdfs_file_exist = False
-  try:
-    ExecuteHadoop(does_hdfs_file_exist_cmd,
-                  user=component_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  bin_dir=params.hadoop_bin_dir
-    )
-    does_hdfs_file_exist = True
-  except Fail:
-    pass
-
-  if not does_hdfs_file_exist:
-    source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
-    return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
-  return 1


Mime
View raw message