helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zzh...@apache.org
Subject [1/3] remove test scripts for helix-agent. disable helix-agent
Date Thu, 02 May 2013 01:26:44 GMT
Updated Branches:
  refs/heads/master d5289aebc -> 4ebc0fad9


http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/script/utility.py
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/script/utility.py b/helix-agent/src/main/scripts/integration-test/script/utility.py
deleted file mode 100644
index 0454f00..0000000
--- a/helix-agent/src/main/scripts/integration-test/script/utility.py
+++ /dev/null
@@ -1,813 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-'''
-====  utilities
-'''
-
-import sys, os, subprocess
-import socket, pdb, re
-import urllib, errno
-import time, shutil
-import tempfile
-import random
-import socket
- 
-sys_call_debug=False
-enable_sys_call_debug=False
-debug_enabled=False
-host_name_global = (os.popen("/bin/hostname").read()).split("\n")[0]
-#host_name_global = socket.gethostbyaddr(socket.gethostbyname(socket.gethostname()))[0]
-
-view_root=None
-test_name=None
-#pdb.set_trace()
-this_file_full_path=os.path.abspath(__file__)
-# use logical pwd so symlink can be done from root
-this_file_dirname=os.path.dirname(this_file_full_path)
-this_file_name=os.path.basename(this_file_full_path)
-#this_file_dirname="PWD" in os.environ and os.environ["PWD"] or os.path.dirname(this_file_full_path)
-
-work_dir=None
-log_dir=None
-var_dir=None
-var_dir_template="%s/integration-test/var"
-testcase_dir=None
-testcase_dir_template="%s/integration-test/testcases"
-cwd_dir=os.getcwd()
-import getpass
-username=getpass.getuser()
-# used to run cmd, can combine multiple command
-components=[
-     "test_relay"
-    ,"test_bootstrap_producer"
-    ,"bootstrap_server"
-    ,"bootstrap_consumer"
-    ,"profile_relay"
-    ,"profile_consumer"
-]
-
-def dbg_print(in_str):
-    #import pdb
-    #pdb.set_trace()
-    if debug_enabled:
-        print ("== " + sys._getframe(1).f_code.co_name + " == " + str(in_str))
-
-def sys_pipe_call(cmd):
-    dbg_print("%s:%s" % (os.getcwd(),cmd))
-    if sys_call_debug:
-        print("cmd = %s " % cmd)
-        if re.search("svn (log|info)",cmd): return os.popen(cmd).read()
-        return ""
-    return os.popen(cmd).read()               
-
-def get_this_file_dirname(): return this_file_dirname
-def get_this_file_name(): return this_file_name
-#handle the json import 
-if sys.version_info[0]==2 and sys.version_info[1]<6:
-  try:
-    import simplejson as json
-  except: 
-    out=sys_pipe_call(os.path.join(get_this_file_dirname(),"install_python_packages.sh"))
-    #print("install json = %s " % out)
-    import simplejson as json
-else:
-  import json
-
-# functions
-def setup_view_root():
-    global view_root
-    if "VIEW_ROOT" in os.environ: view_root = os.environ["VIEW_ROOT"]
-    else: view_root= os.path.abspath("%s/../../" % this_file_dirname)
-    #print("view_root = %s" % view_root)
-    #print("test_name=%s" % test_name)
-    os.chdir(view_root)
-    os.environ["VIEW_ROOT"]=view_root
-
-def get_view_root(): return view_root
-
-def setup_work_dir():
-    global var_dir, work_dir, log_dir, test_name
-    var_dir= var_dir_template % (view_root)
-    import distutils.dir_util
-    distutils.dir_util.mkpath(var_dir, verbose=1)
-
-    if "TEST_NAME" in os.environ: test_name=os.environ["TEST_NAME"]
-    else: assert False, "TEST NAME Not Defined"
-    if "WORK_SUB_DIR" in os.environ: work_dir=os.path.join(var_dir,os.environ["WORK_SUB_DIR"],test_name)
-    else: assert False, "Work Dir Not Defined"
-    if "LOG_SUB_DIR" in os.environ: log_dir=os.path.join(var_dir, os.environ["LOG_SUB_DIR"],
test_name)
-    else: assert False, "Work Dir Not Defined"
-    distutils.dir_util.mkpath(work_dir, verbose=1)  
-    distutils.dir_util.mkpath(log_dir, verbose=1)  
-
-def get_test_name(): return test_name
-def get_work_dir(): return work_dir
-def get_log_dir(): return log_dir
-def get_var_dir(): return var_dir
-def get_script_dir(): return get_this_file_dirname()
-def get_testcase_dir(): return testcase_dir
-def get_cwd(): return cwd_dir
-def get_username(): return username
-
-def my_exit(ret):
-  # close all the file descriptors
-  os.close(1)  # stdin
-  os.close(2)  # stdout
-  os.close(3)  # stderr
-  sys.exit(ret)
-
-def file_exists(file):  # test both 
-    ''' return the abs path of the file if exists '''
-    if os.path.isabs(file): 
-      if os.path.exists(file): return file
-      else: return None
-    tmp_file=os.path.join(view_root, file)
-    if os.path.exists(tmp_file): return tmp_file
-    tmp_file=os.path.join(cwd_dir,file)
-    if os.path.exists(tmp_file): return tmp_file
-    return None
-  
-def set_debug(flag): 
-    global debug_enabled
-    debug_enabled=flag
-
-def set_sys_call_debug(flag): 
-    global enable_sys_call_debug
-    enable_sys_call_debug=flag
-
-def sys_call_debug_begin():
-    if not enable_sys_call_debug: return
-    global sys_call_debug
-    sys_call_debug=True
-    
-def sys_call_debug_end():
-    if not enable_sys_call_debug: return
-    global sys_call_debug
-    sys_call_debug=False
-
-def sys_call(cmd):
-    dbg_print("%s:%s" % (os.getcwd(),cmd))
-    if sys_call_debug:
-        print("cmd = %s " % cmd)
-        return
-    return os.system(cmd)
-
-def subprocess_call_1(cmd, outfp=None):
-    dbg_print("%s:%s" % (os.getcwd(), cmd))
-    if not sys_call_debug:
-        if outfp:
-          p = subprocess.Popen(cmd, shell=True, stdout=outfp, stderr=outfp, close_fds=True)
-        else:
-          #pdb.set_trace()
-          p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
close_fds=True)
-        dbg_print("subprocess pid = %s" % p.pid)
-        return p
-    else:
-        print("cmd = %s " % cmd)
-        return None
-
-def sys_pipe_call_4(cmd):
-    dbg_print("%s:%s" % (os.getcwd(), cmd))
-    if not sys_call_debug:
-        p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, close_fds=True)
-        dbg_print("subprocess pid = %s" % p.pid)
-        return p.stdout
-    else:
-        None
-
-def sys_pipe_call_3(cmd):
-    dbg_print("%s:%s" % (os.getcwd(), cmd))
-    if not sys_call_debug:
-        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, close_fds=True)
-        dbg_print("subprocess pid = %s" % p.pid)
-        #p = os.popen(cmd)
-        return (p.stdout, p.pid)
-    else:
-        None
-
-def sys_pipe_call_5(cmd):
-    ''' return both stdin, stdout and pid '''
-    dbg_print("%s:%s" % (os.getcwd(), cmd))
-    if not sys_call_debug:
-        p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
close_fds=True)
-        dbg_print("subprocess pid = %s" % p.pid)
-        #p = os.popen(cmd)
-        return (p.stdout, p.stderr, p.pid)
-    else:
-        None
-
-def sys_pipe_call_21(input, cmd):
-    ''' call with input pipe to the cmd '''
-    dbg_print("%s:%s:%s" % (os.getcwd(),input, cmd))
-    if not sys_call_debug:
-      return  subprocess.Popen(cmd.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, close_fds=True).communicate(input)[0]
-    else:
-      return  ""
-
-def sys_pipe_call_2(input, cmd):
-    ''' call with input pipe to the cmd '''
-    dbg_print("%s:%s:%s" % (os.getcwd(),input, cmd))
-    if not sys_call_debug:
-      return  subprocess.Popen(cmd.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE,
close_fds=True).communicate(input)[0]
-    else:
-      return  ""
-
-def sys_pipe_call_1(cmd):
-    ''' also return the errors '''
-    dbg_print("%s:%s" % (os.getcwd(),cmd))
-    if not sys_call_debug:
-      p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
close_fds=True)
-      return p.stdout.readlines()
-    else:
-      return  ""
-
-#    dbg_print("%s:%s" % (os.getcwd(),cmd))
-#    return os.popen4(cmd)[1].read()
-
-def sys_call_env(cmd):
-    cmds=cmd.split()
-    dbg_print("cmds= %s " % cmds)
-    os.spawnv( os.P_WAIT, cmds[0], cmds[1:])
-
-def whoami():
-    return sys._getframe(1).f_code.co_name
-
-def my_error(s):
-    if debug_enabled: assert False, "Error: %s" % s
-    else: 
-      print "Error: %s" % s
-      my_exit(1)
-
-def my_warning(s):
-    if debug_enabled:
-        print ("== " + sys._getframe(1).f_code.co_name + " == " + str(s))
-    else: 
-      print "WARNING: %s" % s
-
-def enter_func():
-    dbg_print ("Entering == " + sys._getframe(1).f_code.co_name + " == ")
-
-def get_time():
-    return float("%0.4f" % time.time())   # keep 2 digits
-
-def isOpen(ip,port):
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    try:
-      s.connect((ip, int(port)))
-      s.shutdown(2)
-      return True
-    except:
-      return False
-
-def next_available_port(ip,port):
-    port_num = int(port)
-    while (isOpen(ip, port_num)): port_num +=1
-    return port_num
-
-def find_open_port(host, start_port, seq_num):
-    ''' find the seq_num th port starting from start_port ''' 
-    limit = 100
-    start_port_num = int(start_port)
-    seq = 0
-    for i in range(limit):
-      port_num = start_port_num + i
-      if isOpen(host, port_num): seq += 1
-      if seq == seq_num: return port_num
-    return None
-
-def process_exist(pid, host=None):
-    if not host:
-      try:
-	  os.kill(int(pid), 0)
-      except OSError, err:
-	  if err.errno == errno.ESRCH: return False # not running
-	  elif err.errno == errno.EPERM: return True  # own by others by running
-	  else: my_error("Unknown error")
-      else:
-          return True # running
-    else:  # remote run
-      process_cnt = sys_pipe_call("ssh %s@%s 'ps -ef | grep %s | wc -l" % (username, host,
pid)).split("\n")[0]
-      return process_cnt > 0
-
-# remote execute related, by default remote execution is off
-setup_view_root()
-config_dir="%s/integration-test/config" % view_root
-remote_config_file="%s/remote_execute_on.cfg" % config_dir
-remote_run=False    # this is to indicate use of remote_config
-remote_launch=False  # this is to indicate remote ssh recursive call
-remote_run_config={}
-remote_view_root=None
-def get_remote_view_root(): return remote_view_root
-def set_remote_view_root(v_root): 
-    global remote_view_root
-    remote_view_root = v_root
-def get_remote_log_dir(): 
-    return os.path.join(var_dir_template % remote_view_root, "log")
-def get_remote_work_dir(): 
-    return os.path.join(var_dir_template % remote_view_root, "work")
-
-import ConfigParser
-
-def check_remote_config(remote_config_parser):
-  allowed_options=["host","port","view_root"]
-  section_names = remote_config_parser.sections() # returns a list of strings of section
names
-  for section in section_names:
-    if not [x for x in components if re.search(x, section)]:
-      my_error("Invalid section %s in config file " % (section))
-    if [x for x in ["test_relay, profile_realy, bootstrap_server"] if re.search(x, section)]:
-      if not remote_config_parser.has_option(section, "host"):   # set the default host
-        remote_config_parser.set(section, "host",host_name_global) 
-
-def parse_config_cfg(remote_config_file):
-  remote_config_parser = ConfigParser.SafeConfigParser()
-  remote_config_parser.read(remote_config_file)
-  #check_remote_config(remote_config_parser)   # do not check for now
-  for section in remote_config_parser.sections(): # returns a list of strings of section
names
-    remote_run_config[section]={}
-    for option in remote_config_parser.options(section):
-      remote_run_config[section][option]=remote_config_parser.get(section,option)
-  return remote_run_config
-
-def parse_config_json(remote_config_file):
-  return json.load(open(remote_config_file))
-
-def parse_config(remote_config_file_input):
-  global remote_run_config, remote_run
-  remote_config_file = file_exists(remote_config_file_input)
-  if not remote_config_file: my_error("remote_config_file %s does not existi!!" % remote_config_file_input)
-  file_type = os.path.splitext(remote_config_file)[1].lower()
-  if file_type not in (".cfg",".json"): my_error("remote_config_file type %s is not .json
or .cfg file" % file_type)
-  file_type = file_type.lstrip(".")
-  remote_run_config = globals()["parse_config_%s" % file_type](remote_config_file)
-  remote_run = True
-
-def is_remote_run(): return remote_run
-def is_remote_launch(): return remote_launch
-def set_remote_launch(): 
-  global remote_launch
-  remote_launch=True
-def get_remote_run_config(): return remote_run_config
-
-if "REMOTE_CONFIG_FILE" in os.environ:   # can be set from env or from a file
-  parse_config(os.environ["REMOTE_CONFIG_FILE"])
-  remote_launch = True # env will not replicated across, so set env will enable launch
- 
-# url utilities
-def quote_json(in_str):
-    ret = re.sub('([{,])(\w+)(:)','\\1"\\2"\\3', in_str)
-    dbg_print("ret = %s" % ret)
-    return ret
-
-def send_url(url_str):
-    dbg_print("url_str = %s" % url_str)
-    usock = urllib.urlopen(url_str)
-    output = usock.read()
-    dbg_print("output = %s" % output)
-    usock.close()
-    return output
-
-# sqlplus
-default_db_port=1521
-conn_str_template="%%s/%%s@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=%%s)(PORT=%s)))(CONNECT_DATA=(SERVICE_NAME=%%s)))"
% default_db_port
-sqlplus_cmd="sqlplus"
-#sqlplus_cmd="NLS_LANG=_.UTF8 sqlplus"   # handle utf8
-sqlplus_heading='''
-set echo off
-set pages 50000
-set long 2000000000
-set linesize 5000
-column xml format A5000
-set colsep ,     
-set trimspool on 
-set heading off
-set headsep off  
-set feedback 0
--- set datetime format
-ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MON-DD HH24:MI:SS';
-ALTER SESSION SET NLS_TIMESTAMP_FORMAT='YYYY-MON-DD HH24:MI:SS.FF3';
-'''
-# use milliseconds
-
-def exec_sql_one_row(qry, user, passwd, sid, host):
-    return exec_sql(qry, user, passwd, sid, host, True)[0]
-
-def exec_sql_split_results(result_line):
-    dbg_print("result_line = %s" % result_line)
-    # validate to see if there are errors
-    err_pattern = re.compile("ORA-\d+|SP2-\d+")
-    is_err=err_pattern.search(result_line)
-    if is_err: return [["DBERROR","|".join([r.lstrip() for r in result_line.split("\n") if
r != ""])]]
-    else: return [[c.strip() for c in r.split(",")] for r in result_line.split("\n") if r
!= ""]
-
-def exec_sql(qry, user, passwd, sid, host, do_split=False):
-    ''' returns an list of results '''
-    dbg_print("qry = %s" % (qry)) 
-    sqlplus_input="%s \n %s; \n exit \n" % (sqlplus_heading, qry)
-    #(user, passwd, sid, host) = tuple(area_conn_info[options.area])
-    dbg_print("conn info= %s %s %s %s" % (user, passwd, sid, host))
-    sqlplus_call="%s -S %s" % (sqlplus_cmd, conn_str_template % (user, passwd, host, sid))
-    os.environ["NLS_LANG"]=".UTF8"  # handle utf8
-    ret_str = sys_pipe_call_2(sqlplus_input, sqlplus_call) 
-    dbg_print("ret_str = %s" % ret_str)
-    # may skip this
-    if do_split: return exec_sql_split_results(ret_str)
-    else: return ret_str
-
-def parse_db_conf_file(db_config_file, db_src_ids_str=""):
-    global db_conn_user_id, db_sid, db_host, db_conn_user_passwd, db_src_info, db_src_ids
-    db_src_info={}
-    db_sources=json.load(open(db_config_file))
-    uri = db_sources["uri"]
-    db_conn_user_id = (uri.split("/")[0]).split(":")[-1]
-    db_conn_user_passwd = (uri.split("@")[0]).split("/")[-1]
-    db_host= (uri.split("@")[1]).split(":")[0]
-    tmp = uri.split("@")[1]
-    if tmp.find("/") != -1: db_sid = tmp.split("/")[-1]
-    else: db_sid = tmp.split(":")[-1]
-    dbg_print("db_conn_user_id = %s, db_conn_user_passwd = %s, db_host = %s, db_sid = %s"
% (db_conn_user_id, db_conn_user_passwd, db_host, db_sid))
-
-    schema_registry_dir=os.path.join(get_view_root(),"schemas_registry")
-    schema_registry_list=os.listdir(schema_registry_dir)
-    schema_registry_list.sort()
-    sources={}
-    for src in db_sources["sources"]: sources[src["id"]]=src
-    if db_src_ids_str: 
-      if db_src_ids_str=="all": db_src_ids=sources.keys()
-      else: db_src_ids = [int(x) for x in db_src_ids_str.split(",")]
-    else: db_src_ids=[]
-    for src_id in db_src_ids:
-      if src_id not in sources: 
-        my_error("source id %s not in config file %s. Available source ids are %s" % (src_id,
db_config_file, sources.keys()))
-      src_info = sources[src_id]
-      src_name = src_info["name"].split(".")[-1]
-      db_avro_file_path = os.path.join(schema_registry_dir,[x for x in schema_registry_list
if re.search("%s.*avsc" % src_name,x)][-1])
-      if not os.path.exists(db_avro_file_path): my_error("Schema file %s does not exist"
% db_avro_file_path)
-      db_user_id = db_conn_user_id
-      src_uri = src_info["uri"]
-      if src_uri.find(".") != -1: db_user_id = src_uri.split(".")[0]
-      db_src_info[src_id] = {"src_name":src_name,"db_user_id":db_user_id, "db_avro_file_path":db_avro_file_path,
"uri":src_uri}
-      dbg_print("db_src_info for src_id %s = %s" % (src_id, db_src_info[src_id]))
-    return db_conn_user_id, db_sid, db_host, db_conn_user_passwd, db_src_info, db_src_ids
-
-''' mysql related stuff '''
-mysql_cmd="mysql"
-def get_mysql_call(dbname, user, passwd, host):
-    conn_str = "%s -s " % mysql_cmd
-    if dbname: conn_str += "-D%s " % dbname
-    if user: conn_str += "-u%s " % user
-    if passwd: conn_str += "-p%s " % passwd
-    if host: conn_str += "-P%s " % host
-    return conn_str
-
-def mysql_exec_sql_one_row(qry, dbname=None, user=None, passwd=None, host=None):
-    ret = mysql_exec_sql(qry, dbname, user, passwd, host, True)
-    dbg_print("ret = %s" % ret)
-    if ret: return ret[0]
-    else: return None
-
-def mysql_exec_sql_split_results(result_line):
-    dbg_print("result_line = %s" % result_line)
-    # validate to see if there are errors
-    err_pattern = re.compile("ERROR \d+")
-    is_err=err_pattern.search(result_line)
-    if is_err: return [["DBERROR","|".join([r.lstrip() for r in result_line.split("\n") if
r != ""])]]
-    else: return [[c.strip() for c in r.split("\t")] for r in result_line.split("\n") if
r != ""]
-
-def mysql_exec_sql(qry, dbname=None, user=None, passwd=None, host=None, do_split=False):
-    ''' returns an list of results '''
-    dbg_print("qry = %s" % (qry)) 
-    mysql_input=" %s; \n exit \n" % (qry)
-    dbg_print("conn info= %s %s %s %s" % (dbname, user, passwd, host))
-    mysql_call=get_mysql_call(dbname, user, passwd, host)
-    dbg_print("mysql_call= %s" % (mysql_call))
-    #if not re.search("select",qry): return None # test only, select only
-    ret_str = sys_pipe_call_21(mysql_input, mysql_call)   # also returns the error
-    dbg_print("ret_str = %s" % ret_str)
-    # may skip this
-    if do_split: return mysql_exec_sql_split_results(ret_str)
-    else: return ret_str
-
-def get_copy_name(input_file_name):
-    input_f = os.path.basename(input_file_name)
-    input_f_split =  input_f.split(".")
-    append_idx = min(len(input_f_split)-2,0)
-    input_f_split[append_idx] += time.strftime('_%y%m%d_%H%M%S')
-    new_file= os.path.join(work_dir, ".".join(input_f_split))
-    return new_file
-
-def save_copy(input_files):
-    for i in range(len(input_files)):
-      new_file= get_copy_name(input_files[i])
-      dbg_print("Copy %s to %s" % (input_files[i], new_file))
-      if not remote_run:
-        shutil.copy(input_files[i], new_file)
-      else:
-        remote_run_copy(input_files[i], new_file, i)
-      input_files[i] = new_file
-    return input_files
-
-def save_copy_one(input_file):
-    ''' wrapper for save copy '''
-    input_files=[input_file]
-    save_copy(input_files)
-    return input_files[0]
-
-def db_config_detect_host_nomral_open(db_host, db_port, db_user=None, passwd=None, db_sid=None):
-    return isOpen(db_host, db_port)
-
-def db_config_detect_host_oracle_open(db_host, db_port, db_user, passwd, db_sid):
-    ret = exec_sql("exit", db_user, passwd, db_sid, db_host, do_split=False)
-    return not re.search("ERROR:",ret)
-
-def db_config_detect_host(db_host, db_port=default_db_port, detect_oracle=False, db_user=None,
passwd=None, db_sid=None):
-    detect_func = detect_oracle and db_config_detect_host_oracle_open or db_config_detect_host_nomral_open
-    if detect_func(db_host, db_port, db_user, passwd, db_sid): return (db_host, db_port)
 # OK
-    possible_hosts = ["localhost"]        # try local host
-    found_host = False
-    for new_db_host in possible_hosts:
-      if not detect_func(new_db_host, db_port, db_user, passwd, db_sid): continue
-      found_host = True
-      break
-    if not found_host: my_error("db server on %s and possible hosts %s port %s is down" %
(db_host, possible_hosts, db_port))
-    print "Substitue the host %s with %s" % (db_host, new_db_host)
-    return (new_db_host, db_port)
-
-def db_config_change(db_relay_config):
-    ''' if there is a config file, handle the case that db is on on local host '''
-    (db_conn_user_id, db_sid, db_host, db_conn_user_passwd, db_src_info, db_src_ids) = parse_db_conf_file(db_relay_config)
-    (new_db_host, new_db_port) = db_config_detect_host(db_host, detect_oracle=True, db_user=db_conn_user_id,
passwd=db_conn_user_passwd, db_sid=db_sid)
-    if new_db_host == db_host: return db_relay_config
-    new_db_config_file = get_copy_name(db_relay_config)
-    print "New config file is %s" % (new_db_config_file)
-    host_port_re = re.compile("@%s:%s:" % (db_host, default_db_port))
-    new_host_port = "@%s:%s:" % (new_db_host, new_db_port)
-    new_db_config_f = open(new_db_config_file, "w")
-    for line in open(db_relay_config):
-      new_db_config_f.write("%s" % host_port_re.sub(new_host_port, line))
-    return new_db_config_file
-
-# get a certain field in url response
-def http_get_field(url_template, host, port, field_name):
-    out = send_url(url_template % (host, port)).split("\n")[1]
-    dbg_print("out = %s" % out)
-    if re.search("Exception:", out): my_error("Exception getting: %s" % out)
-    # work around the invalid json, with out the quote, DDS-379
-    out=quote_json(out)
-    field_value = json.loads(out)
-    return field_value[field_name]
-
-# wait util
-# Classes
-class RetCode:
-  OK=0
-  ERROR=1
-  TIMEOUT=2
-  DIFF=3
-  ZERO_SIZE=4
-
-# wait utility
-def wait_for_condition_1(cond_func, timeout=60, sleep_interval = 0.1):
-  ''' wait for a certain cond. cond could be a function. 
-     This cannot be in utility. Because it needs to see the cond function '''
-  #dbg_print("cond = %s" % cond)
-  if sys_call_debug: return RetCode.OK
-  sleep_cnt = 0
-  ret = RetCode.TIMEOUT
-  while (sleep_cnt * sleep_interval < timeout):
-    dbg_print("attempt %s " % sleep_cnt)
-    if cond_func():
-      dbg_print("success")
-      ret = RetCode.OK
-      break
-    time.sleep(sleep_interval)
-    sleep_cnt += 1
-  return ret
-
-def wait_for_port(host, port):
-  def test_port_not_open():
-    return not isOpen(host, port) 
-  ret = wait_for_condition_1(test_port_not_open, timeout=20, sleep_interval=2)
-  if ret != RetCode.OK:
-    print "ERROR: host:port %s%s is in use" % (host, port)
-  return ret
-
-# find child pid contains java
-# works with linux
-def find_java_pid(this_pid):
-  cmd = sys_pipe_call("ps -o command --pid %s --noheader" % this_pid).split("\n")[0]
-  dbg_print("cmd = %s" % cmd)
-  cmd_split = cmd.split()
-  if len(cmd_split) ==0: return None
-  if re.search("java$", cmd_split[0]): return this_pid
-  child_processes = [x for x in sys_pipe_call("ps -o pid --ppid %s --noheader" % this_pid).split("\n")
if x!=""]
-  for child_process in child_processes:
-    dbg_print("child_process = %s" % child_process)
-    child_pid = child_process.split()[0]
-    java_pid = find_java_pid(child_pid)
-    if java_pid: return java_pid
-  return None
-
-# pid info
-PROCESS_INFO_FILE_NAME="process_info.json"
-def set_work_dir(dir):
-  global work_dir
-  work_dir = dir
-
-def get_process_info_file(dir=None):
-  if not dir: dir = get_work_dir()
-  return os.path.join(dir,PROCESS_INFO_FILE_NAME)  # need to do this after init
-
-def validate_process_info_file():
-  process_info_file = get_process_info_file()
-  if os.path.exists(process_info_file): 
-    return process_info_file
-  else: my_error("Process info file %s for test '%s' does not exist. Please run setup first
or give correct test name." % (process_info_file, get_test_name()))
-
-def get_process_info(process_info_file=None):
-  if not process_info_file: process_info_file = get_process_info_file()
-  if file_exists(process_info_file): 
-    try: 
-      process_info = json.load(open(process_info_file))
-    except ValueError: 
-      my_error("file %s does not have a valid json. Please remove it." % process_info_file)
-  else: 
-    my_warning("process_info_file %s does not exist" % process_info_file)
-    process_info = {}
-  return process_info
-
-def process_info_get_pid_from_log(log_file):
-  log_file_handle = open(log_file)
-  for i in range(10):
-    line = log_file_handle.readline()
-    m = re.search("^([0-9]+)$", line)
-    if i==0 and m:
-      return m.group(1)
-    m = re.search("## java process pid = ([0-9]+)", line)
-    if m:
-      return m.group(1)
-  my_error("cannot find pid in log_file %s" % log_file)
-
-def get_process_info_key(component,id):
-  return "%s:%s" % (component, id)
-
-def split_process_info_key(key):
-  ''' split into component and id '''
-  return tuple(key.split(":"))
-
-def save_process_info(component, id, port, log_file, host=None, admin_port=None, mysql_port=None):
   
-  # port can be None
-  process_info = get_process_info()
-  key = get_process_info_key (component, id)
-  process_info[key]={}
-  process_info[key]["host"] = host !=None and host or host_name_global
-  process_info[key]["port"] = port 
-  process_info[key]["view_root"] = get_view_root()
-  if not re.search("^mysql", component):
-   process_info[key]["port_byteman"] = port and int(port) + 1000 or random.randint(16000,17000)
-  process_info[key]["pid"] =  process_info_get_pid_from_log(log_file)
-  if admin_port: process_info[key]["port_admin"] =  admin_port
-  if mysql_port: process_info[key]["port_mysql"] =  mysql_port
-  process_info_file = get_process_info_file()
-  process_info_fs = open(process_info_file, "w")
-  json.dump(process_info, process_info_fs ,sort_keys=True,indent=4)
-  process_info_fs.close()
-
-  if key not in process_info: my_error("key %s is not in process_info" % (key, process_info))
-  return process_info
-
-def list_process():
-  process_info_file = validate_process_info_file()
-  print "=== Process info for test '%s': %s ===" % (get_test_name(), process_info_file)
-  print "".join(open(process_info_file).readlines())
-
-def get_down_process():
-  process_info_file = validate_process_info_file()
-  process_info = get_process_info()
-  down_process={}
-  for key in process_info:
-    pid = process_info[key]["pid"]
-    dbg_print("checking (%s:%s)" % (key,pid))
-    if not process_exist(pid,remote_run and process_info[key]["host"] or None):
-      down_process[key] = pid
-  return down_process
-
-def check_process_up():
-  down_process = get_down_process()
-  if down_process:
-    for key in down_process:
-      print  "(%s:%s) is down" % (key, down_process[key])
-    return False
-  else:
-    return True  # all the processes are up
-
-# == remote run
-def get_remote_host(component, id="1", field="host"):
-  if not is_remote_run(): return "localhost"
-  key = get_process_info_key(component,id)
-  if key not in get_remote_run_config():
-    my_error("Cannot find remote host for %s in remote_run_config" % (key))
-  return get_remote_run_config()[key][field]
-
-def get_remote_view_root(component, id="1"):
-  get_remote_host(component,id,"view_root")
-
-def need_remote_run(process_info):
-  for k, v in process_info.items():
-    if not re.search("^mysql",k):  # fiter out mysql
-      if v["host"].split(".")[0] != host_name_global.split(".")[0]:
-         return True   # need remote run 
-  return False
-
-metabuilder_file=".metabuilder.properties"
-def get_bldfwk_dir():
-    bldfwk_file = "%s/%s" % (get_view_root(), metabuilder_file)
-    bldfwk_dir = None
-    if not os.path.exists(bldfwk_file): return None
-    for line in open(bldfwk_file):
-      m = re.search("(bldshared-[0-9]+)",line)
-      if m: 
-        bldfwk_dir= m.group(1)
-        break
-    print "Warning. Cannot find bldshared-dir, run ant -f bootstrap.xml"
-    #assert bldfwk_dir, "Cannot find bldshared-dir, run ant -f bootstrap.xml"
-    return bldfwk_dir
-
-rsync_path="/usr//bin/rsync"
-remote_deploy_cmd_template='''rsync -avz  --exclude=.svn --exclude=var --exclude=test-output
--exclude=lucene-indexes --exclude=mmap --exclude=mmappedBuffer --exclude=eventLog --exclude=cp_com_linkedin_events
--exclude=dist --rsync-path=%s %s/ %s:%s'''
-remote_deploy_bldcmd_template='''rsync -avz  --exclude=.svn --rsync-path=%s %s %s:%s'''
-remote_deploy_change_blddir_cmd_template='''ssh %s "sed 's/%s/%s/' %s > %s_tmp; mv -f
%s_tmp %s" '''
-def do_remote_deploy(reset=False):
-    global rsync_path
-    if not remote_run:  # check
-      my_error("Remote config file is not set. use --remote_config_file or set REMOTE_CONFIG_FILE!")
-    bldfwk_dir = get_bldfwk_dir()
-    view_root = get_view_root()
-    already_copied={}
-    for section in remote_run_config:
-      remote_host = remote_run_config[section]["host"]
-      remote_view_root = remote_run_config[section]["view_root"]
-      key = "%s:%s" % (remote_host, remote_view_root)
-      if key in already_copied: 
-        print "Already copied. Skip: host: %s, view_root: %s" % (remote_host, remote_view_root)
-        continue
-      else: already_copied[key]=1
-      if "rsync_path" in remote_run_config[section]: rsync_path=remote_run_config[section]["rsync_path"]
-      remote_view_root_parent = os.path.dirname(remote_view_root)
-      if reset: sys_call("ssh %s rm -rf %s" % (remote_host, remote_view_root))
-      sys_call("ssh %s mkdir -p %s" % (remote_host, remote_view_root))
-      cmd = remote_deploy_cmd_template % (rsync_path, view_root, remote_host, remote_view_root)
-      sys_call(cmd) 
-      if bldfwk_dir:
-        cmd = remote_deploy_bldcmd_template % (rsync_path, os.path.join(os.path.dirname(view_root),bldfwk_dir),
remote_host, remote_view_root_parent)
-        sys_call(cmd) 
-        # replace the metabuilder, TODO, escape the /
-        metabuilder_full_path = os.path.join(remote_view_root, metabuilder_file)
-        cmd = remote_deploy_change_blddir_cmd_template % (remote_host, view_root.replace("/","\/"),
remote_view_root.replace("/","\/"), metabuilder_full_path,  metabuilder_full_path,  metabuilder_full_path,
 metabuilder_full_path)
-        sys_call(cmd) 
-      # copy gradle cache
-      gradle_cache_template = "%s/.gradle/cache"
-      gradle_cache_dir = gradle_cache_template % os.environ["HOME"] 
-      if remote_host.split(".")[0] != host_name_global.split(".")[0] and gradle_cache_dir:
-        ret = sys_pipe_call("ssh %s pwd" % (remote_host))
-        remote_home = ret.split("\n")[0] 
-        ret = sys_call("ssh %s mkdir -p %s " % (remote_host, (gradle_cache_template % remote_home)))
-        cmd = "rsync -avz --rsync-path=%s %s/ %s:%s" % (rsync_path, gradle_cache_dir , remote_host,
gradle_cache_template % remote_home)
-        sys_call(cmd)
-    return RetCode.OK
-
-def get_remote_host_viewroot_path():
-  ''' returns host, view_root, rsync path for each pair of uniq (host,view_root) '''
-  host_viewroot_dict = {}
-  for component in remote_run_config:
-    remote_host = remote_run_config[component]["host"]
-    remote_view_root = remote_run_config[component]["view_root"]
-    combined_key = "%s,%s" % (remote_host, remote_view_root)
-    if remote_host.split(".")[0] != host_name_global.split(".")[0] and not combined_key in
host_viewroot_dict:
-      host_viewroot_dict[combined_key] =  "rsync_path" in remote_run_config[component] and
remote_run_config[component]["rsync_path"] or rsync_path
-  keys = host_viewroot_dict.keys()
-  ret = []
-  for k in keys: 
-    l = k.split(",")
-    l.append(host_viewroot_dict[k])
-    ret.append(tuple(l))
-  return ret
-
-#====End of Utilties============
-

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/script/utility.pyc
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/script/utility.pyc b/helix-agent/src/main/scripts/integration-test/script/utility.pyc
deleted file mode 100644
index e89aeb8..0000000
Binary files a/helix-agent/src/main/scripts/integration-test/script/utility.pyc and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/setup_env.inc
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/setup_env.inc b/helix-agent/src/main/scripts/integration-test/setup_env.inc
deleted file mode 100644
index 95d0f02..0000000
--- a/helix-agent/src/main/scripts/integration-test/setup_env.inc
+++ /dev/null
@@ -1,37 +0,0 @@
-# set up env, figure out the path
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-ROOT_DIR=../../../
-PATH_PREFIX_DIR=../../../integration-test
-SCRIPT_DIR=$PATH_PREFIX_DIR/script/
-CONFIG_DIR=integration-test/config
-VAR_DIR=$PATH_PREFIX_DIR/var
-LOG_DIR=$VAR_DIR/log
-WORK_DIR=$WORK_DIR/work
-WORK_DIR_FROM_ROOT=integration-test/var/work
-LOG_DIR_FROM_ROOT=integration-test/var/log
-DATA_DIR=$PATH_PREFIX_DIR/data
-# solaris tail
-TAIL_PATH=/usr/xpg4/bin/tail     
-if [ ! -f $TAIL_PATH ]; then
-  TAIL_PATH=tail
-fi
-# over all stats
-all_stat=0

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/testcases/foo_test.py
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/testcases/foo_test.py b/helix-agent/src/main/scripts/integration-test/testcases/foo_test.py
deleted file mode 100755
index d9c24aa..0000000
--- a/helix-agent/src/main/scripts/integration-test/testcases/foo_test.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file 
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file 
-# to you under the Apache License, Version 2.0 (the 
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import os, time, sys
-
-this_file_full_path=os.path.abspath(__file__)
-this_file_dirname=os.path.dirname(this_file_full_path)
-
-# write pid for monitor
-pid=os.getpid()
-
-file=open(os.path.join(this_file_dirname,"../var/log/default/foo_TestDB0_0_pid.txt"), "wb")
-file.write("%s\n" % os.getpid())
-file.close()
-
-# this output tells dds_driver.py to return
-print "start"
-sys.stdout.flush()
-
-print "byebye"
-

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/testcases/report_pass_fail.inc
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/testcases/report_pass_fail.inc
b/helix-agent/src/main/scripts/integration-test/testcases/report_pass_fail.inc
deleted file mode 100644
index e396f58..0000000
--- a/helix-agent/src/main/scripts/integration-test/testcases/report_pass_fail.inc
+++ /dev/null
@@ -1,40 +0,0 @@
-# Report pass or fail
-#   Input:
-#      stat_txt, what to display to report 
-# output $?
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-stat=$?
-num_steps=$(($num_steps+1))
-if [ $stat == 0 ]; then
-  echo $stat_txt SUCCESS
-else
-  echo $stat_txt FAIL
- all_stat=$(($all_stat+1))
-fi
-if [ $final_report ]; then
-   if [ $all_stat -gt 0 ]; then
-     echo test FAIL. There are $all_stat failing steps.
-   else
-     if [ $num_steps -gt 1 ]; then
-       echo ALL $num_steps steps for test SUCCESS.
-     fi
-   fi
-fi
-

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/main/scripts/integration-test/testcases/setup_env.inc
----------------------------------------------------------------------
diff --git a/helix-agent/src/main/scripts/integration-test/testcases/setup_env.inc b/helix-agent/src/main/scripts/integration-test/testcases/setup_env.inc
deleted file mode 100644
index bb2e104..0000000
--- a/helix-agent/src/main/scripts/integration-test/testcases/setup_env.inc
+++ /dev/null
@@ -1,60 +0,0 @@
-# set up env, figure out the path
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-#ROOT_DIR=../../../
-export PATH_PREFIX_DIR=../
-export SCRIPT_DIR=$PATH_PREFIX_DIR/script/
-export VAR_DIR=$PATH_PREFIX_DIR/var
-export DATA_DIR=$PATH_PREFIX_DIR/data
-
-export SCRIPT_DIR=../script
-export CONFIG_DIR=integration-test/config
-#export VAR_DIR=../var
-
-DEFAULT_TEST_NAME=`basename $0`
-
-if [ -z "${TEST_NAME}" -o "${TEST_NAME}" = "-bash" ] ; then
-  TEST_NAME=${DEFAULT_TEST_NAME}
-fi
-
-export TEST_NAME
-export VAR_WORK_DIR=../var/work
-export VAR_WORK_DIR_FROM_ROOT=intergration-test/var/work
-
-if [ -z "${TEST_NAME}" ] ; then
-  export LOG_DIR=../var/log
-  export WORK_DIR=../var/work
-  export WORK_DIR_FROM_ROOT=integration-test/var/work
-  export LOG_DIR_FROM_ROOT=integration-test/var/log
-else
-  export LOG_DIR=../var/log/${TEST_NAME}
-  export WORK_DIR=../var/work/${TEST_NAME}
-  export WORK_DIR_FROM_ROOT=integration-test/var/work/${TEST_NAME}
-  export LOG_DIR_FROM_ROOT=integration-test/var/log/${TEST_NAME}
-fi
-#export DATA_DIR=../data
-export VIEW_ROOT=`cd ../../; echo $PWD`
-# solaris tail
-export TAIL_PATH=/usr/xpg4/bin/tail
-if [ ! -f $TAIL_PATH ]; then
-  export TAIL_PATH=tail
-fi
-# over all stats
-all_stat=0

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/4ebc0fad/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java
----------------------------------------------------------------------
diff --git a/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java b/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java
index 6f5ad6c..daad645 100644
--- a/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java
+++ b/helix-agent/src/test/java/org/apache/helix/agent/TestHelixAgent.java
@@ -40,7 +40,8 @@ import org.testng.annotations.Test;
 
 public class TestHelixAgent extends ZkUnitTestBase {
   
-  @Test
+  // disable this test
+  // @Test
   public void test() throws Exception {
     String className = TestHelper.getTestClassName();
     String methodName = TestHelper.getTestMethodName();


Mime
View raw message