couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject [01/12] couchdb commit: updated refs/heads/1843-feature-bigcouch to 152a21a
Date Thu, 06 Feb 2014 22:12:15 GMT
Updated Branches:
  refs/heads/1843-feature-bigcouch 3069c0134 -> 152a21ad2


Add interim dev/run script

This script assists in booting a basic cluster for development purposes. Further
refinement of the commandline interface will follow.


Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/811eadf4
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/811eadf4
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/811eadf4

Branch: refs/heads/1843-feature-bigcouch
Commit: 811eadf4a7b60e2049e6ced5dadb11b41424e647
Parents: 3069c01
Author: Brian Mitchell <brian@cloudant.com>
Authored: Wed Feb 5 16:37:35 2014 -0500
Committer: Brian Mitchell <brian@cloudant.com>
Committed: Wed Feb 5 17:28:07 2014 -0500

----------------------------------------------------------------------
 dev/boot_node.erl | 125 ++++++++++++++++++++++++++++
 dev/run           | 219 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 344 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb/blob/811eadf4/dev/boot_node.erl
----------------------------------------------------------------------
diff --git a/dev/boot_node.erl b/dev/boot_node.erl
new file mode 100644
index 0000000..cf49b7e
--- /dev/null
+++ b/dev/boot_node.erl
@@ -0,0 +1,125 @@
+-module(boot_node).
+
+-export([start/0]).
+
+
+start() ->
+    monitor_parent(),
+    Apps = load_apps(),
+    Deps = load_deps(Apps),
+    start_all_apps(Deps).
+
+
+monitor_parent() ->
+    {ok, [[PPid]]} = init:get_argument(parent_pid),
+    spawn(fun() -> monitor_parent(PPid) end).
+
+
+monitor_parent(PPid) ->
+    timer:sleep(1000),
+    case os:cmd("kill -0 " ++ PPid) of
+        "" ->
+            monitor_parent(PPid);
+        _Else ->
+            % Assume _Else is a no such process error
+            init:stop()
+    end.
+
+
+load_apps() ->
+    {ok, [[Config]]} = init:get_argument(reltool_config),
+    {ok, Terms} = file:consult(Config),
+    load_apps(Terms).
+
+
+load_apps([]) ->
+    erlang:error(failed_to_load_apps);
+load_apps([{sys, Terms} | _]) ->
+    load_apps(Terms);
+load_apps([{rel, "couchdb", _Vsn, Apps} | _]) ->
+    Apps;
+load_apps([_ | Rest]) ->
+    load_apps(Rest).
+
+
+load_deps(Apps) ->
+    load_deps(Apps, dict:new()).
+
+
+load_deps([], Deps) ->
+    Deps;
+load_deps([App | Rest], Deps) ->
+    load_app(App),
+    case application:get_key(App, applications) of
+        {ok, AppDeps0} ->
+            NewDeps = dict:store(App, AppDeps0, Deps),
+            Filter = fun(A) -> not dict:is_key(A, Deps) end,
+            AppDeps = lists:filter(Filter, AppDeps0),
+            load_deps(AppDeps ++ Rest, NewDeps);
+        _ ->
+            NewDeps = dict:store(App, [], Deps),
+            load_deps(Rest, NewDeps)
+    end.
+
+
+load_app(App) ->
+    case application:load(App) of
+        ok ->
+            case application:get_key(App, modules) of
+                {ok, Modules} ->
+                    lists:foreach(fun(Mod) ->
+                        OK = load_app_module(Mod),
+                        case load_app_module(Mod) of
+                            ok -> ok;
+                            E -> io:format("~p = load_app_module(~p)~n", [E, Mod])
+                        end
+                    end, Modules);
+                undefined ->
+                    ok
+            end;
+        {error, {already_loaded, App}} ->
+            ok;
+        Error ->
+            Error
+    end.
+
+
+load_app_module(Mod) ->
+    case code:is_loaded(Mod) of
+        {file, _} ->
+            ok;
+        _ ->
+            case code:load_file(Mod) of
+                {module, Mod} ->
+                    ok;
+                Error ->
+                    Error
+            end
+    end.
+
+
+start_all_apps(Deps) ->
+    lists:foldl(fun(App, Started) ->
+        start_app(App, Deps, Started)
+    end, [], dict:fetch_keys(Deps)).
+
+
+start_app(App, Deps, Started) ->
+    case lists:member(App, Started) of
+        true ->
+            Started;
+        false ->
+            AppDeps = dict:fetch(App, Deps),
+            NowStarted = lists:foldl(fun(Dep, Acc) ->
+                start_app(Dep, Deps, Acc)
+            end, Started, AppDeps),
+            case application:start(App) of
+                ok ->
+                    [App | NowStarted];
+                {error, {already_started,App}} ->
+                    % Kernel causes this
+                    [App | NowStarted];
+                Else ->
+                    erlang:error(Else)
+            end
+    end.

http://git-wip-us.apache.org/repos/asf/couchdb/blob/811eadf4/dev/run
----------------------------------------------------------------------
diff --git a/dev/run b/dev/run
new file mode 100755
index 0000000..e49040f
--- /dev/null
+++ b/dev/run
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+
+import atexit
+import contextlib as ctx
+import glob
+import optparse as op
+import os
+import re
+import select
+import subprocess as sp
+import sys
+import time
+import urllib
+
+
+USAGE = "%prog [options] [command to run...]"
+DEV_PATH = os.path.dirname(os.path.abspath(__file__))
+COUCHDB = os.path.dirname(DEV_PATH)
+N = 3
+
+
+def init_log_dir():
+    logdir = os.path.join(DEV_PATH, "logs")
+    if not os.path.exists(logdir):
+        os.makedirs(logdir)
+
+
+def init_beams():
+    # Including this for people that forget to run
+    # make dev.
+    for fname in glob.glob(os.path.join(DEV_PATH, "*.erl")):
+        cmd = [
+            "erlc",
+            "-o", DEV_PATH + os.sep,
+            fname
+        ]
+        sp.check_call(cmd)
+
+
+def hack_default_ini(opts, node, args, contents):
+    # Replace log file
+    logfile = os.path.join(DEV_PATH, "logs", "%s.log" % node)
+    repl = "file = %s" % logfile
+    contents = re.sub("(?m)^file.*$", repl, contents)
+
+    # Replace couchjs command
+    couchjs = os.path.join(COUCHDB, "couchjs", "build", "couchjs")
+    mainjs = os.path.join(COUCHDB, "couchjs", "build", "main.js")
+    repl = "javascript = %s %s" % (couchjs, mainjs)
+    contents = re.sub("(?m)^javascript.*$", repl, contents)
+
+    return contents
+
+
+def hack_local_ini(opts, node, args, contents):
+    if opts.admin is None:
+        return contents
+    usr, pwd = opts.admin.split(":", 1)
+    return contents + "\n[admins]\n%s = %s" % (usr, pwd)
+
+
+def write_config(opts, node, args):
+    etc_src = os.path.join(COUCHDB, "rel", "overlay", "etc")
+    etc_tgt = os.path.join(DEV_PATH, "lib", node, "etc")
+    if not os.path.exists(etc_tgt):
+        os.makedirs(etc_tgt)
+
+    etc_files = glob.glob(os.path.join(etc_src, "*"))
+    for fname in etc_files:
+        base = os.path.basename(fname)
+        tgt = os.path.join(etc_tgt, base)
+        with open(fname) as handle:
+            contents = handle.read()
+        for key in args:
+            contents = re.sub("{{%s}}" % key, args[key], contents)
+        if base == "default.ini":
+            contents = hack_default_ini(opts, node, args, contents)
+        elif base == "local.ini":
+            contents = hack_local_ini(opts, node, args, contents)
+        with open(tgt, "w") as handle:
+            handle.write(contents)
+
+
+def write_configs(opts):
+    datadir = os.path.join(DEV_PATH, "data")
+    if not os.path.exists(datadir):
+        os.makedirs(datadir)
+    for i in range(1,4):
+        node = "node%d" % i
+        args = {
+            "prefix": os.path.join(COUCHDB, "rel", "overlay"),
+            "data_dir": os.path.join(DEV_PATH, "lib", node, "data"),
+            "view_dir": os.path.join(DEV_PATH, "lib", node, "data"),
+            "node_name": "-name %s@127.0.0.1" % node,
+            "clouseau_name": "clouseau%d@127.0.0.1" % i,
+            "cluster_port": str((10000 * i) + 5984),
+            "backend_port" : str((10000 * i) + 5986)
+        }
+        if not os.path.exists(args["data_dir"]):
+            os.makedirs(args["data_dir"])
+        write_config(opts, node, args)
+
+
+def all_nodes_alive(n):
+    for i in range(1, n+1):
+        url = "http://127.0.0.1:{0}/".format(local_port(i))
+        while True:
+            try:
+                with ctx.closing(urllib.urlopen(url)) as resp:
+                    print(resp.read())
+            except IOError:
+                time.sleep(0.25)
+                continue
+            break
+    return True
+
+
+def local_port(n):
+    return 10000 * n + 5986
+
+
+def node_port(n):
+    return 10000 * n + 5984
+
+
+def boot_node(node):
+    apps = os.path.join(COUCHDB, "src")
+    env = os.environ.copy()
+    env["ERL_LIBS"] = os.pathsep.join([apps])
+    cmd = [
+        "erl",
+        "-args_file", os.path.join(DEV_PATH, "lib", node, "etc", "vm.args"),
+        "-couch_ini",
+            os.path.join(DEV_PATH, "lib", node, "etc", "default.ini"),
+            os.path.join(DEV_PATH, "lib", node, "etc", "local.ini"),
+        "-reltool_config", os.path.join(COUCHDB, "rel", "reltool.config"),
+        "-parent_pid", str(os.getpid()),
+        "-pa", DEV_PATH,
+        "-pa", os.path.join(COUCHDB, "src", "*"),
+        "-s", "boot_node"
+    ]
+    logfname = os.path.join(DEV_PATH, "logs", "%s.log" % node)
+    log = open(logfname, "w")
+    print ' '.join(cmd)
+    return sp.Popen(
+            cmd,
+            stdin=sp.PIPE,
+            stdout=log,
+            stderr=sp.STDOUT,
+            env=env)
+
+
+def connect_nodes(backdoor):
+    for i in range(1,4):
+        cmd = "curl -s %snodes/node%s@127.0.0.1 -X PUT -d '{}'"
+        cmd = cmd % (backdoor, i)
+        sp.check_call(cmd, shell=True)
+
+
+def run_command(cmd):
+    p = sp.Popen(cmd, shell=True)
+    p.wait()
+    exit(p.returncode)
+
+
+def wait_for_procs(procs):
+    while True:
+        for p in procs:
+            if p.returncode is not None:
+                exit(1)
+        time.sleep(2)
+
+
+def options():
+    return [
+        op.make_option("-a", "--admin", metavar="USER:PASS", default=None,
+            help="Add an admin account to the development cluster")
+    ]
+
+
+def main():
+    parser = op.OptionParser(usage=USAGE, option_list=options())
+    opts, args = parser.parse_args()
+
+    init_log_dir()
+    init_beams()
+    write_configs(opts)
+
+    procs = []
+
+    def kill_procs():
+        for p in procs:
+            if p.returncode is None:
+                p.terminate()
+    atexit.register(kill_procs)
+
+    for i in range(1, 4):
+        p = boot_node("node%d" % i)
+        procs.append(p)
+
+    for i in range(30):
+        if all_nodes_alive(N):
+            break
+        time.sleep(1)
+
+    connect_nodes("http://127.0.0.1:15986/")
+
+    if len(args):
+        run_command(" ".join(args))
+    else:
+        wait_for_procs(procs)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass


Mime
View raw message