Author: cmlenz
Date: Mon Apr 7 14:06:07 2008
New Revision: 645687
URL: http://svn.apache.org/viewvc?rev=645687&view=rev
Log:
mochiweb branch: Merged revisions 642956-642964,642966-643202,643204-643555,643557-644592,644595-644648,644650-644673,644675-645681
via svnmerge from /incubator/couchdb/trunk
Modified:
incubator/couchdb/branches/mochiweb/ (props changed)
incubator/couchdb/branches/mochiweb/share/server/main.js
incubator/couchdb/branches/mochiweb/share/www/browse/index.html
incubator/couchdb/branches/mochiweb/share/www/script/browse.js
incubator/couchdb/branches/mochiweb/share/www/script/couch.js
incubator/couchdb/branches/mochiweb/share/www/script/couch_tests.js
incubator/couchdb/branches/mochiweb/share/www/script/pprint.js
incubator/couchdb/branches/mochiweb/share/www/style/layout.css
incubator/couchdb/branches/mochiweb/src/couchdb/couch_db.erl
incubator/couchdb/branches/mochiweb/src/couchdb/couch_httpd.erl
incubator/couchdb/branches/mochiweb/src/couchdb/couch_query_servers.erl
incubator/couchdb/branches/mochiweb/src/couchdb/couch_rep.erl
incubator/couchdb/branches/mochiweb/src/couchdb/couch_stream.erl
Propchange: incubator/couchdb/branches/mochiweb/
------------------------------------------------------------------------------
--- svnmerge-integrated (original)
+++ svnmerge-integrated Mon Apr 7 14:06:07 2008
@@ -1 +1 @@
-/incubator/couchdb/trunk:1-642955,642965-644443,644445-644674
+/incubator/couchdb/trunk:1-645681
Modified: incubator/couchdb/branches/mochiweb/share/server/main.js
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/server/main.js?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/server/main.js [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/server/main.js [utf-8] Mon Apr 7 14:06:07 2008
@@ -68,7 +68,7 @@
// [
// [["Key","Value"]], <- fun 1 returned 1 key value
// [], <- fun 2 returned 0 key values
- // [["Key1","Value1"],["Key2","Value2"]],<- fun 3 returned 2 key values
+ // [["Key1","Value1"],["Key2","Value2"]] <- fun 3 returned 2 key values
// ]
//
var doc = cmd[1];
Modified: incubator/couchdb/branches/mochiweb/share/www/browse/index.html
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/browse/index.html?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/browse/index.html [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/www/browse/index.html [utf-8] Mon Apr 7 14:06:07
2008
@@ -24,6 +24,7 @@
<script src="../script/jquery.dialog.js"></script>
<script src="../script/couch.js"></script>
<script src="../script/browse.js"></script>
+ <script src="../script/pprint.js"></script>
<script>
var page = new CouchIndexPage();
$(document).ready(function() {
@@ -47,15 +48,16 @@
<thead>
<tr>
<th>Name</th>
- <th>Number of Documents</th>
- <th>Update Seq</th>
+ <th class="size">Size</th>
+ <th class="count">Number of Documents</th>
+ <th class="seq">Update Seq</th>
</tr>
</thead>
<tbody class="content">
</tbody>
<tbody class="footer">
<tr>
- <td colspan="3"></td>
+ <td colspan="4"></td>
</tr>
</tbody>
</table>
Modified: incubator/couchdb/branches/mochiweb/share/www/script/browse.js
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/script/browse.js?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/script/browse.js [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/www/script/browse.js [utf-8] Mon Apr 7 14:06:07
2008
@@ -41,8 +41,9 @@
var info = new CouchDB(dbName).info();
$("#databases tbody.content").append(
"<tr><th><a href='database.html?" + dbName + "'>" +
- dbName + "</a></th><td>" + info.doc_count +"</td><td>"
+
- info.update_seq + "</td></tr>");
+ dbName + "</a></th><td class='size'>" + prettyPrintSize(info.disk_size)
+
+ "</td><td class='count'>" + info.doc_count +
+ "</td><td class='seq'>" + info.update_seq + "</td></tr>");
$("#databases tbody tr:odd").addClass("odd");
$("#databases tbody.footer tr td").text(allDbs.length + " database(s)");
}
Modified: incubator/couchdb/branches/mochiweb/share/www/script/couch.js
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/script/couch.js?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/script/couch.js [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/www/script/couch.js [utf-8] Mon Apr 7 14:06:07
2008
@@ -87,6 +87,9 @@
var result = JSON.parse(req.responseText);
if (req.status != 201)
throw result;
+ for(i in docs) {
+ docs[i]._rev = result.new_revs[i].rev;
+ }
return result;
}
@@ -127,6 +130,14 @@
var req = request("GET", this.uri + "_all_docs" + encodeOptions(options));
var result = JSON.parse(req.responseText);
if (req.status != 200)
+ throw result;
+ return result;
+ }
+
+ this.compact = function() {
+ var req = request("POST", this.uri + "_compact");
+ var result = JSON.parse(req.responseText);
+ if (req.status != 202)
throw result;
return result;
}
Modified: incubator/couchdb/branches/mochiweb/share/www/script/couch_tests.js
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/script/couch_tests.js?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/script/couch_tests.js [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/www/script/couch_tests.js [utf-8] Mon Apr 7
14:06:07 2008
@@ -839,8 +839,33 @@
headers: {"if-match": etag}
});
T(xhr.status == 202)
- }
+ },
+ compact: function(debug) {
+ var db = new CouchDB("test_suite_db");
+ db.deleteDb();
+ db.createDb();
+ if (debug) debugger;
+ var docs = makeDocs(0, 10);
+ var saveResult = db.bulkSave(docs);
+ T(saveResult.ok);
+ var originalsize = db.info().disk_size;
+
+ for(var i in docs) {
+ db.deleteDoc(docs[i]);
+ }
+ var deletesize = db.info().disk_size;
+ T(deletesize > originalsize);
+
+ var xhr = CouchDB.request("POST", "/test_suite_db/_compact");
+ T(xhr.status == 202);
+ //compaction isn't instantaneous, loop until done
+ while(db.info().compact_running) {};
+
+ var compactedsize = db.info().disk_size;
+
+ T(deletesize > originalsize);
+ }
};
function makeDocs(start, end, templateDoc) {
Modified: incubator/couchdb/branches/mochiweb/share/www/script/pprint.js
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/script/pprint.js?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/script/pprint.js [utf-8] (original)
+++ incubator/couchdb/branches/mochiweb/share/www/script/pprint.js [utf-8] Mon Apr 7 14:06:07
2008
@@ -10,7 +10,7 @@
// License for the specific language governing permissions and limitations under
// the License.
-/* JSON pretty printing */
+// JSON pretty printing
function prettyPrintJSON(val, indent, linesep, depth) {
indent = indent != null ? indent : 4;
@@ -57,4 +57,18 @@
return buf.join("");
}
}
+}
+
+// File size pretty printing
+
+function prettyPrintSize(size) {
+ var jump = 512;
+ if (size < jump) return size + " bytes";
+ var units = ["KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+ var i = 0;
+ while (size >= jump && i < units.length) {
+ i += 1;
+ size /= 1024
+ }
+ return size.toFixed(1) + ' ' + units[i - 1];
}
Modified: incubator/couchdb/branches/mochiweb/share/www/style/layout.css
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/share/www/style/layout.css?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/share/www/style/layout.css (original)
+++ incubator/couchdb/branches/mochiweb/share/www/style/layout.css Mon Apr 7 14:06:07 2008
@@ -159,7 +159,8 @@
#footer { background: #ddd; border-top: 1px solid #bbb; color: #000;
font-size: 80%; opacity: .7; padding: 5px 10px; position: absolute; right: 0;
- bottom: 0; height: 10px; width: 190px; text-align: right;
+ bottom: 0; height: 1.3em; width: 190px; text-align: right;
+ white-space: nowrap;
}
#view { position: absolute; left: 0; right: 210px; top: 0; bottom: 0;
height: 100%;
@@ -273,6 +274,13 @@
#viewcode div.bottom button.save { font-weight: bold; }
#viewcode.expanded label { background-position: 0 -96px; }
#viewcode.expanded textarea, #viewcode.expanded div.bottom { display: block; }
+
+/* Database table */
+
+#databases thead th.size, #databases thead th.count, #databases thead th.seq,
+#databases tbody td.size, #databases tbody td.count, #databases tbody td.seq {
+ text-align: right;
+}
/* Documents table */
Modified: incubator/couchdb/branches/mochiweb/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/src/couchdb/couch_db.erl?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/src/couchdb/couch_db.erl (original)
+++ incubator/couchdb/branches/mochiweb/src/couchdb/couch_db.erl Mon Apr 7 14:06:07 2008
@@ -72,7 +72,7 @@
ok = file:rename(Filepath ++ ".compact", Filepath),
Fd0;
{error, enoent} ->
- throw({error, notfound})
+ throw({error, not_found})
end;
Else ->
throw(Else)
@@ -105,6 +105,9 @@
open(DbName, Filepath) ->
start_link(DbName, Filepath, []).
+
+% Compaction still needs work. Right now readers and writers can get an error
+% file compaction changeover. This doesn't need to be the case.
start_compact(MainPid) ->
gen_server:cast(MainPid, start_compact).
@@ -179,8 +182,8 @@
{doc_count, Count},
{doc_del_count, DelCount},
{update_seq, SeqNum},
- {compacting, Compactor/=nil},
- {size, Size}
+ {compact_running, Compactor/=nil},
+ {disk_size, Size}
],
{ok, InfoList}.
@@ -253,6 +256,7 @@
Doc#doc{revs=[integer_to_list(couch_util:rand32()) | Revs]}
end
end, Docs),
+ NewRevs = [NewRev || #doc{revs=[NewRev|_]} <- Docs2],
DocBuckets = group_alike_docs(Docs2),
Ids = [Id || [#doc{id=Id}|_] <- DocBuckets],
Db = get_db(MainPid),
@@ -275,11 +279,17 @@
% flush unwritten binaries to disk.
DocBuckets3 = [[doc_flush_binaries(Doc, Db#db.fd) || Doc <- Bucket] || Bucket <-
DocBuckets2],
-
+
case gen_server:call(MainPid, {update_docs, DocBuckets3, Options}) of
- ok ->
- % return back the new rev ids, in the same order input.
- {ok, [NewRev || #doc{revs=[NewRev|_]} <- Docs2]};
+ ok -> {ok, NewRevs};
+ retry ->
+ Db2 = get_db(MainPid),
+ DocBuckets4 = [[doc_flush_binaries(Doc, Db2#db.fd) || Doc <- Bucket] || Bucket
<- DocBuckets3],
+ % We only retry once
+ case gen_server:call(MainPid, {update_docs, DocBuckets4, Options}) of
+ ok -> {ok, NewRevs};
+ Else -> throw(Else)
+ end;
Else->
throw(Else)
end.
@@ -477,7 +487,7 @@
MainPid ! {initialized, Db2},
update_loop(Db2).
-update_loop(#db{name=Name,filepath=Filepath, main_pid=MainPid}=Db) ->
+update_loop(#db{fd=Fd,name=Name,filepath=Filepath, main_pid=MainPid}=Db) ->
receive
{OrigFrom, update_docs, DocActions, Options} ->
case (catch update_docs_int(Db, DocActions, Options)) of
@@ -486,6 +496,9 @@
gen_server:reply(OrigFrom, ok),
couch_db_update_notifier:notify({updated, Name}),
update_loop(Db2);
+ retry ->
+ gen_server:reply(OrigFrom, retry),
+ update_loop(Db);
conflict ->
gen_server:reply(OrigFrom, conflict),
update_loop(Db);
@@ -519,7 +532,17 @@
doc_count = Db#db.doc_count,
doc_del_count = Db#db.doc_del_count,
filepath = Filepath},
- close_db(Db),
+
+ couch_stream:close(Db#db.summary_stream),
+ % close file handle async.
+ % wait 5 secs before closing, allowing readers to finish
+ unlink(Fd),
+ spawn_link(fun() ->
+ receive after 5000 -> ok end,
+ couch_file:close(Fd),
+ file:delete(Filepath ++ ".old")
+ end),
+
ok = gen_server:call(MainPid, {db_updated, NewDb2}),
couch_log:info("Compaction for db ~p completed.", [Name]),
update_loop(NewDb2#db{compactor_pid=nil});
@@ -651,17 +674,29 @@
flush_trees(_Db, [], AccFlushedTrees) ->
{ok, lists:reverse(AccFlushedTrees)};
-flush_trees(Db, [Unflushed | RestUnflushed], AccFlushed) ->
+flush_trees(#db{fd=Fd}=Db, [Unflushed | RestUnflushed], AccFlushed) ->
Flushed = couch_key_tree:map(
fun(_Rev, Value) ->
case Value of
#doc{attachments=Atts,deleted=IsDeleted}=Doc ->
% this node value is actually an unwritten document summary,
% write to disk.
-
- % convert bins, removing the FD.
- % All bins should have been flushed to disk already.
- Bins = [{BinName, {BinType, BinSp, BinLen}} || {BinName, {BinType, {_Fd,
BinSp, BinLen}}} <- Atts],
+ % make sure the Fd in the written bins is the same Fd we are.
+ Bins =
+ case Atts of
+ [] -> [];
+ [{_BName, {_Type, {BinFd, _Sp, _Len}}} | _ ] when BinFd == Fd ->
+ % convert bins, removing the FD.
+ % All bins should have been flushed to disk already.
+ [{BinName, {BinType, BinSp, BinLen}}
+ || {BinName, {BinType, {_Fd, BinSp, BinLen}}}
+ <- Atts];
+ _ ->
+ % BinFd must not equal our Fd. This can happen when a database
+ % is being updated during a compaction
+ couch_log:debug("File where the attachments are written has changed.
Possibly retrying."),
+ throw(retry)
+ end,
{ok, NewSummaryPointer} = couch_stream:write_term(Db#db.summary_stream, {Doc#doc.body,
Bins}),
{IsDeleted, NewSummaryPointer};
_ ->
@@ -880,7 +915,7 @@
fun(#doc_info{update_seq=Seq}=DocInfo, _Offset, {AccNewDb, AccUncopied}) ->
case couch_util:should_flush() of
true ->
- NewDb2 = copy_docs(Db, AccNewDb, lists:reverse(AccUncopied, DocInfo)),
+ NewDb2 = copy_docs(Db, AccNewDb, lists:reverse([DocInfo | AccUncopied])),
{ok, {commit_data(NewDb2#db{update_seq=Seq}), []}};
false ->
{ok, {AccNewDb, [DocInfo | AccUncopied]}}
Modified: incubator/couchdb/branches/mochiweb/src/couchdb/couch_httpd.erl
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/src/couchdb/couch_httpd.erl?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/src/couchdb/couch_httpd.erl (original)
+++ incubator/couchdb/branches/mochiweb/src/couchdb/couch_httpd.erl Mon Apr 7 14:06:07 2008
@@ -218,6 +218,15 @@
handle_db_request(_Req, _Method, {_DbName, _Db, ["_bulk_docs"]}) ->
throw({method_not_allowed, "POST"});
+handle_db_request(Req, 'POST', {_DbName, Db, ["_compact"]}) ->
+ ok = couch_db:start_compact(Db),
+ send_json(Req, 202, {obj, [
+ {ok, true}
+ ]});
+
+handle_db_request(_Req, _Method, {_DbName, _Db, ["_compact"]}) ->
+ throw({method_not_allowed, "POST"});
+
handle_db_request(Req, 'GET', {DbName, _Db, ["_search"]}) ->
case Req:parse_qs() of
[{"q", Query}] when (length(Query) > 0) ->
@@ -227,7 +236,7 @@
throw({no_fulltext_query, "Empty Query String"})
end;
-handle_db_request(_Req, _Method, {_DbName, _Db, ["_full_text"]}) ->
+handle_db_request(_Req, _Method, {_DbName, _Db, ["_search"]}) ->
throw({method_not_allowed, "GET,HEAD"});
% View request handlers
@@ -474,9 +483,9 @@
{_, undefined} ->
[DocRev];
{undefined, _} ->
- Etag;
+ [Etag];
_ when DocRev == Etag ->
- Etag;
+ [Etag];
_ ->
throw({bad_request, "Document rev and etag have different values"})
end,
Modified: incubator/couchdb/branches/mochiweb/src/couchdb/couch_query_servers.erl
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/src/couchdb/couch_query_servers.erl?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/src/couchdb/couch_query_servers.erl (original)
+++ incubator/couchdb/branches/mochiweb/src/couchdb/couch_query_servers.erl Mon Apr 7 14:06:07
2008
@@ -36,8 +36,6 @@
readline(Port, []).
readline(Port, Acc) ->
- Timer = erlang:send_after(timeout(), self(), timeout),
- Result =
receive
{Port, {data, {noeol, Data}}} ->
readline(Port, [Data|Acc]);
@@ -45,20 +43,11 @@
lists:flatten(lists:reverse(Acc, Data));
{Port, Err} ->
catch port_close(Port),
- erlang:cancel_timer(Timer),
- throw({map_process_error, Err});
- timeout ->
+ throw({map_process_error, Err})
+ after timeout() ->
catch port_close(Port),
throw({map_process_error, "map function timed out"})
- end,
- case erlang:cancel_timer(Timer) of
- false ->
- % message already sent. clear it
- receive timeout -> ok end;
- _ ->
- ok
- end,
- Result.
+ end.
read_json(Port) ->
case cjson:decode(readline(Port)) of
@@ -108,8 +97,7 @@
map_docs({_Lang, Port}, Docs) ->
% send the documents
- Results =
- lists:map(
+ Results = lists:map(
fun(Doc) ->
Json = couch_doc:to_json_obj(Doc, []),
case prompt(Port, {"map_doc", Json}) of
Modified: incubator/couchdb/branches/mochiweb/src/couchdb/couch_rep.erl
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/src/couchdb/couch_rep.erl?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/src/couchdb/couch_rep.erl (original)
+++ incubator/couchdb/branches/mochiweb/src/couchdb/couch_rep.erl Mon Apr 7 14:06:07 2008
@@ -14,7 +14,7 @@
-include("couch_db.hrl").
--export([replicate/2, replicate/3, test/0, test_write_docs/3]).
+-export([replicate/2, replicate/3]).
-record(stats, {
docs_read=0,
@@ -117,8 +117,7 @@
end.
pull_rep(DbTarget, DbSource, SourceSeqNum, Stats) ->
- {ok, NewSeq} =
- enum_docs_since(DbSource, SourceSeqNum,
+ {ok, NewSeq} = enum_docs_since(DbSource, SourceSeqNum,
fun(#doc_info{update_seq=Seq}=SrcDocInfo, _, {_, AccStats}) ->
Stats2 = maybe_save_docs(DbTarget, DbSource, SrcDocInfo, AccStats),
{ok, {Seq, Stats2}}
@@ -136,23 +135,15 @@
[] ->
Stats;
_Else ->
- % the 'ok' below validates no unrecoverable errors (like network failure, etc).
{ok, DocResults} = open_doc_revs(DbSource, Id, MissingRevs, [latest]),
+ % only save successful reads
+ Docs = [RevDoc || {ok, RevDoc} <- DocResults],
+ ok = save_docs(DbTarget, Docs, []),
- Docs = [RevDoc || {ok, RevDoc} <- DocResults], % only match successful loads
-
- Stats2 = Stats#stats{
+ Stats#stats{
docs_read=Stats#stats.docs_read + length(Docs),
- read_errors=Stats#stats.read_errors + length(DocResults) - length(Docs)},
-
- case Docs of
- [] ->
- Stats2;
- _ ->
- % the 'ok' below validates no unrecoverable errors (like network failure, etc).
- ok = save_docs(DbTarget, Docs, []),
- Stats2#stats{docs_copied=Stats2#stats.docs_copied+length(Docs)}
- end
+ read_errors=Stats#stats.read_errors + length(DocResults) - length(Docs),
+ docs_copied=Stats#stats.docs_copied + length(Docs)}
end.
@@ -289,29 +280,3 @@
couch_db:open_doc_revs(Db, DocId, Revs, Options).
-
-
-
-test() ->
- couch_server:start(),
- %{ok, LocalA} = couch_server:open("replica_a"),
- {ok, LocalA} = couch_server:create("replica_a", [overwrite]),
- {ok, _} = couch_server:create("replica_b", [overwrite]),
- %DbA = "replica_a",
- DbA = "http://localhost:5984/replica_a/",
- %DbB = "replica_b",
- DbB = "http://localhost:5984/replica_b/",
- _DocUnids = test_write_docs(10, LocalA, []),
- replicate(DbA, DbB),
- %{ok, _Rev} = couch_db:delete_doc(LocalA, lists:nth(1, DocUnids), any),
- % replicate(DbA, DbB),
- ok.
-
-test_write_docs(0, _Db, Output) ->
- lists:reverse(Output);
-test_write_docs(N, Db, Output) ->
- Doc = #doc{
- id=integer_to_list(N),
- body={obj, [{"foo", integer_to_list(N)}, {"num", N}, {"bar", "blah"}]}},
- couch_db:save_doc(Db, Doc, []),
- test_write_docs(N-1, Db, [integer_to_list(N) | Output]).
Modified: incubator/couchdb/branches/mochiweb/src/couchdb/couch_stream.erl
URL: http://svn.apache.org/viewvc/incubator/couchdb/branches/mochiweb/src/couchdb/couch_stream.erl?rev=645687&r1=645686&r2=645687&view=diff
==============================================================================
--- incubator/couchdb/branches/mochiweb/src/couchdb/couch_stream.erl (original)
+++ incubator/couchdb/branches/mochiweb/src/couchdb/couch_stream.erl Mon Apr 7 14:06:07 2008
@@ -79,7 +79,7 @@
{ok, Bin, Sp2}.
copy_to_new_stream(Src, Sp, Len, DestFd) ->
- Dest = open(DestFd),
+ {ok, Dest} = open(DestFd),
{ok, NewSp} = copy(Src, Sp, Len, Dest),
close(Dest),
{ok, NewSp}.
|