couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gar...@apache.org
Subject couchdb-nmo git commit: Import mongo collections into CouchDB
Date Tue, 12 Jan 2016 11:37:40 GMT
Repository: couchdb-nmo
Updated Branches:
  refs/heads/master 909b275a7 -> dbe4e44b8


Import mongo collections into CouchDB

New command `import-mongo` which imports a mongo collection into a
database.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/commit/dbe4e44b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/tree/dbe4e44b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/diff/dbe4e44b

Branch: refs/heads/master
Commit: dbe4e44b8eabd506f57721ddb7ebf54f5bd466e4
Parents: 909b275
Author: Garren Smith <garren.smith@gmail.com>
Authored: Mon Nov 30 14:41:34 2015 +0200
Committer: Garren Smith <garren.smith@gmail.com>
Committed: Tue Jan 12 13:36:31 2016 +0200

----------------------------------------------------------------------
 doc/api/nmo-import-mongo.md |  11 ++++
 doc/cli/nmo-import-mongo.md |  24 +++++++++
 package.json                |   1 +
 src/import-mongo.js         |  92 +++++++++++++++++++++++++++++++
 src/nmo.js                  |   1 +
 test/import-mongo.js        | 113 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 242 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/doc/api/nmo-import-mongo.md
----------------------------------------------------------------------
diff --git a/doc/api/nmo-import-mongo.md b/doc/api/nmo-import-mongo.md
new file mode 100644
index 0000000..9b0d9b3
--- /dev/null
+++ b/doc/api/nmo-import-mongo.md
@@ -0,0 +1,11 @@
+nmo-import-mongo(3) -- import-mongo
+==============================
+
+## SYNOPSIS
+
+    nmo.commands.import-mongo(<mongourl>, <collection>, [<url> || <cluster>],
<database>)
+
+
+## DESCRIPTION
+
+Bulk import a defined collection from a MongoDB instance into CouchDB.

http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/doc/cli/nmo-import-mongo.md
----------------------------------------------------------------------
diff --git a/doc/cli/nmo-import-mongo.md b/doc/cli/nmo-import-mongo.md
new file mode 100644
index 0000000..408ccfb
--- /dev/null
+++ b/doc/cli/nmo-import-mongo.md
@@ -0,0 +1,24 @@
+nmo-import-mongo(1) -- Bulk import a MongoDB collection
+===========================================
+
+## SYNOPSIS
+
+    nmo import-mongo <clustername> <database> <MongoDB-url> <collection>
+    nmo import-mongo <url> <database> <MongoDB-url> <collection>
+
+
+## DESCRIPTION
+
+Import a MongoDB collection into CouchDB.
+
+Example:
+
+This will import the collection `restaurants` from the MongoDB url `mongodb://localhost:27017/test`
into the database
+`mydatabase` in the cluster `mycluster`.
+
+    nmo import-mongo mycluster mydatabase mongodb://localhost:27017/test restaurants
+
+This will import the collection `restaurants` from the MongoDB url `mongodb://localhost:27017/test`
into the database
+`mydatabase` at the url `http://127.0.0.1:15984`.
+
+    nmo import-mongo http://127.0.0.1:15984 mydatabase mongodb://localhost:27017/test restaurants

http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/package.json
----------------------------------------------------------------------
diff --git a/package.json b/package.json
index a89f3a5..f917a12 100644
--- a/package.json
+++ b/package.json
@@ -33,6 +33,7 @@
     "couchbulkimporter": "^1.0.0",
     "csv-parse": "^1.0.0",
     "ini": "~1.3.3",
+    "mongodb": "^2.0.49",
     "nopt": "~3.0.1",
     "npmlog": "~2.0.0",
     "osenv": "~0.1.0",

http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/src/import-mongo.js
----------------------------------------------------------------------
diff --git a/src/import-mongo.js b/src/import-mongo.js
new file mode 100644
index 0000000..7b03e7f
--- /dev/null
+++ b/src/import-mongo.js
@@ -0,0 +1,92 @@
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import mongo from 'mongodb';
+import BulkBadger from 'bulkbadger';
+import { getUrlFromCluster } from './utils';
+
+export function cli (cluster, database,  mongourl, collection) {
+  if (!cluster || !database || !mongourl || !collection) {
+    const msg = [
+      'Usage:',
+      '',
+      'nmo import-mongo <clustername> <database> <MongoDB-url> <collection>',
+      'nmo import-mongo <url> <database> <MongoDB-url> <collection>'
+    ].join('\n');
+    const err = new Error(msg);
+    err.type = 'EUSAGE';
+    throw err;
+  }
+
+  return importmongo(cluster, database, mongourl, collection);
+}
+
+export function validateMongoUrl (url) {
+  return /mongodb:\/\//.test(url);
+}
+
+
+export default importmongo;
+function importmongo (cluster, database, mongourl, collection) {
+  return new Promise((resolve, reject) => {
+    if (!validateMongoUrl(mongourl)) {
+      const err = new Error('Invalid MongoDB url, url must start with mongodb://');
+      err.type = 'EUSAGE';
+      reject(err);
+      return;
+    }
+
+    mongo.connect(mongourl, function (err, db) {
+      if (err) {
+        reject(err);
+        return;
+      }
+
+      const clusterUrl = getUrlFromCluster(cluster);
+      const col = db.collection(collection);
+      col.count(function (errCount, noOfDocs) {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        if (noOfDocs === 0 || errCount) {
+          const err = new Error([
+            'There are 0 documents in this collection. That could mean that',
+            'the collection does not exist or that the database does not exist.'
+          ].join(''));
+          err.type = 'EUSAGE';
+          reject(err);
+          return;
+        }
+
+        console.log('Migration started!');
+        col.find({}, {})
+          .on('error', function (err) {
+            err.message = 'Error fetching collection - ' + err.message;
+            reject(err);
+          })
+          .pipe(new BulkBadger())
+          .on('error', function (err) {
+            err.message = 'Error - ' + err.message;
+            if (/CouchDB server answered/.test(err.message)) {
+              err.type = 'EUSAGE';
+            }
+            reject(err);
+          })
+          .pipe(new CouchBulkImporter({
+            url: clusterUrl + '/' + database
+          }))
+          .on('error', function (err) {
+            err.message = 'Error migration incomplete - ' + err.message;
+            reject(err);
+          })
+          .on('finish', function () {
+            db.close();
+            console.log('Migration complete!');
+            resolve();
+          });
+      });
+    });
+
+  });
+}

http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/src/nmo.js
----------------------------------------------------------------------
diff --git a/src/nmo.js b/src/nmo.js
index b5124b8..54b96f1 100644
--- a/src/nmo.js
+++ b/src/nmo.js
@@ -14,6 +14,7 @@ const commands = [
   'savetofile',
   'replicate-from',
   'replicate-to',
+  'import-mongo',
   'query'
 ];
 

http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/dbe4e44b/test/import-mongo.js
----------------------------------------------------------------------
diff --git a/test/import-mongo.js b/test/import-mongo.js
new file mode 100644
index 0000000..d6231fa
--- /dev/null
+++ b/test/import-mongo.js
@@ -0,0 +1,113 @@
+import assert from 'assert';
+import nock from 'nock';
+import { createConfigFile } from './common';
+
+import nmo from '../src/nmo.js';
+import importmongo, { validateMongoUrl, cli }  from '../src/import-mongo.js';
+import mongo from 'mongodb';
+import JSONStream from 'JSONStream';
+
+//mongodb mock
+const mongoDocs = [{a: 1}, {b: 2}];
+const col = {
+  find: function () {
+    const stream = JSONStream.stringify();
+    setTimeout(function () {
+      stream.write(mongoDocs);
+      stream.end();
+    }, 10);
+    return stream;
+  },
+  count: function (cb) {
+    cb(null, mongoDocs.length);
+  }
+};
+
+mongo.connect = function (url, cb) {
+  const db = {
+    collection: function () {
+      return col;
+    },
+
+    close: function () {
+
+    }
+  };
+
+  cb(null, db);
+};
+
+describe('import-mongo', () => {
+  createConfigFile();
+  beforeEach(() => {
+    nmo
+      .load({nmoconf: __dirname + '/fixtures/randomini'});
+  });
+
+  describe('cli', () => {
+    it('throws error if no inputs', (done) => {
+
+      try {
+        cli();
+      } catch(e) {
+        assert.deepEqual(e.type, 'EUSAGE');
+      }
+      done();
+    });
+
+  });
+
+  describe('validateMongoUrl', () => {
+    it('returns false for bad url', () => {
+      assert.ok(!validateMongoUrl('bad-url'));
+    });
+
+    it('returns true for valid url', () => {
+      assert.ok(validateMongoUrl('mongodb://localhost:27017/test'));
+    });
+  });
+
+  describe('importmongo', done => {
+
+    it('imports from mongodb to couchdb', () => {
+      nock('http://127.0.0.1')
+        .put('/fake-mongo')
+        .reply(200)
+        .post('/fake-mongo/_bulk_docs', {docs:[ '[\n[{"a":1},{"b":2}]', '\n]\n' ] })
+        .reply(200);
+
+      return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants');
+
+    });
+
+    it('error for bad mongodb url', () => {
+      return importmongo('clusterone', 'fake-mongo', 'bad-mongo-url', 'restaurants')
+      .catch(function (err) {
+        assert.ok(/Invalid MongoDB/.test(err.message));
+      });
+    });
+
+     it('rejects on an error', () => {
+      nock('http://127.0.0.1')
+        .put('/fake-mongo')
+        .reply(200)
+        .post('/fake-mongo/_bulk_docs')
+        .reply(500, 'error with docs');
+
+      return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants')
+      .catch(function (err) {
+        assert.ok(/error with docs/.test(err.message));
+      });
+     });
+
+     it('returns error for no docs in collection', () => {
+      col.count = (cb) => {cb(null, 0);};
+
+      return importmongo('clusterone', 'fake-mongo', 'mongodb://localhost:27017/test', 'restaurants')
+      .catch(function (err) {
+        assert.ok(/There are 0 documents/.test(err.message));
+      });
+
+     });
+  });
+});


Mime
View raw message