avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1718715 - in /avro/trunk: CHANGES.txt lang/js/doc/API.md lang/js/lib/schemas.js lang/js/test/test_schemas.js
Date Tue, 08 Dec 2015 21:34:43 GMT
Author: blue
Date: Tue Dec  8 21:34:42 2015
New Revision: 1718715

URL: http://svn.apache.org/viewvc?rev=1718715&view=rev
Log:
AVRO-1767: Fall back to unwrapped union values. Contributed by Matthieu Monsch.

Previously, union wrapping during `type.clone` calls would be done in
place of normal copy. Now it is done as a backup if a normal copy fails.
This is more consistent with the documentation, and makes it much
simpler to wrap nested unions.

Other changes:

+ Add an optional argument to `type.getName` to expose built-in type
  names (useful to implement an "optional" logical type for example).
+ Document (and rename) `assertLogicalTypes` parsing option.
+ Tweak logical type validity check, enabling error hooks to work even
  when a logical type is valid for a subset of its underlying type's
  values (e.g.  when defining an "even integer" type). Previously the
  path would have been correct but the type would have been that of the
  underlying one.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/js/doc/API.md
    avro/trunk/lang/js/lib/schemas.js
    avro/trunk/lang/js/test/test_schemas.js

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1718715&r1=1718714&r2=1718715&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Tue Dec  8 21:34:42 2015
@@ -63,6 +63,8 @@ Avro 1.8.0 (10 August 2014)
 
     AVRO-1747. JS: Add Javascript IO implementation. (Matthieu Monsch via blue)
 
+    AVRO-1767. JS: Fall back to unwrapped union values. (Matthieu Monsch via blue)
+
   OPTIMIZATIONS
 
   IMPROVEMENTS

Modified: avro/trunk/lang/js/doc/API.md
URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/doc/API.md?rev=1718715&r1=1718714&r2=1718715&view=diff
==============================================================================
--- avro/trunk/lang/js/doc/API.md (original)
+++ avro/trunk/lang/js/doc/API.md Tue Dec  8 21:34:42 2015
@@ -33,12 +33,17 @@ limitations under the License.
     `'./Schema.avsc'`).
   + A decoded schema object (e.g. `{type: 'array', items: 'int'}`).
 + `opts` {Object} Parsing options. The following keys are currently supported:
+  + `namespace` {String} Optional parent namespace.
+  + `registry` {Object} Registry of predefined type names. This can for example
+    be used to override the types used for primitives or to split a schema
+    declaration over multiple files.
   + `logicalTypes` {Object} Optional dictionary of
     [`LogicalType`](#class-logicaltypeattrs-opts-types). This can be used to
     support serialization and deserialization of arbitrary native objects.
-  + `namespace` {String} Optional parent namespace.
-  + `registry` {Object} Optional registry of predefined type names. This can
-    for example be used to override the types used for primitives.
+  + `assertLogicalTypes` {Boolean} The Avro specification mandates that we fall
+    through to the underlying type if a logical type is invalid. When set, this
+    option will override this behavior and throw an error when a logical type
+    can't be applied.
   + `typeHook(attrs, opts)` {Function} Function called before each new type is
     instantiated. The relevant decoded schema is available as first argument
     and the parsing options as second. This function can optionally return a
@@ -336,7 +341,10 @@ evolution can be used to significantly s
 
 Returns a random value of `type`.
 
-##### `type.getName()`
+##### `type.getName([noRef])`
+
++ `noRef` {Boolean} Return built-in names (e.g. `'record'`, `'map'`,
+  `'boolean'`) rather than user-specified references.
 
 Returns `type`'s fully qualified name if it exists, `undefined` otherwise.
 

Modified: avro/trunk/lang/js/lib/schemas.js
URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/lib/schemas.js?rev=1718715&r1=1718714&r2=1718715&view=diff
==============================================================================
--- avro/trunk/lang/js/lib/schemas.js (original)
+++ avro/trunk/lang/js/lib/schemas.js Tue Dec  8 21:34:42 2015
@@ -115,10 +115,7 @@ function createType(attrs, opts) {
       try {
         return new DerivedType(attrs, opts);
       } catch (err) {
-        if (opts.assertLogicalType) {
-          // The spec mandates that we fall through to the underlying type if
-          // the logical type is invalid. We provide this option to ease
-          // debugging.
+        if (opts.assertLogicalTypes) {
           throw err;
         }
         LOGICAL_TYPE = null;
@@ -305,7 +302,9 @@ Type.prototype.compareBuffers = function
   return this._match(new Tap(buf1), new Tap(buf2));
 };
 
-Type.prototype.getName = function () { return this._name; };
+Type.prototype.getName = function (noRef) {
+  return noRef ? getTypeName(this) : this._name;
+};
 
 Type.prototype.getSchema = function (noDeref) {
   // Since JS objects are unordered, this implementation (unfortunately)
@@ -870,19 +869,9 @@ UnionType.prototype._copy = function (va
   if (val === null && this._indices['null'] !== undefined) {
     return null;
   }
+
   var i, l, obj;
-  if (wrap === 1) {
-    // Promote into first match (convenience, slow).
-    i = 0;
-    l = this._types.length;
-    while (i < l && obj === undefined) {
-      try {
-        obj = this._types[i]._copy(val, opts);
-      } catch (err) {
-        i++;
-      }
-    }
-  } else if (typeof val == 'object') {
+  if (typeof val == 'object') {
     var keys = Object.keys(val);
     if (keys.length === 1) {
       var name = keys[0];
@@ -905,6 +894,18 @@ UnionType.prototype._copy = function (va
       }
     }
   }
+  if (wrap === 1 && obj === undefined) {
+    // Try promoting into first match (convenience, slow).
+    i = 0;
+    l = this._types.length;
+    while (i < l && obj === undefined) {
+      try {
+        obj = this._types[i]._copy(val, opts);
+      } catch (err) {
+        i++;
+      }
+    }
+  }
   if (obj !== undefined) {
     return new this._constructors[i](obj);
   }
@@ -1801,23 +1802,31 @@ LogicalType.prototype._read = function (
   return this._fromValue(this._underlyingType._read(tap));
 };
 
-LogicalType.prototype._write = function (tap, val) {
-  this._underlyingType._write(tap, this._toValue(val));
+LogicalType.prototype._write = function (tap, any) {
+  this._underlyingType._write(tap, this._toValue(any));
 };
 
-LogicalType.prototype._check = function (val, cb) {
-  return this._underlyingType._check(this._toValue(val), cb);
+LogicalType.prototype._check = function (any, cb) {
+  try {
+    var val = this._toValue(any);
+  } catch (err) {
+    if (cb) {
+      cb(PATH.slice(), any, this);
+    }
+    return false;
+  }
+  return this._underlyingType._check(val, cb);
 };
 
-LogicalType.prototype._copy = function (val, opts) {
+LogicalType.prototype._copy = function (any, opts) {
   var type = this._underlyingType;
   switch (opts && opts.coerce) {
     case 3: // To string.
-      return type._copy(this._toValue(val), opts);
+      return type._copy(this._toValue(any), opts);
     case 2: // From string.
-      return this._fromValue(type._copy(val, opts));
+      return this._fromValue(type._copy(any, opts));
     default: // Normal copy.
-      return this._fromValue(type._copy(this._toValue(val), opts));
+      return this._fromValue(type._copy(this._toValue(any), opts));
   }
 };
 

Modified: avro/trunk/lang/js/test/test_schemas.js
URL: http://svn.apache.org/viewvc/avro/trunk/lang/js/test/test_schemas.js?rev=1718715&r1=1718714&r2=1718715&view=diff
==============================================================================
--- avro/trunk/lang/js/test/test_schemas.js (original)
+++ avro/trunk/lang/js/test/test_schemas.js Tue Dec  8 21:34:42 2015
@@ -58,6 +58,12 @@ suite('types', function () {
       assert.equal(t.compareBuffers(bt, bt), 0);
     });
 
+    test('get name', function () {
+      var t = new types.BooleanType();
+      assert.strictEqual(t.getName(), undefined);
+      assert.equal(t.getName(true), 'boolean');
+    });
+
   });
 
   suite('IntType', function () {
@@ -1460,6 +1466,22 @@ suite('types', function () {
       assert.throws(function () { v2.createResolver(v1); });
     });
 
+    test('getName', function () {
+      var t = createType({
+        type: 'record',
+        name: 'Person',
+        doc: 'Hi!',
+        namespace: 'earth',
+        aliases: ['Human'],
+        fields: [
+          {name: 'friends', type: {type: 'array', items: 'string'}},
+          {name: 'age', aliases: ['years'], type: {type: 'int'}}
+        ]
+      });
+      assert.strictEqual(t.getName(), 'earth.Person');
+      assert.equal(t.getName(true), 'record');
+    });
+
     test('getSchema', function () {
       var t = createType({
         type: 'record',
@@ -1879,7 +1901,7 @@ suite('types', function () {
       assert.throws(function () {
         createType(attrs, {
           logicalTypes: logicalTypes,
-          assertLogicalType: true
+          assertLogicalTypes: true
         });
       });
     });
@@ -1968,6 +1990,151 @@ suite('types', function () {
       assert.equal(t2.fromBuffer(buf, res), +d);
     });
 
+    suite('union logical types', function () {
+      // Unions are slightly tricky to override with logical types since their
+      // schemas aren't represented as objects.
+
+      var schema = [
+        'null',
+        {
+          name: 'Person',
+          type: 'record',
+          fields: [
+            {name: 'name', type: 'string'},
+            {name: 'age', type: ['null', 'int'], 'default': null}
+          ]
+        }
+      ];
+
+      function createUnionTypeHook(Type) {
+        var visited = [];
+        return function(attrs, opts) {
+          if (attrs instanceof Array && !~visited.indexOf(attrs)) {
+            visited.push(attrs);
+            return new Type(attrs, opts);
+          }
+        };
+      }
+
+      test('unwrapped', function () {
+
+        /**
+        * A generic union type which exposes its values directly.
+        *
+        * This implementation is pretty minimal, we could optimize it by caching
+        * the underlying union's type names for example.
+        *
+        */
+        function UnwrappedUnionType(attrs, opts) {
+          types.LogicalType.call(this, attrs, opts);
+        }
+        util.inherits(UnwrappedUnionType, types.LogicalType);
+
+        UnwrappedUnionType.prototype._fromValue = function (val) {
+          return val === null ? null : val[Object.keys(val)[0]];
+        };
+
+        UnwrappedUnionType.prototype._toValue = function (any) {
+          return this.getUnderlyingType().clone(any, {wrapUnions: true});
+        };
+
+        var t1 = createType(
+          schema,
+          {typeHook: createUnionTypeHook(UnwrappedUnionType)}
+        );
+        var obj = {name: 'Ann', age: 23};
+        assert(t1.isValid(obj));
+        var buf = t1.toBuffer(obj);
+        var t2 = createType(schema);
+        assert.deepEqual(
+          t2.fromBuffer(buf),
+          {Person: {name: 'Ann', age: {'int': 23}}}
+        );
+
+      });
+
+      test('optional', function () {
+
+        /**
+        * A basic optional type.
+        *
+        * It assumes an underlying union of the form `["null", ???]`.
+        *
+        * Enhancements include:
+        *
+        * + Performing a check in the constructor on the underlying type (i.e.
+        *   union with the correct form).
+        * + Code-generating the conversion methods (especially a constructor
+        *   for `_toValue`).
+        *
+        */
+        function OptionalType(attrs, opts) {
+          types.LogicalType.call(this, attrs, opts);
+          var type = this.getUnderlyingType().getTypes()[1];
+          this._name = type.getName() || type.getName(true);
+        }
+        util.inherits(OptionalType, types.LogicalType);
+
+        OptionalType.prototype._fromValue = function (val) {
+          return val === null ? null : val[this._name];
+        };
+
+        OptionalType.prototype._toValue = function (any) {
+          if (any === null) {
+            return null;
+          }
+          var obj = {};
+          obj[this._name] = any;
+          return obj;
+        };
+
+        var t1 = createType(
+          schema,
+          {typeHook: createUnionTypeHook(OptionalType)}
+        );
+        var obj = {name: 'Ann', age: 23};
+        assert(t1.isValid(obj));
+        var buf = t1.toBuffer(obj);
+        var t2 = createType(schema);
+        assert.deepEqual(
+          t2.fromBuffer(buf),
+          {Person: {name: 'Ann', age: {'int': 23}}}
+        );
+
+      });
+
+    });
+
+    test('even integer', function () {
+      function EvenIntType(attrs, opts) {
+        types.LogicalType.call(this, attrs, opts, [types.IntType]);
+      }
+      util.inherits(EvenIntType, types.LogicalType);
+      EvenIntType.prototype._fromValue = function (val) {
+        this._assertValid(val);
+        return val;
+      };
+      EvenIntType.prototype._toValue = EvenIntType.prototype._fromValue;
+      EvenIntType.prototype._assertValid = function (any) {
+        if (any !== (any | 0) || any % 2) {
+          throw new Error('invalid');
+        }
+      };
+
+      var opts = {logicalTypes: {'even-integer': EvenIntType}};
+      var t = createType({type: 'int', logicalType: 'even-integer'}, opts);
+      assert(t.isValid(2));
+      assert(!t.isValid(3));
+      assert(!t.isValid('abc'));
+      assert.equal(t.fromBuffer(new Buffer([4])), 2);
+      assert.equal(t.clone(4), 4);
+      assert.equal(t.fromString('6'), 6);
+      assert.throws(function () { t.clone(3); });
+      assert.throws(function () { t.fromString('5'); });
+      assert.throws(function () { t.toBuffer(3); });
+      assert.throws(function () { t.fromBuffer(new Buffer([2])); });
+    });
+
   });
 
   suite('createType', function  () {
@@ -2205,7 +2372,7 @@ suite('types', function () {
 
   });
 
-  suite('type names', function () {
+  suite('type references', function () {
 
     test('existing', function () {
       var type = createType({



Mime
View raw message