hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [2/7] hive git commit: HIVE-12075 : add analyze command to explictly cache file metadata in HBase metastore (Sergey Shelukhin, reviewed by Alan Gates)
Date Fri, 18 Dec 2015 22:43:07 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php b/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index 34c2205..75f4a47 100644
--- a/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++ b/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -1051,6 +1051,11 @@ interface ThriftHiveMetastoreIf extends \FacebookServiceIf {
    * @return \metastore\ClearFileMetadataResult
    */
   public function clear_file_metadata(\metastore\ClearFileMetadataRequest $req);
+  /**
+   * @param \metastore\CacheFileMetadataRequest $req
+   * @return \metastore\CacheFileMetadataResult
+   */
+  public function cache_file_metadata(\metastore\CacheFileMetadataRequest $req);
 }
 
 class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metastore\ThriftHiveMetastoreIf {
@@ -8559,6 +8564,57 @@ class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metas
     throw new \Exception("clear_file_metadata failed: unknown result");
   }
 
+  public function cache_file_metadata(\metastore\CacheFileMetadataRequest $req)
+  {
+    $this->send_cache_file_metadata($req);
+    return $this->recv_cache_file_metadata();
+  }
+
+  public function send_cache_file_metadata(\metastore\CacheFileMetadataRequest $req)
+  {
+    $args = new \metastore\ThriftHiveMetastore_cache_file_metadata_args();
+    $args->req = $req;
+    $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary');
+    if ($bin_accel)
+    {
+      thrift_protocol_write_binary($this->output_, 'cache_file_metadata', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+    }
+    else
+    {
+      $this->output_->writeMessageBegin('cache_file_metadata', TMessageType::CALL, $this->seqid_);
+      $args->write($this->output_);
+      $this->output_->writeMessageEnd();
+      $this->output_->getTransport()->flush();
+    }
+  }
+
+  public function recv_cache_file_metadata()
+  {
+    $bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_read_binary');
+    if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '\metastore\ThriftHiveMetastore_cache_file_metadata_result', $this->input_->isStrictRead());
+    else
+    {
+      $rseqid = 0;
+      $fname = null;
+      $mtype = 0;
+
+      $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+      if ($mtype == TMessageType::EXCEPTION) {
+        $x = new TApplicationException();
+        $x->read($this->input_);
+        $this->input_->readMessageEnd();
+        throw $x;
+      }
+      $result = new \metastore\ThriftHiveMetastore_cache_file_metadata_result();
+      $result->read($this->input_);
+      $this->input_->readMessageEnd();
+    }
+    if ($result->success !== null) {
+      return $result->success;
+    }
+    throw new \Exception("cache_file_metadata failed: unknown result");
+  }
+
 }
 
 // HELPER FUNCTIONS AND STRUCTURES
@@ -39493,4 +39549,163 @@ class ThriftHiveMetastore_clear_file_metadata_result {
 
 }
 
+class ThriftHiveMetastore_cache_file_metadata_args {
+  static $_TSPEC;
+
+  /**
+   * @var \metastore\CacheFileMetadataRequest
+   */
+  public $req = null;
+
+  public function __construct($vals=null) {
+    if (!isset(self::$_TSPEC)) {
+      self::$_TSPEC = array(
+        1 => array(
+          'var' => 'req',
+          'type' => TType::STRUCT,
+          'class' => '\metastore\CacheFileMetadataRequest',
+          ),
+        );
+    }
+    if (is_array($vals)) {
+      if (isset($vals['req'])) {
+        $this->req = $vals['req'];
+      }
+    }
+  }
+
+  public function getName() {
+    return 'ThriftHiveMetastore_cache_file_metadata_args';
+  }
+
+  public function read($input)
+  {
+    $xfer = 0;
+    $fname = null;
+    $ftype = 0;
+    $fid = 0;
+    $xfer += $input->readStructBegin($fname);
+    while (true)
+    {
+      $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+      if ($ftype == TType::STOP) {
+        break;
+      }
+      switch ($fid)
+      {
+        case 1:
+          if ($ftype == TType::STRUCT) {
+            $this->req = new \metastore\CacheFileMetadataRequest();
+            $xfer += $this->req->read($input);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        default:
+          $xfer += $input->skip($ftype);
+          break;
+      }
+      $xfer += $input->readFieldEnd();
+    }
+    $xfer += $input->readStructEnd();
+    return $xfer;
+  }
+
+  public function write($output) {
+    $xfer = 0;
+    $xfer += $output->writeStructBegin('ThriftHiveMetastore_cache_file_metadata_args');
+    if ($this->req !== null) {
+      if (!is_object($this->req)) {
+        throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+      }
+      $xfer += $output->writeFieldBegin('req', TType::STRUCT, 1);
+      $xfer += $this->req->write($output);
+      $xfer += $output->writeFieldEnd();
+    }
+    $xfer += $output->writeFieldStop();
+    $xfer += $output->writeStructEnd();
+    return $xfer;
+  }
+
+}
+
+class ThriftHiveMetastore_cache_file_metadata_result {
+  static $_TSPEC;
+
+  /**
+   * @var \metastore\CacheFileMetadataResult
+   */
+  public $success = null;
+
+  public function __construct($vals=null) {
+    if (!isset(self::$_TSPEC)) {
+      self::$_TSPEC = array(
+        0 => array(
+          'var' => 'success',
+          'type' => TType::STRUCT,
+          'class' => '\metastore\CacheFileMetadataResult',
+          ),
+        );
+    }
+    if (is_array($vals)) {
+      if (isset($vals['success'])) {
+        $this->success = $vals['success'];
+      }
+    }
+  }
+
+  public function getName() {
+    return 'ThriftHiveMetastore_cache_file_metadata_result';
+  }
+
+  public function read($input)
+  {
+    $xfer = 0;
+    $fname = null;
+    $ftype = 0;
+    $fid = 0;
+    $xfer += $input->readStructBegin($fname);
+    while (true)
+    {
+      $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+      if ($ftype == TType::STOP) {
+        break;
+      }
+      switch ($fid)
+      {
+        case 0:
+          if ($ftype == TType::STRUCT) {
+            $this->success = new \metastore\CacheFileMetadataResult();
+            $xfer += $this->success->read($input);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        default:
+          $xfer += $input->skip($ftype);
+          break;
+      }
+      $xfer += $input->readFieldEnd();
+    }
+    $xfer += $input->readStructEnd();
+    return $xfer;
+  }
+
+  public function write($output) {
+    $xfer = 0;
+    $xfer += $output->writeStructBegin('ThriftHiveMetastore_cache_file_metadata_result');
+    if ($this->success !== null) {
+      if (!is_object($this->success)) {
+        throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+      }
+      $xfer += $output->writeFieldBegin('success', TType::STRUCT, 0);
+      $xfer += $this->success->write($output);
+      $xfer += $output->writeFieldEnd();
+    }
+    $xfer += $output->writeFieldStop();
+    $xfer += $output->writeStructEnd();
+    return $xfer;
+  }
+
+}
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-php/metastore/Types.php b/metastore/src/gen/thrift/gen-php/metastore/Types.php
index e43a13d..fe4c966 100644
--- a/metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -14564,6 +14564,10 @@ class PutFileMetadataRequest {
    * @var string[]
    */
   public $metadata = null;
+  /**
+   * @var int
+   */
+  public $type = null;
 
   public function __construct($vals=null) {
     if (!isset(self::$_TSPEC)) {
@@ -14584,6 +14588,10 @@ class PutFileMetadataRequest {
             'type' => TType::STRING,
             ),
           ),
+        3 => array(
+          'var' => 'type',
+          'type' => TType::I32,
+          ),
         );
     }
     if (is_array($vals)) {
@@ -14593,6 +14601,9 @@ class PutFileMetadataRequest {
       if (isset($vals['metadata'])) {
         $this->metadata = $vals['metadata'];
       }
+      if (isset($vals['type'])) {
+        $this->type = $vals['type'];
+      }
     }
   }
 
@@ -14649,6 +14660,13 @@ class PutFileMetadataRequest {
             $xfer += $input->skip($ftype);
           }
           break;
+        case 3:
+          if ($ftype == TType::I32) {
+            $xfer += $input->readI32($this->type);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
         default:
           $xfer += $input->skip($ftype);
           break;
@@ -14696,6 +14714,11 @@ class PutFileMetadataRequest {
       }
       $xfer += $output->writeFieldEnd();
     }
+    if ($this->type !== null) {
+      $xfer += $output->writeFieldBegin('type', TType::I32, 3);
+      $xfer += $output->writeI32($this->type);
+      $xfer += $output->writeFieldEnd();
+    }
     $xfer += $output->writeFieldStop();
     $xfer += $output->writeStructEnd();
     return $xfer;
@@ -14854,6 +14877,225 @@ class ClearFileMetadataRequest {
 
 }
 
+class CacheFileMetadataResult {
+  static $_TSPEC;
+
+  /**
+   * @var bool
+   */
+  public $isSupported = null;
+
+  public function __construct($vals=null) {
+    if (!isset(self::$_TSPEC)) {
+      self::$_TSPEC = array(
+        1 => array(
+          'var' => 'isSupported',
+          'type' => TType::BOOL,
+          ),
+        );
+    }
+    if (is_array($vals)) {
+      if (isset($vals['isSupported'])) {
+        $this->isSupported = $vals['isSupported'];
+      }
+    }
+  }
+
+  public function getName() {
+    return 'CacheFileMetadataResult';
+  }
+
+  public function read($input)
+  {
+    $xfer = 0;
+    $fname = null;
+    $ftype = 0;
+    $fid = 0;
+    $xfer += $input->readStructBegin($fname);
+    while (true)
+    {
+      $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+      if ($ftype == TType::STOP) {
+        break;
+      }
+      switch ($fid)
+      {
+        case 1:
+          if ($ftype == TType::BOOL) {
+            $xfer += $input->readBool($this->isSupported);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        default:
+          $xfer += $input->skip($ftype);
+          break;
+      }
+      $xfer += $input->readFieldEnd();
+    }
+    $xfer += $input->readStructEnd();
+    return $xfer;
+  }
+
+  public function write($output) {
+    $xfer = 0;
+    $xfer += $output->writeStructBegin('CacheFileMetadataResult');
+    if ($this->isSupported !== null) {
+      $xfer += $output->writeFieldBegin('isSupported', TType::BOOL, 1);
+      $xfer += $output->writeBool($this->isSupported);
+      $xfer += $output->writeFieldEnd();
+    }
+    $xfer += $output->writeFieldStop();
+    $xfer += $output->writeStructEnd();
+    return $xfer;
+  }
+
+}
+
+class CacheFileMetadataRequest {
+  static $_TSPEC;
+
+  /**
+   * @var string
+   */
+  public $dbName = null;
+  /**
+   * @var string
+   */
+  public $tblName = null;
+  /**
+   * @var string
+   */
+  public $partName = null;
+  /**
+   * @var bool
+   */
+  public $isAllParts = null;
+
+  public function __construct($vals=null) {
+    if (!isset(self::$_TSPEC)) {
+      self::$_TSPEC = array(
+        1 => array(
+          'var' => 'dbName',
+          'type' => TType::STRING,
+          ),
+        2 => array(
+          'var' => 'tblName',
+          'type' => TType::STRING,
+          ),
+        3 => array(
+          'var' => 'partName',
+          'type' => TType::STRING,
+          ),
+        4 => array(
+          'var' => 'isAllParts',
+          'type' => TType::BOOL,
+          ),
+        );
+    }
+    if (is_array($vals)) {
+      if (isset($vals['dbName'])) {
+        $this->dbName = $vals['dbName'];
+      }
+      if (isset($vals['tblName'])) {
+        $this->tblName = $vals['tblName'];
+      }
+      if (isset($vals['partName'])) {
+        $this->partName = $vals['partName'];
+      }
+      if (isset($vals['isAllParts'])) {
+        $this->isAllParts = $vals['isAllParts'];
+      }
+    }
+  }
+
+  public function getName() {
+    return 'CacheFileMetadataRequest';
+  }
+
+  public function read($input)
+  {
+    $xfer = 0;
+    $fname = null;
+    $ftype = 0;
+    $fid = 0;
+    $xfer += $input->readStructBegin($fname);
+    while (true)
+    {
+      $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+      if ($ftype == TType::STOP) {
+        break;
+      }
+      switch ($fid)
+      {
+        case 1:
+          if ($ftype == TType::STRING) {
+            $xfer += $input->readString($this->dbName);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        case 2:
+          if ($ftype == TType::STRING) {
+            $xfer += $input->readString($this->tblName);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        case 3:
+          if ($ftype == TType::STRING) {
+            $xfer += $input->readString($this->partName);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        case 4:
+          if ($ftype == TType::BOOL) {
+            $xfer += $input->readBool($this->isAllParts);
+          } else {
+            $xfer += $input->skip($ftype);
+          }
+          break;
+        default:
+          $xfer += $input->skip($ftype);
+          break;
+      }
+      $xfer += $input->readFieldEnd();
+    }
+    $xfer += $input->readStructEnd();
+    return $xfer;
+  }
+
+  public function write($output) {
+    $xfer = 0;
+    $xfer += $output->writeStructBegin('CacheFileMetadataRequest');
+    if ($this->dbName !== null) {
+      $xfer += $output->writeFieldBegin('dbName', TType::STRING, 1);
+      $xfer += $output->writeString($this->dbName);
+      $xfer += $output->writeFieldEnd();
+    }
+    if ($this->tblName !== null) {
+      $xfer += $output->writeFieldBegin('tblName', TType::STRING, 2);
+      $xfer += $output->writeString($this->tblName);
+      $xfer += $output->writeFieldEnd();
+    }
+    if ($this->partName !== null) {
+      $xfer += $output->writeFieldBegin('partName', TType::STRING, 3);
+      $xfer += $output->writeString($this->partName);
+      $xfer += $output->writeFieldEnd();
+    }
+    if ($this->isAllParts !== null) {
+      $xfer += $output->writeFieldBegin('isAllParts', TType::BOOL, 4);
+      $xfer += $output->writeBool($this->isAllParts);
+      $xfer += $output->writeFieldEnd();
+    }
+    $xfer += $output->writeFieldStop();
+    $xfer += $output->writeStructEnd();
+    return $xfer;
+  }
+
+}
+
 class GetAllFunctionsResponse {
   static $_TSPEC;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
index 22d794f..9564ded 100755
--- a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
+++ b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote
@@ -155,6 +155,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print('  GetFileMetadataResult get_file_metadata(GetFileMetadataRequest req)')
   print('  PutFileMetadataResult put_file_metadata(PutFileMetadataRequest req)')
   print('  ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req)')
+  print('  CacheFileMetadataResult cache_file_metadata(CacheFileMetadataRequest req)')
   print('  string getName()')
   print('  string getVersion()')
   print('  fb_status getStatus()')
@@ -1010,6 +1011,12 @@ elif cmd == 'clear_file_metadata':
     sys.exit(1)
   pp.pprint(client.clear_file_metadata(eval(args[0]),))
 
+elif cmd == 'cache_file_metadata':
+  if len(args) != 1:
+    print('cache_file_metadata requires 1 args')
+    sys.exit(1)
+  pp.pprint(client.cache_file_metadata(eval(args[0]),))
+
 elif cmd == 'getName':
   if len(args) != 0:
     print('getName requires 0 args')

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
index 5c72a27..97acb5d 100644
--- a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
+++ b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py
@@ -1073,6 +1073,13 @@ class Iface(fb303.FacebookService.Iface):
     """
     pass
 
+  def cache_file_metadata(self, req):
+    """
+    Parameters:
+     - req
+    """
+    pass
+
 
 class Client(fb303.FacebookService.Client, Iface):
   """
@@ -5873,6 +5880,37 @@ class Client(fb303.FacebookService.Client, Iface):
       return result.success
     raise TApplicationException(TApplicationException.MISSING_RESULT, "clear_file_metadata failed: unknown result")
 
+  def cache_file_metadata(self, req):
+    """
+    Parameters:
+     - req
+    """
+    self.send_cache_file_metadata(req)
+    return self.recv_cache_file_metadata()
+
+  def send_cache_file_metadata(self, req):
+    self._oprot.writeMessageBegin('cache_file_metadata', TMessageType.CALL, self._seqid)
+    args = cache_file_metadata_args()
+    args.req = req
+    args.write(self._oprot)
+    self._oprot.writeMessageEnd()
+    self._oprot.trans.flush()
+
+  def recv_cache_file_metadata(self):
+    iprot = self._iprot
+    (fname, mtype, rseqid) = iprot.readMessageBegin()
+    if mtype == TMessageType.EXCEPTION:
+      x = TApplicationException()
+      x.read(iprot)
+      iprot.readMessageEnd()
+      raise x
+    result = cache_file_metadata_result()
+    result.read(iprot)
+    iprot.readMessageEnd()
+    if result.success is not None:
+      return result.success
+    raise TApplicationException(TApplicationException.MISSING_RESULT, "cache_file_metadata failed: unknown result")
+
 
 class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
   def __init__(self, handler):
@@ -6008,6 +6046,7 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
     self._processMap["get_file_metadata"] = Processor.process_get_file_metadata
     self._processMap["put_file_metadata"] = Processor.process_put_file_metadata
     self._processMap["clear_file_metadata"] = Processor.process_clear_file_metadata
+    self._processMap["cache_file_metadata"] = Processor.process_cache_file_metadata
 
   def process(self, iprot, oprot):
     (name, type, seqid) = iprot.readMessageBegin()
@@ -9263,6 +9302,25 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor):
     oprot.writeMessageEnd()
     oprot.trans.flush()
 
+  def process_cache_file_metadata(self, seqid, iprot, oprot):
+    args = cache_file_metadata_args()
+    args.read(iprot)
+    iprot.readMessageEnd()
+    result = cache_file_metadata_result()
+    try:
+      result.success = self._handler.cache_file_metadata(args.req)
+      msg_type = TMessageType.REPLY
+    except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
+      raise
+    except Exception as ex:
+      msg_type = TMessageType.EXCEPTION
+      logging.exception(ex)
+      result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+    oprot.writeMessageBegin("cache_file_metadata", msg_type, seqid)
+    result.write(oprot)
+    oprot.writeMessageEnd()
+    oprot.trans.flush()
+
 
 # HELPER FUNCTIONS AND STRUCTURES
 
@@ -31884,3 +31942,134 @@ class clear_file_metadata_result:
 
   def __ne__(self, other):
     return not (self == other)
+
+class cache_file_metadata_args:
+  """
+  Attributes:
+   - req
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRUCT, 'req', (CacheFileMetadataRequest, CacheFileMetadataRequest.thrift_spec), None, ), # 1
+  )
+
+  def __init__(self, req=None,):
+    self.req = req
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRUCT:
+          self.req = CacheFileMetadataRequest()
+          self.req.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('cache_file_metadata_args')
+    if self.req is not None:
+      oprot.writeFieldBegin('req', TType.STRUCT, 1)
+      self.req.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    return
+
+
+  def __hash__(self):
+    value = 17
+    value = (value * 31) ^ hash(self.req)
+    return value
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class cache_file_metadata_result:
+  """
+  Attributes:
+   - success
+  """
+
+  thrift_spec = (
+    (0, TType.STRUCT, 'success', (CacheFileMetadataResult, CacheFileMetadataResult.thrift_spec), None, ), # 0
+  )
+
+  def __init__(self, success=None,):
+    self.success = success
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 0:
+        if ftype == TType.STRUCT:
+          self.success = CacheFileMetadataResult()
+          self.success.read(iprot)
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('cache_file_metadata_result')
+    if self.success is not None:
+      oprot.writeFieldBegin('success', TType.STRUCT, 0)
+      self.success.write(oprot)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    return
+
+
+  def __hash__(self):
+    value = 17
+    value = (value * 31) ^ hash(self.success)
+    return value
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index ba525ed..fbeab5e 100644
--- a/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -10266,17 +10266,20 @@ class PutFileMetadataRequest:
   Attributes:
    - fileIds
    - metadata
+   - type
   """
 
   thrift_spec = (
     None, # 0
     (1, TType.LIST, 'fileIds', (TType.I64,None), None, ), # 1
     (2, TType.LIST, 'metadata', (TType.STRING,None), None, ), # 2
+    (3, TType.I32, 'type', None, None, ), # 3
   )
 
-  def __init__(self, fileIds=None, metadata=None,):
+  def __init__(self, fileIds=None, metadata=None, type=None,):
     self.fileIds = fileIds
     self.metadata = metadata
+    self.type = type
 
   def read(self, iprot):
     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -10307,6 +10310,11 @@ class PutFileMetadataRequest:
           iprot.readListEnd()
         else:
           iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.I32:
+          self.type = iprot.readI32()
+        else:
+          iprot.skip(ftype)
       else:
         iprot.skip(ftype)
       iprot.readFieldEnd()
@@ -10331,6 +10339,10 @@ class PutFileMetadataRequest:
         oprot.writeString(iter507)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
+    if self.type is not None:
+      oprot.writeFieldBegin('type', TType.I32, 3)
+      oprot.writeI32(self.type)
+      oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
@@ -10346,6 +10358,7 @@ class PutFileMetadataRequest:
     value = 17
     value = (value * 31) ^ hash(self.fileIds)
     value = (value * 31) ^ hash(self.metadata)
+    value = (value * 31) ^ hash(self.type)
     return value
 
   def __repr__(self):
@@ -10480,6 +10493,181 @@ class ClearFileMetadataRequest:
   def __ne__(self, other):
     return not (self == other)
 
+class CacheFileMetadataResult:
+  """
+  Attributes:
+   - isSupported
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.BOOL, 'isSupported', None, None, ), # 1
+  )
+
+  def __init__(self, isSupported=None,):
+    self.isSupported = isSupported
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.BOOL:
+          self.isSupported = iprot.readBool()
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('CacheFileMetadataResult')
+    if self.isSupported is not None:
+      oprot.writeFieldBegin('isSupported', TType.BOOL, 1)
+      oprot.writeBool(self.isSupported)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.isSupported is None:
+      raise TProtocol.TProtocolException(message='Required field isSupported is unset!')
+    return
+
+
+  def __hash__(self):
+    value = 17
+    value = (value * 31) ^ hash(self.isSupported)
+    return value
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
+class CacheFileMetadataRequest:
+  """
+  Attributes:
+   - dbName
+   - tblName
+   - partName
+   - isAllParts
+  """
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.STRING, 'dbName', None, None, ), # 1
+    (2, TType.STRING, 'tblName', None, None, ), # 2
+    (3, TType.STRING, 'partName', None, None, ), # 3
+    (4, TType.BOOL, 'isAllParts', None, None, ), # 4
+  )
+
+  def __init__(self, dbName=None, tblName=None, partName=None, isAllParts=None,):
+    self.dbName = dbName
+    self.tblName = tblName
+    self.partName = partName
+    self.isAllParts = isAllParts
+
+  def read(self, iprot):
+    if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+      fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+      return
+    iprot.readStructBegin()
+    while True:
+      (fname, ftype, fid) = iprot.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      if fid == 1:
+        if ftype == TType.STRING:
+          self.dbName = iprot.readString()
+        else:
+          iprot.skip(ftype)
+      elif fid == 2:
+        if ftype == TType.STRING:
+          self.tblName = iprot.readString()
+        else:
+          iprot.skip(ftype)
+      elif fid == 3:
+        if ftype == TType.STRING:
+          self.partName = iprot.readString()
+        else:
+          iprot.skip(ftype)
+      elif fid == 4:
+        if ftype == TType.BOOL:
+          self.isAllParts = iprot.readBool()
+        else:
+          iprot.skip(ftype)
+      else:
+        iprot.skip(ftype)
+      iprot.readFieldEnd()
+    iprot.readStructEnd()
+
+  def write(self, oprot):
+    if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+      oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStructBegin('CacheFileMetadataRequest')
+    if self.dbName is not None:
+      oprot.writeFieldBegin('dbName', TType.STRING, 1)
+      oprot.writeString(self.dbName)
+      oprot.writeFieldEnd()
+    if self.tblName is not None:
+      oprot.writeFieldBegin('tblName', TType.STRING, 2)
+      oprot.writeString(self.tblName)
+      oprot.writeFieldEnd()
+    if self.partName is not None:
+      oprot.writeFieldBegin('partName', TType.STRING, 3)
+      oprot.writeString(self.partName)
+      oprot.writeFieldEnd()
+    if self.isAllParts is not None:
+      oprot.writeFieldBegin('isAllParts', TType.BOOL, 4)
+      oprot.writeBool(self.isAllParts)
+      oprot.writeFieldEnd()
+    oprot.writeFieldStop()
+    oprot.writeStructEnd()
+
+  def validate(self):
+    if self.dbName is None:
+      raise TProtocol.TProtocolException(message='Required field dbName is unset!')
+    if self.tblName is None:
+      raise TProtocol.TProtocolException(message='Required field tblName is unset!')
+    return
+
+
+  def __hash__(self):
+    value = 17
+    value = (value * 31) ^ hash(self.dbName)
+    value = (value * 31) ^ hash(self.tblName)
+    value = (value * 31) ^ hash(self.partName)
+    value = (value * 31) ^ hash(self.isAllParts)
+    return value
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, value)
+      for key, value in self.__dict__.iteritems()]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+  def __ne__(self, other):
+    return not (self == other)
+
 class GetAllFunctionsResponse:
   """
   Attributes:

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index f943f2d..a7b1e86 100644
--- a/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -2358,10 +2358,12 @@ class PutFileMetadataRequest
   include ::Thrift::Struct, ::Thrift::Struct_Union
   FILEIDS = 1
   METADATA = 2
+  TYPE = 3
 
   FIELDS = {
     FILEIDS => {:type => ::Thrift::Types::LIST, :name => 'fileIds', :element => {:type => ::Thrift::Types::I64}},
-    METADATA => {:type => ::Thrift::Types::LIST, :name => 'metadata', :element => {:type => ::Thrift::Types::STRING, :binary => true}}
+    METADATA => {:type => ::Thrift::Types::LIST, :name => 'metadata', :element => {:type => ::Thrift::Types::STRING, :binary => true}},
+    TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :optional => true, :enum_class => ::FileMetadataExprType}
   }
 
   def struct_fields; FIELDS; end
@@ -2369,6 +2371,9 @@ class PutFileMetadataRequest
   def validate
     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field fileIds is unset!') unless @fileIds
     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field metadata is unset!') unless @metadata
+    unless @type.nil? || ::FileMetadataExprType::VALID_VALUES.include?(@type)
+      raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
+    end
   end
 
   ::Thrift::Struct.generate_accessors self
@@ -2406,6 +2411,47 @@ class ClearFileMetadataRequest
   ::Thrift::Struct.generate_accessors self
 end
 
+class CacheFileMetadataResult
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  ISSUPPORTED = 1
+
+  FIELDS = {
+    ISSUPPORTED => {:type => ::Thrift::Types::BOOL, :name => 'isSupported'}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field isSupported is unset!') if @isSupported.nil?
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
+class CacheFileMetadataRequest
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  DBNAME = 1
+  TBLNAME = 2
+  PARTNAME = 3
+  ISALLPARTS = 4
+
+  FIELDS = {
+    DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+    TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+    PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partName', :optional => true},
+    ISALLPARTS => {:type => ::Thrift::Types::BOOL, :name => 'isAllParts', :optional => true}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
 class GetAllFunctionsResponse
   include ::Thrift::Struct, ::Thrift::Struct_Union
   FUNCTIONS = 1

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
----------------------------------------------------------------------
diff --git a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
index 5fe54b5..a90a180 100644
--- a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
+++ b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
@@ -2199,6 +2199,21 @@ module ThriftHiveMetastore
       raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'clear_file_metadata failed: unknown result')
     end
 
+    def cache_file_metadata(req)
+      send_cache_file_metadata(req)
+      return recv_cache_file_metadata()
+    end
+
+    def send_cache_file_metadata(req)
+      send_message('cache_file_metadata', Cache_file_metadata_args, :req => req)
+    end
+
+    def recv_cache_file_metadata()
+      result = receive_message(Cache_file_metadata_result)
+      return result.success unless result.success.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'cache_file_metadata failed: unknown result')
+    end
+
   end
 
   class Processor < ::FacebookService::Processor 
@@ -3853,6 +3868,13 @@ module ThriftHiveMetastore
       write_result(result, oprot, 'clear_file_metadata', seqid)
     end
 
+    def process_cache_file_metadata(seqid, iprot, oprot)
+      args = read_args(iprot, Cache_file_metadata_args)
+      result = Cache_file_metadata_result.new()
+      result.success = @handler.cache_file_metadata(args.req)
+      write_result(result, oprot, 'cache_file_metadata', seqid)
+    end
+
   end
 
   # HELPER FUNCTIONS AND STRUCTURES
@@ -8831,5 +8853,37 @@ module ThriftHiveMetastore
     ::Thrift::Struct.generate_accessors self
   end
 
+  class Cache_file_metadata_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    REQ = 1
+
+    FIELDS = {
+      REQ => {:type => ::Thrift::Types::STRUCT, :name => 'req', :class => ::CacheFileMetadataRequest}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Cache_file_metadata_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::CacheFileMetadataResult}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
 end
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
new file mode 100644
index 0000000..ec0be2b
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+
+/**
+ * Same as PartitionExpressionProxy, but for file format specific methods for metadata cache.
+ */
+public interface FileFormatProxy {
+
+  /**
+   * Applies SARG to file metadata, and produces some result for this file.
+   * @param sarg SARG
+   * @param byteBuffer File metadata from metastore cache.
+   * @return The result to return to client for this file, or null if file is eliminated.
+   * @throws IOException
+   */
+  ByteBuffer applySargToMetadata(SearchArgument sarg, ByteBuffer byteBuffer) throws IOException;
+
+  /**
+   * @param fs The filesystem of the file.
+   * @param path The file path.
+   * @param addedVals Output parameter; additional column values for columns returned by
+   *                  getAddedColumnsToCache to cache in MS.
+   * @return The ORC file metadata for a given file.
+   */
+  ByteBuffer getMetadataToCache(
+      FileSystem fs, Path path, ByteBuffer[] addedVals) throws IOException;
+
+  /**
+   * @return Additional column names to cache in MS for this format.
+   */
+  ByteBuffer[] getAddedColumnsToCache();
+
+  /**
+   * @param metadata File metadatas.
+   * @return Additional values for columns returned by getAddedColumnsToCache to cache in MS
+   *         for respective metadatas.
+   */
+  ByteBuffer[][] getAddedValuesToCache(List<ByteBuffer> metadata);
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java
index 7c3525a..bd4e188 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java
@@ -22,9 +22,89 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 
-public interface FileMetadataHandler {
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.hbase.MetadataStore;
 
-  void getFileMetadataByExpr(List<Long> fileIds, byte[] expr,
+/**
+ * The base implementation of a file metadata handler for a specific file type.
+ * There are currently two classes for each file type (of 1), this one, which is very simple due
+ * to the fact that it just calls the proxy class for most calls; and the proxy class, that
+ * contains the actual implementation that depends on some stuff in QL (for ORC).
+ */
+public abstract class FileMetadataHandler {
+  static final Log LOG = LogFactory.getLog(FileMetadataHandler.class);
+
+  private Configuration conf;
+  private PartitionExpressionProxy expressionProxy;
+  private FileFormatProxy fileFormatProxy;
+  private MetadataStore store;
+
+  /**
+   * Same as RawStore.getFileMetadataByExpr.
+   */
+  public abstract void getFileMetadataByExpr(List<Long> fileIds, byte[] expr,
       ByteBuffer[] metadatas, ByteBuffer[] results, boolean[] eliminated) throws IOException;
 
+  protected abstract FileMetadataExprType getType();
+
+  protected PartitionExpressionProxy getExpressionProxy() {
+    return expressionProxy;
+  }
+
+  protected FileFormatProxy getFileFormatProxy() {
+    return fileFormatProxy;
+  }
+
+  protected MetadataStore getStore() {
+    return store;
+  }
+
+  /**
+   * Configures the handler. Called once before use.
+   * @param conf Config.
+   * @param expressionProxy Expression proxy to access ql stuff.
+   * @param store Storage interface to manipulate the metadata.
+   */
+  public void configure(
+      Configuration conf, PartitionExpressionProxy expressionProxy, MetadataStore store) {
+    this.conf = conf;
+    this.expressionProxy = expressionProxy;
+    this.store = store;
+    this.fileFormatProxy = expressionProxy.getFileFormatProxy(getType());
+  }
+
+  /**
+   * Caches the file metadata for a particular file.
+   * @param fileId File id.
+   * @param fs The filesystem of the file.
+   * @param path Path to the file.
+   */
+  public void cacheFileMetadata(long fileId, FileSystem fs, Path path)
+      throws IOException, InterruptedException {
+    // ORC is in ql, so we cannot do anything here. For now, all the logic is in the proxy.
+    ByteBuffer[] cols = fileFormatProxy.getAddedColumnsToCache();
+    ByteBuffer[] vals = (cols == null) ? null : new ByteBuffer[cols.length];
+    ByteBuffer metadata = fileFormatProxy.getMetadataToCache(fs, path, vals);
+    LOG.info("Caching file metadata for " + path + ", size " + metadata.remaining());
+    store.storeFileMetadata(fileId, metadata, cols, vals);
+  }
+
+  /**
+   * @return the added column names to be cached in metastore with the metadata for this type.
+   */
+  public ByteBuffer[] createAddedCols() {
+    return fileFormatProxy.getAddedColumnsToCache();
+  }
+
+  /**
+   * @return the values for the added columns returned by createAddedCols for respective metadatas.
+   */
+  public ByteBuffer[][] createAddedColVals(List<ByteBuffer> metadata) {
+    return fileFormatProxy.getAddedValuesToCache(metadata);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
new file mode 100644
index 0000000..9b43328
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataManager.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.fs.LocatedFileStatus;
+
+import org.apache.hadoop.fs.RemoteIterator;
+
+import java.util.LinkedList;
+
+import java.util.Queue;
+
+import java.util.ArrayList;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.io.HdfsUtils;
+import org.apache.hadoop.hive.metastore.HiveMetaStore.ThreadLocalRawStore;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+public class FileMetadataManager {
+  private static final Log LOG = LogFactory.getLog(FileMetadataManager.class);
+  private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
+
+  private final ThreadLocalRawStore tlms;
+  private final ExecutorService threadPool;
+  private final HiveConf conf;
+
+  private final class CacheUpdateRequest implements Callable<Void> {
+    FileMetadataExprType type;
+    String location;
+
+    public CacheUpdateRequest(FileMetadataExprType type, String location) {
+      this.type = type;
+      this.location = location;
+    }
+
+    @Override
+    public Void call() throws Exception {
+      try {
+        cacheMetadata(type, location);
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+      } catch (Exception ex) {
+        // Nobody can see this exception on the threadpool; just log it.
+        LOG.error("Failed to cache file metadata in background for " + type + ", " + location, ex);
+      }
+      return null;
+    }
+  }
+
+  public FileMetadataManager(ThreadLocalRawStore tlms, HiveConf conf) {
+    this.tlms = tlms;
+    this.conf = conf;
+    int numThreads = HiveConf.getIntVar(conf, ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS);
+    this.threadPool = Executors.newFixedThreadPool(numThreads,
+        new ThreadFactoryBuilder().setNameFormat("File-Metadata-%d").setDaemon(true).build());
+  }
+
+  public void queueCacheMetadata(String location, FileMetadataExprType type) {
+    threadPool.submit(new CacheUpdateRequest(type, location));
+  }
+
+  private void cacheMetadata(FileMetadataExprType type, String location)
+      throws MetaException, IOException, InterruptedException {
+    Path path = new Path(location);
+    FileSystem fs = path.getFileSystem(conf);
+    List<Path> files;
+    if (!fs.isDirectory(path)) {
+      files = Lists.newArrayList(path);
+    } else {
+      files = new ArrayList<>();
+      RemoteIterator<LocatedFileStatus> iter = fs.listFiles(path, true);
+      while (iter.hasNext()) {
+        // TODO: use fileId right from the list after HDFS-7878; or get dfs client and do it
+        LocatedFileStatus lfs = iter.next();
+        if (lfs.isDirectory()) continue;
+        files.add(lfs.getPath());
+      }
+    }
+    for (Path file : files) {
+      long fileId;
+      try {
+        fileId = SHIMS.getFileId(fs, Path.getPathWithoutSchemeAndAuthority(file).toString());
+      } catch (UnsupportedOperationException ex) {
+        LOG.error("Cannot cache file metadata for " + location + "; "
+            + fs.getClass().getCanonicalName() + " does not support fileId");
+        return;
+      }
+      LOG.info("Caching file metadata for " + file + " (file ID " + fileId + ")");
+      file = HdfsUtils.getFileIdPath(fs, file, fileId);
+      tlms.getMS().getFileMetadataHandler(type).cacheFileMetadata(fileId, fs, file);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 0940fd7..81d7128 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.hive.metastore.events.PreEventContext;
 import org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent;
 import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.PreReadTableEvent;
+import org.apache.hadoop.hive.metastore.filemeta.OrcFileMetadataHandler;
 import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
 import org.apache.hadoop.hive.metastore.txn.TxnHandler;
 import org.apache.hadoop.hive.serde2.Deserializer;
@@ -196,13 +197,25 @@ public class HiveMetaStore extends ThriftHiveMetastore {
     }
   }
 
-  public static class HMSHandler extends FacebookBase implements IHMSHandler {
+  /**
+   * An ugly interface because everything about this file is ugly. RawStore is threadlocal so this
+   * thread-local disease propagates everywhere, and FileMetadataManager cannot just get a RawStore
+   * or handlers to use; it will need to have this method to make thread-local handlers and a
+   * thread-local RawStore.
+   */
+  public interface ThreadLocalRawStore {
+    RawStore getMS() throws MetaException;
+  }
+
+  public static class HMSHandler extends FacebookBase implements IHMSHandler, ThreadLocalRawStore {
     public static final Logger LOG = HiveMetaStore.LOG;
     private String rawStoreClassName;
     private final HiveConf hiveConf; // stores datastore (jpox) properties,
                                      // right now they come from jpox.properties
 
     private static String currentUrl;
+    private FileMetadataManager fileMetadataManager;
+    private PartitionExpressionProxy expressionProxy;
 
     //For Metrics
     private int initDatabaseCount, initTableCount, initPartCount;
@@ -444,6 +457,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         Timer cleaner = new Timer("Metastore Events Cleaner Thread", true);
         cleaner.schedule(new EventCleanerTask(this), cleanFreq, cleanFreq);
       }
+
+      expressionProxy = PartFilterExprUtil.createExpressionProxy(hiveConf);
+      fileMetadataManager = new FileMetadataManager((ThreadLocalRawStore)this, hiveConf);
     }
 
     private String addPrefix(String s) {
@@ -510,6 +526,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
      */
     @InterfaceAudience.LimitedPrivate({"HCATALOG"})
     @InterfaceStability.Evolving
+    @Override
     public RawStore getMS() throws MetaException {
       RawStore ms = threadLocalMS.get();
       if (ms == null) {
@@ -1527,9 +1544,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
 
       boolean trashEnabled = false;
       try {
-	trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
+  trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
       } catch(NumberFormatException ex) {
-	// nothing to do
+  // nothing to do
       }
 
       if (trashEnabled) {
@@ -5763,7 +5780,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
     public PutFileMetadataResult put_file_metadata(PutFileMetadataRequest req) throws TException {
       RawStore ms = getMS();
       if (ms.isFileMetadataSupported()) {
-        ms.putFileMetadata(req.getFileIds(), req.getMetadata());
+        ms.putFileMetadata(req.getFileIds(), req.getMetadata(), req.getType());
       }
       return new PutFileMetadataResult();
     }
@@ -5771,10 +5788,109 @@ public class HiveMetaStore extends ThriftHiveMetastore {
     @Override
     public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req)
         throws TException {
-      getMS().putFileMetadata(req.getFileIds(), null);
+      getMS().putFileMetadata(req.getFileIds(), null, null);
       return new ClearFileMetadataResult();
     }
 
+    @Override
+    public CacheFileMetadataResult cache_file_metadata(
+        CacheFileMetadataRequest req) throws TException {
+      RawStore ms = getMS();
+      if (!ms.isFileMetadataSupported()) {
+        return new CacheFileMetadataResult(false);
+      }
+      String dbName = req.getDbName(), tblName = req.getTblName(),
+          partName = req.isSetPartName() ? req.getPartName() : null;
+      boolean isAllPart = req.isSetIsAllParts() && req.isIsAllParts();
+      ms.openTransaction();
+      boolean success = false;
+      try {
+        Table tbl = ms.getTable(dbName, tblName);
+        if (tbl == null) {
+          throw new NoSuchObjectException(dbName + "." + tblName + " not found");
+        }
+        boolean isPartitioned = tbl.isSetPartitionKeys() && tbl.getPartitionKeysSize() > 0;
+        String tableInputFormat = tbl.isSetSd() ? tbl.getSd().getInputFormat() : null;
+        if (!isPartitioned) {
+          if (partName != null || isAllPart) {
+            throw new MetaException("Table is not partitioned");
+          }
+          if (!tbl.isSetSd() || !tbl.getSd().isSetLocation()) {
+            throw new MetaException(
+                "Table does not have storage location; this operation is not supported on views");
+          }
+          FileMetadataExprType type = expressionProxy.getMetadataType(tableInputFormat);
+          if (type == null) {
+            throw new MetaException("The operation is not supported for " + tableInputFormat);
+          }
+          fileMetadataManager.queueCacheMetadata(tbl.getSd().getLocation(), type);
+          success = true;
+        } else {
+          List<String> partNames = null;
+          if (partName != null) {
+            partNames = Lists.newArrayList(partName);
+          } else if (isAllPart) {
+            partNames = ms.listPartitionNames(dbName, tblName, (short)-1);
+          } else {
+            throw new MetaException("Table is partitioned");
+          }
+          int batchSize = HiveConf.getIntVar(
+              hiveConf, ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX);
+          int index = 0;
+          int successCount = 0, failCount = 0;
+          HashSet<String> failFormats = null;
+          while (index < partNames.size()) {
+            int currentBatchSize = Math.min(batchSize, partNames.size() - index);
+            List<String> nameBatch = partNames.subList(index, index + currentBatchSize);
+            index += currentBatchSize;
+            List<Partition> parts = ms.getPartitionsByNames(dbName, tblName, nameBatch);
+            for (Partition part : parts) {
+              if (!part.isSetSd() || !part.getSd().isSetLocation()) {
+                throw new MetaException("Partition does not have storage location;" +
+                    " this operation is not supported on views");
+              }
+              String inputFormat = part.getSd().isSetInputFormat()
+                  ? part.getSd().getInputFormat() : tableInputFormat;
+              FileMetadataExprType type = expressionProxy.getMetadataType(inputFormat);
+              if (type == null) {
+                ++failCount;
+                if (failFormats == null) {
+                  failFormats = new HashSet<>();
+                }
+                failFormats.add(inputFormat);
+              } else {
+                ++successCount;
+                fileMetadataManager.queueCacheMetadata(part.getSd().getLocation(), type);
+              }
+            }
+          }
+          success = true; // Regardless of the following exception
+          if (failCount > 0) {
+            String errorMsg = "The operation failed for " + failCount + " partitions and "
+                + "succeeded for " + successCount + " partitions; unsupported formats: ";
+            boolean isFirst = true;
+            for (String s : failFormats) {
+              if (!isFirst) {
+                errorMsg += ", ";
+              }
+              isFirst = false;
+              errorMsg += s;
+            }
+            throw new MetaException(errorMsg);
+          }
+        }
+      } finally {
+        if (success) {
+          if (!ms.commitTransaction()) {
+            throw new MetaException("Failed to commit");
+          }
+        } else {
+          ms.rollbackTransaction();
+        }
+      }
+      return new CacheFileMetadataResult(true);
+    }
+
     @VisibleForTesting
     public void updateMetrics() throws MetaException {
       initTableCount = getMS().getTableCount();
@@ -6284,4 +6400,18 @@ public class HiveMetaStore extends ThriftHiveMetastore {
         ".  Root Cause: ", ex);
     }
   }
+
+  public static Map<FileMetadataExprType, FileMetadataHandler> createHandlerMap() {
+    Map<FileMetadataExprType, FileMetadataHandler> fmHandlers = new HashMap<>();
+    for (FileMetadataExprType v : FileMetadataExprType.values()) {
+      switch (v) {
+      case ORC_SARG:
+        fmHandlers.put(v, new OrcFileMetadataHandler());
+        break;
+      default:
+        throw new AssertionError("Unsupported type " + v);
+      }
+    }
+    return fmHandlers;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 178796d..a17c6d8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -63,6 +63,8 @@ import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
 import org.apache.hadoop.hive.metastore.api.AddPartitionsResult;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.CacheFileMetadataRequest;
+import org.apache.hadoop.hive.metastore.api.CacheFileMetadataResult;
 import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
 import org.apache.hadoop.hive.metastore.api.ClearFileMetadataRequest;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -2261,4 +2263,19 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
   public boolean isSameConfObj(HiveConf c) {
     return conf == c;
   }
+
+  @Override
+  public boolean cacheFileMetadata(
+      String dbName, String tableName, String partName, boolean allParts) throws TException {
+    CacheFileMetadataRequest req = new CacheFileMetadataRequest();
+    req.setDbName(dbName);
+    req.setTblName(tableName);
+    if (partName != null) {
+      req.setPartName(partName);
+    } else {
+      req.setIsAllParts(allParts);
+    }
+    CacheFileMetadataResult result = client.cache_file_metadata(req);
+    return result.isIsSupported();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index aa96f77..25e0d38 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -1509,4 +1509,7 @@ public interface IMetaStoreClient {
   void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) throws TException;
 
   boolean isSameConfObj(HiveConf c);
+
+  boolean cacheFileMetadata(String dbName, String tableName, String partName,
+      boolean allParts) throws TException;
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 23068f8..432f7d0 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1516,10 +1516,12 @@ public class MetaStoreUtils {
     return listeners;
   }
 
-  public static Class<?> getClass(String rawStoreClassName)
+  @SuppressWarnings("unchecked")
+  public static Class<? extends RawStore> getClass(String rawStoreClassName)
       throws MetaException {
     try {
-      return Class.forName(rawStoreClassName, true, JavaUtils.getClassLoader());
+      return (Class<? extends RawStore>)
+          Class.forName(rawStoreClassName, true, JavaUtils.getClassLoader());
     } catch (ClassNotFoundException e) {
       throw new MetaException(rawStoreClassName + " class not found");
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index abfe2b8..05d5b57 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -7756,18 +7756,19 @@ public class ObjectStore implements RawStore, Configurable {
   }
 
   @Override
-  public ByteBuffer[] getFileMetadata(List<Long> fileIds) {
-    throw new UnsupportedOperationException();
+  public boolean isFileMetadataSupported() {
+    return false;
   }
 
   @Override
-  public void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) {
+  public ByteBuffer[] getFileMetadata(List<Long> fileIds) {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public boolean isFileMetadataSupported() {
-    return false;
+  public void putFileMetadata(
+      List<Long> fileIds, List<ByteBuffer> metadata, FileMetadataExprType type) {
+    throw new UnsupportedOperationException();
   }
 
   @Override
@@ -7776,6 +7777,11 @@ public class ObjectStore implements RawStore, Configurable {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public FileMetadataHandler getFileMetadataHandler(FileMetadataExprType type) {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * Removed cached classloaders from DataNucleus
    * DataNucleus caches classloaders in NucleusContext.
@@ -7803,5 +7809,4 @@ public class ObjectStore implements RawStore, Configurable {
       }
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
index ed59829..8d6dc12 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
@@ -18,17 +18,17 @@
 
 package org.apache.hadoop.hive.metastore;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.List;
 
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 
 /**
- * The proxy interface that metastore uses to manipulate and apply
- * serialized filter expressions coming from client.
+ * The proxy interface that metastore uses for variety of QL operations (metastore can't depend
+ * on QL because QL depends on metastore; creating metastore-client module would be a proper way
+ * to solve this problem).
  */
 public interface PartitionExpressionProxy {
 
@@ -48,24 +48,28 @@ public interface PartitionExpressionProxy {
    * @param partitionNames Partition names; the list is modified in place.
    * @return Whether there were any unknown partitions preserved in the name list.
    */
-  public boolean filterPartitionsByExpr(List<String> partColumnNames,
+  boolean filterPartitionsByExpr(List<String> partColumnNames,
       List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr,
       String defaultPartitionName, List<String> partitionNames) throws MetaException;
 
   /**
-   * Creates SARG from serialized representation.
-   * @param expr SARG, serialized as Kryo.
-   * @return SARG.
+   * Determines the file metadata type from input format of the source table or partition.
+   * @param inputFormat Input format name.
+   * @return The file metadata type.
    */
-  public SearchArgument createSarg(byte[] expr);
+  FileMetadataExprType getMetadataType(String inputFormat);
 
   /**
-   * Applies SARG to file metadata, and produces some result for this file.
-   * @param sarg SARG
-   * @param byteBuffer File metadata from metastore cache.
-   * @return The result to return to client for this file, or null if file is eliminated.
-   * @throws IOException
+   * Gets a separate proxy that can be used to call file-format-specific methods.
+   * @param type The file metadata type.
+   * @return The proxy.
+   */
+  FileFormatProxy getFileFormatProxy(FileMetadataExprType type);
+
+  /**
+   * Creates SARG from serialized representation.
+   * @param expr SARG, serialized as Kryo.
+   * @return SARG.
    */
-  public ByteBuffer applySargToFileMetadata(SearchArgument sarg, ByteBuffer byteBuffer)
-      throws IOException;
+  SearchArgument createSarg(byte[] expr);
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
index e118a3b..d228f24 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -610,8 +610,10 @@ public interface RawStore extends Configurable {
   /**
    * @param fileIds List of file IDs from the filesystem.
    * @param metadata Metadata buffers corresponding to fileIds in the list.
+   * @param type The type; determines the class that can do additiona processing for metadata.
    */
-  void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) throws MetaException;
+  void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata,
+      FileMetadataExprType type) throws MetaException;
 
   /**
    * @return Whether file metadata cache is supported by this implementation.
@@ -635,6 +637,9 @@ public interface RawStore extends Configurable {
       ByteBuffer[] metadatas, ByteBuffer[] exprResults, boolean[] eliminated)
           throws MetaException;
 
+  /** Gets file metadata handler for the corresponding type. */
+  FileMetadataHandler getFileMetadataHandler(FileMetadataExprType type);
+
   /**
    * Gets total number of tables.
    */

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java
index 14189da..1b388aa 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java
@@ -22,42 +22,34 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.FileMetadataHandler;
-import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
-import org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 
-public class OrcFileMetadataHandler implements FileMetadataHandler {
-  private final Configuration conf;
-  private final PartitionExpressionProxy expressionProxy;
-  private final HBaseReadWrite hbase;
+public class OrcFileMetadataHandler extends FileMetadataHandler {
 
-  public OrcFileMetadataHandler(Configuration conf,
-      PartitionExpressionProxy expressionProxy, HBaseReadWrite hbase) {
-    this.conf = conf;
-    this.expressionProxy = expressionProxy;
-    this.hbase = hbase;
+  @Override
+  protected FileMetadataExprType getType() {
+    return FileMetadataExprType.ORC_SARG;
   }
 
   @Override
   public void getFileMetadataByExpr(List<Long> fileIds, byte[] expr,
       ByteBuffer[] metadatas, ByteBuffer[] results, boolean[] eliminated) throws IOException {
-    SearchArgument sarg = expressionProxy.createSarg(expr);
+    SearchArgument sarg = getExpressionProxy().createSarg(expr);
     // For now, don't push anything into HBase, nor store anything special in HBase
     if (metadatas == null) {
       // null means don't return metadata; we'd need the array anyway for now.
       metadatas = new ByteBuffer[results.length];
     }
-    hbase.getFileMetadata(fileIds, metadatas);
+    getStore().getFileMetadata(fileIds, metadatas);
     for (int i = 0; i < metadatas.length;  ++i) {
       if (metadatas[i] == null) continue;
-      ByteBuffer result = expressionProxy.applySargToFileMetadata(sarg, metadatas[i]);
+      ByteBuffer result = getFileFormatProxy().applySargToMetadata(sarg, metadatas[i]);
       eliminated[i] = (result == null);
       if (!eliminated[i]) {
         results[i] = result;
       }
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index 287394e..81f1324 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -86,7 +86,7 @@ import java.util.Set;
 /**
  * Class to manage storing object in and reading them from HBase.
  */
-public class HBaseReadWrite {
+public class HBaseReadWrite implements MetadataStore {
 
   final static String AGGR_STATS_TABLE = "HBMS_AGGR_STATS";
   final static String DB_TABLE = "HBMS_DBS";
@@ -2154,6 +2154,7 @@ public class HBaseReadWrite {
    * @param fileIds file ID list.
    * @return Serialized file metadata.
    */
+  @Override
   public void getFileMetadata(List<Long> fileIds, ByteBuffer[] result) throws IOException {
     byte[][] keys = new byte[fileIds.size()][];
     for (int i = 0; i < fileIds.size(); ++i) {
@@ -2164,15 +2165,64 @@ public class HBaseReadWrite {
 
   /**
    * @param fileIds file ID list.
-   * @param metadata Serialized file metadata.
-   */
-  void storeFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata)
+   * @param metadataBuffers Serialized file metadatas, one per file ID.
+   * @param addedCols The column names for additional columns created by file-format-specific
+   *                  metadata handler, to be stored in the cache.
+   * @param addedVals The values for addedCols; one value per file ID per added column.
+   */
+  @Override
+  public void storeFileMetadata(List<Long> fileIds, List<ByteBuffer> metadataBuffers,
+      ByteBuffer[] addedCols, ByteBuffer[][] addedVals)
       throws IOException, InterruptedException {
     byte[][] keys = new byte[fileIds.size()][];
     for (int i = 0; i < fileIds.size(); ++i) {
       keys[i] = HBaseUtils.makeLongKey(fileIds.get(i));
     }
-    multiModify(FILE_METADATA_TABLE, keys, CATALOG_CF, CATALOG_COL, metadata);
+    // HBase APIs are weird. To supply bytebuffer value, you have to also have bytebuffer
+    // column name, but not column family. So there. Perhaps we should add these to constants too.
+    ByteBuffer colNameBuf = ByteBuffer.wrap(CATALOG_COL);
+    @SuppressWarnings("deprecation")
+    HTableInterface htab = conn.getHBaseTable(FILE_METADATA_TABLE);
+    List<Row> actions = new ArrayList<>(keys.length);
+    for (int keyIx = 0; keyIx < keys.length; ++keyIx) {
+      ByteBuffer value = (metadataBuffers != null) ? metadataBuffers.get(keyIx) : null;
+      ByteBuffer[] av = addedVals == null ? null : addedVals[keyIx];
+      if (value == null) {
+        actions.add(new Delete(keys[keyIx]));
+        assert av == null;
+      } else {
+        Put p = new Put(keys[keyIx]);
+        p.addColumn(CATALOG_CF, colNameBuf, HConstants.LATEST_TIMESTAMP, value);
+        if (av != null) {
+          assert av.length == addedCols.length;
+          for (int colIx = 0; colIx < addedCols.length; ++colIx) {
+            p.addColumn(STATS_CF, addedCols[colIx], HConstants.LATEST_TIMESTAMP, av[colIx]);
+          }
+        }
+        actions.add(p);
+      }
+    }
+    Object[] results = new Object[keys.length];
+    htab.batch(actions, results);
+    // TODO: should we check results array? we don't care about partial results
+    conn.flush(htab);
+  }
+
+  @Override
+  public void storeFileMetadata(long fileId, ByteBuffer metadata,
+      ByteBuffer[] addedCols, ByteBuffer[] addedVals) throws IOException, InterruptedException {
+    @SuppressWarnings("deprecation")
+    HTableInterface htab = conn.getHBaseTable(FILE_METADATA_TABLE);
+    Put p = new Put(HBaseUtils.makeLongKey(fileId));
+    p.addColumn(CATALOG_CF, ByteBuffer.wrap(CATALOG_COL), HConstants.LATEST_TIMESTAMP, metadata);
+    assert (addedCols == null && addedVals == null) || (addedCols.length == addedVals.length);
+    if (addedCols != null) {
+      for (int i = 0; i < addedCols.length; ++i) {
+        p.addColumn(STATS_CF, addedCols[i], HConstants.LATEST_TIMESTAMP, addedVals[i]);
+      }
+    }
+    htab.put(p);
+    conn.flush(htab);
   }
 
   /**********************************************************************************************

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index b9509ab..fcf983f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -67,7 +67,6 @@ import org.apache.hadoop.hive.metastore.api.Type;
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
 import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
 import org.apache.hadoop.hive.metastore.api.UnknownTableException;
-import org.apache.hadoop.hive.metastore.filemeta.OrcFileMetadataHandler;
 import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
 import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
@@ -80,6 +79,7 @@ import org.apache.thrift.TException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -99,7 +99,7 @@ public class HBaseStore implements RawStore {
   private Configuration conf;
   private int txnNestLevel = 0;
   private PartitionExpressionProxy expressionProxy = null;
-  private Map<FileMetadataExprType, FileMetadataHandler> fmHandlers = new HashMap<>();
+  private Map<FileMetadataExprType, FileMetadataHandler> fmHandlers;
 
   public HBaseStore() {
   }
@@ -2308,26 +2308,32 @@ public class HBaseStore implements RawStore {
     // setConf is being called with new configuration object (though that
     // is not expected to happen, doing it just for safety)
     // TODO: why not re-intialize HBaseReadWrite?
-    if (expressionProxy == null || conf != configuration) {
-      expressionProxy = PartFilterExprUtil.createExpressionProxy(configuration);
-    }
+    Configuration oldConf = conf;
     conf = configuration;
-    createFileMetadataHandlers();
+    if (expressionProxy != null && conf != oldConf) {
+      LOG.warn("Unexpected setConf when we were already configured");
+    }
+    if (expressionProxy == null || conf != oldConf) {
+      expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
+    }
+    if (conf != oldConf) {
+      fmHandlers = HiveMetaStore.createHandlerMap();
+      configureFileMetadataHandlers(fmHandlers.values());
+    }
   }
 
-  private void createFileMetadataHandlers() {
-    for (FileMetadataExprType v : FileMetadataExprType.values()) {
-      switch (v) {
-      case ORC_SARG:
-        fmHandlers.put(v, new OrcFileMetadataHandler(conf, expressionProxy, getHBase()));
-        break;
-      default:
-        throw new AssertionError("Unsupported type " + v);
-      }
+  private void configureFileMetadataHandlers(Collection<FileMetadataHandler> fmHandlers) {
+    for (FileMetadataHandler fmh : fmHandlers) {
+      fmh.configure(conf, expressionProxy, getHBase());
     }
   }
 
   @Override
+  public FileMetadataHandler getFileMetadataHandler(FileMetadataExprType type) {
+    return fmHandlers.get(type);
+  }
+
+  @Override
   public Configuration getConf() {
     return conf;
 
@@ -2476,11 +2482,21 @@ public class HBaseStore implements RawStore {
   }
 
   @Override
-  public void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) throws MetaException {
+  public void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata,
+      FileMetadataExprType type) throws MetaException {
     openTransaction();
     boolean commit = false;
     try {
-      getHBase().storeFileMetadata(fileIds, metadata);
+      ByteBuffer[][] addedVals = null;
+      ByteBuffer[] addedCols = null;
+      if (type != null) {
+        FileMetadataHandler fmh = fmHandlers.get(type);
+        addedCols = fmh.createAddedCols();
+        if (addedCols != null) {
+          addedVals = fmh.createAddedColVals(metadata);
+        }
+      }
+      getHBase().storeFileMetadata(fileIds, metadata, addedCols, addedVals);
       commit = true;
     } catch (IOException | InterruptedException e) {
       LOG.error("Unable to store file metadata", e);

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java
new file mode 100644
index 0000000..0382e8a
--- /dev/null
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hbase;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public interface MetadataStore {
+  /**
+   * @param fileIds file ID list.
+   * @param result The ref parameter, used to return the serialized file metadata.
+   */
+  void getFileMetadata(List<Long> fileIds, ByteBuffer[] result) throws IOException;
+
+  /**
+   * @param fileIds file ID list.
+   * @param metadataBuffers Serialized file metadata, one per file ID.
+   * @param addedCols The column names for additional columns created by file-format-specific
+   *                  metadata handler, to be stored in the cache.
+   * @param addedVals The values for addedCols; one value per file ID per added column.
+   */
+  void storeFileMetadata(List<Long> fileIds, List<ByteBuffer> metadataBuffers,
+      ByteBuffer[] addedCols, ByteBuffer[][] addedVals) throws IOException, InterruptedException;
+
+  /**
+   * @param fileId The file ID.
+   * @param metadataBuffers Serialized file metadata.
+   * @param addedCols The column names for additional columns created by file-format-specific
+   *                  metadata handler, to be stored in the cache.
+   * @param addedVals The values for addedCols; one value per added column.
+   */
+  void storeFileMetadata(long fileId, ByteBuffer metadata, ByteBuffer[] addedCols,
+      ByteBuffer[] addedVals) throws IOException, InterruptedException;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index c1156b3..477a3be 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -775,7 +775,8 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
   }
 
   @Override
-  public void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) {
+  public void putFileMetadata(
+      List<Long> fileIds, List<ByteBuffer> metadata, FileMetadataExprType type) {
   }
 
   @Override
@@ -803,4 +804,9 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
   public int getDatabaseCount() throws MetaException {
     return objectStore.getDatabaseCount();
   }
+
+  @Override
+  public FileMetadataHandler getFileMetadataHandler(FileMetadataExprType type) {
+    return null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index bf20e99..ecabd5d 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -792,7 +792,8 @@ public class DummyRawStoreForJdoConnection implements RawStore {
   }
 
   @Override
-  public void putFileMetadata(List<Long> fileIds, List<ByteBuffer> metadata) {
+  public void putFileMetadata(
+      List<Long> fileIds, List<ByteBuffer> metadata, FileMetadataExprType type) {
   }
 
   @Override
@@ -819,6 +820,11 @@ public class DummyRawStoreForJdoConnection implements RawStore {
   public int getDatabaseCount() throws MetaException {
     return 0;
   }
+
+  @Override
+  public FileMetadataHandler getFileMetadataHandler(FileMetadataExprType type) {
+    return null;
+  }
 }
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7df62023/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
index d72bf76..032af69 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
@@ -18,11 +18,11 @@
 
 package org.apache.hadoop.hive.metastore;
 
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 
-import java.nio.ByteBuffer;
 import java.util.List;
 
 /**
@@ -42,12 +42,17 @@ public class MockPartitionExpressionForMetastore implements PartitionExpressionP
   }
 
   @Override
+  public FileMetadataExprType getMetadataType(String inputFormat) {
+    return null;
+  }
+
+  @Override
   public SearchArgument createSarg(byte[] expr) {
     return null;
   }
 
   @Override
-  public ByteBuffer applySargToFileMetadata(SearchArgument sarg, ByteBuffer byteBuffer) {
+  public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) {
     return null;
   }
 }


Mime
View raw message