hadoop-hive-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Min Zhou (JIRA)" <j...@apache.org>
Subject [jira] Issue Comment Edited: (HIVE-338) Executing cli commands into thrift server
Date Wed, 13 May 2009 11:42:47 GMT

    [ https://issues.apache.org/jira/browse/HIVE-338?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=12681621#action_12681621
] 

Min Zhou edited comment on HIVE-338 at 5/13/09 4:42 AM:
--------------------------------------------------------

support add file/jar now


python example
{code:python}
#!/usr/bin/env python

import sys

from hive import ThriftHive
from hive.ttypes import HiveServerException
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

try:
    transport = TSocket.TSocket('localhost', 10000)
    transport = TTransport.TBufferedTransport(transport)
    protocol = TBinaryProtocol.TBinaryProtocol(transport)

    client = ThriftHive.Client(protocol)
    transport.open()

    client.execute('ADD FILE /home/zhoumin/py/foo')
    client.execute('ADD FILE /home/zhoumin/py/streaming.py')
    query = '''
        INSERT OVERWRITE TABLE streaming_pokes
        MAP (pokes.foo, pokes.bar)         
            USING 'streaming.py'
        AS new_foo, new_bar
        FROM pokes                                '''

    client.execute(query)
    row = client.fetchOne()
    print row
    
    transport.close()

except Thrift.TException, tx:
    print '%s' % (tx.message)

{code}

java example:
{code:java}
package zhoumin.example;

import java.util.ArrayList;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.service.HiveClient;
import org.apache.hadoop.hive.service.HiveServerException;

import com.facebook.thrift.TException;
import com.facebook.thrift.protocol.TBinaryProtocol;
import com.facebook.thrift.protocol.TProtocol;
import com.facebook.thrift.transport.TSocket;
import com.facebook.thrift.transport.TTransport;
import com.facebook.thrift.transport.TTransportException;


public class MyClient {
  public static final int THREADS_NUMBER = 10;
  
  public static class Worker implements Callable<String> {
    
    TTransport transport;
    TProtocol protocol;
    HiveClient client;
    
    public Worker() {
      transport = new TSocket("localhost", 10000);
      protocol = new TBinaryProtocol(transport);
      client = new HiveClient(protocol); 
    }

    public String call() throws Exception {
      transport.open();
      client.execute("add jar /home/zhoumin//hadoop/mapreduce/zhoumin/dist/zhoumin-0.00.1.jar");
      client.execute("CREATE TEMPORARY FUNCTION strlen AS 'hadoop.hive.udf.UdfStringLength'");
      client.execute("select strlen(mid) from log_data");
      String row = client.fetchOne();
      transport.close();
      return row;
    }
    
  }
  
  public static void main(String[] args) throws TTransportException,
      TException, HiveServerException, MetaException {
    ExecutorService exec = Executors.newCachedThreadPool();
    
    ArrayList<Future<String>> results = new ArrayList<Future<String>>();
    for(int i = 0; i < THREADS_NUMBER; i++) {
      results.add(exec.submit(new Worker()));
    }
    
    for(Future<String> fs : results) {
      try {
        System.out.println(fs.get());
      }catch (InterruptedException e) {
        System.out.println(e);
      } catch(ExecutionException e) {
        System.out.println(e);
      } finally {
        exec.shutdown();
      }
    }
  }
}
{code}

the add jar command is also supported on CLI now.

      was (Author: coderplay):
    support add file/jar now


--------------------------------------------------------------------------------------------------------------------
python example

#!/usr/bin/env python

import sys

from hive import ThriftHive
from hive.ttypes import HiveServerException
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol

try:
    transport = TSocket.TSocket('localhost', 10000)
    transport = TTransport.TBufferedTransport(transport)
    protocol = TBinaryProtocol.TBinaryProtocol(transport)

    client = ThriftHive.Client(protocol)
    transport.open()

    client.execute('ADD FILE /home/zhoumin/py/foo')
    client.execute('ADD FILE /home/zhoumin/py/streaming.py')
    query = '''
        INSERT OVERWRITE TABLE streaming_pokes
        MAP (pokes.foo, pokes.bar)         
            USING 'streaming.py'
        AS new_foo, new_bar
        FROM pokes                                '''

    client.execute(query)
    row = client.fetchOne()
    print row
    
    transport.close()

except Thrift.TException, tx:
    print '%s' % (tx.message)


--------------------------------------------------------------------------------------------------------------------
java example:

package zhoumin.example;

import java.util.ArrayList;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.service.HiveClient;
import org.apache.hadoop.hive.service.HiveServerException;

import com.facebook.thrift.TException;
import com.facebook.thrift.protocol.TBinaryProtocol;
import com.facebook.thrift.protocol.TProtocol;
import com.facebook.thrift.transport.TSocket;
import com.facebook.thrift.transport.TTransport;
import com.facebook.thrift.transport.TTransportException;


public class MyClient {
  public static final int THREADS_NUMBER = 10;
  
  public static class Worker implements Callable<String> {
    
    TTransport transport;
    TProtocol protocol;
    HiveClient client;
    
    public Worker() {
      transport = new TSocket("localhost", 10000);
      protocol = new TBinaryProtocol(transport);
      client = new HiveClient(protocol); 
    }

    public String call() throws Exception {
      transport.open();
      client.execute("add jar /home/zhoumin//hadoop/mapreduce/zhoumin/dist/zhoumin-0.00.1.jar");
      client.execute("CREATE TEMPORARY FUNCTION strlen AS 'hadoop.hive.udf.UdfStringLength'");
      client.execute("select strlen(mid) from log_data");
      String row = client.fetchOne();
      transport.close();
      return row;
    }
    
  }
  
  public static void main(String[] args) throws TTransportException,
      TException, HiveServerException, MetaException {
    ExecutorService exec = Executors.newCachedThreadPool();
    
    ArrayList<Future<String>> results = new ArrayList<Future<String>>();
    for(int i = 0; i < THREADS_NUMBER; i++) {
      results.add(exec.submit(new Worker()));
    }
    
    for(Future<String> fs : results) {
      try {
        System.out.println(fs.get());
      }catch (InterruptedException e) {
        System.out.println(e);
      } catch(ExecutionException e) {
        System.out.println(e);
      } finally {
        exec.shutdown();
      }
    }
  }
}


the add jar command is also supported on CLI now.
  
> Executing cli commands into thrift server
> -----------------------------------------
>
>                 Key: HIVE-338
>                 URL: https://issues.apache.org/jira/browse/HIVE-338
>             Project: Hadoop Hive
>          Issue Type: Improvement
>          Components: Server Infrastructure
>    Affects Versions: 0.3.0
>            Reporter: Min Zhou
>         Attachments: hiveserver-v1.patch
>
>
> Let thrift server support set, add/delete file/jar and normal HSQL query.

-- 
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.


Mime
View raw message