1

我已经通过这些方法，但我似乎无法找出如何推断 Hive 中工作的完成百分比（如 eventListener！）。请帮忙！编辑 - 我认为您可以从客户端获得“我已完成映射......所以我已完成 50%”（如果我要提交命令 OVERWRITE EXTERNAL TABLE）。带有 Brisk 的 OpsCenter（由 Datastax 提供）就是这样做的。

import java.util.List;

import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.service.HiveServerException;
import org.apache.hadoop.hive.service.ThriftHive;
import org.apache.hadoop.hive.service.ThriftHive.Client;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.transport.TSocket;

public class Hive {

    static Client client;
    static TSocket transport;

    public static void main(String args[]) throws HiveServerException,
            TException, MetaException {

        transport = new TSocket("hiveserver",
                10000);
        transport.setTimeout(999999999);

        TBinaryProtocol protocol = new TBinaryProtocol(transport);
        client = new ThriftHive.Client(protocol);
        transport.open();

        System.out.println("Starting map job...");

        Thread mapReduceThread = new Thread(new HiveQuery(
                "SELECT COUNT(*) FROM myHiveTable"));
        mapReduceThread.start();

        System.out.println("Waiting on map...");
    }

    private static class HiveQuery implements Runnable {

        private String hql;

        public HiveQuery(String hql) {
            this.setHql(hql);
        }

        public void run() {
            long start = System.currentTimeMillis();

            // Blocking
            try {
                client.execute(this.getHql());
            } catch (HiveServerException e) {
                e.printStackTrace();
            } catch (TException e) {
                e.printStackTrace();
            }

            List<String> responseList = null;
            try {
                responseList = client.fetchAll();
            } catch (HiveServerException e) {
                e.printStackTrace();
            } catch (TException e) {
                e.printStackTrace();
            }

            long elapsedTimeMillis = System.currentTimeMillis() - start;
            float elapsedTime = elapsedTimeMillis / 1000F;

            System.out.println("Job took: " + elapsedTime + " seconds");

            for (String response : responseList) {
                System.out.println("Response: " + response);
            }

            transport.close();
            System.out.println("Closed transport");
            System.exit(0);
        }

        public void setHql(String hql) {
            this.hql = hql;
        }

        public String getHql() {
            return hql;
        }
    }

}
4

1 回答 1

0

你说的外推是什么意思？

我认为您无法通过 Hadoop 中的已完成百分比数字推断出任何结果，除非它上升了，那么就会发生更多的工作。你不能说“哦，那意味着我还有 20 秒”或类似的话。

于 2011-06-29T12:10:29.727 回答