FAQ
Repository: hbase
Updated Branches:
   refs/heads/HBASE-14850 3b71d0019 -> 7b1786e8f


HBASE-15771 Document all the public classes

Summary: Add on a bunch of documentation around implementation so far.

Test Plan:
     make doc
Doxygen has no warnings

Differential Revision: https://reviews.facebook.net/D57753


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bce08a58
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bce08a58
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bce08a58

Branch: refs/heads/HBASE-14850
Commit: bce08a5855e112042e12f815db73aacc6c92abf8
Parents: 3b71d00
Author: Elliott Clark <eclark@apache.org>
Authored: Thu May 5 13:14:20 2016 -0700
Committer: Elliott Clark <eclark@apache.org>
Committed: Fri May 6 09:07:28 2016 -0700

----------------------------------------------------------------------
  hbase-native-client/BUILDING.md | 18 +++++-
  .../connection/client-dispatcher.h | 11 +++-
  .../connection/client-handler.cc | 37 +++++-------
  hbase-native-client/connection/client-handler.h | 36 +++++++-----
  .../connection/connection-factory.h | 18 ++++++
  .../connection/connection-pool.cc | 5 +-
  .../connection/connection-pool.h | 44 +++++++++++++-
  hbase-native-client/connection/pipeline.h | 18 ++++++
  hbase-native-client/connection/request.h | 32 ++++++----
  hbase-native-client/connection/response.h | 33 +++++++++--
  hbase-native-client/core/BUCK | 1 -
  hbase-native-client/core/client.h | 12 ++++
  hbase-native-client/core/connection.cc | 20 -------
  hbase-native-client/core/connection.h | 26 ---------
  hbase-native-client/core/location-cache.cc | 14 ++++-
  hbase-native-client/core/location-cache.h | 36 ++++++++++--
  hbase-native-client/core/meta-utils.h | 12 ++++
  hbase-native-client/core/put.cc | 21 -------
  hbase-native-client/core/put.h | 27 ---------
  hbase-native-client/core/region-location.h | 41 +++++++++++++
  hbase-native-client/serde/rpc.cc | 1 -
  hbase-native-client/serde/rpc.h | 61 +++++++++++++++++++-
  hbase-native-client/serde/zk.h | 14 +++++
  hbase-native-client/utils/user-util.h | 18 ++++++
  24 files changed, 387 insertions(+), 169 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/BUILDING.md
----------------------------------------------------------------------
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
index 20ef2a0..4c06776 100644
--- a/hbase-native-client/BUILDING.md
+++ b/hbase-native-client/BUILDING.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
  under the License.
  -->

-#Building HBase native client
+# Building HBase native client

  The HBase native client build using buck and produces a linux library.

@@ -58,6 +58,20 @@ buck build //core:simple-client
  ```

  That will build the library, then build and test everything, then build
-the simple-client binary. Buck will find all modules used, and compile
+the simple-client binary. Buck will find all modules used, and compile
  them in parallel, caching the results. Output from buck is in the buck-out
  foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
+
+
+# Make
+
+If learning buck isn't your thing there is a Makefile wrapper for your
+convenience.
+
+```
+make help
+make check
+make clean
+make all
+make build
+```

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 4435a1b..4bfb35d 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -30,17 +30,22 @@
  #include "connection/response.h"

  namespace hbase {
+/**
+ * Dispatcher that assigns a call_id and then routes the response back to the future.
+ */
  class ClientDispatcher
      : public wangle::ClientDispatcherBase<SerializePipeline,
                                            std::unique_ptr<Request>, Response> {
  public:
+ /** Create a new ClientDispatcher */
    ClientDispatcher();
- ~ClientDispatcher() {
- LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
- }
+ /** Read a response off the pipeline. */
    void read(Context *ctx, Response in) override;
+ /** Take a request as a call and send it down the pipeline. */
    folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
+ /** Close the dispatcher and the associated pipeline. */
    folly::Future<folly::Unit> close(Context *ctx) override;
+ /** Close the dispatcher and the associated pipeline. */
    folly::Future<folly::Unit> close() override;

  private:

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 4fdb7ae..cae03c7 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
  using google::protobuf::Message;

  ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), header_info_(),
+ : user_name_(user_name), serde_(), once_flag_(),
        resp_msgs_(
            make_unique<folly::AtomicHashMap<
                uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
@@ -75,36 +75,25 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
        used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
        // Make sure that bytes were parsed.
        CHECK(used_bytes == buf->length());
- received.set_response(resp_msg);
+ received.set_resp_msg(resp_msg);
      }
      ctx->fireRead(std::move(received));
    }
  }

  Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
- // Keep track of if we have sent the header.
- //
- // even though the bool is atomic we can load it lazily here.
- if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
-
- // Grab the lock.
- // We need to make sure that no one gets past here without there being a
- // hearder sent.
- std::lock_guard<std::mutex> lock(header_info_->mutex_);
-
- // Now see if we are the first thread to get through.
- //
- // If this is the first thread to get through then the
- // need_send_header will have been true before this.
- if (header_info_->need_.exchange(false)) {
- auto pre = serde_.Preamble();
- auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
- }
- }
-
+ // We need to send the header once.
+ // So use call_once to make sure that only one thread wins this.
+ std::call_once((*once_flag_), [ctx, this]() {
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ });
+
+ // Now store the call id to response.
    resp_msgs_->insert(r->call_id(), r->resp_msg());
+ // Send the data down the pipeline.
    return ctx->fireWrite(
        serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
  }

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index 1a4275f..984a947 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -41,17 +41,38 @@ class Message;

  namespace hbase {

+/**
+ * wangle::Handler implementation to convert hbase::Request to IOBuf and
+ * convert IOBuf to hbase::Response.
+ *
+ * This class deals with sending the connection header and preamble
+ * on first request.
+ */
  class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
                                               Response, std::unique_ptr<Request>,
                                               std::unique_ptr<folly::IOBuf>> {
  public:
+ /**
+ * Create the handler
+ * @param user_name the user name of the user running this process.
+ */
    ClientHandler(std::string user_name);
+
+ /**
+ * Get bytes from the wire.
+ * This should be the full message as the length field decoder should be
+ * in the pipeline before this.
+ */
    void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+
+ /**
+ * Write the data down the wire.
+ */
    folly::Future<folly::Unit> write(Context *ctx,
                                     std::unique_ptr<Request> r) override;

  private:
- std::unique_ptr<HeaderInfo> header_info_;
+ std::unique_ptr<std::once_flag> once_flag_;
    std::string user_name_;
    RpcSerde serde_;

@@ -60,17 +81,4 @@ private:
        uint32_t, std::shared_ptr<google::protobuf::Message>>>
        resp_msgs_;
  };
-
-/**
- * Class to contain the info about if the connection header and preamble has
- * been sent.
- *
- * We use a serperate class here so that ClientHandler is relocatable.
- */
-class HeaderInfo {
-public:
- HeaderInfo() : need_(true), mutex_() {}
- std::atomic<bool> need_;
- std::mutex mutex_;
-};
  } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 2284a7c..8b6d8d8 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -28,14 +28,32 @@
  #include "connection/service.h"

  namespace hbase {
+
+/**
+ * Class to create a ClientBootstrap and turn it into a connected
+ * pipeline.
+ */
  class ConnectionFactory {
  public:
+ /**
+ * Constructor.
+ * There should only be one ConnectionFactory per client.
+ */
    ConnectionFactory();
+ /** Default Desctructor */
    virtual ~ConnectionFactory() = default;

+ /**
+ * Create a BootStrap from which a connection can be made.
+ */
    virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
    MakeBootstrap();

+ /**
+ * Connect a ClientBootstrap to a server and return the pipeline.
+ *
+ * This is mostly visible so that mocks can override socket connections.
+ */
    virtual std::shared_ptr<HBaseService>
    Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
            const std::string &hostname, int port);

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index 6ed5ad9..75f343e 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -26,11 +26,11 @@ using std::mutex;
  using std::unique_ptr;
  using std::shared_ptr;
  using hbase::pb::ServerName;
+using hbase::ConnectionPool;
+using hbase::HBaseService;
  using folly::SharedMutexWritePriority;
  using folly::SocketAddress;

-namespace hbase {
-
  ConnectionPool::ConnectionPool()
      : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
        map_mutex_() {}
@@ -75,4 +75,3 @@ void ConnectionPool::close(const ServerName &sn) {
    auto service = found->second;
    connections_.erase(found);
  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 907afdb..5edd407 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -18,6 +18,7 @@
   */
  #pragma once

+#include <boost/functional/hash.hpp>
  #include <folly/SharedMutex.h>
  #include <mutex>
  #include <unordered_map>
@@ -27,26 +28,63 @@
  #include "if/HBase.pb.h"

  namespace hbase {
+
+/** Equals function for server name that ignores start time */
  struct ServerNameEquals {
+
+ /** equals */
    bool operator()(const hbase::pb::ServerName &lhs,
                    const hbase::pb::ServerName &rhs) const {
      return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
    }
  };
+
+/** Hash for ServerName that ignores the start time. */
  struct ServerNameHash {
+ /** hash */
    std::size_t operator()(hbase::pb::ServerName const &s) const {
- std::size_t h1 = std::hash<std::string>()(s.host_name());
- std::size_t h2 = std::hash<uint32_t>()(s.port());
- return h1 ^ (h2 << 2);
+ std::size_t h = 0;
+ boost::hash_combine(h, s.host_name());
+ boost::hash_combine(h, s.port());
+ return h;
    }
  };

+/**
+ * @brief Connection pooling for HBase rpc connection.
+ *
+ * This is a thread safe connection pool. It allows getting
+ * a shared connection to HBase by server name. This is
+ * useful for keeping a single connection no matter how many regions a
+ * regionserver has on it.
+ */
  class ConnectionPool {
  public:
+ /** Create connection pool wit default connection factory */
    ConnectionPool();
+
+ /**
+ * Desctructor.
+ * All connections will be close.
+ * All connections will be released
+ */
    ~ConnectionPool();
+
+ /**
+ * Constructor that allows specifiying the connetion factory.
+ * This is useful for testing.
+ */
    explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
+
+ /**
+ * Get a connection to the server name. Start time is ignored.
+ * This can be a blocking operation for a short time.
+ */
    std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+
+ /**
+ * Close/remove a connection.
+ */
    void close(const hbase::pb::ServerName &sn);

  private:

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
index e90cb93..88d7845 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -26,12 +26,30 @@
  #include "utils/user-util.h"

  namespace hbase {
+
+/** Pipeline to turn IOBuf into requests */
  using SerializePipeline =
      wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;

+/**
+ * Factory to create new pipelines for HBase RPC's.
+ */
  class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
  public:
+ /**
+ * Constructor. This will create user util.
+ */
    RpcPipelineFactory();
+
+ /**
+ * Create a new pipeline.
+ * The pipeline will be:
+ *
+ * - Async Socke Handler
+ * - Event Base Handler
+ * - Length Field Based Frame Decoder
+ * - Client Handler
+ */
    SerializePipeline::Ptr
    newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;


http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
index 743c469..634d1ba 100644
--- a/hbase-native-client/connection/request.h
+++ b/hbase-native-client/connection/request.h
@@ -25,30 +25,42 @@
  #include <string>

  namespace hbase {
+
+/**
+ * Main request class.
+ * This holds the request object and the un-filled in approriatley typed
+ * response object.
+ */
  class Request {
  public:
+ /** Create a request object for a get */
    static std::unique_ptr<Request> get();
+ /** Create a request object for a mutate */
    static std::unique_ptr<Request> mutate();
+ /** Create a request object for a scan */
    static std::unique_ptr<Request> scan();

+ /**
+ * This should be private. Do not use this.
+ *
+ *
+ * Constructor that's public for make_unique. This sets all the messages and
+ * method name.
+ */
    Request(std::shared_ptr<google::protobuf::Message> req,
            std::shared_ptr<google::protobuf::Message> resp, std::string method);

+ /** Get the call id. */
    uint32_t call_id() { return call_id_; }
+ /** Set the call id. This should only be set once. */
    void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
+ /** Get the backing request protobuf message. */
    std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
+ /** Get the backing response protobuf message. */
    std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
-
- void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
- req_msg_ = msg;
- }
- void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
- resp_msg_ = msg;
- }
-
+ /** Get the method name. This is used to the the receiving rpc server what
+ * method type to decode. */
    std::string method() { return method_; }
- void set_method(std::string method) { method_ = method; }

  private:
    uint32_t call_id_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/connection/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
index d2eb19a..aac57c2 100644
--- a/hbase-native-client/connection/response.h
+++ b/hbase-native-client/connection/response.h
@@ -29,20 +29,41 @@ class Message;

  namespace hbase {

+/**
+ * @brief Class representing a rpc response
+ *
+ * This is the class sent to a service.
+ */
  class Response {
  public:
- Response() : call_id_(0), response_(nullptr) {}
+ /**
+ * Constructor.
+ * Initinalizes the call id to 0. 0 should never be a valid call id.
+ */
+ Response() : call_id_(0), resp_msg_(nullptr) {}
+
+ /** Get the call_id */
    uint32_t call_id() { return call_id_; }
+
+ /** Set the call_id */
    void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- std::shared_ptr<google::protobuf::Message> response() const {
- return response_;
+
+ /**
+ * Get the response message.
+ * The caller is reponsible for knowing the type. In practice the call id is
+ * used to figure out the type.
+ */
+ std::shared_ptr<google::protobuf::Message> resp_msg() const {
+ return resp_msg_;
    }
- void set_response(std::shared_ptr<google::protobuf::Message> response) {
- response_ = std::move(response);
+
+ /** Set the response message. */
+ void set_resp_msg(std::shared_ptr<google::protobuf::Message> response) {
+ resp_msg_ = std::move(response);
    }

  private:
    uint32_t call_id_;
- std::shared_ptr<google::protobuf::Message> response_;
+ std::shared_ptr<google::protobuf::Message> resp_msg_;
  };
  } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef8c2f8..485f9ba 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,7 +20,6 @@ cxx_library(
      name="core",
      exported_headers=[
          "client.h",
- "connection.h",
          "hbase_macros.h",
          "region-location.h",
          "location-cache.h",

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 4bed751..4a6d23b 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -28,8 +28,20 @@
  #include "if/Cell.pb.h"

  namespace hbase {
+
+/**
+ * Client.
+ *
+ * This is the class that provides access to an HBase cluster.
+ * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
+ */
  class Client {
  public:
+
+ /**
+ * Create a new client.
+ * @param quorum_spec Where to connect to get Zookeeper bootstrap information.
+ */
    explicit Client(std::string quorum_spec);

  private:

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.cc b/hbase-native-client/core/connection.cc
deleted file mode 100644
index e9a28eb..0000000
--- a/hbase-native-client/core/connection.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/connection.h"
-
-void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.h b/hbase-native-client/core/connection.h
deleted file mode 100644
index 5c9d6b8..0000000
--- a/hbase-native-client/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-class Connection {
- char *zk_quorum;
-
-public:
- void set_zk_quorum(char *zk_q);
-};

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index e2a6251..583d305 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -128,15 +128,24 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
        });
  }

+/**
+ * Filter to remove a service from the location cache and the connection cache on errors
+ * or on cloase.
+ */
  class RemoveServiceFilter
      : public ServiceFilter<std::unique_ptr<Request>, Response> {

  public:
+
+ /** Create a new filter. */
    RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
                        ConnectionPool &cp)
        : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
          cp_(cp) {}

+ /**
+ * Close will remove the connection from all caches.
+ */
    folly::Future<folly::Unit> close() override {
      if (!released.exchange(true)) {
        return this->service_->close().then([this]() {
@@ -148,10 +157,13 @@ public:
      }
    }

+
+ /** Has this been closed */
    virtual bool isAvailable() override {
      return !released && service_->isAvailable();
    }

+ /** Send the message. */
    folly::Future<Response> operator()(unique_ptr<Request> req) override {
      // TODO(eclark): add in an on error handler that will
      // remove the region location from the cache if needed.
@@ -168,7 +180,7 @@ private:

  std::shared_ptr<RegionLocation>
  LocationCache::CreateLocation(const Response &resp) {
- auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
    auto &results = resp_msg->results().Get(0);
    auto &cells = results.cell();
    auto ri = folly::to<RegionInfo>(cells.Get(0).value());

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 7f76428..88bec18 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -33,25 +33,48 @@
  #include "serde/table-name.h"

  namespace hbase {
-
+// Forward
  class Request;
  class Response;
  namespace pb {
  class ServerName;
  }

+/**
+ * Class that can look up and cache locations.
+ */
  class LocationCache {
  public:
- explicit LocationCache(std::string quorum_spec,
- std::shared_ptr<folly::Executor> executor);
+ /**
+ * Constructor.
+ * @param quorum_spec Where to connect for Zookeeper.
+ * @param executor The cpu executor to run on.
+ */
+ LocationCache(std::string quorum_spec,
+ std::shared_ptr<folly::Executor> executor);
+ /**
+ * Destructor.
+ * This will clean up the zookeeper connections.
+ */
    ~LocationCache();
- // Meta Related Methods.
- // These are only public until testing is complete
+
+ /**
+ * Where is meta hosted.
+ *
+ * TODO: This should be a RegionLocation.
+ */
    folly::Future<hbase::pb::ServerName> LocateMeta();
+
+ /**
+ * Go read meta and find out where a region is located.
+ */
    folly::Future<std::shared_ptr<RegionLocation>>
    LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
+
+ /**
+ * Remove the cached location of meta.
+ */
    void InvalidateMeta();
- ConnectionPool cp_;

  private:
    void RefreshMetaLocation();
@@ -63,6 +86,7 @@ private:
    std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
    std::mutex meta_lock_;
    MetaUtil meta_util_;
+ ConnectionPool cp_;

    // TODO: migrate this to a smart pointer with a deleter.
    zhandle_t *zk_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index dfef065..f6cc84f 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -26,10 +26,22 @@

  namespace hbase {

+/**
+ * @brief Utility for meta operations.
+ */
  class MetaUtil {
  public:
+ /**
+ * Given a table and a row give the row key from which to start a scan to find
+ * region locations.
+ */
    std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
                                   const std::string &row) const;
+
+ /**
+ * Given a row we're trying to access create a request to look up the
+ * location.
+ */
    std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
                                         const std::string &row) const;
  };

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/put.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.cc b/hbase-native-client/core/put.cc
deleted file mode 100644
index 806a478..0000000
--- a/hbase-native-client/core/put.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/put.h"
-
-Put::~Put() {}

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/put.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.h b/hbase-native-client/core/put.h
deleted file mode 100644
index d34aa0d..0000000
--- a/hbase-native-client/core/put.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/mutation.h"
-
-class Put : public Mutation {
-public:
- ~Put();
-};

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/core/region-location.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
index 7887526..a3d832e 100644
--- a/hbase-native-client/core/region-location.h
+++ b/hbase-native-client/core/region-location.h
@@ -25,19 +25,60 @@

  namespace hbase {

+/**
+ * @brief class to hold where a region is located.
+ *
+ * This class holds where a region is located, the information about it, the
+ * region name, and a connection to the service used for connecting to it.
+ */
  class RegionLocation {
  public:
+ /**
+ * Constructor.
+ * @param region_name The region name of this region.
+ * @param ri The decoded RegionInfo of this region.
+ * @param sn The server name of the HBase regionserver thought to be hosting
+ * this region.
+ * @param service the connected service to the regionserver.
+ */
    RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
                   hbase::pb::ServerName sn,
                   std::shared_ptr<HBaseService> service)
        : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}

+ /**
+ * Get a reference to the regio info
+ */
    const hbase::pb::RegionInfo &region_info() { return ri_; }
+
+ /**
+ * Get a reference to the server name
+ */
    const hbase::pb::ServerName &server_name() { return sn_; }
+
+ /**
+ * Get a reference to the region name.
+ */
    const std::string &region_name() { return region_name_; }
+
+ /**
+ * Get a service. This could be closed or null. It's the caller's
+ * responsibility to check.
+ */
    std::shared_ptr<HBaseService> service() { return service_; }
+
+ /**
+ * Set the service.
+ * This should be used if the region moved or if the connection is thought to
+ * be bad and a new tcp connection needs to be made.
+ */
    void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }

+ /**
+ * Set the servername if the region has moved.
+ */
+ void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
+
  private:
    std::string region_name_;
    hbase::pb::RegionInfo ri_;

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/serde/rpc.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.cc b/hbase-native-client/serde/rpc.cc
index 4c3c999..b573738 100644
--- a/hbase-native-client/serde/rpc.cc
+++ b/hbase-native-client/serde/rpc.cc
@@ -85,7 +85,6 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
  }

  RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
-RpcSerde::~RpcSerde() {}

  unique_ptr<IOBuf> RpcSerde::Preamble() {
    auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/serde/rpc.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.h b/hbase-native-client/serde/rpc.h
index cefb583..b28b4a3 100644
--- a/hbase-native-client/serde/rpc.h
+++ b/hbase-native-client/serde/rpc.h
@@ -32,22 +32,81 @@ class Message;
  }

  namespace hbase {
+
+/**
+ * @brief Class for serializing a deserializing rpc formatted data.
+ *
+ * RpcSerde is the one stop shop for reading/writing data to HBase daemons.
+ * It should throw exceptions if anything goes wrong.
+ */
  class RpcSerde {
  public:
+ /**
+ * Constructor assumes the default auth type.
+ */
    RpcSerde();
- virtual ~RpcSerde();
+
+ /**
+ * Destructor. This is provided just for testing purposes.
+ */
+ virtual ~RpcSerde() = default;
+
+ /**
+ * Pase a message in the delimited format.
+ *
+ * A message in delimited format consists of the following:
+ *
+ * - a protobuf var int32.
+ * - A protobuf object serialized.
+ */
    int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+
+ /**
+ * Create a new connection preamble in a new IOBuf.
+ */
    std::unique_ptr<folly::IOBuf> Preamble();
+
+ /**
+ * Create the header protobuf object and serialize it to a new IOBuf.
+ * Header is in the following format:
+ *
+ * - Big endian length
+ * - ConnectionHeader object serialized out.
+ */
    std::unique_ptr<folly::IOBuf> Header(const std::string &user);
+
+ /**
+ * Serialize a request message into a protobuf.
+ * Request consists of:
+ *
+ * - Big endian length
+ * - RequestHeader object
+ * - The passed in Message object
+ */
    std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
                                          const std::string &method,
                                          const google::protobuf::Message *msg);
+
+ /**
+ * Serialize a message in the delimited format.
+ * Delimited format consists of the following:
+ *
+ * - A protobuf var int32
+ * - The message object seriailized after that.
+ */
    std::unique_ptr<folly::IOBuf>
    SerializeDelimited(const google::protobuf::Message &msg);

+ /**
+ * Serilalize a message. This does not add any length prepend.
+ */
    std::unique_ptr<folly::IOBuf>
    SerializeMessage(const google::protobuf::Message &msg);

+ /**
+ * Prepend a length IOBuf to the given IOBuf chain.
+ * This involves no copies or moves of the passed in data.
+ */
    std::unique_ptr<folly::IOBuf>
    PrependLength(std::unique_ptr<folly::IOBuf> msg);


http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/serde/zk.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk.h b/hbase-native-client/serde/zk.h
index b672bf4..fa3de5b 100644
--- a/hbase-native-client/serde/zk.h
+++ b/hbase-native-client/serde/zk.h
@@ -28,8 +28,22 @@ class IOBuf;
  }

  namespace hbase {
+
+/** @brief A class to convert data from ZooKeeper to other formats.
+ *
+ * This class will convert data to and from Zookeeper into protobuf objects.
+ *
+ */
  class ZkDeserializer {
  public:
+ /**
+ * Merge the data from a buffer into a given message.
+ *
+ * @param buf Naked pointer to iobuf containing data read from zookeeper.
+ * @param out Naked pointer into which the data will be merged. The message
+ * should be the correct type.
+ * @return returns true if the parsing was successful.
+ */
    bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
  };
  } // namespace hbase

http://git-wip-us.apache.org/repos/asf/hbase/blob/bce08a58/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
index da95d78..0b4cc73 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -24,12 +24,30 @@
  #include <mutex>

  namespace hbase {
+
+/**
+ * @brief Class to help with user/group information.
+ *
+ * This class will get the current user, and information about them. It caches
+ * the user information after the first invocation.
+ */
  class UserUtil {
  public:
+ /**
+ * Constructor.
+ */
    UserUtil();
+
+ /**
+ * Get the username of the user owning this process. This is thread safe and
+ * lockless for every invocation other than the first one.
+ */
    std::string user_name();

  private:
+ /**
+ * Compute the username. This will block.
+ */
    void compute_user_name();
    std::atomic<bool> init_;
    std::string user_name_;

Search Discussions

  • Eclark at May 6, 2016 at 4:11 pm
    HBASE-15761 Add on more server name tests


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7b1786e8
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7b1786e8
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7b1786e8

    Branch: refs/heads/HBASE-14850
    Commit: 7b1786e8fb92c75ca9f37fb388b3ba8c4e9c9709
    Parents: bce08a5
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed May 4 12:04:18 2016 -0700
    Committer: Elliott Clark <eclark@apache.org>
    Committed: Fri May 6 09:11:38 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/serde/server-name-test.cc | 18 ++++++++++++++++++
      hbase-native-client/serde/server-name.h | 4 +++-
      2 files changed, 21 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b1786e8/hbase-native-client/serde/server-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
    index 35dcbc1..2281fa2 100644
    --- a/hbase-native-client/serde/server-name-test.cc
    +++ b/hbase-native-client/serde/server-name-test.cc
    @@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
        ASSERT_EQ("test", sn.host_name());
        ASSERT_EQ(123, sn.port());
      }
    +
    +TEST(TestServerName, TestIps) {
    + auto sn = folly::to<ServerName>("127.0.0.1:999");
    + ASSERT_EQ("127.0.0.1", sn.host_name());
    + ASSERT_EQ(999, sn.port());
    +}
    +
    +TEST(TestServerName, TestThrow) {
    + ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here"));
    +}
    +
    +TEST(TestServerName, TestIPV6) {
    + auto sn = folly::to<ServerName>("[::::1]:123");
    +
    + ASSERT_EQ("[::::1]", sn.host_name());
    + ASSERT_EQ(123, sn.port());
    +
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b1786e8/hbase-native-client/serde/server-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
    index bdba087..9844465 100644
    --- a/hbase-native-client/serde/server-name.h
    +++ b/hbase-native-client/serde/server-name.h
    @@ -12,7 +12,9 @@ template <class String> void parseTo(String in, ServerName &out) {
        std::string s = folly::to<std::string>(in);

        auto delim = s.rfind(":");
    - DCHECK(delim != std::string::npos);
    + if (delim == std::string::npos) {
    + throw std::runtime_error("Couldn't parse server name");
    + }
        out.set_host_name(s.substr(0, delim));
        // Now keep everything after the : (delim + 1) to the end.
        out.set_port(folly::to<int>(s.substr(delim + 1)));
  • Eclark at May 12, 2016 at 5:29 pm
    Repository: hbase
    Updated Branches:
       refs/heads/HBASE-14850 a72c2db64 -> b434d0eaf


    HBASE-15774 Fix Upgrade lock usage in connection pool.

    Summary:
    Use upgrade lock better.
    Fix a clang warning around initializing the UpgradeHolder incorrectly.
    Remove dead code. ( We'l need to add it back when there's a better plan)
    Add on more comments.

    Test Plan: buck test --all

    Differential Revision: https://reviews.facebook.net/D58005


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b434d0ea
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b434d0ea
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b434d0ea

    Branch: refs/heads/HBASE-14850
    Commit: b434d0eafbc84b8f5dacfbd20a927a8cb6dd6372
    Parents: a72c2db
    Author: Elliott Clark <eclark@apache.org>
    Authored: Tue May 10 17:44:41 2016 -0700
    Committer: Elliott Clark <eclark@apache.org>
    Committed: Thu May 12 10:29:50 2016 -0700

    ----------------------------------------------------------------------
      .../connection/connection-pool-test.cc | 12 ++--
      .../connection/connection-pool.cc | 47 +++++++++++---
      .../connection/connection-pool.h | 6 +-
      hbase-native-client/core/client.h | 2 +-
      hbase-native-client/core/location-cache.cc | 68 ++++----------------
      5 files changed, 62 insertions(+), 73 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/b434d0ea/hbase-native-client/connection/connection-pool-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
    index c0c346f..bd2d585 100644
    --- a/hbase-native-client/connection/connection-pool-test.cc
    +++ b/hbase-native-client/connection/connection-pool-test.cc
    @@ -79,9 +79,9 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
        sn.set_host_name(hostname);
        sn.set_port(port);

    - auto result = cp.get(sn);
    + auto result = cp.Get(sn);
        ASSERT_TRUE(result != nullptr);
    - result = cp.get(sn);
    + result = cp.Get(sn);
      }

      TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
    @@ -102,13 +102,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
        ConnectionPool cp{mock_cf};

        {
    - auto result_one = cp.get(folly::to<ServerName>(
    + auto result_one = cp.Get(folly::to<ServerName>(
              hostname_one + ":" + folly::to<std::string>(port)));
    - auto result_two = cp.get(folly::to<ServerName>(
    + auto result_two = cp.Get(folly::to<ServerName>(
              hostname_two + ":" + folly::to<std::string>(port)));
        }
    - auto result_one = cp.get(
    + auto result_one = cp.Get(
            folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
    - auto result_two = cp.get(
    + auto result_two = cp.Get(
            folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b434d0ea/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index 90e2056..aa3d094 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -48,25 +48,54 @@ ConnectionPool::~ConnectionPool() {
        clients_.clear();
      }

    -std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
    - // Create a read lock.
    - SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
    +std::shared_ptr<HBaseService> ConnectionPool::Get(const ServerName &sn) {
    + // Try and get th cached connection.
    + auto found_ptr = GetCached(sn);

    + // If there's no connection then create it.
    + if (found_ptr == nullptr) {
    + found_ptr = GetNew(sn);
    + }
    + return found_ptr;
    +}
    +
    +std::shared_ptr<HBaseService> ConnectionPool::GetCached(const ServerName &sn) {
    + SharedMutexWritePriority::ReadHolder holder(map_mutex_);
        auto found = connections_.find(sn);
    - if (found == connections_.end() || found->second == nullptr) {
    - // Move the upgradable lock into the write lock if the connection
    - // hasn't been found.
    - SharedMutexWritePriority::WriteHolder holder(std::move(holder));
    + if (found == connections_.end()) {
    + return nullptr;
    + }
    + return found->second;
    +}
    +
    +std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
    + // Grab the upgrade lock. While we are double checking other readers can
    + // continue on
    + SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
    +
    + // Now check if someone else created the connection before we got the lock
    + // This is safe since we hold the upgrade lock.
    + // upgrade lock is more power than the reader lock.
    + auto found = connections_.find(sn);
    + if (found != connections_.end() && found->second != nullptr) {
    + return found->second;
    + } else {
    + // Yeah it looks a lot like there's no connection
    + SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
    +
    + // Make double sure there are not stale connections hanging around.
    + connections_.erase(sn);
    +
    + // Nope we are the ones who should create the new connection.
          auto client = cf_->MakeBootstrap();
          auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
          clients_.insert(std::make_pair(sn, client));
          connections_.insert(std::make_pair(sn, dispatcher));
          return dispatcher;
        }
    - return found->second;
      }

    -void ConnectionPool::close(const ServerName &sn) {
    +void ConnectionPool::Close(const ServerName &sn) {
        SharedMutexWritePriority::WriteHolder holder{map_mutex_};

        auto found = connections_.find(sn);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b434d0ea/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 60f00de..605a81b 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -80,14 +80,16 @@ public:
         * Get a connection to the server name. Start time is ignored.
         * This can be a blocking operation for a short time.
         */
    - std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
    + std::shared_ptr<HBaseService> Get(const hbase::pb::ServerName &sn);

        /**
         * Close/remove a connection.
         */
    - void close(const hbase::pb::ServerName &sn);
    + void Close(const hbase::pb::ServerName &sn);

      private:
    + std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
    + std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
                           ServerNameHash, ServerNameEquals>
            connections_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b434d0ea/hbase-native-client/core/client.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
    index 4db82c4..ba24bb9 100644
    --- a/hbase-native-client/core/client.h
    +++ b/hbase-native-client/core/client.h
    @@ -48,9 +48,9 @@ public:
        ~Client();

      private:
    - LocationCache location_cache_;
        std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
        std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
    + LocationCache location_cache_;
      };

      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b434d0ea/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 9f2a0ef..6ba8add 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -115,7 +115,7 @@ Future<std::shared_ptr<RegionLocation>>
      LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
        return this->LocateMeta()
            .via(cpu_executor_.get())
    - .then([this](ServerName sn) { return this->cp_.get(sn); })
    + .then([this](ServerName sn) { return this->cp_.Get(sn); })
          .then([tn, row, this](std::shared_ptr<HBaseService> service) {
              return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
    @@ -134,67 +134,25 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
            })
            .then([this](std::shared_ptr<RegionLocation> rl) {
              // Now fill out the connection.
    - rl->set_service(cp_.get(rl->server_name()));
    + rl->set_service(cp_.Get(rl->server_name()));
              return rl;
            });
      }

    -/**
    - * Filter to remove a service from the location cache and the connection cache
    - * on errors
    - * or on cloase.
    - */
    -class RemoveServiceFilter
    - : public ServiceFilter<std::unique_ptr<Request>, Response> {
    -
    -public:
    - /** Create a new filter. */
    - RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    - ConnectionPool &cp)
    - : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
    - cp_(cp) {}
    -
    - /**
    - * Close will remove the connection from all caches.
    - */
    - folly::Future<folly::Unit> close() override {
    - if (!released.exchange(true)) {
    - return this->service_->close().then([this]() {
    - // TODO(eclark): remove the service from the meta cache.
    - this->cp_.close(this->sn_);
    - });
    - } else {
    - return folly::makeFuture();
    - }
    - }
    -
    - /** Has this been closed */
    - virtual bool isAvailable() override {
    - return !released && service_->isAvailable();
    - }
    -
    - /** Send the message. */
    - folly::Future<Response> operator()(unique_ptr<Request> req) override {
    - // TODO(eclark): add in an on error handler that will
    - // remove the region location from the cache if needed.
    - // Also close the connection if this is likely to be an error
    - // that needs to get a new connection.
    - return (*this->service_)(std::move(req));
    - }
    -
    -private:
    - std::atomic<bool> released{false};
    - hbase::pb::ServerName sn_;
    - ConnectionPool &cp_;
    -};
    -
      std::shared_ptr<RegionLocation>
      LocationCache::CreateLocation(const Response &resp) {
        auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
        auto &results = resp_msg->results().Get(0);
        auto &cells = results.cell();
    - auto ri = folly::to<RegionInfo>(cells.Get(0).value());
    - auto sn = folly::to<ServerName>(cells.Get(1).value());
    - return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
    - nullptr);
    +
    + // TODO(eclark): There should probably be some better error
    + // handling around this.
    + auto cell_zero = cells.Get(0).value();
    + auto cell_one = cells.Get(1).value();
    + auto row = cells.Get(0).row();
    +
    + auto region_info = folly::to<RegionInfo>(cell_zero);
    + auto server_name = folly::to<ServerName>(cell_one);
    + return std::make_shared<RegionLocation>(row, std::move(region_info),
    + server_name, nullptr);
      }
  • Eclark at May 17, 2016 at 4:07 pm
    Repository: hbase
    Updated Branches:
       refs/heads/HBASE-14850 20f43d2f5 -> 8c1b82b27


    HBASE-15821 Document TestUtil


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8c1b82b2
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8c1b82b2
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8c1b82b2

    Branch: refs/heads/HBASE-14850
    Commit: 8c1b82b273ba47826d25072243ddb16d53be6a21
    Parents: 20f43d2
    Author: Elliott Clark <eclark@apache.org>
    Authored: Fri May 13 09:17:49 2016 -0700
    Committer: Elliott Clark <eclark@apache.org>
    Committed: Tue May 17 09:07:46 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/bin/format-code.sh | 2 +-
      .../connection/connection-pool.h | 4 ++--
      hbase-native-client/core/location-cache.cc | 2 +-
      hbase-native-client/core/location-cache.h | 2 +-
      hbase-native-client/test-util/test-util.cc | 20 +++++++++++++-------
      hbase-native-client/test-util/test-util.h | 20 +++++++++++++++-----
      6 files changed, 33 insertions(+), 17 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/bin/format-code.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
    index 36a114d..cc8b368 100755
    --- a/hbase-native-client/bin/format-code.sh
    +++ b/hbase-native-client/bin/format-code.sh
    @@ -19,5 +19,5 @@ set -euo pipefail
      IFS=$'\n\t'


    -find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
    +find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
      find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 605a81b..b8c950b 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -88,8 +88,8 @@ public:
        void Close(const hbase::pb::ServerName &sn);

      private:
    - std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
    - std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
    + std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
    + std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
                           ServerNameHash, ServerNameEquals>
            connections_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 6ba8add..efd2210 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
        return this->LocateMeta()
            .via(cpu_executor_.get())
            .then([this](ServerName sn) { return this->cp_.Get(sn); })
    - .then([tn, row, this](std::shared_ptr<HBaseService> service) {
    + .then([tn, row, this](std::shared_ptr<HBaseService> service) {
              return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
            .then([this](Response resp) {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index d435530..830cd96 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -55,7 +55,7 @@ public:
         * @param io_executor executor used to talk to the network
         */
        LocationCache(std::string quorum_spec,
    - std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
    + std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
                      std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
        /**
         * Destructor.

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/test-util/test-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc
    index e5fba48..88ce7c8 100644
    --- a/hbase-native-client/test-util/test-util.cc
    +++ b/hbase-native-client/test-util/test-util.cc
    @@ -24,13 +24,18 @@
      using hbase::TestUtil;
      using folly::Random;

    -const static int STR_LEN = 32;
    -
    -std::string TestUtil::RandString() {
    - auto s = std::string(STR_LEN, 'z');
    -
    - for (int i = 0; i < STR_LEN; i++) {
    +std::string TestUtil::RandString(int len) {
    + // Create the whole string.
    + // Filling everything with z's
    + auto s = std::string(len, 'z');
    +
    + // Now pick a bunch of random numbers
    + for (int i = 0; i < len; i++) {
    + // use Folly's random to get the numbers
    + // as I don't want to have to learn
    + // all the cpp rand invocation magic.
          auto r = Random::rand32('a', 'z');
    + // Cast that to ascii.
          s[i] = static_cast<char>(r);
        }
        return s;
    @@ -42,12 +47,13 @@ TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
        auto res_code = std::system(cmd.c_str());
        CHECK(res_code == 0);
      }
    +
      TestUtil::~TestUtil() {
        auto res_code = std::system("bin/stop-local-hbase.sh");
        CHECK(res_code == 0);
      }

    -void TestUtil::RunShellCmd(const std::string& command) {
    +void TestUtil::RunShellCmd(const std::string &command) {
        auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
        auto res_code = std::system(cmd_string.c_str());
        CHECK(res_code == 0);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8c1b82b2/hbase-native-client/test-util/test-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h
    index 395b157..20e4981 100644
    --- a/hbase-native-client/test-util/test-util.h
    +++ b/hbase-native-client/test-util/test-util.h
    @@ -30,20 +30,30 @@ namespace hbase {
       */
      class TestUtil {
      public:
    -
        /**
         * Creating a TestUtil will spin up a cluster.
         */
        TestUtil();
    +
        /**
    - * Destroying a TestUtil will spin down a cluster.
    + * Destroying a TestUtil will spin down a cluster and remove the test dir.
         */
        ~TestUtil();
    +
    + /**
    + * Run a command in the hbase shell. Command should not include any double
    + * quotes.
    + *
    + * This should only be used until there is a good Admin support from the
    + * native client
    + */
    + void RunShellCmd(const std::string &command);
    +
        /**
    - * Run a command in the hbase shell.
    + * Create a random string. This random string is all letters, as such it is
    + * very good for use as a directory name.
         */
    - void RunShellCmd(const std::string& command);
    - static std::string RandString();
    + static std::string RandString(int len = 32);

      private:
        folly::test::TemporaryDirectory temp_dir_;
  • Ramkrishna at May 18, 2016 at 7:11 am
    Repository: hbase
    Updated Branches:
       refs/heads/master b2b3b1fa4 -> 7b5d5394c


    HBASE-15784 Misuse core/maxPoolSize of LinkedBlockingQueue in
    ThreadPoolExecutor (Jingcheng Du)


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7b5d5394
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7b5d5394
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7b5d5394

    Branch: refs/heads/master
    Commit: 7b5d5394c08e4d5a18c8f9fc62b45930bf892f41
    Parents: b2b3b1f
    Author: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
    Authored: Wed May 18 12:40:43 2016 +0530
    Committer: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
    Committed: Wed May 18 12:40:43 2016 +0530

    ----------------------------------------------------------------------
      .../hadoop/hbase/client/ConnectionImplementation.java | 10 ++++++----
      hbase-client/src/test/resources/hbase-site.xml | 4 ----
      .../hbase/replication/regionserver/HFileReplicator.java | 2 +-
      .../regionserver/RegionReplicaReplicationEndpoint.java | 6 +-----
      .../org/apache/hadoop/hbase/util/MultiHConnection.java | 6 +-----
      .../org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java | 1 -
      .../org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java | 1 -
      .../apache/hadoop/hbase/util/TestHBaseFsckReplicas.java | 1 -
      .../org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java | 1 -
      hbase-server/src/test/resources/hbase-site.xml | 4 ----
      .../hadoop/hbase/thrift/TBoundedThreadPoolServer.java | 10 +++++++---
      .../apache/hadoop/hbase/thrift/ThriftServerRunner.java | 6 ++++--
      12 files changed, 20 insertions(+), 32 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    index 9a7dfc7..429e47d 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    @@ -352,8 +352,8 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
          if (batchPool == null) {
            synchronized (this) {
              if (batchPool == null) {
    - this.batchPool = getThreadPool(conf.getInt("hbase.hconnection.threads.max", 256),
    - conf.getInt("hbase.hconnection.threads.core", 256), "-shared", null);
    + int threads = conf.getInt("hbase.hconnection.threads.max", 256);
    + this.batchPool = getThreadPool(threads, threads, "-shared", null);
                this.cleanupPool = true;
              }
            }
    @@ -377,6 +377,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
              new LinkedBlockingQueue<Runnable>(maxThreads *
                  conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                      HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
    + coreThreads = maxThreads;
          }
          ThreadPoolExecutor tpe = new ThreadPoolExecutor(
              coreThreads,
    @@ -397,9 +398,10 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
                //To start with, threads.max.core threads can hit the meta (including replicas).
                //After that, requests will get queued up in the passed queue, and only after
                //the queue is full, a new thread will be started
    + int threads = conf.getInt("hbase.hconnection.meta.lookup.threads.max", 128);
                this.metaLookupPool = getThreadPool(
    - conf.getInt("hbase.hconnection.meta.lookup.threads.max", 128),
    - conf.getInt("hbase.hconnection.meta.lookup.threads.core", 10),
    + threads,
    + threads,
                   "-metaLookup-shared-", new LinkedBlockingQueue<Runnable>());
              }
            }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-client/src/test/resources/hbase-site.xml
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml
    index 5788238..99d2ab8 100644
    --- a/hbase-client/src/test/resources/hbase-site.xml
    +++ b/hbase-client/src/test/resources/hbase-site.xml
    @@ -26,10 +26,6 @@
          <value>true</value>
        </property>
        <property>
    - <name>hbase.hconnection.meta.lookup.threads.core</name>
    - <value>4</value>
    - </property>
    - <property>
          <name>hbase.hconnection.threads.keepalivetime</name>
          <value>3</value>
        </property>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    index 1a1044d..9893e7e 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    @@ -107,7 +107,7 @@ public class HFileReplicator {
          ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
          builder.setNameFormat("HFileReplicationCallable-%1$d");
          this.exec =
    - new ThreadPoolExecutor(1, maxCopyThreads, 60, TimeUnit.SECONDS,
    + new ThreadPoolExecutor(maxCopyThreads, maxCopyThreads, 60, TimeUnit.SECONDS,
                  new LinkedBlockingQueue<Runnable>(), builder.build());
          this.exec.allowCoreThreadTimeOut(true);
          this.copiesPerThread =

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    index da37cfa..3611608 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    @@ -235,20 +235,16 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
         */
        private ExecutorService getDefaultThreadPool(Configuration conf) {
          int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256);
    - int coreThreads = conf.getInt("hbase.region.replica.replication.threads.core", 16);
          if (maxThreads == 0) {
            maxThreads = Runtime.getRuntime().availableProcessors() * 8;
          }
    - if (coreThreads == 0) {
    - coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    - }
          long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60);
          LinkedBlockingQueue<Runnable> workQueue =
              new LinkedBlockingQueue<Runnable>(maxThreads *
                  conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                    HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
          ThreadPoolExecutor tpe = new ThreadPoolExecutor(
    - coreThreads,
    + maxThreads,
            maxThreads,
            keepAliveTime,
            TimeUnit.SECONDS,

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    index ed72ea2..6efb10c 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    @@ -135,20 +135,16 @@ public class MultiHConnection {
        private void createBatchPool(Configuration conf) {
          // Use the same config for keep alive as in ConnectionImplementation.getBatchPool();
          int maxThreads = conf.getInt("hbase.multihconnection.threads.max", 256);
    - int coreThreads = conf.getInt("hbase.multihconnection.threads.core", 256);
          if (maxThreads == 0) {
            maxThreads = Runtime.getRuntime().availableProcessors() * 8;
          }
    - if (coreThreads == 0) {
    - coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    - }
          long keepAliveTime = conf.getLong("hbase.multihconnection.threads.keepalivetime", 60);
          LinkedBlockingQueue<Runnable> workQueue =
              new LinkedBlockingQueue<Runnable>(maxThreads
                  * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                    HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
          ThreadPoolExecutor tpe =
    - new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue,
    + new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue,
                  Threads.newDaemonThreadFactory("MultiHConnection" + "-shared-"));
          tpe.allowCoreThreadTimeOut(true);
          this.batchPool = tpe;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    index 8e96f83..b04689c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    @@ -57,7 +57,6 @@ public class TestHBaseFsckMOB extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(1);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    index d5869ed..a7c0c55 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    @@ -109,7 +109,6 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(1);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    index a859f78..2aa436c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    @@ -64,7 +64,6 @@ public class TestHBaseFsckReplicas extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(3);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    index 4865217..6d0e48c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    @@ -73,7 +73,6 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(2);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/resources/hbase-site.xml
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
    index 1b429b5..bfe8ee1 100644
    --- a/hbase-server/src/test/resources/hbase-site.xml
    +++ b/hbase-server/src/test/resources/hbase-site.xml
    @@ -163,10 +163,6 @@
          </description>
        </property>
        <property>
    - <name>hbase.hconnection.meta.lookup.threads.core</name>
    - <value>4</value>
    - </property>
    - <property>
          <name>hbase.hconnection.threads.keepalivetime</name>
          <value>3</value>
        </property>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    ----------------------------------------------------------------------
    diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    index 84613bd..1b8ebc9 100644
    --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    @@ -132,7 +132,7 @@ public class TBoundedThreadPoolServer extends TServer {
        }

        /** Executor service for handling client connections */
    - private ExecutorService executorService;
    + private ThreadPoolExecutor executorService;

        /** Flag for stopping the server */
        private volatile boolean stopped;
    @@ -142,9 +142,12 @@ public class TBoundedThreadPoolServer extends TServer {
        public TBoundedThreadPoolServer(Args options, ThriftMetrics metrics) {
          super(options);

    + int minWorkerThreads = options.minWorkerThreads;
    + int maxWorkerThreads = options.maxWorkerThreads;
          if (options.maxQueuedRequests > 0) {
            this.callQueue = new CallQueue(
                new LinkedBlockingQueue<Call>(options.maxQueuedRequests), metrics);
    + minWorkerThreads = maxWorkerThreads;
          } else {
            this.callQueue = new CallQueue(new SynchronousQueue<Call>(), metrics);
          }
    @@ -153,9 +156,10 @@ public class TBoundedThreadPoolServer extends TServer {
          tfb.setDaemon(true);
          tfb.setNameFormat("thrift-worker-%d");
          executorService =
    - new ThreadPoolExecutor(options.minWorkerThreads,
    - options.maxWorkerThreads, options.threadKeepAliveTimeSec,
    + new ThreadPoolExecutor(minWorkerThreads,
    + maxWorkerThreads, options.threadKeepAliveTimeSec,
                  TimeUnit.SECONDS, this.callQueue, tfb.build());
    + executorService.allowCoreThreadTimeOut(true);
          serverOptions = options;
        }


    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    ----------------------------------------------------------------------
    diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    index 7d228dc..8767a3c 100644
    --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    @@ -555,7 +555,7 @@ public class ThriftServerRunner implements Runnable {
              CallQueue callQueue =
                  new CallQueue(new LinkedBlockingQueue<Call>(), metrics);
              ExecutorService executorService = createExecutor(
    - callQueue, serverArgs.getMinWorkerThreads(), serverArgs.getMaxWorkerThreads());
    + callQueue, serverArgs.getMaxWorkerThreads(), serverArgs.getMaxWorkerThreads());
              serverArgs.executorService(executorService)
                        .processor(processor)
                        .transportFactory(transportFactory)
    @@ -620,8 +620,10 @@ public class ThriftServerRunner implements Runnable {
          ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
          tfb.setDaemon(true);
          tfb.setNameFormat("thrift-worker-%d");
    - return new ThreadPoolExecutor(minWorkers, maxWorkers,
    + ThreadPoolExecutor threadPool = new ThreadPoolExecutor(minWorkers, maxWorkers,
                  Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build());
    + threadPool.allowCoreThreadTimeOut(true);
    + return threadPool;
        }

        private InetAddress getBindAddress(Configuration conf)
  • Syuanjiang at May 18, 2016 at 2:12 pm
    Repository: hbase
    Updated Branches:
       refs/heads/branch-1.1 ce6f111a3 -> 55212d531


    HBASE-15850 Localize the configuration change in testCheckTableLocks to reduce flakiness of TestHBaseFsck test suite (Stephen Yuan Jiang)


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55212d53
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55212d53
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55212d53

    Branch: refs/heads/branch-1.1
    Commit: 55212d53139d197e1fd499592c2ff9a390ed60c5
    Parents: ce6f111
    Author: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Authored: Wed May 18 07:12:29 2016 -0700
    Committer: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Committed: Wed May 18 07:12:29 2016 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hbase/util/TestHBaseFsck.java | 20 +++++++++++++-------
      1 file changed, 13 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/55212d53/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    index 04ee249..3731a51 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    @@ -2465,20 +2465,26 @@ public class TestHBaseFsck {
          Threads.sleep(300); // wait some more to ensure writeLock.acquire() is called

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK}); // still one expired, one not-expired
    + // still one expired, one not-expired
    + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});

          edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
              TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK}); // both are expired
    -
    - conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1); // reaping from ZKInterProcessWriteLock uses znode cTime,
    - // which is not injectable through EnvironmentEdge
    + // both are expired
    + assertErrors(
    + hbck,
    + new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
    +
    + Configuration localConf = new Configuration(conf);
    + // reaping from ZKInterProcessWriteLock uses znode cTime,
    + // which is not injectable through EnvironmentEdge
    + localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
          Threads.sleep(10);
    - hbck = doFsck(conf, true); // now fix both cases
    + hbck = doFsck(localConf, true); // now fix both cases

    - hbck = doFsck(conf, false);
    + hbck = doFsck(localConf, false);
          assertNoErrors(hbck);

          // ensure that locks are deleted
  • Syuanjiang at May 18, 2016 at 4:23 pm
    Repository: hbase
    Updated Branches:
       refs/heads/branch-1.2 7c0fc0d6c -> d14803534


    HBASE-15850 Localize the configuration change in testCheckTableLocks to reduce flakiness of TestHBaseFsck test suite (Stephen Yuan Jiang)


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d1480353
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d1480353
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d1480353

    Branch: refs/heads/branch-1.2
    Commit: d14803534b93fc393535a7314f3596b3ba4c95c4
    Parents: 7c0fc0d
    Author: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Authored: Wed May 18 07:12:29 2016 -0700
    Committer: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Committed: Wed May 18 09:21:24 2016 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hbase/util/TestHBaseFsck.java | 20 +++++++++++++-------
      1 file changed, 13 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/d1480353/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    index 8f2f2da..322bc74 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
    @@ -2523,20 +2523,26 @@ public class TestHBaseFsck {
          Threads.sleep(300); // wait some more to ensure writeLock.acquire() is called

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK}); // still one expired, one not-expired
    + // still one expired, one not-expired
    + assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});

          edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
              TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK}); // both are expired
    -
    - conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1); // reaping from ZKInterProcessWriteLock uses znode cTime,
    - // which is not injectable through EnvironmentEdge
    + // both are expired
    + assertErrors(
    + hbck,
    + new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
    +
    + Configuration localConf = new Configuration(conf);
    + // reaping from ZKInterProcessWriteLock uses znode cTime,
    + // which is not injectable through EnvironmentEdge
    + localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
          Threads.sleep(10);
    - hbck = doFsck(conf, true); // now fix both cases
    + hbck = doFsck(localConf, true); // now fix both cases

    - hbck = doFsck(conf, false);
    + hbck = doFsck(localConf, false);
          assertNoErrors(hbck);

          // ensure that locks are deleted
  • Enis at May 18, 2016 at 6:57 pm
    Repository: hbase
    Updated Branches:
       refs/heads/branch-1 007852254 -> 49359a4ee
       refs/heads/branch-1.1 e4ef86cbd -> cf0dd5d4f
       refs/heads/branch-1.2 d14803534 -> 98aa826b7
       refs/heads/branch-1.3 d3d50338e -> 8566fd0f1
       refs/heads/master 3bd9220f0 -> c1ada0a37


    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1ada0a3
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1ada0a3
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1ada0a3

    Branch: refs/heads/master
    Commit: c1ada0a373561132a3359b48a27975b2e85978da
    Parents: 3bd9220
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:36:10 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ada0a3/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index 8b16a5b..84367b0 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -144,9 +144,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -450,4 +447,4 @@ public class LocalHBaseCluster {
          connection.close();
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ada0a3/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index 002bdb2..f788bed 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -86,6 +86,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Enis at May 18, 2016 at 6:57 pm
    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8566fd0f
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8566fd0f
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8566fd0f

    Branch: refs/heads/branch-1.3
    Commit: 8566fd0f1ec4ae0bcbadb0550d6b857402ee6f49
    Parents: d3d5033
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:37:40 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/8566fd0f/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index da38775..b98078a 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -143,9 +143,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -447,4 +444,4 @@ public class LocalHBaseCluster {
          }
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/8566fd0f/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index 002bdb2..f788bed 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -86,6 +86,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Enis at May 18, 2016 at 6:57 pm
    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/98aa826b
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/98aa826b
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/98aa826b

    Branch: refs/heads/branch-1.2
    Commit: 98aa826b78ed63f73cb7bde7f62eb75aa813151d
    Parents: d148035
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:39:08 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/98aa826b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index afedbe9..e79d518 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -143,9 +143,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -487,4 +484,4 @@ public class LocalHBaseCluster {
          }
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/98aa826b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index 68f4c13..64470f3 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -87,6 +87,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Enis at May 18, 2016 at 6:57 pm
    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf0dd5d4
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf0dd5d4
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf0dd5d4

    Branch: refs/heads/branch-1.1
    Commit: cf0dd5d4fe292a1a21b6103e4dca901d1e9009c3
    Parents: e4ef86c
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:40:37 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/cf0dd5d4/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index e60f868..f70f73f 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -143,9 +143,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -487,4 +484,4 @@ public class LocalHBaseCluster {
          }
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/cf0dd5d4/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index a3154d4..42f83ca 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -87,6 +87,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Enis at May 18, 2016 at 6:57 pm
    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/49359a4e
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/49359a4e
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/49359a4e

    Branch: refs/heads/branch-1
    Commit: 49359a4ee3713349190e056c1941c290c03fb1dd
    Parents: 0078522
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:36:16 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/49359a4e/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index da38775..b98078a 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -143,9 +143,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -447,4 +444,4 @@ public class LocalHBaseCluster {
          }
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/49359a4e/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index 002bdb2..f788bed 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -86,6 +86,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Jerryjch at May 18, 2016 at 7:24 pm
    Repository: hbase
    Updated Branches:
       refs/heads/master c1ada0a37 -> 393bcd69f


    HBASE-15841 Performance Evaluation tool total rows may not be set correctly


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/393bcd69
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/393bcd69
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/393bcd69

    Branch: refs/heads/master
    Commit: 393bcd69f9a6088e2af7751619f973a43671b7aa
    Parents: c1ada0a
    Author: Jerry He <jerryjch@apache.org>
    Authored: Wed May 18 12:09:13 2016 -0700
    Committer: Jerry He <jerryjch@apache.org>
    Committed: Wed May 18 12:09:13 2016 -0700

    ----------------------------------------------------------------------
      .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 3 +--
      1 file changed, 1 insertion(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/393bcd69/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    index 02b994a..c85607d 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    @@ -2143,8 +2143,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
            // total size in GB specified
            opts.totalRows = (int) opts.size * rowsPerGB;
            opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
    - } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
    - // number of rows specified
    + } else {
            opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
            opts.size = opts.totalRows / rowsPerGB;
          }
  • Jerryjch at May 18, 2016 at 7:24 pm
    Repository: hbase
    Updated Branches:
       refs/heads/branch-1 49359a4ee -> a9972355c


    HBASE-15841 Performance Evaluation tool total rows may not be set correctly


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a9972355
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a9972355
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a9972355

    Branch: refs/heads/branch-1
    Commit: a9972355c4c374f9350be37249b2d7acf84fcf5f
    Parents: 49359a4
    Author: Jerry He <jerryjch@apache.org>
    Authored: Wed May 18 12:15:18 2016 -0700
    Committer: Jerry He <jerryjch@apache.org>
    Committed: Wed May 18 12:15:18 2016 -0700

    ----------------------------------------------------------------------
      .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 3 +--
      1 file changed, 1 insertion(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/a9972355/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    index 7dced80..12f5073 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    @@ -2074,8 +2074,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
                // total size in GB specified
                opts.totalRows = (int) opts.size * rowsPerGB;
                opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
    - } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
    - // number of rows specified
    + } else {
                opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
                opts.size = opts.totalRows / rowsPerGB;
              }
  • Jerryjch at May 18, 2016 at 7:25 pm
    Repository: hbase
    Updated Branches:
       refs/heads/branch-1.3 8566fd0f1 -> 4b1221398


    HBASE-15841 Performance Evaluation tool total rows may not be set correctly


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b122139
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b122139
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b122139

    Branch: refs/heads/branch-1.3
    Commit: 4b1221398b18aaf749d522fb4f7b1594a7468aad
    Parents: 8566fd0
    Author: Jerry He <jerryjch@apache.org>
    Authored: Wed May 18 12:15:18 2016 -0700
    Committer: Jerry He <jerryjch@apache.org>
    Committed: Wed May 18 12:18:20 2016 -0700

    ----------------------------------------------------------------------
      .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 3 +--
      1 file changed, 1 insertion(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/4b122139/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    index 7dced80..12f5073 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    @@ -2074,8 +2074,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
                // total size in GB specified
                opts.totalRows = (int) opts.size * rowsPerGB;
                opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
    - } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
    - // number of rows specified
    + } else {
                opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
                opts.size = opts.totalRows / rowsPerGB;
              }
  • Eclark at May 18, 2016 at 10:49 pm
    Repository: hbase
    Updated Branches:
       refs/heads/HBASE-14850 eb4b234a1 -> f49f262f3 (forced update)


    Revert "HBASE-15593 Time limit of scanning should be offered by client (Phil Yang)"

    This reverts commit e47bfb90786df6fa794af844028a4a0f50b3433c.


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b2b3b1fa
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b2b3b1fa
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b2b3b1fa

    Branch: refs/heads/HBASE-14850
    Commit: b2b3b1fa4d061b242970b335bbb87892537e5d99
    Parents: 98fa263
    Author: zhangduo <zhangduo@apache.org>
    Authored: Wed May 18 13:08:50 2016 +0800
    Committer: zhangduo <zhangduo@apache.org>
    Committed: Wed May 18 13:08:50 2016 +0800

    ----------------------------------------------------------------------
      .../hbase/protobuf/generated/RPCProtos.java | 106 ++-----------------
      hbase-protocol/src/main/protobuf/RPC.proto | 1 -
      .../org/apache/hadoop/hbase/ipc/CallRunner.java | 2 +-
      .../org/apache/hadoop/hbase/ipc/RpcServer.java | 43 ++------
      .../hadoop/hbase/ipc/RpcServerInterface.java | 5 -
      .../hbase/regionserver/RSRpcServices.java | 9 --
      .../TestScannerHeartbeatMessages.java | 19 ++--
      7 files changed, 27 insertions(+), 158 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    ----------------------------------------------------------------------
    diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    index d05eb57..2d4a430 100644
    --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    @@ -3895,16 +3895,6 @@ public final class RPCProtos {
           * </pre>
           */
          int getPriority();
    -
    - // optional uint32 timeout = 7;
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - boolean hasTimeout();
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - int getTimeout();
        }
        /**
         * Protobuf type {@code hbase.pb.RequestHeader}
    @@ -4007,11 +3997,6 @@ public final class RPCProtos {
                    priority_ = input.readUInt32();
                    break;
                  }
    - case 56: {
    - bitField0_ |= 0x00000040;
    - timeout_ = input.readUInt32();
    - break;
    - }
                }
              }
            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
    @@ -4225,22 +4210,6 @@ public final class RPCProtos {
            return priority_;
          }

    - // optional uint32 timeout = 7;
    - public static final int TIMEOUT_FIELD_NUMBER = 7;
    - private int timeout_;
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public boolean hasTimeout() {
    - return ((bitField0_ & 0x00000040) == 0x00000040);
    - }
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public int getTimeout() {
    - return timeout_;
    - }
    -
          private void initFields() {
            callId_ = 0;
            traceInfo_ = org.apache.hadoop.hbase.protobuf.generated.TracingProtos.RPCTInfo.getDefaultInstance();
    @@ -4248,7 +4217,6 @@ public final class RPCProtos {
            requestParam_ = false;
            cellBlockMeta_ = org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta.getDefaultInstance();
            priority_ = 0;
    - timeout_ = 0;
          }
          private byte memoizedIsInitialized = -1;
          public final boolean isInitialized() {
    @@ -4280,9 +4248,6 @@ public final class RPCProtos {
            if (((bitField0_ & 0x00000020) == 0x00000020)) {
              output.writeUInt32(6, priority_);
            }
    - if (((bitField0_ & 0x00000040) == 0x00000040)) {
    - output.writeUInt32(7, timeout_);
    - }
            getUnknownFields().writeTo(output);
          }

    @@ -4316,10 +4281,6 @@ public final class RPCProtos {
              size += com.google.protobuf.CodedOutputStream
                .computeUInt32Size(6, priority_);
            }
    - if (((bitField0_ & 0x00000040) == 0x00000040)) {
    - size += com.google.protobuf.CodedOutputStream
    - .computeUInt32Size(7, timeout_);
    - }
            size += getUnknownFields().getSerializedSize();
            memoizedSerializedSize = size;
            return size;
    @@ -4373,11 +4334,6 @@ public final class RPCProtos {
              result = result && (getPriority()
                  == other.getPriority());
            }
    - result = result && (hasTimeout() == other.hasTimeout());
    - if (hasTimeout()) {
    - result = result && (getTimeout()
    - == other.getTimeout());
    - }
            result = result &&
                getUnknownFields().equals(other.getUnknownFields());
            return result;
    @@ -4415,10 +4371,6 @@ public final class RPCProtos {
              hash = (37 * hash) + PRIORITY_FIELD_NUMBER;
              hash = (53 * hash) + getPriority();
            }
    - if (hasTimeout()) {
    - hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
    - hash = (53 * hash) + getTimeout();
    - }
            hash = (29 * hash) + getUnknownFields().hashCode();
            memoizedHashCode = hash;
            return hash;
    @@ -4554,8 +4506,6 @@ public final class RPCProtos {
              bitField0_ = (bitField0_ & ~0x00000010);
              priority_ = 0;
              bitField0_ = (bitField0_ & ~0x00000020);
    - timeout_ = 0;
    - bitField0_ = (bitField0_ & ~0x00000040);
              return this;
            }

    @@ -4616,10 +4566,6 @@ public final class RPCProtos {
                to_bitField0_ |= 0x00000020;
              }
              result.priority_ = priority_;
    - if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
    - to_bitField0_ |= 0x00000040;
    - }
    - result.timeout_ = timeout_;
              result.bitField0_ = to_bitField0_;
              onBuilt();
              return result;
    @@ -4656,9 +4602,6 @@ public final class RPCProtos {
              if (other.hasPriority()) {
                setPriority(other.getPriority());
              }
    - if (other.hasTimeout()) {
    - setTimeout(other.getTimeout());
    - }
              this.mergeUnknownFields(other.getUnknownFields());
              return this;
            }
    @@ -5181,39 +5124,6 @@ public final class RPCProtos {
              return this;
            }

    - // optional uint32 timeout = 7;
    - private int timeout_ ;
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public boolean hasTimeout() {
    - return ((bitField0_ & 0x00000040) == 0x00000040);
    - }
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public int getTimeout() {
    - return timeout_;
    - }
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public Builder setTimeout(int value) {
    - bitField0_ |= 0x00000040;
    - timeout_ = value;
    - onChanged();
    - return this;
    - }
    - /**
    - * <code>optional uint32 timeout = 7;</code>
    - */
    - public Builder clearTimeout() {
    - bitField0_ = (bitField0_ & ~0x00000040);
    - timeout_ = 0;
    - onChanged();
    - return this;
    - }
    -
            // @@protoc_insertion_point(builder_scope:hbase.pb.RequestHeader)
          }

    @@ -6231,17 +6141,17 @@ public final class RPCProtos {
            "llBlockMeta\022\016\n\006length\030\001 \001(\r\"|\n\021Exception" +
            "Response\022\034\n\024exception_class_name\030\001 \001(\t\022\023",
            "\n\013stack_trace\030\002 \001(\t\022\020\n\010hostname\030\003 \001(\t\022\014\n" +
    - "\004port\030\004 \001(\005\022\024\n\014do_not_retry\030\005 \001(\010\"\311\001\n\rRe" +
    + "\004port\030\004 \001(\005\022\024\n\014do_not_retry\030\005 \001(\010\"\270\001\n\rRe" +
            "questHeader\022\017\n\007call_id\030\001 \001(\r\022&\n\ntrace_in" +
            "fo\030\002 \001(\0132\022.hbase.pb.RPCTInfo\022\023\n\013method_n" +
            "ame\030\003 \001(\t\022\025\n\rrequest_param\030\004 \001(\010\0220\n\017cell" +
            "_block_meta\030\005 \001(\0132\027.hbase.pb.CellBlockMe" +
    - "ta\022\020\n\010priority\030\006 \001(\r\022\017\n\007timeout\030\007 \001(\r\"\203\001" +
    - "\n\016ResponseHeader\022\017\n\007call_id\030\001 \001(\r\022.\n\texc" +
    - "eption\030\002 \001(\0132\033.hbase.pb.ExceptionRespons" +
    - "e\0220\n\017cell_block_meta\030\003 \001(\0132\027.hbase.pb.Ce",
    - "llBlockMetaB<\n*org.apache.hadoop.hbase.p" +
    - "rotobuf.generatedB\tRPCProtosH\001\240\001\001"
    + "ta\022\020\n\010priority\030\006 \001(\r\"\203\001\n\016ResponseHeader\022" +
    + "\017\n\007call_id\030\001 \001(\r\022.\n\texception\030\002 \001(\0132\033.hb" +
    + "ase.pb.ExceptionResponse\0220\n\017cell_block_m" +
    + "eta\030\003 \001(\0132\027.hbase.pb.CellBlockMetaB<\n*or",
    + "g.apache.hadoop.hbase.protobuf.generated" +
    + "B\tRPCProtosH\001\240\001\001"
          };
          com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
            new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
    @@ -6277,7 +6187,7 @@ public final class RPCProtos {
                internal_static_hbase_pb_RequestHeader_fieldAccessorTable = new
                  com.google.protobuf.GeneratedMessage.FieldAccessorTable(
                    internal_static_hbase_pb_RequestHeader_descriptor,
    - new java.lang.String[] { "CallId", "TraceInfo", "MethodName", "RequestParam", "CellBlockMeta", "Priority", "Timeout", });
    + new java.lang.String[] { "CallId", "TraceInfo", "MethodName", "RequestParam", "CellBlockMeta", "Priority", });
                internal_static_hbase_pb_ResponseHeader_descriptor =
                  getDescriptor().getMessageTypes().get(5);
                internal_static_hbase_pb_ResponseHeader_fieldAccessorTable = new

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-protocol/src/main/protobuf/RPC.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-protocol/src/main/protobuf/RPC.proto b/hbase-protocol/src/main/protobuf/RPC.proto
    index 8413d25..59bb03d 100644
    --- a/hbase-protocol/src/main/protobuf/RPC.proto
    +++ b/hbase-protocol/src/main/protobuf/RPC.proto
    @@ -125,7 +125,6 @@ message RequestHeader {
        // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
        // See HConstants.
        optional uint32 priority = 6;
    - optional uint32 timeout = 7;
      }

      message ResponseHeader {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    index 00e08c9..3514245 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    @@ -114,7 +114,7 @@ public class CallRunner {
              }
              // make the call
              resultPair = this.rpcServer.call(call.service, call.md, call.param, call.cellScanner,
    - call.timestamp, this.status, call.timeout);
    + call.timestamp, this.status);
            } catch (Throwable e) {
              RpcServer.LOG.debug(Thread.currentThread().getName() + ": " + call.toShortString(), e);
              errorThrowable = e;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    index 483ce86..b9a9b26 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    @@ -264,13 +264,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
        private static final String WARN_RESPONSE_TIME = "hbase.ipc.warn.response.time";
        private static final String WARN_RESPONSE_SIZE = "hbase.ipc.warn.response.size";

    - /**
    - * Minimum allowable timeout (in milliseconds) in rpc request's header. This
    - * configuration exists to prevent the rpc service regarding this request as timeout immediately.
    - */
    - private static final String MIN_CLIENT_REQUEST_TIMEOUT = "hbase.ipc.min.client.request.timeout";
    - private static final int DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT = 20;
    -
        /** Default value for above params */
        private static final int DEFAULT_MAX_REQUEST_SIZE = DEFAULT_MAX_CALLQUEUE_SIZE / 4; // 256M
        private static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds
    @@ -281,9 +274,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
        private final int maxRequestSize;
        private final int warnResponseTime;
        private final int warnResponseSize;
    -
    - private final int minClientRequestTimeout;
    -
        private final Server server;
        private final List<BlockingServiceAndInterface> services;

    @@ -312,7 +302,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          protected Connection connection; // connection to client
          protected long timestamp; // the time received when response is null
                                         // the time served when response is not null
    - protected int timeout;
          /**
           * Chain of buffers to send as response.
           */
    @@ -336,7 +325,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
              justification="Can't figure why this complaint is happening... see below")
          Call(int id, final BlockingService service, final MethodDescriptor md, RequestHeader header,
               Message param, CellScanner cellScanner, Connection connection, Responder responder,
    - long size, TraceInfo tinfo, final InetAddress remoteAddress, int timeout) {
    + long size, TraceInfo tinfo, final InetAddress remoteAddress) {
            this.id = id;
            this.service = service;
            this.md = md;
    @@ -354,7 +343,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            this.remoteAddress = remoteAddress;
            this.retryImmediatelySupported =
                connection == null? null: connection.retryImmediatelySupported;
    - this.timeout = timeout;
          }

          /**
    @@ -1287,13 +1275,13 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          // Fake 'call' for failed authorization response
          private static final int AUTHORIZATION_FAILED_CALLID = -1;
          private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALLID, null, null, null,
    - null, null, this, null, 0, null, null, 0);
    + null, null, this, null, 0, null, null);
          private ByteArrayOutputStream authFailedResponse =
              new ByteArrayOutputStream();
          // Fake 'call' for SASL context setup
          private static final int SASL_CALLID = -33;
          private final Call saslCall = new Call(SASL_CALLID, null, null, null, null, null, this, null,
    - 0, null, null, 0);
    + 0, null, null);

          // was authentication allowed with a fallback to simple auth
          private boolean authenticatedWithFallback;
    @@ -1711,7 +1699,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {

          private int doBadPreambleHandling(final String msg, final Exception e) throws IOException {
            LOG.warn(msg);
    - Call fakeCall = new Call(-1, null, null, null, null, null, this, responder, -1, null, null,0);
    + Call fakeCall = new Call(-1, null, null, null, null, null, this, responder, -1, null, null);
            setupResponse(null, fakeCall, e, msg);
            responder.doRespond(fakeCall);
            // Returning -1 closes out the connection.
    @@ -1886,7 +1874,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            if ((totalRequestSize + callQueueSize.get()) > maxQueueSize) {
              final Call callTooBig =
                new Call(id, this.service, null, null, null, null, this,
    - responder, totalRequestSize, null, null, 0);
    + responder, totalRequestSize, null, null);
              ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
              metrics.exception(CALL_QUEUE_TOO_BIG_EXCEPTION);
              setupResponse(responseBuffer, callTooBig, CALL_QUEUE_TOO_BIG_EXCEPTION,
    @@ -1935,7 +1923,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {

              final Call readParamsFailedCall =
                new Call(id, this.service, null, null, null, null, this,
    - responder, totalRequestSize, null, null, 0);
    + responder, totalRequestSize, null, null);
              ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
              setupResponse(responseBuffer, readParamsFailedCall, t,
                msg + "; " + t.getMessage());
    @@ -1946,12 +1934,8 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            TraceInfo traceInfo = header.hasTraceInfo()
                ? new TraceInfo(header.getTraceInfo().getTraceId(), header.getTraceInfo().getParentId())
                : null;
    - int timeout = 0;
    - if (header.hasTimeout()){
    - timeout = Math.max(minClientRequestTimeout, header.getTimeout());
    - }
            Call call = new Call(id, this.service, md, header, param, cellScanner, this, responder,
    - totalRequestSize, traceInfo, this.addr, timeout);
    + totalRequestSize, traceInfo, this.addr);

            if (!scheduler.dispatch(new CallRunner(RpcServer.this, call))) {
              callQueueSize.add(-1 * call.getSize());
    @@ -2103,8 +2087,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
          this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME, DEFAULT_WARN_RESPONSE_TIME);
          this.warnResponseSize = conf.getInt(WARN_RESPONSE_SIZE, DEFAULT_WARN_RESPONSE_SIZE);
    - this.minClientRequestTimeout = conf.getInt(MIN_CLIENT_REQUEST_TIMEOUT,
    - DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT);
    +
          this.maxRequestSize = conf.getInt(MAX_REQUEST_SIZE, DEFAULT_MAX_REQUEST_SIZE);

          // Start the listener here and let it bind to the port
    @@ -2245,12 +2228,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          this.secretManager = (SecretManager<TokenIdentifier>) secretManager;
        }

    - public Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    - Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
    - throws IOException {
    - return call(service, md, param, cellScanner, receiveTime, status, 0);
    - }
    -
        /**
         * This is a server side method, which is invoked over RPC. On success
         * the return response has protobuf response payload. On failure, the
    @@ -2258,8 +2235,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
         */
        @Override
        public Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    - Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status,
    - int timeout)
    + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
        throws IOException {
          try {
            status.setRPC(md.getName(), new Object[]{param}, receiveTime);
    @@ -2269,7 +2245,6 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            //get an instance of the method arg type
            long startTime = System.currentTimeMillis();
            PayloadCarryingRpcController controller = new PayloadCarryingRpcController(cellScanner);
    - controller.setCallTimeout(timeout);
            Message result = service.callBlockingMethod(md, controller, param);
            long endTime = System.currentTimeMillis();
            int processingTime = (int) (endTime - startTime);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    index dd7e584..ab8b485 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    @@ -52,11 +52,6 @@ public interface RpcServerInterface {
          Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
        throws IOException, ServiceException;

    - Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    - Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status,
    - int timeout)
    - throws IOException, ServiceException;
    -
        void setErrorHandler(HBaseRPCErrorHandler handler);
        HBaseRPCErrorHandler getErrorHandler();


    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    index ebd85bd..95ab36d 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    @@ -88,7 +88,6 @@ import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
      import org.apache.hadoop.hbase.ipc.RpcServerInterface;
      import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
      import org.apache.hadoop.hbase.ipc.ServerRpcController;
    -import org.apache.hadoop.hbase.ipc.TimeLimitedRpcController;
      import org.apache.hadoop.hbase.master.MasterRpcServices;
      import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
      import org.apache.hadoop.hbase.protobuf.RequestConverter;
    @@ -2699,14 +2698,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
                          timeLimitDelta =
                              scannerLeaseTimeoutPeriod > 0 ? scannerLeaseTimeoutPeriod : rpcTimeout;
                        }
    - if (controller instanceof TimeLimitedRpcController) {
    - TimeLimitedRpcController timeLimitedRpcController =
    - (TimeLimitedRpcController)controller;
    - if (timeLimitedRpcController.getCallTimeout() > 0) {
    - timeLimitDelta = Math.min(timeLimitDelta,
    - timeLimitedRpcController.getCallTimeout());
    - }
    - }
                        // Use half of whichever timeout value was more restrictive... But don't allow
                        // the time limit to be less than the allowable minimum (could cause an
                        // immediatate timeout before scanning any data).

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b2b3b1fa/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    index a958ee0..b8eacc7 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    @@ -18,7 +18,6 @@
      package org.apache.hadoop.hbase.regionserver;

      import static org.junit.Assert.assertEquals;
    -import static org.junit.Assert.assertNull;
      import static org.junit.Assert.assertTrue;
      import static org.junit.Assert.fail;

    @@ -103,16 +102,16 @@ public class TestScannerHeartbeatMessages {
        private static int VALUE_SIZE = 128;
        private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);

    -
    - private static int SERVER_TIMEOUT = 6000;
    -
        // Time, in milliseconds, that the client will wait for a response from the server before timing
        // out. This value is used server side to determine when it is necessary to send a heartbeat
        // message to the client
    - private static int CLIENT_TIMEOUT = SERVER_TIMEOUT / 3;
    + private static int CLIENT_TIMEOUT = 2000;
    +
    + // The server limits itself to running for half of the CLIENT_TIMEOUT value.
    + private static int SERVER_TIME_LIMIT = CLIENT_TIMEOUT / 2;

        // By default, at most one row's worth of cells will be retrieved before the time limit is reached
    - private static int DEFAULT_ROW_SLEEP_TIME = CLIENT_TIMEOUT / 5;
    + private static int DEFAULT_ROW_SLEEP_TIME = SERVER_TIME_LIMIT / 2;
        // By default, at most cells for two column families are retrieved before the time limit is
        // reached
        private static int DEFAULT_CF_SLEEP_TIME = DEFAULT_ROW_SLEEP_TIME / NUM_FAMILIES;
    @@ -125,8 +124,8 @@ public class TestScannerHeartbeatMessages {

          conf.setStrings(HConstants.REGION_IMPL, HeartbeatHRegion.class.getName());
          conf.setStrings(HConstants.REGION_SERVER_IMPL, HeartbeatHRegionServer.class.getName());
    - conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SERVER_TIMEOUT);
    - conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, SERVER_TIMEOUT);
    + conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CLIENT_TIMEOUT);
    + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, CLIENT_TIMEOUT);
          conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 1);

          // Check the timeout condition after every cell
    @@ -141,7 +140,7 @@ public class TestScannerHeartbeatMessages {
          Table ht = TEST_UTIL.createTable(name, families);
          List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
          ht.put(puts);
    - ht.getConfiguration().setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CLIENT_TIMEOUT);
    +
          return ht;
        }

    @@ -283,7 +282,7 @@ public class TestScannerHeartbeatMessages {
          @Override
          public ReturnCode filterKeyValue(Cell v) throws IOException {
            try {
    - Thread.sleep(CLIENT_TIMEOUT/2 + 10);
    + Thread.sleep(SERVER_TIME_LIMIT + 10);
            } catch (InterruptedException e) {
              Thread.currentThread().interrupt();
            }
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15784 Misuse core/maxPoolSize of LinkedBlockingQueue in
    ThreadPoolExecutor (Jingcheng Du)


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7b5d5394
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7b5d5394
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7b5d5394

    Branch: refs/heads/HBASE-14850
    Commit: 7b5d5394c08e4d5a18c8f9fc62b45930bf892f41
    Parents: b2b3b1f
    Author: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
    Authored: Wed May 18 12:40:43 2016 +0530
    Committer: Ramkrishna <ramkrishna.s.vasudevan@intel.com>
    Committed: Wed May 18 12:40:43 2016 +0530

    ----------------------------------------------------------------------
      .../hadoop/hbase/client/ConnectionImplementation.java | 10 ++++++----
      hbase-client/src/test/resources/hbase-site.xml | 4 ----
      .../hbase/replication/regionserver/HFileReplicator.java | 2 +-
      .../regionserver/RegionReplicaReplicationEndpoint.java | 6 +-----
      .../org/apache/hadoop/hbase/util/MultiHConnection.java | 6 +-----
      .../org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java | 1 -
      .../org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java | 1 -
      .../apache/hadoop/hbase/util/TestHBaseFsckReplicas.java | 1 -
      .../org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java | 1 -
      hbase-server/src/test/resources/hbase-site.xml | 4 ----
      .../hadoop/hbase/thrift/TBoundedThreadPoolServer.java | 10 +++++++---
      .../apache/hadoop/hbase/thrift/ThriftServerRunner.java | 6 ++++--
      12 files changed, 20 insertions(+), 32 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    index 9a7dfc7..429e47d 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
    @@ -352,8 +352,8 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
          if (batchPool == null) {
            synchronized (this) {
              if (batchPool == null) {
    - this.batchPool = getThreadPool(conf.getInt("hbase.hconnection.threads.max", 256),
    - conf.getInt("hbase.hconnection.threads.core", 256), "-shared", null);
    + int threads = conf.getInt("hbase.hconnection.threads.max", 256);
    + this.batchPool = getThreadPool(threads, threads, "-shared", null);
                this.cleanupPool = true;
              }
            }
    @@ -377,6 +377,7 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
              new LinkedBlockingQueue<Runnable>(maxThreads *
                  conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                      HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
    + coreThreads = maxThreads;
          }
          ThreadPoolExecutor tpe = new ThreadPoolExecutor(
              coreThreads,
    @@ -397,9 +398,10 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
                //To start with, threads.max.core threads can hit the meta (including replicas).
                //After that, requests will get queued up in the passed queue, and only after
                //the queue is full, a new thread will be started
    + int threads = conf.getInt("hbase.hconnection.meta.lookup.threads.max", 128);
                this.metaLookupPool = getThreadPool(
    - conf.getInt("hbase.hconnection.meta.lookup.threads.max", 128),
    - conf.getInt("hbase.hconnection.meta.lookup.threads.core", 10),
    + threads,
    + threads,
                   "-metaLookup-shared-", new LinkedBlockingQueue<Runnable>());
              }
            }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-client/src/test/resources/hbase-site.xml
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/test/resources/hbase-site.xml b/hbase-client/src/test/resources/hbase-site.xml
    index 5788238..99d2ab8 100644
    --- a/hbase-client/src/test/resources/hbase-site.xml
    +++ b/hbase-client/src/test/resources/hbase-site.xml
    @@ -26,10 +26,6 @@
          <value>true</value>
        </property>
        <property>
    - <name>hbase.hconnection.meta.lookup.threads.core</name>
    - <value>4</value>
    - </property>
    - <property>
          <name>hbase.hconnection.threads.keepalivetime</name>
          <value>3</value>
        </property>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    index 1a1044d..9893e7e 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/HFileReplicator.java
    @@ -107,7 +107,7 @@ public class HFileReplicator {
          ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
          builder.setNameFormat("HFileReplicationCallable-%1$d");
          this.exec =
    - new ThreadPoolExecutor(1, maxCopyThreads, 60, TimeUnit.SECONDS,
    + new ThreadPoolExecutor(maxCopyThreads, maxCopyThreads, 60, TimeUnit.SECONDS,
                  new LinkedBlockingQueue<Runnable>(), builder.build());
          this.exec.allowCoreThreadTimeOut(true);
          this.copiesPerThread =

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    index da37cfa..3611608 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RegionReplicaReplicationEndpoint.java
    @@ -235,20 +235,16 @@ public class RegionReplicaReplicationEndpoint extends HBaseReplicationEndpoint {
         */
        private ExecutorService getDefaultThreadPool(Configuration conf) {
          int maxThreads = conf.getInt("hbase.region.replica.replication.threads.max", 256);
    - int coreThreads = conf.getInt("hbase.region.replica.replication.threads.core", 16);
          if (maxThreads == 0) {
            maxThreads = Runtime.getRuntime().availableProcessors() * 8;
          }
    - if (coreThreads == 0) {
    - coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    - }
          long keepAliveTime = conf.getLong("hbase.region.replica.replication.threads.keepalivetime", 60);
          LinkedBlockingQueue<Runnable> workQueue =
              new LinkedBlockingQueue<Runnable>(maxThreads *
                  conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                    HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
          ThreadPoolExecutor tpe = new ThreadPoolExecutor(
    - coreThreads,
    + maxThreads,
            maxThreads,
            keepAliveTime,
            TimeUnit.SECONDS,

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    index ed72ea2..6efb10c 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MultiHConnection.java
    @@ -135,20 +135,16 @@ public class MultiHConnection {
        private void createBatchPool(Configuration conf) {
          // Use the same config for keep alive as in ConnectionImplementation.getBatchPool();
          int maxThreads = conf.getInt("hbase.multihconnection.threads.max", 256);
    - int coreThreads = conf.getInt("hbase.multihconnection.threads.core", 256);
          if (maxThreads == 0) {
            maxThreads = Runtime.getRuntime().availableProcessors() * 8;
          }
    - if (coreThreads == 0) {
    - coreThreads = Runtime.getRuntime().availableProcessors() * 8;
    - }
          long keepAliveTime = conf.getLong("hbase.multihconnection.threads.keepalivetime", 60);
          LinkedBlockingQueue<Runnable> workQueue =
              new LinkedBlockingQueue<Runnable>(maxThreads
                  * conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
                    HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS));
          ThreadPoolExecutor tpe =
    - new ThreadPoolExecutor(coreThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue,
    + new ThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, workQueue,
                  Threads.newDaemonThreadFactory("MultiHConnection" + "-shared-"));
          tpe.allowCoreThreadTimeOut(true);
          this.batchPool = tpe;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    index 8e96f83..b04689c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
    @@ -57,7 +57,6 @@ public class TestHBaseFsckMOB extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(1);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    index d5869ed..a7c0c55 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    @@ -109,7 +109,6 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(1);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    index a859f78..2aa436c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java
    @@ -64,7 +64,6 @@ public class TestHBaseFsckReplicas extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(3);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    index 4865217..6d0e48c 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckTwoRS.java
    @@ -73,7 +73,6 @@ public class TestHBaseFsckTwoRS extends BaseTestHBaseFsck {

          conf.setInt("hbase.htable.threads.max", POOL_SIZE);
          conf.setInt("hbase.hconnection.threads.max", 2 * POOL_SIZE);
    - conf.setInt("hbase.hconnection.threads.core", POOL_SIZE);
          conf.setInt("hbase.hbck.close.timeout", 2 * REGION_ONLINE_TIMEOUT);
          conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 8 * REGION_ONLINE_TIMEOUT);
          TEST_UTIL.startMiniCluster(2);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-server/src/test/resources/hbase-site.xml
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
    index 1b429b5..bfe8ee1 100644
    --- a/hbase-server/src/test/resources/hbase-site.xml
    +++ b/hbase-server/src/test/resources/hbase-site.xml
    @@ -163,10 +163,6 @@
          </description>
        </property>
        <property>
    - <name>hbase.hconnection.meta.lookup.threads.core</name>
    - <value>4</value>
    - </property>
    - <property>
          <name>hbase.hconnection.threads.keepalivetime</name>
          <value>3</value>
        </property>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    ----------------------------------------------------------------------
    diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    index 84613bd..1b8ebc9 100644
    --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/TBoundedThreadPoolServer.java
    @@ -132,7 +132,7 @@ public class TBoundedThreadPoolServer extends TServer {
        }

        /** Executor service for handling client connections */
    - private ExecutorService executorService;
    + private ThreadPoolExecutor executorService;

        /** Flag for stopping the server */
        private volatile boolean stopped;
    @@ -142,9 +142,12 @@ public class TBoundedThreadPoolServer extends TServer {
        public TBoundedThreadPoolServer(Args options, ThriftMetrics metrics) {
          super(options);

    + int minWorkerThreads = options.minWorkerThreads;
    + int maxWorkerThreads = options.maxWorkerThreads;
          if (options.maxQueuedRequests > 0) {
            this.callQueue = new CallQueue(
                new LinkedBlockingQueue<Call>(options.maxQueuedRequests), metrics);
    + minWorkerThreads = maxWorkerThreads;
          } else {
            this.callQueue = new CallQueue(new SynchronousQueue<Call>(), metrics);
          }
    @@ -153,9 +156,10 @@ public class TBoundedThreadPoolServer extends TServer {
          tfb.setDaemon(true);
          tfb.setNameFormat("thrift-worker-%d");
          executorService =
    - new ThreadPoolExecutor(options.minWorkerThreads,
    - options.maxWorkerThreads, options.threadKeepAliveTimeSec,
    + new ThreadPoolExecutor(minWorkerThreads,
    + maxWorkerThreads, options.threadKeepAliveTimeSec,
                  TimeUnit.SECONDS, this.callQueue, tfb.build());
    + executorService.allowCoreThreadTimeOut(true);
          serverOptions = options;
        }


    http://git-wip-us.apache.org/repos/asf/hbase/blob/7b5d5394/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    ----------------------------------------------------------------------
    diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    index 7d228dc..8767a3c 100644
    --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServerRunner.java
    @@ -555,7 +555,7 @@ public class ThriftServerRunner implements Runnable {
              CallQueue callQueue =
                  new CallQueue(new LinkedBlockingQueue<Call>(), metrics);
              ExecutorService executorService = createExecutor(
    - callQueue, serverArgs.getMinWorkerThreads(), serverArgs.getMaxWorkerThreads());
    + callQueue, serverArgs.getMaxWorkerThreads(), serverArgs.getMaxWorkerThreads());
              serverArgs.executorService(executorService)
                        .processor(processor)
                        .transportFactory(transportFactory)
    @@ -620,8 +620,10 @@ public class ThriftServerRunner implements Runnable {
          ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
          tfb.setDaemon(true);
          tfb.setNameFormat("thrift-worker-%d");
    - return new ThreadPoolExecutor(minWorkers, maxWorkers,
    + ThreadPoolExecutor threadPool = new ThreadPoolExecutor(minWorkers, maxWorkers,
                  Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build());
    + threadPool.allowCoreThreadTimeOut(true);
    + return threadPool;
        }

        private InetAddress getBindAddress(Configuration conf)
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15593 Time limit of scanning should be offered by client

    Signed-off-by: zhangduo <zhangduo@apache.org>


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3bd9220f
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3bd9220f
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3bd9220f

    Branch: refs/heads/HBASE-14850
    Commit: 3bd9220f0ca315822ba2f5652add772b8c518f71
    Parents: 7b5d539
    Author: Phil Yang <ud1937@gmail.com>
    Authored: Wed May 18 13:25:15 2016 +0800
    Committer: zhangduo <zhangduo@apache.org>
    Committed: Wed May 18 19:13:15 2016 +0800

    ----------------------------------------------------------------------
      .../hadoop/hbase/ipc/AsyncRpcChannelImpl.java | 2 +
      .../apache/hadoop/hbase/ipc/RpcClientImpl.java | 1 +
      .../hbase/protobuf/generated/RPCProtos.java | 106 +++++++++++++++++--
      hbase-protocol/src/main/protobuf/RPC.proto | 1 +
      .../org/apache/hadoop/hbase/ipc/CallRunner.java | 2 +-
      .../org/apache/hadoop/hbase/ipc/RpcServer.java | 43 ++++++--
      .../hadoop/hbase/ipc/RpcServerInterface.java | 5 +
      .../hbase/regionserver/RSRpcServices.java | 9 ++
      .../TestScannerHeartbeatMessages.java | 19 ++--
      9 files changed, 161 insertions(+), 27 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
    index cd61b61..2b9000a 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannelImpl.java
    @@ -397,6 +397,8 @@ public class AsyncRpcChannelImpl implements AsyncRpcChannel {
            if (call.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) {
              requestHeaderBuilder.setPriority(call.getPriority());
            }
    + requestHeaderBuilder.setTimeout(call.rpcTimeout > Integer.MAX_VALUE ?
    + Integer.MAX_VALUE : (int)call.rpcTimeout);

            RPCProtos.RequestHeader rh = requestHeaderBuilder.build();


    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
    index a5d2482..00eea7a 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
    @@ -907,6 +907,7 @@ public class RpcClientImpl extends AbstractRpcClient {
            if (priority != PayloadCarryingRpcController.PRIORITY_UNSET) {
              builder.setPriority(priority);
            }
    + builder.setTimeout(call.timeout);
            RequestHeader requestHeader = builder.build();

            setupIOstreams();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    ----------------------------------------------------------------------
    diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    index 2d4a430..d05eb57 100644
    --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
    @@ -3895,6 +3895,16 @@ public final class RPCProtos {
           * </pre>
           */
          int getPriority();
    +
    + // optional uint32 timeout = 7;
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + boolean hasTimeout();
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + int getTimeout();
        }
        /**
         * Protobuf type {@code hbase.pb.RequestHeader}
    @@ -3997,6 +4007,11 @@ public final class RPCProtos {
                    priority_ = input.readUInt32();
                    break;
                  }
    + case 56: {
    + bitField0_ |= 0x00000040;
    + timeout_ = input.readUInt32();
    + break;
    + }
                }
              }
            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
    @@ -4210,6 +4225,22 @@ public final class RPCProtos {
            return priority_;
          }

    + // optional uint32 timeout = 7;
    + public static final int TIMEOUT_FIELD_NUMBER = 7;
    + private int timeout_;
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public boolean hasTimeout() {
    + return ((bitField0_ & 0x00000040) == 0x00000040);
    + }
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public int getTimeout() {
    + return timeout_;
    + }
    +
          private void initFields() {
            callId_ = 0;
            traceInfo_ = org.apache.hadoop.hbase.protobuf.generated.TracingProtos.RPCTInfo.getDefaultInstance();
    @@ -4217,6 +4248,7 @@ public final class RPCProtos {
            requestParam_ = false;
            cellBlockMeta_ = org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta.getDefaultInstance();
            priority_ = 0;
    + timeout_ = 0;
          }
          private byte memoizedIsInitialized = -1;
          public final boolean isInitialized() {
    @@ -4248,6 +4280,9 @@ public final class RPCProtos {
            if (((bitField0_ & 0x00000020) == 0x00000020)) {
              output.writeUInt32(6, priority_);
            }
    + if (((bitField0_ & 0x00000040) == 0x00000040)) {
    + output.writeUInt32(7, timeout_);
    + }
            getUnknownFields().writeTo(output);
          }

    @@ -4281,6 +4316,10 @@ public final class RPCProtos {
              size += com.google.protobuf.CodedOutputStream
                .computeUInt32Size(6, priority_);
            }
    + if (((bitField0_ & 0x00000040) == 0x00000040)) {
    + size += com.google.protobuf.CodedOutputStream
    + .computeUInt32Size(7, timeout_);
    + }
            size += getUnknownFields().getSerializedSize();
            memoizedSerializedSize = size;
            return size;
    @@ -4334,6 +4373,11 @@ public final class RPCProtos {
              result = result && (getPriority()
                  == other.getPriority());
            }
    + result = result && (hasTimeout() == other.hasTimeout());
    + if (hasTimeout()) {
    + result = result && (getTimeout()
    + == other.getTimeout());
    + }
            result = result &&
                getUnknownFields().equals(other.getUnknownFields());
            return result;
    @@ -4371,6 +4415,10 @@ public final class RPCProtos {
              hash = (37 * hash) + PRIORITY_FIELD_NUMBER;
              hash = (53 * hash) + getPriority();
            }
    + if (hasTimeout()) {
    + hash = (37 * hash) + TIMEOUT_FIELD_NUMBER;
    + hash = (53 * hash) + getTimeout();
    + }
            hash = (29 * hash) + getUnknownFields().hashCode();
            memoizedHashCode = hash;
            return hash;
    @@ -4506,6 +4554,8 @@ public final class RPCProtos {
              bitField0_ = (bitField0_ & ~0x00000010);
              priority_ = 0;
              bitField0_ = (bitField0_ & ~0x00000020);
    + timeout_ = 0;
    + bitField0_ = (bitField0_ & ~0x00000040);
              return this;
            }

    @@ -4566,6 +4616,10 @@ public final class RPCProtos {
                to_bitField0_ |= 0x00000020;
              }
              result.priority_ = priority_;
    + if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
    + to_bitField0_ |= 0x00000040;
    + }
    + result.timeout_ = timeout_;
              result.bitField0_ = to_bitField0_;
              onBuilt();
              return result;
    @@ -4602,6 +4656,9 @@ public final class RPCProtos {
              if (other.hasPriority()) {
                setPriority(other.getPriority());
              }
    + if (other.hasTimeout()) {
    + setTimeout(other.getTimeout());
    + }
              this.mergeUnknownFields(other.getUnknownFields());
              return this;
            }
    @@ -5124,6 +5181,39 @@ public final class RPCProtos {
              return this;
            }

    + // optional uint32 timeout = 7;
    + private int timeout_ ;
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public boolean hasTimeout() {
    + return ((bitField0_ & 0x00000040) == 0x00000040);
    + }
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public int getTimeout() {
    + return timeout_;
    + }
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public Builder setTimeout(int value) {
    + bitField0_ |= 0x00000040;
    + timeout_ = value;
    + onChanged();
    + return this;
    + }
    + /**
    + * <code>optional uint32 timeout = 7;</code>
    + */
    + public Builder clearTimeout() {
    + bitField0_ = (bitField0_ & ~0x00000040);
    + timeout_ = 0;
    + onChanged();
    + return this;
    + }
    +
            // @@protoc_insertion_point(builder_scope:hbase.pb.RequestHeader)
          }

    @@ -6141,17 +6231,17 @@ public final class RPCProtos {
            "llBlockMeta\022\016\n\006length\030\001 \001(\r\"|\n\021Exception" +
            "Response\022\034\n\024exception_class_name\030\001 \001(\t\022\023",
            "\n\013stack_trace\030\002 \001(\t\022\020\n\010hostname\030\003 \001(\t\022\014\n" +
    - "\004port\030\004 \001(\005\022\024\n\014do_not_retry\030\005 \001(\010\"\270\001\n\rRe" +
    + "\004port\030\004 \001(\005\022\024\n\014do_not_retry\030\005 \001(\010\"\311\001\n\rRe" +
            "questHeader\022\017\n\007call_id\030\001 \001(\r\022&\n\ntrace_in" +
            "fo\030\002 \001(\0132\022.hbase.pb.RPCTInfo\022\023\n\013method_n" +
            "ame\030\003 \001(\t\022\025\n\rrequest_param\030\004 \001(\010\0220\n\017cell" +
            "_block_meta\030\005 \001(\0132\027.hbase.pb.CellBlockMe" +
    - "ta\022\020\n\010priority\030\006 \001(\r\"\203\001\n\016ResponseHeader\022" +
    - "\017\n\007call_id\030\001 \001(\r\022.\n\texception\030\002 \001(\0132\033.hb" +
    - "ase.pb.ExceptionResponse\0220\n\017cell_block_m" +
    - "eta\030\003 \001(\0132\027.hbase.pb.CellBlockMetaB<\n*or",
    - "g.apache.hadoop.hbase.protobuf.generated" +
    - "B\tRPCProtosH\001\240\001\001"
    + "ta\022\020\n\010priority\030\006 \001(\r\022\017\n\007timeout\030\007 \001(\r\"\203\001" +
    + "\n\016ResponseHeader\022\017\n\007call_id\030\001 \001(\r\022.\n\texc" +
    + "eption\030\002 \001(\0132\033.hbase.pb.ExceptionRespons" +
    + "e\0220\n\017cell_block_meta\030\003 \001(\0132\027.hbase.pb.Ce",
    + "llBlockMetaB<\n*org.apache.hadoop.hbase.p" +
    + "rotobuf.generatedB\tRPCProtosH\001\240\001\001"
          };
          com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
            new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
    @@ -6187,7 +6277,7 @@ public final class RPCProtos {
                internal_static_hbase_pb_RequestHeader_fieldAccessorTable = new
                  com.google.protobuf.GeneratedMessage.FieldAccessorTable(
                    internal_static_hbase_pb_RequestHeader_descriptor,
    - new java.lang.String[] { "CallId", "TraceInfo", "MethodName", "RequestParam", "CellBlockMeta", "Priority", });
    + new java.lang.String[] { "CallId", "TraceInfo", "MethodName", "RequestParam", "CellBlockMeta", "Priority", "Timeout", });
                internal_static_hbase_pb_ResponseHeader_descriptor =
                  getDescriptor().getMessageTypes().get(5);
                internal_static_hbase_pb_ResponseHeader_fieldAccessorTable = new

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-protocol/src/main/protobuf/RPC.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-protocol/src/main/protobuf/RPC.proto b/hbase-protocol/src/main/protobuf/RPC.proto
    index 59bb03d..8413d25 100644
    --- a/hbase-protocol/src/main/protobuf/RPC.proto
    +++ b/hbase-protocol/src/main/protobuf/RPC.proto
    @@ -125,6 +125,7 @@ message RequestHeader {
        // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
        // See HConstants.
        optional uint32 priority = 6;
    + optional uint32 timeout = 7;
      }

      message ResponseHeader {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    index 3514245..00e08c9 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/CallRunner.java
    @@ -114,7 +114,7 @@ public class CallRunner {
              }
              // make the call
              resultPair = this.rpcServer.call(call.service, call.md, call.param, call.cellScanner,
    - call.timestamp, this.status);
    + call.timestamp, this.status, call.timeout);
            } catch (Throwable e) {
              RpcServer.LOG.debug(Thread.currentThread().getName() + ": " + call.toShortString(), e);
              errorThrowable = e;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    index b9a9b26..483ce86 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
    @@ -264,6 +264,13 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
        private static final String WARN_RESPONSE_TIME = "hbase.ipc.warn.response.time";
        private static final String WARN_RESPONSE_SIZE = "hbase.ipc.warn.response.size";

    + /**
    + * Minimum allowable timeout (in milliseconds) in rpc request's header. This
    + * configuration exists to prevent the rpc service regarding this request as timeout immediately.
    + */
    + private static final String MIN_CLIENT_REQUEST_TIMEOUT = "hbase.ipc.min.client.request.timeout";
    + private static final int DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT = 20;
    +
        /** Default value for above params */
        private static final int DEFAULT_MAX_REQUEST_SIZE = DEFAULT_MAX_CALLQUEUE_SIZE / 4; // 256M
        private static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds
    @@ -274,6 +281,9 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
        private final int maxRequestSize;
        private final int warnResponseTime;
        private final int warnResponseSize;
    +
    + private final int minClientRequestTimeout;
    +
        private final Server server;
        private final List<BlockingServiceAndInterface> services;

    @@ -302,6 +312,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          protected Connection connection; // connection to client
          protected long timestamp; // the time received when response is null
                                         // the time served when response is not null
    + protected int timeout;
          /**
           * Chain of buffers to send as response.
           */
    @@ -325,7 +336,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
              justification="Can't figure why this complaint is happening... see below")
          Call(int id, final BlockingService service, final MethodDescriptor md, RequestHeader header,
               Message param, CellScanner cellScanner, Connection connection, Responder responder,
    - long size, TraceInfo tinfo, final InetAddress remoteAddress) {
    + long size, TraceInfo tinfo, final InetAddress remoteAddress, int timeout) {
            this.id = id;
            this.service = service;
            this.md = md;
    @@ -343,6 +354,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            this.remoteAddress = remoteAddress;
            this.retryImmediatelySupported =
                connection == null? null: connection.retryImmediatelySupported;
    + this.timeout = timeout;
          }

          /**
    @@ -1275,13 +1287,13 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          // Fake 'call' for failed authorization response
          private static final int AUTHORIZATION_FAILED_CALLID = -1;
          private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALLID, null, null, null,
    - null, null, this, null, 0, null, null);
    + null, null, this, null, 0, null, null, 0);
          private ByteArrayOutputStream authFailedResponse =
              new ByteArrayOutputStream();
          // Fake 'call' for SASL context setup
          private static final int SASL_CALLID = -33;
          private final Call saslCall = new Call(SASL_CALLID, null, null, null, null, null, this, null,
    - 0, null, null);
    + 0, null, null, 0);

          // was authentication allowed with a fallback to simple auth
          private boolean authenticatedWithFallback;
    @@ -1699,7 +1711,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {

          private int doBadPreambleHandling(final String msg, final Exception e) throws IOException {
            LOG.warn(msg);
    - Call fakeCall = new Call(-1, null, null, null, null, null, this, responder, -1, null, null);
    + Call fakeCall = new Call(-1, null, null, null, null, null, this, responder, -1, null, null,0);
            setupResponse(null, fakeCall, e, msg);
            responder.doRespond(fakeCall);
            // Returning -1 closes out the connection.
    @@ -1874,7 +1886,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            if ((totalRequestSize + callQueueSize.get()) > maxQueueSize) {
              final Call callTooBig =
                new Call(id, this.service, null, null, null, null, this,
    - responder, totalRequestSize, null, null);
    + responder, totalRequestSize, null, null, 0);
              ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
              metrics.exception(CALL_QUEUE_TOO_BIG_EXCEPTION);
              setupResponse(responseBuffer, callTooBig, CALL_QUEUE_TOO_BIG_EXCEPTION,
    @@ -1923,7 +1935,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {

              final Call readParamsFailedCall =
                new Call(id, this.service, null, null, null, null, this,
    - responder, totalRequestSize, null, null);
    + responder, totalRequestSize, null, null, 0);
              ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
              setupResponse(responseBuffer, readParamsFailedCall, t,
                msg + "; " + t.getMessage());
    @@ -1934,8 +1946,12 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            TraceInfo traceInfo = header.hasTraceInfo()
                ? new TraceInfo(header.getTraceInfo().getTraceId(), header.getTraceInfo().getParentId())
                : null;
    + int timeout = 0;
    + if (header.hasTimeout()){
    + timeout = Math.max(minClientRequestTimeout, header.getTimeout());
    + }
            Call call = new Call(id, this.service, md, header, param, cellScanner, this, responder,
    - totalRequestSize, traceInfo, this.addr);
    + totalRequestSize, traceInfo, this.addr, timeout);

            if (!scheduler.dispatch(new CallRunner(RpcServer.this, call))) {
              callQueueSize.add(-1 * call.getSize());
    @@ -2087,7 +2103,8 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
          this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME, DEFAULT_WARN_RESPONSE_TIME);
          this.warnResponseSize = conf.getInt(WARN_RESPONSE_SIZE, DEFAULT_WARN_RESPONSE_SIZE);
    -
    + this.minClientRequestTimeout = conf.getInt(MIN_CLIENT_REQUEST_TIMEOUT,
    + DEFAULT_MIN_CLIENT_REQUEST_TIMEOUT);
          this.maxRequestSize = conf.getInt(MAX_REQUEST_SIZE, DEFAULT_MAX_REQUEST_SIZE);

          // Start the listener here and let it bind to the port
    @@ -2228,6 +2245,12 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
          this.secretManager = (SecretManager<TokenIdentifier>) secretManager;
        }

    + public Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
    + throws IOException {
    + return call(service, md, param, cellScanner, receiveTime, status, 0);
    + }
    +
        /**
         * This is a server side method, which is invoked over RPC. On success
         * the return response has protobuf response payload. On failure, the
    @@ -2235,7 +2258,8 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
         */
        @Override
        public Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    - Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
    + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status,
    + int timeout)
        throws IOException {
          try {
            status.setRPC(md.getName(), new Object[]{param}, receiveTime);
    @@ -2245,6 +2269,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
            //get an instance of the method arg type
            long startTime = System.currentTimeMillis();
            PayloadCarryingRpcController controller = new PayloadCarryingRpcController(cellScanner);
    + controller.setCallTimeout(timeout);
            Message result = service.callBlockingMethod(md, controller, param);
            long endTime = System.currentTimeMillis();
            int processingTime = (int) (endTime - startTime);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    index ab8b485..dd7e584 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
    @@ -52,6 +52,11 @@ public interface RpcServerInterface {
          Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
        throws IOException, ServiceException;

    + Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
    + Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status,
    + int timeout)
    + throws IOException, ServiceException;
    +
        void setErrorHandler(HBaseRPCErrorHandler handler);
        HBaseRPCErrorHandler getErrorHandler();


    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    index 95ab36d..ebd85bd 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
    @@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface;
      import org.apache.hadoop.hbase.ipc.RpcServerInterface;
      import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
      import org.apache.hadoop.hbase.ipc.ServerRpcController;
    +import org.apache.hadoop.hbase.ipc.TimeLimitedRpcController;
      import org.apache.hadoop.hbase.master.MasterRpcServices;
      import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
      import org.apache.hadoop.hbase.protobuf.RequestConverter;
    @@ -2698,6 +2699,14 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
                          timeLimitDelta =
                              scannerLeaseTimeoutPeriod > 0 ? scannerLeaseTimeoutPeriod : rpcTimeout;
                        }
    + if (controller instanceof TimeLimitedRpcController) {
    + TimeLimitedRpcController timeLimitedRpcController =
    + (TimeLimitedRpcController)controller;
    + if (timeLimitedRpcController.getCallTimeout() > 0) {
    + timeLimitDelta = Math.min(timeLimitDelta,
    + timeLimitedRpcController.getCallTimeout());
    + }
    + }
                        // Use half of whichever timeout value was more restrictive... But don't allow
                        // the time limit to be less than the allowable minimum (could cause an
                        // immediatate timeout before scanning any data).

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3bd9220f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    index b8eacc7..a958ee0 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java
    @@ -18,6 +18,7 @@
      package org.apache.hadoop.hbase.regionserver;

      import static org.junit.Assert.assertEquals;
    +import static org.junit.Assert.assertNull;
      import static org.junit.Assert.assertTrue;
      import static org.junit.Assert.fail;

    @@ -102,16 +103,16 @@ public class TestScannerHeartbeatMessages {
        private static int VALUE_SIZE = 128;
        private static byte[] VALUE = Bytes.createMaxByteArray(VALUE_SIZE);

    +
    + private static int SERVER_TIMEOUT = 6000;
    +
        // Time, in milliseconds, that the client will wait for a response from the server before timing
        // out. This value is used server side to determine when it is necessary to send a heartbeat
        // message to the client
    - private static int CLIENT_TIMEOUT = 2000;
    -
    - // The server limits itself to running for half of the CLIENT_TIMEOUT value.
    - private static int SERVER_TIME_LIMIT = CLIENT_TIMEOUT / 2;
    + private static int CLIENT_TIMEOUT = SERVER_TIMEOUT / 3;

        // By default, at most one row's worth of cells will be retrieved before the time limit is reached
    - private static int DEFAULT_ROW_SLEEP_TIME = SERVER_TIME_LIMIT / 2;
    + private static int DEFAULT_ROW_SLEEP_TIME = CLIENT_TIMEOUT / 5;
        // By default, at most cells for two column families are retrieved before the time limit is
        // reached
        private static int DEFAULT_CF_SLEEP_TIME = DEFAULT_ROW_SLEEP_TIME / NUM_FAMILIES;
    @@ -124,8 +125,8 @@ public class TestScannerHeartbeatMessages {

          conf.setStrings(HConstants.REGION_IMPL, HeartbeatHRegion.class.getName());
          conf.setStrings(HConstants.REGION_SERVER_IMPL, HeartbeatHRegionServer.class.getName());
    - conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CLIENT_TIMEOUT);
    - conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, CLIENT_TIMEOUT);
    + conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SERVER_TIMEOUT);
    + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, SERVER_TIMEOUT);
          conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 1);

          // Check the timeout condition after every cell
    @@ -140,7 +141,7 @@ public class TestScannerHeartbeatMessages {
          Table ht = TEST_UTIL.createTable(name, families);
          List<Put> puts = createPuts(rows, families, qualifiers, cellValue);
          ht.put(puts);
    -
    + ht.getConfiguration().setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, CLIENT_TIMEOUT);
          return ht;
        }

    @@ -282,7 +283,7 @@ public class TestScannerHeartbeatMessages {
          @Override
          public ReturnCode filterKeyValue(Cell v) throws IOException {
            try {
    - Thread.sleep(SERVER_TIME_LIMIT + 10);
    + Thread.sleep(CLIENT_TIMEOUT/2 + 10);
            } catch (InterruptedException e) {
              Thread.currentThread().interrupt();
            }
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15824 LocalHBaseCluster gets bind exception in master info port - ADDENDUM


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1ada0a3
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1ada0a3
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1ada0a3

    Branch: refs/heads/HBASE-14850
    Commit: c1ada0a373561132a3359b48a27975b2e85978da
    Parents: 3bd9220
    Author: Enis Soztutar <enis@apache.org>
    Authored: Wed May 18 11:36:10 2016 -0700
    Committer: Enis Soztutar <enis@apache.org>
    Committed: Wed May 18 11:36:10 2016 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java | 5 +----
      .../src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java | 3 +++
      2 files changed, 4 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ada0a3/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    index 8b16a5b..84367b0 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    @@ -144,9 +144,6 @@ public class LocalHBaseCluster {
          // Always have masters and regionservers come up on port '0' so we don't
          // clash over default ports.
          conf.set(HConstants.MASTER_PORT, "0");
    - if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    - conf.set(HConstants.MASTER_INFO_PORT, "0");
    - }
          conf.set(HConstants.REGIONSERVER_PORT, "0");
          if (conf.getInt(HConstants.REGIONSERVER_INFO_PORT, 0) != -1) {
            conf.set(HConstants.REGIONSERVER_INFO_PORT, "0");
    @@ -450,4 +447,4 @@ public class LocalHBaseCluster {
          connection.close();
          cluster.shutdown();
        }
    -}
    \ No newline at end of file
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/c1ada0a3/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    index 002bdb2..f788bed 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java
    @@ -86,6 +86,9 @@ public class MiniHBaseCluster extends HBaseCluster {
            throws IOException, InterruptedException {
          super(conf);
          conf.set(HConstants.MASTER_PORT, "0");
    + if (conf.getInt(HConstants.MASTER_INFO_PORT, 0) != -1) {
    + conf.set(HConstants.MASTER_INFO_PORT, "0");
    + }

          // Hadoop 2
          CompatibilityFactory.getInstance(MetricsAssertHelper.class).init();
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15841 Performance Evaluation tool total rows may not be set correctly


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/393bcd69
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/393bcd69
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/393bcd69

    Branch: refs/heads/HBASE-14850
    Commit: 393bcd69f9a6088e2af7751619f973a43671b7aa
    Parents: c1ada0a
    Author: Jerry He <jerryjch@apache.org>
    Authored: Wed May 18 12:09:13 2016 -0700
    Committer: Jerry He <jerryjch@apache.org>
    Committed: Wed May 18 12:09:13 2016 -0700

    ----------------------------------------------------------------------
      .../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 3 +--
      1 file changed, 1 insertion(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/393bcd69/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    index 02b994a..c85607d 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
    @@ -2143,8 +2143,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
            // total size in GB specified
            opts.totalRows = (int) opts.size * rowsPerGB;
            opts.perClientRunRows = opts.totalRows / opts.numClientThreads;
    - } else if (opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows) {
    - // number of rows specified
    + } else {
            opts.totalRows = opts.perClientRunRows * opts.numClientThreads;
            opts.size = opts.totalRows / rowsPerGB;
          }
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15843 Replace RegionState.getRegionInTransition() Map with a Set


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5ac54e6a
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5ac54e6a
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5ac54e6a

    Branch: refs/heads/HBASE-14850
    Commit: 5ac54e6abb4e14f3d2fb2e013a7688d7043a60d3
    Parents: 393bcd6
    Author: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
    Authored: Wed May 18 12:48:26 2016 -0700
    Committer: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
    Committed: Wed May 18 12:48:26 2016 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hbase/ClusterStatus.java | 9 ++--
      .../hadoop/hbase/protobuf/ProtobufUtil.java | 15 +++----
      .../hbase/rsgroup/RSGroupAdminServer.java | 2 +-
      .../master/AssignmentManagerStatusTmpl.jamon | 43 +++++++++----------
      .../hadoop/hbase/master/AssignmentManager.java | 8 ++--
      .../org/apache/hadoop/hbase/master/HMaster.java | 4 +-
      .../hadoop/hbase/master/MasterDumpServlet.java | 9 ++--
      .../hadoop/hbase/master/RegionStates.java | 44 +++++++++-----------
      .../org/apache/hadoop/hbase/util/HBaseFsck.java | 14 +++----
      .../hadoop/hbase/util/HBaseFsckRepair.java | 12 ++++--
      .../hbase/coprocessor/TestMasterObserver.java | 6 +--
      .../hbase/master/TestMasterStatusServlet.java | 19 ++++-----
      .../TestRegionMergeTransactionOnCluster.java | 2 +-
      .../TestSplitTransactionOnCluster.java | 8 ++--
      .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 17 ++++----
      hbase-shell/src/main/ruby/hbase/admin.rb | 2 +-
      16 files changed, 103 insertions(+), 111 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
    index bc97a95..ffeb51a 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClusterStatus.java
    @@ -25,6 +25,7 @@ import java.util.Collection;
      import java.util.Collections;
      import java.util.HashMap;
      import java.util.Map;
    +import java.util.Set;

      import org.apache.hadoop.hbase.classification.InterfaceAudience;
      import org.apache.hadoop.hbase.classification.InterfaceStability;
    @@ -80,7 +81,7 @@ public class ClusterStatus extends VersionedWritable {
        private Collection<ServerName> deadServers;
        private ServerName master;
        private Collection<ServerName> backupMasters;
    - private Map<String, RegionState> intransition;
    + private Set<RegionState> intransition;
        private String clusterId;
        private String[] masterCoprocessors;
        private Boolean balancerOn;
    @@ -90,7 +91,7 @@ public class ClusterStatus extends VersionedWritable {
            final Collection<ServerName> deadServers,
            final ServerName master,
            final Collection<ServerName> backupMasters,
    - final Map<String, RegionState> rit,
    + final Set<RegionState> rit,
            final String[] masterCoprocessors,
            final Boolean balancerOn) {
          this.hbaseVersion = hbaseVersion;
    @@ -261,7 +262,7 @@ public class ClusterStatus extends VersionedWritable {
        }

        @InterfaceAudience.Private
    - public Map<String, RegionState> getRegionsInTransition() {
    + public Set<RegionState> getRegionsInTransition() {
          return this.intransition;
        }

    @@ -340,7 +341,7 @@ public class ClusterStatus extends VersionedWritable {
          int ritSize = (intransition != null) ? intransition.size() : 0;
          sb.append("\nNumber of regions in transition: " + ritSize);
          if (ritSize > 0) {
    - for (RegionState state: intransition.values()) {
    + for (RegionState state: intransition) {
              sb.append("\n " + state.toDescriptiveString());
            }
          }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
    ----------------------------------------------------------------------
    diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
    index bbc13ab..e16a3a3 100644
    --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
    +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
    @@ -31,10 +31,12 @@ import java.security.PrivilegedExceptionAction;
      import java.util.ArrayList;
      import java.util.Collection;
      import java.util.HashMap;
    +import java.util.HashSet;
      import java.util.List;
      import java.util.Map;
      import java.util.Map.Entry;
      import java.util.NavigableSet;
    +import java.util.Set;
      import java.util.concurrent.TimeUnit;

      import org.apache.hadoop.conf.Configuration;
    @@ -3528,12 +3530,11 @@ public final class ProtobufUtil {
            backupMasters.add(ProtobufUtil.toServerName(sn));
          }

    - Map<String, RegionState> rit = null;
    - rit = new HashMap<String, RegionState>(proto.getRegionsInTransitionList().size());
    + Set<RegionState> rit = null;
    + rit = new HashSet<RegionState>(proto.getRegionsInTransitionList().size());
          for (RegionInTransition region : proto.getRegionsInTransitionList()) {
    - String key = new String(region.getSpec().getValue().toByteArray());
            RegionState value = RegionState.convert(region.getRegionState());
    - rit.put(key, value);
    + rit.add(value);
          }

          String[] masterCoprocessors = null;
    @@ -3577,11 +3578,11 @@ public final class ProtobufUtil {
          }

          if (status.getRegionsInTransition() != null) {
    - for (Map.Entry<String, RegionState> rit : status.getRegionsInTransition().entrySet()) {
    - ClusterStatusProtos.RegionState rs = rit.getValue().convert();
    + for (RegionState rit : status.getRegionsInTransition()) {
    + ClusterStatusProtos.RegionState rs = rit.convert();
              RegionSpecifier.Builder spec =
                  RegionSpecifier.newBuilder().setType(RegionSpecifierType.REGION_NAME);
    - spec.setValue(ByteStringer.wrap(Bytes.toBytes(rit.getKey())));
    + spec.setValue(ByteStringer.wrap(rit.getRegion().getRegionName()));

              RegionInTransition pbRIT =
                  RegionInTransition.newBuilder().setSpec(spec.build()).setRegionState(rs).build();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
    ----------------------------------------------------------------------
    diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
    index 8725781..7aea464 100644
    --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
    +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java
    @@ -191,7 +191,7 @@ public class RSGroupAdminServer extends RSGroupAdmin {
                    }
                  }
                  for (RegionState state :
    - master.getAssignmentManager().getRegionStates().getRegionsInTransition().values()) {
    + master.getAssignmentManager().getRegionStates().getRegionsInTransition()) {
                    if (state.getServerName().getHostPort().equals(rs)) {
                      regions.add(state.getRegion());
                    }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
    index 42334ff..e2ae09d 100644
    --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
    +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon
    @@ -23,38 +23,34 @@ org.apache.hadoop.hbase.master.RegionState;
      org.apache.hadoop.conf.Configuration;
      org.apache.hadoop.hbase.HBaseConfiguration;
      org.apache.hadoop.hbase.HConstants;
    -java.util.Iterator;
    -java.util.Map;
    -java.util.List;
    -java.util.ArrayList;
    -java.util.Map.Entry;
    -java.util.Arrays;
    +java.util.HashSet;
    +java.util.SortedSet;
      </%import>
      <%args>
      AssignmentManager assignmentManager;
      int limit = 100;
      </%args>

    -<%java Map<String, RegionState> rit = assignmentManager
    +<%java SortedSet<RegionState> rit = assignmentManager
        .getRegionStates().getRegionsInTransitionOrderedByTimestamp(); %>

      <%if !rit.isEmpty() %>
      <%java>
    -List<String> ritsOverThreshold = new ArrayList<>();
    -List<String> ritsTwiceThreshold = new ArrayList<>();
    +HashSet<String> ritsOverThreshold = new HashSet<String>();
    +HashSet<String> ritsTwiceThreshold = new HashSet<String>();
      // process the map to find region in transition details
      Configuration conf = HBaseConfiguration.create();
      int ritThreshold = conf.getInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 60000);
      int numOfRITOverThreshold = 0;
      long currentTime = System.currentTimeMillis();
    -for (Map.Entry<String, RegionState> e : rit.entrySet()) {
    - long ritTime = currentTime - e.getValue().getStamp();
    +for (RegionState rs : rit) {
    + long ritTime = currentTime - rs.getStamp();
        if(ritTime > (ritThreshold * 2)) {
           numOfRITOverThreshold++;
    - ritsTwiceThreshold.add(e.getKey());
    + ritsTwiceThreshold.add(rs.getRegion().getEncodedName());
        } else if (ritTime > ritThreshold) {
           numOfRITOverThreshold++;
    - ritsOverThreshold.add(e.getKey());
    + ritsOverThreshold.add(rs.getRegion().getEncodedName());
        }
      }

    @@ -64,7 +60,7 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage);
      </%java>
          <section>
          <h2>Regions in Transition</h2>
    - <p><% numOfRITs %> region(s) in transition.
    + <p><% numOfRITs %> region(s) in transition.
           <%if !ritsTwiceThreshold.isEmpty() %>
               <span class="label label-danger" style="font-size:100%;font-weight:normal">
           <%elseif !ritsOverThreshold.isEmpty() %>
    @@ -72,14 +68,14 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage);
           <%else>
               <span>
           </%if>
    - <% numOfRITOverThreshold %> region(s) in transition for
    + <% numOfRITOverThreshold %> region(s) in transition for
                   more than <% ritThreshold %> milliseconds.
               </span>
           </p>
           <div class="tabbable">
               <div class="tab-content">
               <%java int recordItr = 0; %>
    - <%for Map.Entry<String, RegionState> entry : rit.entrySet() %>
    + <%for RegionState rs : rit %>
                   <%if (recordItr % ritsPerPage) == 0 %>
                       <%if recordItr == 0 %>
                   <div class="tab-pane active" id="tab_rits<% (recordItr / ritsPerPage) + 1 %>">
    @@ -89,18 +85,17 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage);
                       <table class="table table-striped" style="margin-bottom:0px;"><tr><th>Region</th>
                           <th>State</th><th>RIT time (ms)</th></tr>
                   </%if>
    -
    - <%if ritsOverThreshold.contains(entry.getKey()) %>
    +
    + <%if ritsOverThreshold.contains(rs.getRegion().getEncodedName()) %>
                           <tr class="alert alert-warning" role="alert">
    - <%elseif ritsTwiceThreshold.contains(entry.getKey()) %>
    + <%elseif ritsTwiceThreshold.contains(rs.getRegion().getEncodedName()) %>
                           <tr class="alert alert-danger" role="alert">
                  <%else>
                          <tr>
                  </%if>
    - <td><% entry.getKey() %></td><td>
    - <% HRegionInfo.getDescriptiveNameFromRegionStateForDisplay(
    - entry.getValue(), conf) %></td>
    - <td><% (currentTime - entry.getValue().getStamp()) %> </td>
    + <td><% rs.getRegion().getEncodedName() %></td><td>
    + <% HRegionInfo.getDescriptiveNameFromRegionStateForDisplay(rs, conf) %></td>
    + <td><% (currentTime - rs.getStamp()) %> </td>
                           </tr>
                           <%java recordItr++; %>
                   <%if (recordItr % ritsPerPage) == 0 %>
    @@ -108,7 +103,7 @@ int numOfPages = (int) Math.ceil(numOfRITs * 1.0 / ritsPerPage);
                   </div>
               </%if>
               </%for>
    -
    +
               <%if (recordItr % ritsPerPage) != 0 %>
                   <%for ; (recordItr % ritsPerPage) != 0 ; recordItr++ %>
                   <tr><td colspan="3" style="height:61px"></td></tr>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    index bcf7b7a..f16463f 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    @@ -471,9 +471,9 @@ public class AssignmentManager {
          }
          if (!failover) {
            // If any region except meta is in transition on a live server, it's a failover.
    - Map<String, RegionState> regionsInTransition = regionStates.getRegionsInTransition();
    + Set<RegionState> regionsInTransition = regionStates.getRegionsInTransition();
            if (!regionsInTransition.isEmpty()) {
    - for (RegionState regionState: regionsInTransition.values()) {
    + for (RegionState regionState: regionsInTransition) {
                ServerName serverName = regionState.getServerName();
                if (!regionState.getRegion().isMetaRegion()
                    && serverName != null && onlineServers.contains(serverName)) {
    @@ -542,7 +542,7 @@ public class AssignmentManager {
                }
              }
            }
    - processRegionsInTransition(regionStates.getRegionsInTransition().values());
    + processRegionsInTransition(regionStates.getRegionsInTransition());
          }

          // Now we can safely claim failover cleanup completed and enable
    @@ -2010,7 +2010,7 @@ public class AssignmentManager {
          long oldestRITTime = 0;
          int ritThreshold = this.server.getConfiguration().
            getInt(HConstants.METRICS_RIT_STUCK_WARNING_THRESHOLD, 60000);
    - for (RegionState state: regionStates.getRegionsInTransition().values()) {
    + for (RegionState state: regionStates.getRegionsInTransition()) {
            totalRITs++;
            long ritTime = currentTime - state.getStamp();
            if (ritTime > ritThreshold) { // more than the threshold

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    index 109097b..f8d0003 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    @@ -1254,7 +1254,7 @@ public class HMaster extends HRegionServer implements MasterServices {
            if (!this.loadBalancerTracker.isBalancerOn()) return false;
            // Only allow one balance run at at time.
            if (this.assignmentManager.getRegionStates().isRegionsInTransition()) {
    - Map<String, RegionState> regionsInTransition =
    + Set<RegionState> regionsInTransition =
                this.assignmentManager.getRegionStates().getRegionsInTransition();
              // if hbase:meta region is in transition, result of assignment cannot be recorded
              // ignore the force flag in that case
    @@ -2152,7 +2152,7 @@ public class HMaster extends HRegionServer implements MasterServices {

          String clusterId = fileSystemManager != null ?
            fileSystemManager.getClusterId().toString() : null;
    - Map<String, RegionState> regionsInTransition = assignmentManager != null ?
    + Set<RegionState> regionsInTransition = assignmentManager != null ?
            assignmentManager.getRegionStates().getRegionsInTransition() : null;
          String[] coprocessors = cpHost != null ? getMasterCoprocessors() : null;
          boolean balancerOn = loadBalancerTracker != null ?

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
    index fc5b474..a921ab5 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
    @@ -24,6 +24,7 @@ import java.io.PrintStream;
      import java.io.PrintWriter;
      import java.util.Date;
      import java.util.Map;
    +import java.util.Set;

      import javax.servlet.http.HttpServletRequest;
      import javax.servlet.http.HttpServletResponse;
    @@ -116,11 +117,9 @@ public class MasterDumpServlet extends StateDumpServlet {
            return;
          }

    - Map<String, RegionState> regionsInTransition =
    - am.getRegionStates().getRegionsInTransition();
    - for (Map.Entry<String, RegionState> e : regionsInTransition.entrySet()) {
    - String rid = e.getKey();
    - RegionState rs = e.getValue();
    + Set<RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    + for (RegionState rs : regionsInTransition) {
    + String rid = rs.getRegion().getRegionNameAsString();
            out.println("Region " + rid + ": " + rs.toDescriptiveString());
          }
        }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
    index be9758a..9da8033 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
    @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.master;
      import java.io.IOException;
      import java.util.ArrayList;
      import java.util.Collection;
    +import java.util.Collections;
    +import java.util.Comparator;
      import java.util.HashMap;
      import java.util.HashSet;
      import java.util.Iterator;
    @@ -28,9 +30,9 @@ import java.util.LinkedList;
      import java.util.List;
      import java.util.Map;
      import java.util.Set;
    -import java.util.Collections;
    -import java.util.Comparator;
    +import java.util.SortedSet;
      import java.util.TreeMap;
    +import java.util.TreeSet;

      import com.google.common.annotations.VisibleForTesting;
      import com.google.common.base.Preconditions;
    @@ -63,6 +65,15 @@ import org.apache.hadoop.hbase.util.Pair;
      public class RegionStates {
        private static final Log LOG = LogFactory.getLog(RegionStates.class);

    + public final static RegionStateStampComparator REGION_STATE_COMPARATOR =
    + new RegionStateStampComparator();
    + private static class RegionStateStampComparator implements Comparator<RegionState> {
    + @Override
    + public int compare(RegionState l, RegionState r) {
    + return Long.compare(l.getStamp(), r.getStamp());
    + }
    + }
    +
        /**
         * Regions currently in transition.
         */
    @@ -205,31 +216,16 @@ public class RegionStates {
        /**
         * Get regions in transition and their states
         */
    - @SuppressWarnings("unchecked")
    - public synchronized Map<String, RegionState> getRegionsInTransition() {
    - return (Map<String, RegionState>)regionsInTransition.clone();
    + public synchronized Set<RegionState> getRegionsInTransition() {
    + return new HashSet<RegionState>(regionsInTransition.values());
        }

    - @SuppressWarnings("unchecked")
    - public synchronized Map<String, RegionState> getRegionsInTransitionOrderedByTimestamp() {
    - Map<String, RegionState> rit = (Map<String, RegionState>)regionsInTransition.clone();
    - List<Map.Entry<String, RegionState>> list = new LinkedList<>(rit.entrySet());
    -
    - // Compare the RITs' timestamps for ordering.
    - Comparator<Map.Entry<String, RegionState>> c =
    - new Comparator<Map.Entry<String, RegionState>>() {
    - @Override
    - public int compare(Map.Entry<String, RegionState> o1, Map.Entry<String, RegionState> o2) {
    - return ((Long)o1.getValue().getStamp()).compareTo((Long)o2.getValue().getStamp());
    - }
    - };
    -
    - Collections.sort(list, c);
    - Map<String, RegionState> result = new LinkedHashMap<>();
    - for (Map.Entry<String, RegionState> entry : list) {
    - result.put(entry.getKey(), entry.getValue());
    + public synchronized SortedSet<RegionState> getRegionsInTransitionOrderedByTimestamp() {
    + final TreeSet<RegionState> rit = new TreeSet<RegionState>(REGION_STATE_COMPARATOR);
    + for (RegionState rs: regionsInTransition.values()) {
    + rit.add(rs);
          }
    - return result;
    + return rit;
        }

        /**

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    index eec5ea4..8666ee6 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    @@ -547,10 +547,10 @@ public class HBaseFsck extends Configured implements Closeable {
          errors.print("Number of requests: " + status.getRequestsCount());
          errors.print("Number of regions: " + status.getRegionsCount());

    - Map<String, RegionState> rits = status.getRegionsInTransition();
    + Set<RegionState> rits = status.getRegionsInTransition();
          errors.print("Number of regions in transition: " + rits.size());
          if (details) {
    - for (RegionState state: rits.values()) {
    + for (RegionState state: rits) {
              errors.print(" " + state.toDescriptiveString());
            }
          }
    @@ -722,7 +722,7 @@ public class HBaseFsck extends Configured implements Closeable {
          checkAndFixTableLocks();

          checkAndFixReplication();
    -
    +
          // Remove the hbck lock
          unlockHbck();

    @@ -3283,7 +3283,7 @@ public class HBaseFsck extends Configured implements Closeable {
            checker.fixExpiredTableLocks();
          }
        }
    -
    +
        private void checkAndFixReplication() throws IOException {
          ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
          checker.checkUnDeletedQueues();
    @@ -4214,7 +4214,7 @@ public class HBaseFsck extends Configured implements Closeable {
        public boolean shouldDisableSplitAndMerge() {
          return fixAny || disableSplitAndMerge;
        }
    -
    +
        /**
         * Set summary mode.
         * Print only summary of the tables and status (OK or INCONSISTENT)
    @@ -4246,7 +4246,7 @@ public class HBaseFsck extends Configured implements Closeable {
          fixTableLocks = shouldFix;
          fixAny |= shouldFix;
        }
    -
    +
        /**
         * Set replication fix mode.
         */
    @@ -4517,7 +4517,7 @@ public class HBaseFsck extends Configured implements Closeable {
          out.println("");
          out.println(" Replication options");
          out.println(" -fixReplication Deletes replication queues for removed peers");
    -
    +
          out.flush();
          errors.reportError(ERROR_CODE.WRONG_USAGE, sw.toString());


    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
    index e681789..819816e 100644
    --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
    +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
    @@ -124,10 +124,14 @@ public class HBaseFsckRepair {
          long expiration = timeout + EnvironmentEdgeManager.currentTime();
          while (EnvironmentEdgeManager.currentTime() < expiration) {
            try {
    - Map<String, RegionState> rits=
    - admin.getClusterStatus().getRegionsInTransition();
    -
    - if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
    + boolean inTransition = false;
    + for (RegionState rs: admin.getClusterStatus().getRegionsInTransition()) {
    + if (rs.getRegion().equals(region)) {
    + inTransition = true;
    + break;
    + }
    + }
    + if (!inTransition) {
                // yay! no longer RIT
                return;
              }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
    index 73917cd..e1c4a1f 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
    @@ -1916,8 +1916,7 @@ public class TestMasterObserver {

            // wait for assignments to finish, if any
            AssignmentManager mgr = master.getAssignmentManager();
    - Collection<RegionState> transRegions =
    - mgr.getRegionStates().getRegionsInTransition().values();
    + Set<RegionState> transRegions = mgr.getRegionStates().getRegionsInTransition();
            for (RegionState state : transRegions) {
              mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
            }
    @@ -1953,8 +1952,7 @@ public class TestMasterObserver {
        private void waitForRITtoBeZero(HMaster master) throws Exception {
          // wait for assignments to finish
          AssignmentManager mgr = master.getAssignmentManager();
    - Collection<RegionState> transRegions =
    - mgr.getRegionStates().getRegionsInTransition().values();
    + Set<RegionState> transRegions = mgr.getRegionStates().getRegionsInTransition();
          for (RegionState state : transRegions) {
            mgr.getRegionStates().waitOnRegionToClearRegionsInTransition(state.getRegion());
          }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
    index 135aae5..02f01c4 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java
    @@ -24,8 +24,8 @@ import java.io.IOException;
      import java.io.StringWriter;
      import java.util.HashSet;
      import java.util.List;
    -import java.util.NavigableMap;
      import java.util.Set;
    +import java.util.TreeSet;
      import java.util.regex.Matcher;
      import java.util.regex.Pattern;

    @@ -90,10 +90,8 @@ public class TestMasterStatusServlet {
          // Fake AssignmentManager and RIT
          AssignmentManager am = Mockito.mock(AssignmentManager.class);
          RegionStates rs = Mockito.mock(RegionStates.class);
    - NavigableMap<String, RegionState> regionsInTransition =
    - Maps.newTreeMap();
    - regionsInTransition.put("r1",
    - new RegionState(FAKE_HRI, RegionState.State.CLOSING, 12345L, FAKE_HOST));
    + Set<RegionState> regionsInTransition = new HashSet<RegionState>();
    + regionsInTransition.add(new RegionState(FAKE_HRI, RegionState.State.CLOSING, 12345L, FAKE_HOST));
          Mockito.doReturn(rs).when(am).getRegionStates();
          Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition();
          Mockito.doReturn(am).when(master).getAssignmentManager();
    @@ -165,19 +163,18 @@ public class TestMasterStatusServlet {
          RegionStates rs = Mockito.mock(RegionStates.class);

          // Add 100 regions as in-transition
    - NavigableMap<String, RegionState> regionsInTransition =
    - Maps.newTreeMap();
    + TreeSet<RegionState> regionsInTransition = new TreeSet<RegionState>(
    + RegionStates.REGION_STATE_COMPARATOR);
          for (byte i = 0; i < 100; i++) {
            HRegionInfo hri = new HRegionInfo(FAKE_TABLE.getTableName(),
                new byte[]{i}, new byte[]{(byte) (i+1)});
    - regionsInTransition.put(hri.getEncodedName(),
    + regionsInTransition.add(
              new RegionState(hri, RegionState.State.CLOSING, 12345L, FAKE_HOST));
          }
          // Add hbase:meta in transition as well
    - regionsInTransition.put(
    - HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(),
    + regionsInTransition.add(
              new RegionState(HRegionInfo.FIRST_META_REGIONINFO,
    - RegionState.State.CLOSING, 12345L, FAKE_HOST));
    + RegionState.State.CLOSING, 123L, FAKE_HOST));
          Mockito.doReturn(rs).when(am).getRegionStates();
          Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransition();
          Mockito.doReturn(regionsInTransition).when(rs).getRegionsInTransitionOrderedByTimestamp();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
    index cd4410f..8499335 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
    @@ -544,7 +544,7 @@ public class TestRegionMergeTransactionOnCluster {
            if (enabled.get() && req.getTransition(0).getTransitionCode()
                == TransitionCode.READY_TO_MERGE && !resp.hasErrorMessage()) {
              RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
    - for (RegionState regionState: regionStates.getRegionsInTransition().values()) {
    + for (RegionState regionState: regionStates.getRegionsInTransition()) {
                // Find the merging_new region and remove it
                if (regionState.isMergingNew()) {
                  regionStates.deleteRegion(regionState.getRegion());

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
    index fe620e7..e6578bb 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
    @@ -31,6 +31,7 @@ import java.io.InterruptedIOException;
      import java.util.Collection;
      import java.util.List;
      import java.util.Map;
    +import java.util.Set;
      import java.util.concurrent.CountDownLatch;
      import java.util.concurrent.atomic.AtomicBoolean;

    @@ -213,8 +214,7 @@ public class TestSplitTransactionOnCluster {
              @Override
              public boolean evaluate() throws Exception {
                RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
    - Map<String, RegionState> rit = regionStates.getRegionsInTransition();
    - return !rit.containsKey(hri.getEncodedName());
    + return !regionStates.isRegionInTransition(hri.getEncodedName());
              }
            });
          } finally {
    @@ -657,7 +657,7 @@ public class TestSplitTransactionOnCluster {
            tableExists = MetaTableAccessor.tableExists(regionServer.getConnection(),
              tableName);
            assertEquals("The specified table should present.", true, tableExists);
    - Map<String, RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
    + Set<RegionState> rit = cluster.getMaster().getAssignmentManager().getRegionStates()
                .getRegionsInTransition();
            assertTrue(rit.size() == 3);
            cluster.getMaster().getAssignmentManager().regionOffline(st.getFirstDaughter());
    @@ -1311,7 +1311,7 @@ public class TestSplitTransactionOnCluster {
            if (enabled.get() && req.getTransition(0).getTransitionCode().equals(
                TransitionCode.READY_TO_SPLIT) && !resp.hasErrorMessage()) {
              RegionStates regionStates = myMaster.getAssignmentManager().getRegionStates();
    - for (RegionState regionState: regionStates.getRegionsInTransition().values()) {
    + for (RegionState regionState: regionStates.getRegionsInTransition()) {
                // Find the merging_new region and remove it
                if (regionState.isSplittingNew()) {
                  regionStates.deleteRegion(regionState.getRegion());

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    index a7c0c55..165fea6 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    @@ -80,6 +80,7 @@ import java.util.LinkedList;
      import java.util.List;
      import java.util.HashMap;
      import java.util.Map;
    +import java.util.Set;
      import java.util.concurrent.Callable;
      import java.util.concurrent.CountDownLatch;
      import java.util.concurrent.ExecutorService;
    @@ -609,7 +610,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
            hbck.close();
          }
        }
    -
    +
        @Test (timeout=180000)
        public void testHbckAfterRegionMerge() throws Exception {
          TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
    @@ -1527,7 +1528,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          // check no errors
          HBaseFsck hbck = doFsck(conf, false);
          assertNoErrors(hbck);
    -
    +
          // create peer
          ReplicationAdmin replicationAdmin = new ReplicationAdmin(conf);
          Assert.assertEquals(0, replicationAdmin.getPeersCount());
    @@ -1538,7 +1539,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          replicationAdmin.addPeer("1", rpc, null);
          replicationAdmin.getPeersCount();
          Assert.assertEquals(1, replicationAdmin.getPeersCount());
    -
    +
          // create replicator
          ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "Test Hbase Fsck", connection);
          ReplicationQueues repQueues =
    @@ -1550,7 +1551,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          Assert.assertEquals(2, repQueues.getAllQueues().size());
          hbck = doFsck(conf, false);
          assertNoErrors(hbck);
    -
    +
          // queues for removed peer
          repQueues.addLog("2", "file1");
          repQueues.addLog("2-server2", "file1");
    @@ -1559,7 +1560,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
              HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE,
              HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE });
    -
    +
          // fix the case
          hbck = doFsck(conf, true);
          hbck = doFsck(conf, false);
    @@ -1568,7 +1569,7 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          Assert.assertEquals(2, repQueues.getAllQueues().size());
          Assert.assertNull(repQueues.getLogsInQueue("2"));
          Assert.assertNull(repQueues.getLogsInQueue("2-sever2"));
    -
    +
          replicationAdmin.removePeer("1");
          repQueues.removeAllQueues();
          zkw.close();
    @@ -1678,8 +1679,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
            st.prepare();
            st.stepsBeforePONR(regionServer, regionServer, false);
            AssignmentManager am = cluster.getMaster().getAssignmentManager();
    - Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    - for (RegionState state : regionsInTransition.values()) {
    + Set<RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition();
    + for (RegionState state : regionsInTransition) {
              am.regionOffline(state.getRegion());
            }
            Map<HRegionInfo, ServerName> regionsMap = new HashMap<HRegionInfo, ServerName>();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/5ac54e6a/hbase-shell/src/main/ruby/hbase/admin.rb
    ----------------------------------------------------------------------
    diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
    index 88a6598..d272e62 100644
    --- a/hbase-shell/src/main/ruby/hbase/admin.rb
    +++ b/hbase-shell/src/main/ruby/hbase/admin.rb
    @@ -720,7 +720,7 @@ module Hbase
              puts("version %s" % [ status.getHBaseVersion() ])
              # Put regions in transition first because usually empty
              puts("%d regionsInTransition" % status.getRegionsInTransition().size())
    - for k, v in status.getRegionsInTransition()
    + for v in status.getRegionsInTransition()
                puts(" %s" % [v])
              end
              master = status.getMaster()
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15731 Add on a connection pool

    Summary:
    Add on a connection pool protected by read write mutex.
    Add on a service filter that will remove a connection from a connection pool when closed

    Test Plan: Need to add on tests.

    Differential Revision: https://reviews.facebook.net/D57411


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59f96123
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59f96123
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59f96123

    Branch: refs/heads/HBASE-14850
    Commit: 59f961237233384399d6d65736579f48fed63cd5
    Parents: 74ff76a
    Author: Elliott Clark <eclark@apache.org>
    Authored: Thu Apr 28 19:53:42 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/connection/BUCK | 5 +
      .../connection/connection-factory.cc | 4 +-
      .../connection/connection-factory.h | 5 +-
      .../connection/connection-pool-test.cc | 77 ++++++++++++++
      .../connection/connection-pool.cc | 89 ++++++++++++++++
      .../connection/connection-pool.h | 59 +++++++++++
      hbase-native-client/connection/service.h | 4 +-
      hbase-native-client/core/BUCK | 4 -
      hbase-native-client/core/client.h | 3 -
      hbase-native-client/core/get-request.cc | 19 ----
      hbase-native-client/core/get-request.h | 35 -------
      hbase-native-client/core/get-result.cc | 19 ----
      hbase-native-client/core/get-result.h | 32 ------
      hbase-native-client/core/location-cache.cc | 1 -
      hbase-native-client/core/simple-client.cc | 105 +++++++++++--------
      15 files changed, 297 insertions(+), 164 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
    index d393885..96f2136 100644
    --- a/hbase-native-client/connection/BUCK
    +++ b/hbase-native-client/connection/BUCK
    @@ -22,6 +22,7 @@ cxx_library(name="connection",
                      "client-dispatcher.h",
                      "client-handler.h",
                      "connection-factory.h",
    + "connection-pool.h",
                      "pipeline.h",
                      "request.h",
                      "response.h",
    @@ -31,6 +32,7 @@ cxx_library(name="connection",
                      "client-dispatcher.cc",
                      "client-handler.cc",
                      "connection-factory.cc",
    + "connection-pool.cc",
                      "pipeline.cc",
                      "request.cc",
                  ],
    @@ -42,3 +44,6 @@ cxx_library(name="connection",
                      "//third-party:wangle",
                  ],
                  visibility=['//core/...', ], )
    +cxx_test(name="connection-pool-test",
    + srcs=["connection-pool-test.cc", ],
    + deps=[":connection", ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/connection-factory.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
    index 7073f9d..b546269 100644
    --- a/hbase-native-client/connection/connection-factory.cc
    +++ b/hbase-native-client/connection/connection-factory.cc
    @@ -45,8 +45,8 @@ ConnectionFactory::ConnectionFactory() {
        bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
      }

    -std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
    -ConnectionFactory::make_connection(std::string host, int port) {
    +std::shared_ptr<HBaseService>
    +ConnectionFactory::make_connection(const std::string &host, int port) {
        // Connect to a given server
        // Then when connected create a ClientDispactcher.
        auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/connection-factory.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
    index 8d1d2f0..5a45316 100644
    --- a/hbase-native-client/connection/connection-factory.h
    +++ b/hbase-native-client/connection/connection-factory.h
    @@ -31,8 +31,9 @@ namespace hbase {
      class ConnectionFactory {
      public:
        ConnectionFactory();
    - std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
    - make_connection(std::string host, int port);
    +
    + virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
    + int port);

      private:
        wangle::ClientBootstrap<SerializePipeline> bootstrap_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/connection-pool-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
    new file mode 100644
    index 0000000..975bc5e
    --- /dev/null
    +++ b/hbase-native-client/connection/connection-pool-test.cc
    @@ -0,0 +1,77 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "connection/connection-pool.h"
    +
    +#include <folly/Logging.h>
    +#include <gmock/gmock.h>
    +
    +#include "connection/connection-factory.h"
    +#include "if/HBase.pb.h"
    +
    +using namespace hbase;
    +
    +using hbase::pb::ServerName;
    +using ::testing::Return;
    +using ::testing::_;
    +
    +class MockConnectionFactory : public ConnectionFactory {
    +public:
    + MOCK_METHOD2(make_connection,
    + std::shared_ptr<HBaseService>(const std::string &hostname,
    + int port));
    +};
    +
    +class MockServiceBase : public HBaseService {
    +public:
    + folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
    + return do_operation(req.get());
    + }
    + virtual folly::Future<Response> do_operation(Request *req) {
    + return folly::makeFuture<Response>(Response{});
    + }
    +};
    +
    +class MockService : public MockServiceBase {
    +public:
    + MOCK_METHOD1(do_operation, folly::Future<Response>(Request *));
    +};
    +
    +TEST(TestConnectionPool, TestOnlyCreateOnce) {
    + std::string hostname{"hostname"};
    + auto mock_service = std::make_shared<MockService>();
    + uint32_t port{999};
    +
    + LOG(ERROR) << "About to make a MockConnectionFactory";
    + auto mock_cf = std::make_shared<MockConnectionFactory>();
    + EXPECT_CALL((*mock_cf), make_connection(_, _))
    + .Times(1)
    + .WillRepeatedly(Return(mock_service));
    + ConnectionPool cp{mock_cf};
    +
    + LOG(ERROR) << "Created ConnectionPool";
    +
    + ServerName sn;
    + sn.set_host_name(hostname);
    + sn.set_port(port);
    +
    + auto result = cp.get(sn);
    + ASSERT_TRUE(result != nullptr);
    + result = cp.get(sn);
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    new file mode 100644
    index 0000000..a967df2
    --- /dev/null
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -0,0 +1,89 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "connection/connection-pool.h"
    +
    +#include <wangle/service/Service.h>
    +
    +using std::mutex;
    +using std::unique_ptr;
    +using std::shared_ptr;
    +using hbase::pb::ServerName;
    +using wangle::ServiceFilter;
    +using folly::SharedMutexWritePriority;
    +
    +namespace hbase {
    +
    +class RemoveServiceFilter
    + : public ServiceFilter<unique_ptr<Request>, Response> {
    +
    +public:
    + RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    + ConnectionPool *cp)
    + : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
    + cp_(cp) {}
    +
    + folly::Future<folly::Unit> close() override {
    + if (!released.exchange(true)) {
    + return this->service_->close().then(
    + [this]() { this->cp_->close(this->sn_); });
    + } else {
    + return folly::makeFuture();
    + }
    + }
    +
    + virtual bool isAvailable() override { return service_->isAvailable(); }
    +
    + folly::Future<Response> operator()(unique_ptr<Request> req) override {
    + return (*this->service_)(std::move(req));
    + }
    +
    +private:
    + std::atomic<bool> released{false};
    + hbase::pb::ServerName sn_;
    + ConnectionPool *cp_;
    +};
    +
    +ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
    +ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
    + : cf_(cf) {}
    +
    +std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
    + SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
    + auto found = connections_.find(sn);
    + if (found == connections_.end() || found->second == nullptr) {
    + SharedMutexWritePriority::WriteHolder holder(std::move(holder));
    + auto new_con = cf_->make_connection(sn.host_name(), sn.port());
    + auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
    + connections_[sn] = wrapped;
    + return new_con;
    + }
    + return found->second;
    +}
    +void ConnectionPool::close(ServerName sn) {
    + SharedMutexWritePriority::WriteHolder holder(map_mutex_);
    +
    + auto found = connections_.find(sn);
    + if (found == connections_.end() || found->second == nullptr) {
    + return;
    + }
    + auto service = found->second;
    + connections_.erase(found);
    +}
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    new file mode 100644
    index 0000000..394cd71
    --- /dev/null
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -0,0 +1,59 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +#include <folly/SharedMutex.h>
    +#include <mutex>
    +#include <unordered_map>
    +
    +#include "connection/connection-factory.h"
    +#include "connection/service.h"
    +#include "if/HBase.pb.h"
    +
    +namespace hbase {
    +struct MyServerNameEquals {
    + bool operator()(const hbase::pb::ServerName &lhs,
    + const hbase::pb::ServerName &rhs) const {
    + return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
    + }
    +};
    +struct MyServerNameHash {
    + std::size_t operator()(hbase::pb::ServerName const &s) const {
    + std::size_t h1 = std::hash<std::string>()(s.host_name());
    + std::size_t h2 = std::hash<uint32_t>()(s.port());
    + return h1 ^ (h2 << 1);
    + }
    +};
    +
    +class ConnectionPool {
    +public:
    + ConnectionPool();
    + explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
    + std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
    + void close(hbase::pb::ServerName sn);
    +
    +private:
    + std::shared_ptr<ConnectionFactory> cf_;
    + std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
    + MyServerNameHash, MyServerNameEquals>
    + connections_;
    + folly::SharedMutexWritePriority map_mutex_;
    +};
    +
    +} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/connection/service.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/service.h b/hbase-native-client/connection/service.h
    index feb14ec..79f087d 100644
    --- a/hbase-native-client/connection/service.h
    +++ b/hbase-native-client/connection/service.h
    @@ -18,9 +18,11 @@
       */
      #pragma once

    +#include <wangle/service/Service.h>
    +
      #include "connection/request.h"
      #include "connection/response.h"

      namespace hbase {
    -using HBaseService = wangle::Service<Request, Response>;
    +using HBaseService = wangle::Service<std::unique_ptr<Request>, Response>;
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index 9db6fda..e555ba4 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -21,8 +21,6 @@ cxx_library(
          exported_headers=[
              "client.h",
              "connection.h",
    - "get-request.h",
    - "get-result.h",
              "hbase_macros.h",
              "location-cache.h",
              "table-name.h",
    @@ -32,8 +30,6 @@ cxx_library(
          ],
          srcs=[
              "client.cc",
    - "get-request.cc",
    - "get-result.cc",
              "location-cache.cc",
              "meta-utils.cc",
              "table-name.cc",

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/client.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
    index b583285..4bed751 100644
    --- a/hbase-native-client/core/client.h
    +++ b/hbase-native-client/core/client.h
    @@ -24,8 +24,6 @@

      #include <string>

    -#include "core/get-request.h"
    -#include "core/get-result.h"
      #include "core/location-cache.h"
      #include "if/Cell.pb.h"

    @@ -33,7 +31,6 @@ namespace hbase {
      class Client {
      public:
        explicit Client(std::string quorum_spec);
    - folly::Future<GetResult> get(const GetRequest &get_request);

      private:
        LocationCache location_cache_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/get-request.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/get-request.cc b/hbase-native-client/core/get-request.cc
    deleted file mode 100644
    index e927ccc..0000000
    --- a/hbase-native-client/core/get-request.cc
    +++ /dev/null
    @@ -1,19 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#include "core/get-request.h"

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/get-request.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/get-request.h b/hbase-native-client/core/get-request.h
    deleted file mode 100644
    index bb755c5..0000000
    --- a/hbase-native-client/core/get-request.h
    +++ /dev/null
    @@ -1,35 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -#include <string>
    -
    -#include "core/table-name.h"
    -
    -namespace hbase {
    -
    -class GetRequest {
    -public:
    - GetRequest(TableName table_name, std::string key);
    -
    -private:
    - TableName table_name_;
    - std::string key_;
    -};
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/get-result.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/get-result.cc b/hbase-native-client/core/get-result.cc
    deleted file mode 100644
    index 7eea483..0000000
    --- a/hbase-native-client/core/get-result.cc
    +++ /dev/null
    @@ -1,19 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#include "core/get-result.h"

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/get-result.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/get-result.h b/hbase-native-client/core/get-result.h
    deleted file mode 100644
    index a49ad98..0000000
    --- a/hbase-native-client/core/get-result.h
    +++ /dev/null
    @@ -1,32 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -#include <string>
    -
    -namespace hbase {
    -
    -class GetResult {
    -public:
    - explicit GetResult(std::string key);
    -
    -private:
    - std::string key_;
    -};
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 5925f4a..c81deba 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -77,7 +77,6 @@ ServerName LocationCache::ReadMetaLocation() {
        int zk_result =
            zoo_get(this->zk_, META_ZNODE_NAME, 0,
                    reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
    - LOG(ERROR) << "len = " << len;
        if (zk_result != ZOK || len < 9) {
          LOG(ERROR) << "Error getting meta location.";
          throw runtime_error("Error getting meta location");

    http://git-wip-us.apache.org/repos/asf/hbase/blob/59f96123/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index 2cb6200..11dcd68 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -25,7 +25,7 @@
      #include <chrono>
      #include <iostream>

    -#include "connection/connection-factory.h"
    +#include "connection/connection-pool.h"
      #include "core/client.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
    @@ -33,13 +33,20 @@
      using namespace folly;
      using namespace std;
      using namespace std::chrono;
    -using namespace hbase;
    -using namespace hbase::pb;
    -using namespace google::protobuf;
    +using hbase::Response;
    +using hbase::Request;
    +using hbase::HBaseService;
    +using hbase::LocationCache;
    +using hbase::ConnectionPool;
    +using hbase::pb::ServerName;
    +using hbase::pb::RegionSpecifier_RegionSpecifierType;
    +using hbase::pb::GetRequest;
    +using hbase::pb::GetResponse;

      // TODO(eclark): remove the need for this.
      DEFINE_string(region, "1588230740", "What region to send a get to");
      DEFINE_string(row, "test", "What row to get");
    +DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");

      int main(int argc, char *argv[]) {
        google::SetUsageMessage(
    @@ -48,46 +55,52 @@ int main(int argc, char *argv[]) {
        google::InitGoogleLogging(argv[0]);

        // Create a connection factory
    - ConnectionFactory cf;
    -
    - LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
    -
    - auto result = cache.LocateMeta().get();
    -
    - // Create a connection to the local host
    - auto conn = cf.make_connection(result.host_name(), result.port());
    -
    - // Send the request
    - auto r = Request::get();
    -
    - // This is a get request so make that
    - auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());
    -
    - // Set what region
    - req_msg->mutable_region()->set_value(FLAGS_region);
    - // It's always this.
    - req_msg->mutable_region()->set_type(
    - RegionSpecifier_RegionSpecifierType::
    - RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
    -
    - // What row.
    - req_msg->mutable_get()->set_row(FLAGS_row);
    -
    - // Send it.
    - auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
    -
    - auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
    - cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
    - if (get_resp->has_result()) {
    - auto &r = get_resp->result();
    - cout << "Result cell_size = " << r.cell_size() << endl;
    - for (auto &cell : r.cell()) {
    - cout << "\trow = " << cell.row() << " family = " << cell.family()
    - << " qualifier = " << cell.qualifier()
    - << " timestamp = " << cell.timestamp() << " value = " << cell.value()
    - << endl;
    - }
    - }
    -
    - return 0;
    + ConnectionPool cp;
    + auto cpu_ex = wangle::getCPUExecutor();
    + LocationCache cache{FLAGS_zookeeper, cpu_ex};
    + auto result =
    + cache.LocateMeta()
    + .then([&cp = cp](ServerName sn) { return cp.get(sn); })
    + .then([](shared_ptr<HBaseService> con) {
    + // Send the request
    + auto r = Request::get();
    + // This is a get request so make that
    + auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
    + // Set what region
    + req_msg->mutable_region()->set_value(FLAGS_region);
    + // It's always this.
    + req_msg->mutable_region()->set_type(
    + RegionSpecifier_RegionSpecifierType::
    + RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
    +
    + // What row.
    + req_msg->mutable_get()->set_row(FLAGS_row);
    +
    + return (*con)(std::move(r));
    + })
    + .then([](Response resp) {
    + return static_pointer_cast<GetResponse>(resp.response());
    + })
    + .via(cpu_ex.get())
    + .then([](shared_ptr<GetResponse> get_resp) {
    + cout << "GetResponse has_result = " << get_resp->has_result()
    + << '\n';
    + if (get_resp->has_result()) {
    + auto &r = get_resp->result();
    + cout << "Result cell_size = " << r.cell_size() << endl;
    + for (auto &cell : r.cell()) {
    + cout << "\trow = " << cell.row()
    + << " family = " << cell.family()
    + << " qualifier = " << cell.qualifier()
    + << " timestamp = " << cell.timestamp()
    + << " value = " << cell.value() << endl;
    + }
    + return 0;
    + }
    +
    + return 1;
    + })
    + .get(milliseconds(5000));
    +
    + return result;
      }
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15774 Fix Upgrade lock usage in connection pool.

    Summary:
    Use upgrade lock better.
    Fix a clang warning around initializing the UpgradeHolder incorrectly.
    Remove dead code. ( We'l need to add it back when there's a better plan)
    Add on more comments.

    Test Plan: buck test --all

    Differential Revision: https://reviews.facebook.net/D58005


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aac4e430
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aac4e430
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aac4e430

    Branch: refs/heads/HBASE-14850
    Commit: aac4e43040119e16047d217d812a6750a01d8476
    Parents: 2b0fb0b
    Author: Elliott Clark <eclark@apache.org>
    Authored: Tue May 10 17:44:41 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      .../connection/connection-pool-test.cc | 12 ++--
      .../connection/connection-pool.cc | 47 +++++++++++---
      .../connection/connection-pool.h | 6 +-
      hbase-native-client/core/client.h | 2 +-
      hbase-native-client/core/location-cache.cc | 68 ++++----------------
      5 files changed, 62 insertions(+), 73 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e430/hbase-native-client/connection/connection-pool-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
    index c0c346f..bd2d585 100644
    --- a/hbase-native-client/connection/connection-pool-test.cc
    +++ b/hbase-native-client/connection/connection-pool-test.cc
    @@ -79,9 +79,9 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
        sn.set_host_name(hostname);
        sn.set_port(port);

    - auto result = cp.get(sn);
    + auto result = cp.Get(sn);
        ASSERT_TRUE(result != nullptr);
    - result = cp.get(sn);
    + result = cp.Get(sn);
      }

      TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
    @@ -102,13 +102,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
        ConnectionPool cp{mock_cf};

        {
    - auto result_one = cp.get(folly::to<ServerName>(
    + auto result_one = cp.Get(folly::to<ServerName>(
              hostname_one + ":" + folly::to<std::string>(port)));
    - auto result_two = cp.get(folly::to<ServerName>(
    + auto result_two = cp.Get(folly::to<ServerName>(
              hostname_two + ":" + folly::to<std::string>(port)));
        }
    - auto result_one = cp.get(
    + auto result_one = cp.Get(
            folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
    - auto result_two = cp.get(
    + auto result_two = cp.Get(
            folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e430/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index 90e2056..aa3d094 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -48,25 +48,54 @@ ConnectionPool::~ConnectionPool() {
        clients_.clear();
      }

    -std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
    - // Create a read lock.
    - SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
    +std::shared_ptr<HBaseService> ConnectionPool::Get(const ServerName &sn) {
    + // Try and get th cached connection.
    + auto found_ptr = GetCached(sn);

    + // If there's no connection then create it.
    + if (found_ptr == nullptr) {
    + found_ptr = GetNew(sn);
    + }
    + return found_ptr;
    +}
    +
    +std::shared_ptr<HBaseService> ConnectionPool::GetCached(const ServerName &sn) {
    + SharedMutexWritePriority::ReadHolder holder(map_mutex_);
        auto found = connections_.find(sn);
    - if (found == connections_.end() || found->second == nullptr) {
    - // Move the upgradable lock into the write lock if the connection
    - // hasn't been found.
    - SharedMutexWritePriority::WriteHolder holder(std::move(holder));
    + if (found == connections_.end()) {
    + return nullptr;
    + }
    + return found->second;
    +}
    +
    +std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
    + // Grab the upgrade lock. While we are double checking other readers can
    + // continue on
    + SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
    +
    + // Now check if someone else created the connection before we got the lock
    + // This is safe since we hold the upgrade lock.
    + // upgrade lock is more power than the reader lock.
    + auto found = connections_.find(sn);
    + if (found != connections_.end() && found->second != nullptr) {
    + return found->second;
    + } else {
    + // Yeah it looks a lot like there's no connection
    + SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
    +
    + // Make double sure there are not stale connections hanging around.
    + connections_.erase(sn);
    +
    + // Nope we are the ones who should create the new connection.
          auto client = cf_->MakeBootstrap();
          auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
          clients_.insert(std::make_pair(sn, client));
          connections_.insert(std::make_pair(sn, dispatcher));
          return dispatcher;
        }
    - return found->second;
      }

    -void ConnectionPool::close(const ServerName &sn) {
    +void ConnectionPool::Close(const ServerName &sn) {
        SharedMutexWritePriority::WriteHolder holder{map_mutex_};

        auto found = connections_.find(sn);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e430/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 60f00de..605a81b 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -80,14 +80,16 @@ public:
         * Get a connection to the server name. Start time is ignored.
         * This can be a blocking operation for a short time.
         */
    - std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
    + std::shared_ptr<HBaseService> Get(const hbase::pb::ServerName &sn);

        /**
         * Close/remove a connection.
         */
    - void close(const hbase::pb::ServerName &sn);
    + void Close(const hbase::pb::ServerName &sn);

      private:
    + std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
    + std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
                           ServerNameHash, ServerNameEquals>
            connections_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e430/hbase-native-client/core/client.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
    index 4db82c4..ba24bb9 100644
    --- a/hbase-native-client/core/client.h
    +++ b/hbase-native-client/core/client.h
    @@ -48,9 +48,9 @@ public:
        ~Client();

      private:
    - LocationCache location_cache_;
        std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
        std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
    + LocationCache location_cache_;
      };

      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e430/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 9f2a0ef..6ba8add 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -115,7 +115,7 @@ Future<std::shared_ptr<RegionLocation>>
      LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
        return this->LocateMeta()
            .via(cpu_executor_.get())
    - .then([this](ServerName sn) { return this->cp_.get(sn); })
    + .then([this](ServerName sn) { return this->cp_.Get(sn); })
          .then([tn, row, this](std::shared_ptr<HBaseService> service) {
              return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
    @@ -134,67 +134,25 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
            })
            .then([this](std::shared_ptr<RegionLocation> rl) {
              // Now fill out the connection.
    - rl->set_service(cp_.get(rl->server_name()));
    + rl->set_service(cp_.Get(rl->server_name()));
              return rl;
            });
      }

    -/**
    - * Filter to remove a service from the location cache and the connection cache
    - * on errors
    - * or on cloase.
    - */
    -class RemoveServiceFilter
    - : public ServiceFilter<std::unique_ptr<Request>, Response> {
    -
    -public:
    - /** Create a new filter. */
    - RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    - ConnectionPool &cp)
    - : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
    - cp_(cp) {}
    -
    - /**
    - * Close will remove the connection from all caches.
    - */
    - folly::Future<folly::Unit> close() override {
    - if (!released.exchange(true)) {
    - return this->service_->close().then([this]() {
    - // TODO(eclark): remove the service from the meta cache.
    - this->cp_.close(this->sn_);
    - });
    - } else {
    - return folly::makeFuture();
    - }
    - }
    -
    - /** Has this been closed */
    - virtual bool isAvailable() override {
    - return !released && service_->isAvailable();
    - }
    -
    - /** Send the message. */
    - folly::Future<Response> operator()(unique_ptr<Request> req) override {
    - // TODO(eclark): add in an on error handler that will
    - // remove the region location from the cache if needed.
    - // Also close the connection if this is likely to be an error
    - // that needs to get a new connection.
    - return (*this->service_)(std::move(req));
    - }
    -
    -private:
    - std::atomic<bool> released{false};
    - hbase::pb::ServerName sn_;
    - ConnectionPool &cp_;
    -};
    -
      std::shared_ptr<RegionLocation>
      LocationCache::CreateLocation(const Response &resp) {
        auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
        auto &results = resp_msg->results().Get(0);
        auto &cells = results.cell();
    - auto ri = folly::to<RegionInfo>(cells.Get(0).value());
    - auto sn = folly::to<ServerName>(cells.Get(1).value());
    - return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
    - nullptr);
    +
    + // TODO(eclark): There should probably be some better error
    + // handling around this.
    + auto cell_zero = cells.Get(0).value();
    + auto cell_one = cells.Get(1).value();
    + auto row = cells.Get(0).row();
    +
    + auto region_info = folly::to<RegionInfo>(cell_zero);
    + auto server_name = folly::to<ServerName>(cell_one);
    + return std::make_shared<RegionLocation>(row, std::move(region_info),
    + server_name, nullptr);
      }
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15724 Use explicit docker image

    Summary:
    Set the tag for cpp docker image
    Don't set the env on command line

    Test Plan: Build it.

    Differential Revision: https://reviews.facebook.net/D57315


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fef51e2a
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fef51e2a
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fef51e2a

    Branch: refs/heads/HBASE-14850
    Commit: fef51e2a2ec7fd4768e7d626b2dc74d9b99acde8
    Parents: a6a2878
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed Apr 27 10:51:05 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/Dockerfile | 4 +++-
      hbase-native-client/bin/start-docker.sh | 1 -
      2 files changed, 3 insertions(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/fef51e2a/hbase-native-client/Dockerfile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
    index 1524c56..8e33e74 100644
    --- a/hbase-native-client/Dockerfile
    +++ b/hbase-native-client/Dockerfile
    @@ -15,13 +15,15 @@
      # See the License for the specific language governing permissions and
      # limitations under the License.

    -FROM pjameson/buck-folly-watchman
    +FROM pjameson/buck-folly-watchman:20160425

      ARG CC=/usr/bin/gcc-5
      ARG CXX=/usr/bin/g++-5
      ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
      ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"

    +ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
    +
      RUN apt-get install -y vim maven inetutils-ping python-pip && \
            pip install yapf && \
            apt-get -qq clean && \

    http://git-wip-us.apache.org/repos/asf/hbase/blob/fef51e2a/hbase-native-client/bin/start-docker.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
    index 9cbd8b7..5c03eb9 100755
    --- a/hbase-native-client/bin/start-docker.sh
    +++ b/hbase-native-client/bin/start-docker.sh
    @@ -53,7 +53,6 @@ docker build -t hbase_native .

      # After the image is built run the thing
      docker run -p 16010:16010/tcp \
    - -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
                 -v ${BASE_DIR}/..:/usr/src/hbase \
                 -v ~/.m2:/root/.m2 \
                 -it hbase_native /bin/bash
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.

    Added check ~/.m2 folder exists; moved scripts to ./bin


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e1a422f1
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e1a422f1
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e1a422f1

    Branch: refs/heads/HBASE-14850
    Commit: e1a422f1f05151045e2ae8323c13464370ec165f
    Parents: 369fb30
    Author: Mikhail Antonov <antonov@apache.org>
    Authored: Wed Jan 6 15:08:21 2016 -0800
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/.buckconfig | 2 +-
      hbase-native-client/Dockerfile | 2 +-
      hbase-native-client/bin/start-docker.sh | 10 ++++-
      .../bin/start_local_hbase_and_wait.sh | 26 ++++++++++++
      .../bin/stop_local_hbase_and_wait.sh | 26 ++++++++++++
      hbase-native-client/core/BUCK | 15 +++++++
      .../core/HBaseNativeClientTestEnv.cc | 42 ++++++++++++++++++++
      .../core/SampleNativeClientTest.cc | 28 +++++++++++++
      hbase-native-client/core/test_env.h | 30 ++++++++++++++
      9 files changed, 178 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/.buckconfig
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/.buckconfig b/hbase-native-client/.buckconfig
    index 3227a2a..402ef27 100644
    --- a/hbase-native-client/.buckconfig
    +++ b/hbase-native-client/.buckconfig
    @@ -1,2 +1,2 @@
      [cxx]
    - gtest_dep = //third-party/googletest/googletest:google-test
    + gtest_dep = //third-party:google-test

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/Dockerfile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
    index 70e823b..5f17f04 100644
    --- a/hbase-native-client/Dockerfile
    +++ b/hbase-native-client/Dockerfile
    @@ -17,6 +17,6 @@

      FROM pjameson/buck-folly-watchman

    -RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
    +RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim maven inetutils-ping

      WORKDIR /usr/local/src/hbase/hbase-native-client

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/bin/start-docker.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
    index 1c9b02e..bf38912 100755
    --- a/hbase-native-client/bin/start-docker.sh
    +++ b/hbase-native-client/bin/start-docker.sh
    @@ -28,5 +28,13 @@ if [[ ! -d third-party/googletest ]]; then
              git clone https://github.com/google/googletest.git third-party/googletest
      fi

    +if [[ ! -d ~/.m2 ]]; then
    + echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
    + exit 1
    +fi;

    -docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native /bin/bash
    +docker run -p 16010:16010/tcp \
    + -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
    + -v ${PWD}/..:/usr/local/src/hbase \
    + -v ~/.m2:/root/.m2 \
    + -it hbase_native /bin/bash

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/bin/start_local_hbase_and_wait.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh b/hbase-native-client/bin/start_local_hbase_and_wait.sh
    new file mode 100755
    index 0000000..64d0b68
    --- /dev/null
    +++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
    @@ -0,0 +1,26 @@
    +#!/usr/bin/env bash
    +
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
    +
    +$PWD/../bin/start-hbase.sh
    +
    +until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
    +do
    + printf "Waiting for local HBase cluster to start\n"
    + sleep 1
    +done

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/bin/stop_local_hbase_and_wait.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
    new file mode 100755
    index 0000000..4e89334
    --- /dev/null
    +++ b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
    @@ -0,0 +1,26 @@
    +#!/usr/bin/env bash
    +
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
    +
    +$PWD/../bin/stop-hbase.sh
    +
    +while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
    +do
    + printf "Waiting for local HBase cluster to stop\n"
    + sleep 1
    +done

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index a7eaa9e..2e4e755 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -48,3 +48,18 @@ cxx_binary(
          'PUBLIC',
        ],
      )
    +
    +cxx_test(
    + name = "core_test",
    + headers = [
    + "test_env.h",
    + ],
    + srcs = [
    + "HBaseNativeClientTestEnv.cc",
    + "SampleNativeClientTest.cc",
    + ],
    + deps = [
    + ":core",
    + ],
    + run_test_separately = True,
    +)

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/HBaseNativeClientTestEnv.cc b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    new file mode 100644
    index 0000000..b8cb8db
    --- /dev/null
    +++ b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    @@ -0,0 +1,42 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include <gtest/gtest.h>
    +#include <core/test_env.h>
    +
    +namespace {
    +
    +class HBaseNativeClientTestEnv : public ::testing::Environment {
    + public:
    + void SetUp() override {
    + init_test_env();
    + }
    +
    + void TearDown() override {
    + clean_test_env();
    + }
    +};
    +
    +} // anonymous
    +
    +int main(int argc, char** argv) {
    + testing::InitGoogleTest(&argc, argv);
    + ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
    + return RUN_ALL_TESTS();
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/core/SampleNativeClientTest.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/SampleNativeClientTest.cc b/hbase-native-client/core/SampleNativeClientTest.cc
    new file mode 100644
    index 0000000..ef564f7
    --- /dev/null
    +++ b/hbase-native-client/core/SampleNativeClientTest.cc
    @@ -0,0 +1,28 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "gtest/gtest.h"
    +
    +/**
    + * Sample test.
    + */
    +TEST(SampleTest, sample) {
    + EXPECT_TRUE(true);
    +}
    +

    http://git-wip-us.apache.org/repos/asf/hbase/blob/e1a422f1/hbase-native-client/core/test_env.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/test_env.h b/hbase-native-client/core/test_env.h
    new file mode 100644
    index 0000000..5796ae1
    --- /dev/null
    +++ b/hbase-native-client/core/test_env.h
    @@ -0,0 +1,30 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include <cstdlib>
    +
    +inline void init_test_env() {
    + // start local HBase cluster to be reused by all tests
    + system("scripts/start_local_hbase_and_wait.sh");
    +}
    +
    +inline void clean_test_env() {
    + // shutdown local HBase cluster
    + system("scripts/stop_local_hbase_and_wait.sh");
    +}
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15826 Clean up ASF license issues


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6bc1ab39
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6bc1ab39
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6bc1ab39

    Branch: refs/heads/HBASE-14850
    Commit: 6bc1ab39e89f1d43dc6501e9127b55f9165b77cd
    Parents: aac4e43
    Author: Elliott Clark <eclark@apache.org>
    Authored: Fri May 13 14:53:47 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/Makefile | 17 +++++++++++++++++
      hbase-native-client/bin/copy-protobuf.sh | 16 ++++++++++++++++
      hbase-native-client/bin/format-code.sh | 16 ++++++++++++++++
      hbase-native-client/bin/start-local-hbase.sh | 1 -
      hbase-native-client/hbase.doxygen | 22 +++++++++++++++++++---
      hbase-native-client/serde/server-name.h | 19 +++++++++++++++++++
      6 files changed, 87 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/Makefile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
    index 90ec6ea..826233f 100644
    --- a/hbase-native-client/Makefile
    +++ b/hbase-native-client/Makefile
    @@ -1,3 +1,20 @@
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
    +
      build:
       $(shell buck build core/... )


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/bin/copy-protobuf.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/copy-protobuf.sh b/hbase-native-client/bin/copy-protobuf.sh
    index c9e70f2..36541cf 100755
    --- a/hbase-native-client/bin/copy-protobuf.sh
    +++ b/hbase-native-client/bin/copy-protobuf.sh
    @@ -1,4 +1,20 @@
      #!/usr/bin/env bash
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
      set -euo pipefail
      IFS=$'\n\t'


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/bin/format-code.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
    index 24d6515..36a114d 100755
    --- a/hbase-native-client/bin/format-code.sh
    +++ b/hbase-native-client/bin/format-code.sh
    @@ -1,4 +1,20 @@
      #!/usr/bin/env bash
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
      set -euo pipefail
      IFS=$'\n\t'


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/bin/start-local-hbase.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start-local-hbase.sh b/hbase-native-client/bin/start-local-hbase.sh
    index 49637d6..12069c2 100755
    --- a/hbase-native-client/bin/start-local-hbase.sh
    +++ b/hbase-native-client/bin/start-local-hbase.sh
    @@ -1,5 +1,4 @@
      #!/usr/bin/env bash
    -
      ##
      # Licensed to the Apache Software Foundation (ASF) under one
      # or more contributor license agreements. See the NOTICE file

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/hbase.doxygen
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/hbase.doxygen b/hbase-native-client/hbase.doxygen
    index 22e2d21..fef404c 100644
    --- a/hbase-native-client/hbase.doxygen
    +++ b/hbase-native-client/hbase.doxygen
    @@ -1,4 +1,20 @@
      # Doxyfile 1.8.11
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.

      #---------------------------------------------------------------------------
      # Project related configuration options
    @@ -103,14 +119,14 @@ WARN_LOGFILE =
      #---------------------------------------------------------------------------
      INPUT =
      INPUT_ENCODING = UTF-8
    -FILE_PATTERNS =
    +FILE_PATTERNS =
      RECURSIVE = YES
      EXCLUDE = third-party/ buck-out/
      EXCLUDE_SYMLINKS = NO
      EXCLUDE_PATTERNS = /*-test.cc
      EXCLUDE_SYMBOLS =
      EXAMPLE_PATH =
    -EXAMPLE_PATTERNS =
    +EXAMPLE_PATTERNS =
      EXAMPLE_RECURSIVE = NO
      IMAGE_PATH =
      INPUT_FILTER =
    @@ -142,7 +158,7 @@ IGNORE_PREFIX =
      # Configuration options related to the HTML output
      #---------------------------------------------------------------------------
      GENERATE_HTML = YES
    -HTML_OUTPUT = html
    +HTML_OUTPUT = html
      HTML_FILE_EXTENSION = .html
      HTML_HEADER =
      HTML_FOOTER =

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6bc1ab39/hbase-native-client/serde/server-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
    index 9844465..6551fdc 100644
    --- a/hbase-native-client/serde/server-name.h
    +++ b/hbase-native-client/serde/server-name.h
    @@ -1,3 +1,22 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
      #pragma once

      #include "if/HBase.pb.h"
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15761 Add on more server name tests


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/35b32b04
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/35b32b04
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/35b32b04

    Branch: refs/heads/HBASE-14850
    Commit: 35b32b049bec24407e095afa2413b0cd92caf674
    Parents: 361a395
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed May 4 12:04:18 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/serde/server-name-test.cc | 18 ++++++++++++++++++
      hbase-native-client/serde/server-name.h | 4 +++-
      2 files changed, 21 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/35b32b04/hbase-native-client/serde/server-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
    index 35dcbc1..2281fa2 100644
    --- a/hbase-native-client/serde/server-name-test.cc
    +++ b/hbase-native-client/serde/server-name-test.cc
    @@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
        ASSERT_EQ("test", sn.host_name());
        ASSERT_EQ(123, sn.port());
      }
    +
    +TEST(TestServerName, TestIps) {
    + auto sn = folly::to<ServerName>("127.0.0.1:999");
    + ASSERT_EQ("127.0.0.1", sn.host_name());
    + ASSERT_EQ(999, sn.port());
    +}
    +
    +TEST(TestServerName, TestThrow) {
    + ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here"));
    +}
    +
    +TEST(TestServerName, TestIPV6) {
    + auto sn = folly::to<ServerName>("[::::1]:123");
    +
    + ASSERT_EQ("[::::1]", sn.host_name());
    + ASSERT_EQ(123, sn.port());
    +
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/35b32b04/hbase-native-client/serde/server-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
    index bdba087..9844465 100644
    --- a/hbase-native-client/serde/server-name.h
    +++ b/hbase-native-client/serde/server-name.h
    @@ -12,7 +12,9 @@ template <class String> void parseTo(String in, ServerName &out) {
        std::string s = folly::to<std::string>(in);

        auto delim = s.rfind(":");
    - DCHECK(delim != std::string::npos);
    + if (delim == std::string::npos) {
    + throw std::runtime_error("Couldn't parse server name");
    + }
        out.set_host_name(s.substr(0, delim));
        // Now keep everything after the : (delim + 1) to the end.
        out.set_port(folly::to<int>(s.substr(delim + 1)));
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15730 Add on script to format all .h,.cc, and BUCK files.


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/74ff76aa
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/74ff76aa
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/74ff76aa

    Branch: refs/heads/HBASE-14850
    Commit: 74ff76aa533a85f5dea2bd9c1159d4c84249b303
    Parents: 3a98ebe
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed Apr 27 15:52:27 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/bin/format-code.sh | 7 +++++++
      hbase-native-client/connection/request.cc | 14 +++++++-------
      hbase-native-client/utils/BUCK | 8 ++++----
      hbase-native-client/utils/user-util-test.cc | 2 +-
      hbase-native-client/utils/user-util.cc | 4 ++--
      hbase-native-client/utils/user-util.h | 2 +-
      6 files changed, 22 insertions(+), 15 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/bin/format-code.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
    new file mode 100755
    index 0000000..24d6515
    --- /dev/null
    +++ b/hbase-native-client/bin/format-code.sh
    @@ -0,0 +1,7 @@
    +#!/usr/bin/env bash
    +set -euo pipefail
    +IFS=$'\n\t'
    +
    +
    +find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
    +find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/connection/request.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/request.cc b/hbase-native-client/connection/request.cc
    index 50ea029..d4a4603 100644
    --- a/hbase-native-client/connection/request.cc
    +++ b/hbase-native-client/connection/request.cc
    @@ -30,16 +30,16 @@ Request::Request(std::shared_ptr<google::protobuf::Message> req,

      std::unique_ptr<Request> Request::get() {
        return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
    - std::make_shared<hbase::pb::GetResponse>(),
    - "Get");
    + std::make_shared<hbase::pb::GetResponse>(),
    + "Get");
      }
      std::unique_ptr<Request> Request::mutate() {
    - return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
    - std::make_shared<hbase::pb::MutateResponse>(),
    - "Mutate");
    + return std::make_unique<Request>(
    + std::make_shared<hbase::pb::MutateRequest>(),
    + std::make_shared<hbase::pb::MutateResponse>(), "Mutate");
      }
      std::unique_ptr<Request> Request::scan() {
        return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
    - std::make_shared<hbase::pb::ScanResponse>(),
    - "Scan");
    + std::make_shared<hbase::pb::ScanResponse>(),
    + "Scan");
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/utils/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
    index 2b65b12..ad12f7d 100644
    --- a/hbase-native-client/utils/BUCK
    +++ b/hbase-native-client/utils/BUCK
    @@ -23,18 +23,18 @@ cxx_library(name="utils",
                      "user-util.cc",
                  ],
                  deps=[
    - '//third-party:folly',
    + '//third-party:folly',
                  ],
                  visibility=[
                      'PUBLIC',
                  ],
                  tests=[
    - ":user-util-test"
    - ],)
    + ":user-util-test"
    + ], )
      cxx_test(name="user-util-test",
               srcs=[
                   "user-util-test.cc",
               ],
               deps=[
                   ":utils",
    - ],)
    + ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/utils/user-util-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
    index 1e72bcc..7c11d8c 100644
    --- a/hbase-native-client/utils/user-util-test.cc
    +++ b/hbase-native-client/utils/user-util-test.cc
    @@ -17,9 +17,9 @@
       *
       */

    +#include <folly/Logging.h>
      #include <gtest/gtest.h>
      #include <string>
    -#include <folly/Logging.h>

      #include "utils/user-util.h"


    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/utils/user-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
    index c4427e3..b801537 100644
    --- a/hbase-native-client/utils/user-util.cc
    +++ b/hbase-native-client/utils/user-util.cc
    @@ -19,10 +19,10 @@

      #include "utils/user-util.h"

    +#include <folly/Logging.h>
      #include <pwd.h>
    -#include <unistd.h>
      #include <sys/types.h>
    -#include <folly/Logging.h>
    +#include <unistd.h>

      using namespace hbase;
      using namespace std;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/74ff76aa/hbase-native-client/utils/user-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
    index 4f74948..9ca28e3 100644
    --- a/hbase-native-client/utils/user-util.h
    +++ b/hbase-native-client/utils/user-util.h
    @@ -19,8 +19,8 @@

      #pragma once

    -#include <memory>
      #include <atomic>
    +#include <memory>
      #include <mutex>

      namespace hbase {
  • Eclark at May 18, 2016 at 10:49 pm
    HBASE-15771 Document all the public classes

    Summary: Add on a bunch of documentation around implementation so far.

    Test Plan:
         make doc
    Doxygen has no warnings

    Differential Revision: https://reviews.facebook.net/D57753


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/361a3952
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/361a3952
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/361a3952

    Branch: refs/heads/HBASE-14850
    Commit: 361a395299d5fc2affc96694a3861474a3f5ed17
    Parents: 7d7bc06
    Author: Elliott Clark <eclark@apache.org>
    Authored: Thu May 5 13:14:20 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/BUILDING.md | 18 +++++-
      .../connection/client-dispatcher.h | 11 +++-
      .../connection/client-handler.cc | 37 +++++-------
      hbase-native-client/connection/client-handler.h | 36 +++++++-----
      .../connection/connection-factory.h | 18 ++++++
      .../connection/connection-pool.cc | 5 +-
      .../connection/connection-pool.h | 44 +++++++++++++-
      hbase-native-client/connection/pipeline.h | 18 ++++++
      hbase-native-client/connection/request.h | 32 ++++++----
      hbase-native-client/connection/response.h | 33 +++++++++--
      hbase-native-client/core/BUCK | 1 -
      hbase-native-client/core/client.h | 12 ++++
      hbase-native-client/core/connection.cc | 20 -------
      hbase-native-client/core/connection.h | 26 ---------
      hbase-native-client/core/location-cache.cc | 14 ++++-
      hbase-native-client/core/location-cache.h | 36 ++++++++++--
      hbase-native-client/core/meta-utils.h | 12 ++++
      hbase-native-client/core/put.cc | 21 -------
      hbase-native-client/core/put.h | 27 ---------
      hbase-native-client/core/region-location.h | 41 +++++++++++++
      hbase-native-client/serde/rpc.cc | 1 -
      hbase-native-client/serde/rpc.h | 61 +++++++++++++++++++-
      hbase-native-client/serde/zk.h | 14 +++++
      hbase-native-client/utils/user-util.h | 18 ++++++
      24 files changed, 387 insertions(+), 169 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/BUILDING.md
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
    index 20ef2a0..4c06776 100644
    --- a/hbase-native-client/BUILDING.md
    +++ b/hbase-native-client/BUILDING.md
    @@ -17,7 +17,7 @@ specific language governing permissions and limitations
      under the License.
      -->

    -#Building HBase native client
    +# Building HBase native client

      The HBase native client build using buck and produces a linux library.

    @@ -58,6 +58,20 @@ buck build //core:simple-client
      ```

      That will build the library, then build and test everything, then build
    -the simple-client binary. Buck will find all modules used, and compile
    +the simple-client binary. Buck will find all modules used, and compile
      them in parallel, caching the results. Output from buck is in the buck-out
      foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
    +
    +
    +# Make
    +
    +If learning buck isn't your thing there is a Makefile wrapper for your
    +convenience.
    +
    +```
    +make help
    +make check
    +make clean
    +make all
    +make build
    +```

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/client-dispatcher.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
    index 4435a1b..4bfb35d 100644
    --- a/hbase-native-client/connection/client-dispatcher.h
    +++ b/hbase-native-client/connection/client-dispatcher.h
    @@ -30,17 +30,22 @@
      #include "connection/response.h"

      namespace hbase {
    +/**
    + * Dispatcher that assigns a call_id and then routes the response back to the future.
    + */
      class ClientDispatcher
          : public wangle::ClientDispatcherBase<SerializePipeline,
                                                std::unique_ptr<Request>, Response> {
      public:
    + /** Create a new ClientDispatcher */
        ClientDispatcher();
    - ~ClientDispatcher() {
    - LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
    - }
    + /** Read a response off the pipeline. */
        void read(Context *ctx, Response in) override;
    + /** Take a request as a call and send it down the pipeline. */
        folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
    + /** Close the dispatcher and the associated pipeline. */
        folly::Future<folly::Unit> close(Context *ctx) override;
    + /** Close the dispatcher and the associated pipeline. */
        folly::Future<folly::Unit> close() override;

      private:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index 4fdb7ae..cae03c7 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
      using google::protobuf::Message;

      ClientHandler::ClientHandler(std::string user_name)
    - : user_name_(user_name), serde_(), header_info_(),
    + : user_name_(user_name), serde_(), once_flag_(),
            resp_msgs_(
                make_unique<folly::AtomicHashMap<
                    uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
    @@ -75,36 +75,25 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
            used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
            // Make sure that bytes were parsed.
            CHECK(used_bytes == buf->length());
    - received.set_response(resp_msg);
    + received.set_resp_msg(resp_msg);
          }
          ctx->fireRead(std::move(received));
        }
      }

      Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
    - // Keep track of if we have sent the header.
    - //
    - // even though the bool is atomic we can load it lazily here.
    - if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
    -
    - // Grab the lock.
    - // We need to make sure that no one gets past here without there being a
    - // hearder sent.
    - std::lock_guard<std::mutex> lock(header_info_->mutex_);
    -
    - // Now see if we are the first thread to get through.
    - //
    - // If this is the first thread to get through then the
    - // need_send_header will have been true before this.
    - if (header_info_->need_.exchange(false)) {
    - auto pre = serde_.Preamble();
    - auto header = serde_.Header(user_name_);
    - pre->appendChain(std::move(header));
    - ctx->fireWrite(std::move(pre));
    - }
    - }
    -
    + // We need to send the header once.
    + // So use call_once to make sure that only one thread wins this.
    + std::call_once((*once_flag_), [ctx, this]() {
    + auto pre = serde_.Preamble();
    + auto header = serde_.Header(user_name_);
    + pre->appendChain(std::move(header));
    + ctx->fireWrite(std::move(pre));
    + });
    +
    + // Now store the call id to response.
        resp_msgs_->insert(r->call_id(), r->resp_msg());
    + // Send the data down the pipeline.
        return ctx->fireWrite(
            serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index 1a4275f..984a947 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -41,17 +41,38 @@ class Message;

      namespace hbase {

    +/**
    + * wangle::Handler implementation to convert hbase::Request to IOBuf and
    + * convert IOBuf to hbase::Response.
    + *
    + * This class deals with sending the connection header and preamble
    + * on first request.
    + */
      class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
                                                   Response, std::unique_ptr<Request>,
                                                   std::unique_ptr<folly::IOBuf>> {
      public:
    + /**
    + * Create the handler
    + * @param user_name the user name of the user running this process.
    + */
        ClientHandler(std::string user_name);
    +
    + /**
    + * Get bytes from the wire.
    + * This should be the full message as the length field decoder should be
    + * in the pipeline before this.
    + */
        void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
    +
    + /**
    + * Write the data down the wire.
    + */
        folly::Future<folly::Unit> write(Context *ctx,
                                         std::unique_ptr<Request> r) override;

      private:
    - std::unique_ptr<HeaderInfo> header_info_;
    + std::unique_ptr<std::once_flag> once_flag_;
        std::string user_name_;
        RpcSerde serde_;

    @@ -60,17 +81,4 @@ private:
            uint32_t, std::shared_ptr<google::protobuf::Message>>>
            resp_msgs_;
      };
    -
    -/**
    - * Class to contain the info about if the connection header and preamble has
    - * been sent.
    - *
    - * We use a serperate class here so that ClientHandler is relocatable.
    - */
    -class HeaderInfo {
    -public:
    - HeaderInfo() : need_(true), mutex_() {}
    - std::atomic<bool> need_;
    - std::mutex mutex_;
    -};
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/connection-factory.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
    index 2284a7c..8b6d8d8 100644
    --- a/hbase-native-client/connection/connection-factory.h
    +++ b/hbase-native-client/connection/connection-factory.h
    @@ -28,14 +28,32 @@
      #include "connection/service.h"

      namespace hbase {
    +
    +/**
    + * Class to create a ClientBootstrap and turn it into a connected
    + * pipeline.
    + */
      class ConnectionFactory {
      public:
    + /**
    + * Constructor.
    + * There should only be one ConnectionFactory per client.
    + */
        ConnectionFactory();
    + /** Default Desctructor */
        virtual ~ConnectionFactory() = default;

    + /**
    + * Create a BootStrap from which a connection can be made.
    + */
        virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
        MakeBootstrap();

    + /**
    + * Connect a ClientBootstrap to a server and return the pipeline.
    + *
    + * This is mostly visible so that mocks can override socket connections.
    + */
        virtual std::shared_ptr<HBaseService>
        Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
                const std::string &hostname, int port);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index 6ed5ad9..75f343e 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -26,11 +26,11 @@ using std::mutex;
      using std::unique_ptr;
      using std::shared_ptr;
      using hbase::pb::ServerName;
    +using hbase::ConnectionPool;
    +using hbase::HBaseService;
      using folly::SharedMutexWritePriority;
      using folly::SocketAddress;

    -namespace hbase {
    -
      ConnectionPool::ConnectionPool()
          : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
            map_mutex_() {}
    @@ -75,4 +75,3 @@ void ConnectionPool::close(const ServerName &sn) {
        auto service = found->second;
        connections_.erase(found);
      }
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 907afdb..5edd407 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -18,6 +18,7 @@
       */
      #pragma once

    +#include <boost/functional/hash.hpp>
      #include <folly/SharedMutex.h>
      #include <mutex>
      #include <unordered_map>
    @@ -27,26 +28,63 @@
      #include "if/HBase.pb.h"

      namespace hbase {
    +
    +/** Equals function for server name that ignores start time */
      struct ServerNameEquals {
    +
    + /** equals */
        bool operator()(const hbase::pb::ServerName &lhs,
                        const hbase::pb::ServerName &rhs) const {
          return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
        }
      };
    +
    +/** Hash for ServerName that ignores the start time. */
      struct ServerNameHash {
    + /** hash */
        std::size_t operator()(hbase::pb::ServerName const &s) const {
    - std::size_t h1 = std::hash<std::string>()(s.host_name());
    - std::size_t h2 = std::hash<uint32_t>()(s.port());
    - return h1 ^ (h2 << 2);
    + std::size_t h = 0;
    + boost::hash_combine(h, s.host_name());
    + boost::hash_combine(h, s.port());
    + return h;
        }
      };

    +/**
    + * @brief Connection pooling for HBase rpc connection.
    + *
    + * This is a thread safe connection pool. It allows getting
    + * a shared connection to HBase by server name. This is
    + * useful for keeping a single connection no matter how many regions a
    + * regionserver has on it.
    + */
      class ConnectionPool {
      public:
    + /** Create connection pool wit default connection factory */
        ConnectionPool();
    +
    + /**
    + * Desctructor.
    + * All connections will be close.
    + * All connections will be released
    + */
        ~ConnectionPool();
    +
    + /**
    + * Constructor that allows specifiying the connetion factory.
    + * This is useful for testing.
    + */
        explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
    +
    + /**
    + * Get a connection to the server name. Start time is ignored.
    + * This can be a blocking operation for a short time.
    + */
        std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
    +
    + /**
    + * Close/remove a connection.
    + */
        void close(const hbase::pb::ServerName &sn);

      private:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/pipeline.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
    index e90cb93..88d7845 100644
    --- a/hbase-native-client/connection/pipeline.h
    +++ b/hbase-native-client/connection/pipeline.h
    @@ -26,12 +26,30 @@
      #include "utils/user-util.h"

      namespace hbase {
    +
    +/** Pipeline to turn IOBuf into requests */
      using SerializePipeline =
          wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;

    +/**
    + * Factory to create new pipelines for HBase RPC's.
    + */
      class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
      public:
    + /**
    + * Constructor. This will create user util.
    + */
        RpcPipelineFactory();
    +
    + /**
    + * Create a new pipeline.
    + * The pipeline will be:
    + *
    + * - Async Socke Handler
    + * - Event Base Handler
    + * - Length Field Based Frame Decoder
    + * - Client Handler
    + */
        SerializePipeline::Ptr
        newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;


    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/request.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
    index 743c469..634d1ba 100644
    --- a/hbase-native-client/connection/request.h
    +++ b/hbase-native-client/connection/request.h
    @@ -25,30 +25,42 @@
      #include <string>

      namespace hbase {
    +
    +/**
    + * Main request class.
    + * This holds the request object and the un-filled in approriatley typed
    + * response object.
    + */
      class Request {
      public:
    + /** Create a request object for a get */
        static std::unique_ptr<Request> get();
    + /** Create a request object for a mutate */
        static std::unique_ptr<Request> mutate();
    + /** Create a request object for a scan */
        static std::unique_ptr<Request> scan();

    + /**
    + * This should be private. Do not use this.
    + *
    + *
    + * Constructor that's public for make_unique. This sets all the messages and
    + * method name.
    + */
        Request(std::shared_ptr<google::protobuf::Message> req,
                std::shared_ptr<google::protobuf::Message> resp, std::string method);

    + /** Get the call id. */
        uint32_t call_id() { return call_id_; }
    + /** Set the call id. This should only be set once. */
        void set_call_id(uint32_t call_id) { call_id_ = call_id; }
    -
    + /** Get the backing request protobuf message. */
        std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
    + /** Get the backing response protobuf message. */
        std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
    -
    - void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
    - req_msg_ = msg;
    - }
    - void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
    - resp_msg_ = msg;
    - }
    -
    + /** Get the method name. This is used to the the receiving rpc server what
    + * method type to decode. */
        std::string method() { return method_; }
    - void set_method(std::string method) { method_ = method; }

      private:
        uint32_t call_id_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/connection/response.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
    index d2eb19a..aac57c2 100644
    --- a/hbase-native-client/connection/response.h
    +++ b/hbase-native-client/connection/response.h
    @@ -29,20 +29,41 @@ class Message;

      namespace hbase {

    +/**
    + * @brief Class representing a rpc response
    + *
    + * This is the class sent to a service.
    + */
      class Response {
      public:
    - Response() : call_id_(0), response_(nullptr) {}
    + /**
    + * Constructor.
    + * Initinalizes the call id to 0. 0 should never be a valid call id.
    + */
    + Response() : call_id_(0), resp_msg_(nullptr) {}
    +
    + /** Get the call_id */
        uint32_t call_id() { return call_id_; }
    +
    + /** Set the call_id */
        void set_call_id(uint32_t call_id) { call_id_ = call_id; }
    - std::shared_ptr<google::protobuf::Message> response() const {
    - return response_;
    +
    + /**
    + * Get the response message.
    + * The caller is reponsible for knowing the type. In practice the call id is
    + * used to figure out the type.
    + */
    + std::shared_ptr<google::protobuf::Message> resp_msg() const {
    + return resp_msg_;
        }
    - void set_response(std::shared_ptr<google::protobuf::Message> response) {
    - response_ = std::move(response);
    +
    + /** Set the response message. */
    + void set_resp_msg(std::shared_ptr<google::protobuf::Message> response) {
    + resp_msg_ = std::move(response);
        }

      private:
        uint32_t call_id_;
    - std::shared_ptr<google::protobuf::Message> response_;
    + std::shared_ptr<google::protobuf::Message> resp_msg_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index ef8c2f8..485f9ba 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -20,7 +20,6 @@ cxx_library(
          name="core",
          exported_headers=[
              "client.h",
    - "connection.h",
              "hbase_macros.h",
              "region-location.h",
              "location-cache.h",

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/client.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
    index 4bed751..4a6d23b 100644
    --- a/hbase-native-client/core/client.h
    +++ b/hbase-native-client/core/client.h
    @@ -28,8 +28,20 @@
      #include "if/Cell.pb.h"

      namespace hbase {
    +
    +/**
    + * Client.
    + *
    + * This is the class that provides access to an HBase cluster.
    + * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
    + */
      class Client {
      public:
    +
    + /**
    + * Create a new client.
    + * @param quorum_spec Where to connect to get Zookeeper bootstrap information.
    + */
        explicit Client(std::string quorum_spec);

      private:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/connection.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/connection.cc b/hbase-native-client/core/connection.cc
    deleted file mode 100644
    index e9a28eb..0000000
    --- a/hbase-native-client/core/connection.cc
    +++ /dev/null
    @@ -1,20 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -#include "core/connection.h"
    -
    -void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/connection.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/connection.h b/hbase-native-client/core/connection.h
    deleted file mode 100644
    index 5c9d6b8..0000000
    --- a/hbase-native-client/core/connection.h
    +++ /dev/null
    @@ -1,26 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -
    -#pragma once
    -
    -class Connection {
    - char *zk_quorum;
    -
    -public:
    - void set_zk_quorum(char *zk_q);
    -};

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index e2a6251..583d305 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -128,15 +128,24 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
            });
      }

    +/**
    + * Filter to remove a service from the location cache and the connection cache on errors
    + * or on cloase.
    + */
      class RemoveServiceFilter
          : public ServiceFilter<std::unique_ptr<Request>, Response> {

      public:
    +
    + /** Create a new filter. */
        RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
                            ConnectionPool &cp)
            : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
              cp_(cp) {}

    + /**
    + * Close will remove the connection from all caches.
    + */
        folly::Future<folly::Unit> close() override {
          if (!released.exchange(true)) {
            return this->service_->close().then([this]() {
    @@ -148,10 +157,13 @@ public:
          }
        }

    +
    + /** Has this been closed */
        virtual bool isAvailable() override {
          return !released && service_->isAvailable();
        }

    + /** Send the message. */
        folly::Future<Response> operator()(unique_ptr<Request> req) override {
          // TODO(eclark): add in an on error handler that will
          // remove the region location from the cache if needed.
    @@ -168,7 +180,7 @@ private:

      std::shared_ptr<RegionLocation>
      LocationCache::CreateLocation(const Response &resp) {
    - auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
    + auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
        auto &results = resp_msg->results().Get(0);
        auto &cells = results.cell();
        auto ri = folly::to<RegionInfo>(cells.Get(0).value());

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index 7f76428..88bec18 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -33,25 +33,48 @@
      #include "serde/table-name.h"

      namespace hbase {
    -
    +// Forward
      class Request;
      class Response;
      namespace pb {
      class ServerName;
      }

    +/**
    + * Class that can look up and cache locations.
    + */
      class LocationCache {
      public:
    - explicit LocationCache(std::string quorum_spec,
    - std::shared_ptr<folly::Executor> executor);
    + /**
    + * Constructor.
    + * @param quorum_spec Where to connect for Zookeeper.
    + * @param executor The cpu executor to run on.
    + */
    + LocationCache(std::string quorum_spec,
    + std::shared_ptr<folly::Executor> executor);
    + /**
    + * Destructor.
    + * This will clean up the zookeeper connections.
    + */
        ~LocationCache();
    - // Meta Related Methods.
    - // These are only public until testing is complete
    +
    + /**
    + * Where is meta hosted.
    + *
    + * TODO: This should be a RegionLocation.
    + */
        folly::Future<hbase::pb::ServerName> LocateMeta();
    +
    + /**
    + * Go read meta and find out where a region is located.
    + */
        folly::Future<std::shared_ptr<RegionLocation>>
        LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
    +
    + /**
    + * Remove the cached location of meta.
    + */
        void InvalidateMeta();
    - ConnectionPool cp_;

      private:
        void RefreshMetaLocation();
    @@ -63,6 +86,7 @@ private:
        std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
        std::mutex meta_lock_;
        MetaUtil meta_util_;
    + ConnectionPool cp_;

        // TODO: migrate this to a smart pointer with a deleter.
        zhandle_t *zk_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/meta-utils.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
    index dfef065..f6cc84f 100644
    --- a/hbase-native-client/core/meta-utils.h
    +++ b/hbase-native-client/core/meta-utils.h
    @@ -26,10 +26,22 @@

      namespace hbase {

    +/**
    + * @brief Utility for meta operations.
    + */
      class MetaUtil {
      public:
    + /**
    + * Given a table and a row give the row key from which to start a scan to find
    + * region locations.
    + */
        std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
                                       const std::string &row) const;
    +
    + /**
    + * Given a row we're trying to access create a request to look up the
    + * location.
    + */
        std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
                                             const std::string &row) const;
      };

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/put.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/put.cc b/hbase-native-client/core/put.cc
    deleted file mode 100644
    index 806a478..0000000
    --- a/hbase-native-client/core/put.cc
    +++ /dev/null
    @@ -1,21 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -
    -#include "core/put.h"
    -
    -Put::~Put() {}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/put.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/put.h b/hbase-native-client/core/put.h
    deleted file mode 100644
    index d34aa0d..0000000
    --- a/hbase-native-client/core/put.h
    +++ /dev/null
    @@ -1,27 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#pragma once
    -
    -#include "core/mutation.h"
    -
    -class Put : public Mutation {
    -public:
    - ~Put();
    -};

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/core/region-location.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
    index 7887526..a3d832e 100644
    --- a/hbase-native-client/core/region-location.h
    +++ b/hbase-native-client/core/region-location.h
    @@ -25,19 +25,60 @@

      namespace hbase {

    +/**
    + * @brief class to hold where a region is located.
    + *
    + * This class holds where a region is located, the information about it, the
    + * region name, and a connection to the service used for connecting to it.
    + */
      class RegionLocation {
      public:
    + /**
    + * Constructor.
    + * @param region_name The region name of this region.
    + * @param ri The decoded RegionInfo of this region.
    + * @param sn The server name of the HBase regionserver thought to be hosting
    + * this region.
    + * @param service the connected service to the regionserver.
    + */
        RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
                       hbase::pb::ServerName sn,
                       std::shared_ptr<HBaseService> service)
            : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}

    + /**
    + * Get a reference to the regio info
    + */
        const hbase::pb::RegionInfo &region_info() { return ri_; }
    +
    + /**
    + * Get a reference to the server name
    + */
        const hbase::pb::ServerName &server_name() { return sn_; }
    +
    + /**
    + * Get a reference to the region name.
    + */
        const std::string &region_name() { return region_name_; }
    +
    + /**
    + * Get a service. This could be closed or null. It's the caller's
    + * responsibility to check.
    + */
        std::shared_ptr<HBaseService> service() { return service_; }
    +
    + /**
    + * Set the service.
    + * This should be used if the region moved or if the connection is thought to
    + * be bad and a new tcp connection needs to be made.
    + */
        void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }

    + /**
    + * Set the servername if the region has moved.
    + */
    + void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
    +
      private:
        std::string region_name_;
        hbase::pb::RegionInfo ri_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/serde/rpc.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/rpc.cc b/hbase-native-client/serde/rpc.cc
    index 4c3c999..b573738 100644
    --- a/hbase-native-client/serde/rpc.cc
    +++ b/hbase-native-client/serde/rpc.cc
    @@ -85,7 +85,6 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
      }

      RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
    -RpcSerde::~RpcSerde() {}

      unique_ptr<IOBuf> RpcSerde::Preamble() {
        auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/serde/rpc.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/rpc.h b/hbase-native-client/serde/rpc.h
    index cefb583..b28b4a3 100644
    --- a/hbase-native-client/serde/rpc.h
    +++ b/hbase-native-client/serde/rpc.h
    @@ -32,22 +32,81 @@ class Message;
      }

      namespace hbase {
    +
    +/**
    + * @brief Class for serializing a deserializing rpc formatted data.
    + *
    + * RpcSerde is the one stop shop for reading/writing data to HBase daemons.
    + * It should throw exceptions if anything goes wrong.
    + */
      class RpcSerde {
      public:
    + /**
    + * Constructor assumes the default auth type.
    + */
        RpcSerde();
    - virtual ~RpcSerde();
    +
    + /**
    + * Destructor. This is provided just for testing purposes.
    + */
    + virtual ~RpcSerde() = default;
    +
    + /**
    + * Pase a message in the delimited format.
    + *
    + * A message in delimited format consists of the following:
    + *
    + * - a protobuf var int32.
    + * - A protobuf object serialized.
    + */
        int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
    +
    + /**
    + * Create a new connection preamble in a new IOBuf.
    + */
        std::unique_ptr<folly::IOBuf> Preamble();
    +
    + /**
    + * Create the header protobuf object and serialize it to a new IOBuf.
    + * Header is in the following format:
    + *
    + * - Big endian length
    + * - ConnectionHeader object serialized out.
    + */
        std::unique_ptr<folly::IOBuf> Header(const std::string &user);
    +
    + /**
    + * Serialize a request message into a protobuf.
    + * Request consists of:
    + *
    + * - Big endian length
    + * - RequestHeader object
    + * - The passed in Message object
    + */
        std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
                                              const std::string &method,
                                              const google::protobuf::Message *msg);
    +
    + /**
    + * Serialize a message in the delimited format.
    + * Delimited format consists of the following:
    + *
    + * - A protobuf var int32
    + * - The message object seriailized after that.
    + */
        std::unique_ptr<folly::IOBuf>
        SerializeDelimited(const google::protobuf::Message &msg);

    + /**
    + * Serilalize a message. This does not add any length prepend.
    + */
        std::unique_ptr<folly::IOBuf>
        SerializeMessage(const google::protobuf::Message &msg);

    + /**
    + * Prepend a length IOBuf to the given IOBuf chain.
    + * This involves no copies or moves of the passed in data.
    + */
        std::unique_ptr<folly::IOBuf>
        PrependLength(std::unique_ptr<folly::IOBuf> msg);


    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/serde/zk.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk.h b/hbase-native-client/serde/zk.h
    index b672bf4..fa3de5b 100644
    --- a/hbase-native-client/serde/zk.h
    +++ b/hbase-native-client/serde/zk.h
    @@ -28,8 +28,22 @@ class IOBuf;
      }

      namespace hbase {
    +
    +/** @brief A class to convert data from ZooKeeper to other formats.
    + *
    + * This class will convert data to and from Zookeeper into protobuf objects.
    + *
    + */
      class ZkDeserializer {
      public:
    + /**
    + * Merge the data from a buffer into a given message.
    + *
    + * @param buf Naked pointer to iobuf containing data read from zookeeper.
    + * @param out Naked pointer into which the data will be merged. The message
    + * should be the correct type.
    + * @return returns true if the parsing was successful.
    + */
        bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/361a3952/hbase-native-client/utils/user-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
    index da95d78..0b4cc73 100644
    --- a/hbase-native-client/utils/user-util.h
    +++ b/hbase-native-client/utils/user-util.h
    @@ -24,12 +24,30 @@
      #include <mutex>

      namespace hbase {
    +
    +/**
    + * @brief Class to help with user/group information.
    + *
    + * This class will get the current user, and information about them. It caches
    + * the user information after the first invocation.
    + */
      class UserUtil {
      public:
    + /**
    + * Constructor.
    + */
        UserUtil();
    +
    + /**
    + * Get the username of the user owning this process. This is thread safe and
    + * lockless for every invocation other than the first one.
    + */
        std::string user_name();

      private:
    + /**
    + * Compute the username. This will block.
    + */
        void compute_user_name();
        std::atomic<bool> init_;
        std::string user_name_;
  • Eclark at May 18, 2016 at 10:54 pm
    HBASE-15821 Document TestUtil


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f49f262f
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f49f262f
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f49f262f

    Branch: refs/heads/HBASE-14850
    Commit: f49f262f3c7efcae228713d3371df69d7c49efce
    Parents: 616405a
    Author: Elliott Clark <eclark@apache.org>
    Authored: Fri May 13 09:17:49 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/bin/format-code.sh | 2 +-
      .../connection/connection-pool.h | 4 ++--
      hbase-native-client/core/location-cache.cc | 2 +-
      hbase-native-client/core/location-cache.h | 2 +-
      hbase-native-client/test-util/test-util.cc | 20 +++++++++++++-------
      hbase-native-client/test-util/test-util.h | 20 +++++++++++++++-----
      6 files changed, 33 insertions(+), 17 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/bin/format-code.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
    index 36a114d..cc8b368 100755
    --- a/hbase-native-client/bin/format-code.sh
    +++ b/hbase-native-client/bin/format-code.sh
    @@ -19,5 +19,5 @@ set -euo pipefail
      IFS=$'\n\t'


    -find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
    +find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
      find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 605a81b..b8c950b 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -88,8 +88,8 @@ public:
        void Close(const hbase::pb::ServerName &sn);

      private:
    - std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
    - std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
    + std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
    + std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
                           ServerNameHash, ServerNameEquals>
            connections_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 6ba8add..efd2210 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
        return this->LocateMeta()
            .via(cpu_executor_.get())
            .then([this](ServerName sn) { return this->cp_.Get(sn); })
    - .then([tn, row, this](std::shared_ptr<HBaseService> service) {
    + .then([tn, row, this](std::shared_ptr<HBaseService> service) {
              return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
            .then([this](Response resp) {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index d435530..830cd96 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -55,7 +55,7 @@ public:
         * @param io_executor executor used to talk to the network
         */
        LocationCache(std::string quorum_spec,
    - std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
    + std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
                      std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
        /**
         * Destructor.

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/test-util/test-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc
    index e5fba48..88ce7c8 100644
    --- a/hbase-native-client/test-util/test-util.cc
    +++ b/hbase-native-client/test-util/test-util.cc
    @@ -24,13 +24,18 @@
      using hbase::TestUtil;
      using folly::Random;

    -const static int STR_LEN = 32;
    -
    -std::string TestUtil::RandString() {
    - auto s = std::string(STR_LEN, 'z');
    -
    - for (int i = 0; i < STR_LEN; i++) {
    +std::string TestUtil::RandString(int len) {
    + // Create the whole string.
    + // Filling everything with z's
    + auto s = std::string(len, 'z');
    +
    + // Now pick a bunch of random numbers
    + for (int i = 0; i < len; i++) {
    + // use Folly's random to get the numbers
    + // as I don't want to have to learn
    + // all the cpp rand invocation magic.
          auto r = Random::rand32('a', 'z');
    + // Cast that to ascii.
          s[i] = static_cast<char>(r);
        }
        return s;
    @@ -42,12 +47,13 @@ TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
        auto res_code = std::system(cmd.c_str());
        CHECK(res_code == 0);
      }
    +
      TestUtil::~TestUtil() {
        auto res_code = std::system("bin/stop-local-hbase.sh");
        CHECK(res_code == 0);
      }

    -void TestUtil::RunShellCmd(const std::string& command) {
    +void TestUtil::RunShellCmd(const std::string &command) {
        auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
        auto res_code = std::system(cmd_string.c_str());
        CHECK(res_code == 0);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f49f262f/hbase-native-client/test-util/test-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h
    index 395b157..20e4981 100644
    --- a/hbase-native-client/test-util/test-util.h
    +++ b/hbase-native-client/test-util/test-util.h
    @@ -30,20 +30,30 @@ namespace hbase {
       */
      class TestUtil {
      public:
    -
        /**
         * Creating a TestUtil will spin up a cluster.
         */
        TestUtil();
    +
        /**
    - * Destroying a TestUtil will spin down a cluster.
    + * Destroying a TestUtil will spin down a cluster and remove the test dir.
         */
        ~TestUtil();
    +
    + /**
    + * Run a command in the hbase shell. Command should not include any double
    + * quotes.
    + *
    + * This should only be used until there is a good Admin support from the
    + * native client
    + */
    + void RunShellCmd(const std::string &command);
    +
        /**
    - * Run a command in the hbase shell.
    + * Create a random string. This random string is all letters, as such it is
    + * very good for use as a directory name.
         */
    - void RunShellCmd(const std::string& command);
    - static std::string RandString();
    + static std::string RandString(int len = 32);

      private:
        folly::test::TemporaryDirectory temp_dir_;
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15696 Move region location cache serialization into serde

    Summary: Use IOBuf for zk

    Test Plan: Added a unit test

    Differential Revision: https://reviews.facebook.net/D57147


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/65cd2da5
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/65cd2da5
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/65cd2da5

    Branch: refs/heads/HBASE-14850
    Commit: 65cd2da5f2a81f66635110deba17ccdfa774b3aa
    Parents: eabb568
    Author: Elliott Clark <eclark@apache.org>
    Authored: Fri Apr 22 22:21:45 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/Dockerfile | 7 +-
      .../connection/client-dispatcher.h | 8 ++
      .../connection/client-handler.cc | 7 +-
      hbase-native-client/connection/client-handler.h | 2 +-
      hbase-native-client/connection/pipeline.h | 2 +-
      hbase-native-client/core/client.cc | 4 +-
      hbase-native-client/core/client.h | 4 +-
      hbase-native-client/core/location-cache-test.cc | 2 +-
      hbase-native-client/core/location-cache.cc | 48 +++-----
      hbase-native-client/core/location-cache.h | 4 +-
      hbase-native-client/core/simple-client.cc | 6 +-
      hbase-native-client/serde/BUCK | 10 ++
      .../serde/client-deserializer-test.cc | 4 +-
      .../serde/client-deserializer.cc | 4 +-
      .../serde/client-serializer-test.cc | 2 +-
      hbase-native-client/serde/client-serializer.cc | 2 +-
      hbase-native-client/serde/client-serializer.h | 2 +-
      .../serde/zk-deserializer-test.cc | 123 +++++++++++++++++++
      hbase-native-client/serde/zk-deserializer.cc | 78 ++++++++++++
      hbase-native-client/serde/zk-deserializer.h | 35 ++++++
      hbase-native-client/utils/user-util-test.cc | 1 +
      21 files changed, 297 insertions(+), 58 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/Dockerfile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
    index 9ccc0d3..1524c56 100644
    --- a/hbase-native-client/Dockerfile
    +++ b/hbase-native-client/Dockerfile
    @@ -19,12 +19,11 @@ FROM pjameson/buck-folly-watchman

      ARG CC=/usr/bin/gcc-5
      ARG CXX=/usr/bin/g++-5
    -ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
    -ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
    +ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
    +ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"

    -RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \
    +RUN apt-get install -y vim maven inetutils-ping python-pip && \
            pip install yapf && \
    - ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
            apt-get -qq clean && \
            apt-get -y -qq autoremove && \
            rm -rf /var/lib/{apt,dpkg,cache,log}/ && \

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-dispatcher.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
    index c3987c9..89c7119 100644
    --- a/hbase-native-client/connection/client-dispatcher.h
    +++ b/hbase-native-client/connection/client-dispatcher.h
    @@ -37,6 +37,14 @@ public:

      private:
        std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
    + // Start at some number way above what could
    + // be there for un-initialized call id counters.
    + //
    + // This makes it easier to make sure that the're are
    + // no access to un-initialized variables.
    + //
    + // uint32_t has a max of 4Billion so 10 more or less is
    + // not a big deal.
        uint32_t current_call_id_ = 10;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index 74b23ef..7b7794d 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -19,15 +19,15 @@

      #include "connection/client-handler.h"

    -#include <glog/logging.h>
      #include <folly/Likely.h>
    +#include <glog/logging.h>

      #include <string>

      #include "connection/request.h"
      #include "connection/response.h"
    -#include "if/RPC.pb.h"
      #include "if/Client.pb.h"
    +#include "if/RPC.pb.h"

      using namespace hbase;
      using namespace folly;
    @@ -60,6 +60,9 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
        }
      }

    +
    +// TODO(eclark): Figure out how to handle the
    +// network errors that are going to come.
      Future<Unit> ClientHandler::write(Context *ctx, Request r) {
        // Keep track of if we have sent the header.
        if (UNLIKELY(need_send_header_)) {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index 38c5725..dbaf5a0 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -22,8 +22,8 @@

      #include <string>

    -#include "serde/client-serializer.h"
      #include "serde/client-deserializer.h"
    +#include "serde/client-serializer.h"

      // Forward decs.
      namespace hbase {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/connection/pipeline.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
    index 68ade48..8114fab 100644
    --- a/hbase-native-client/connection/pipeline.h
    +++ b/hbase-native-client/connection/pipeline.h
    @@ -18,8 +18,8 @@
       */
      #pragma once

    -#include <wangle/service/Service.h>
      #include <folly/io/IOBufQueue.h>
    +#include <wangle/service/Service.h>

      #include "connection/request.h"
      #include "connection/response.h"

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
    index 893894f..266c239 100644
    --- a/hbase-native-client/core/client.cc
    +++ b/hbase-native-client/core/client.cc
    @@ -21,8 +21,8 @@

      #include <folly/Logging.h>
      #include <folly/Random.h>
    -#include <glog/logging.h>
      #include <gflags/gflags.h>
    +#include <glog/logging.h>
      #include <wangle/concurrent/GlobalExecutor.h>

      #include <string>
    @@ -36,5 +36,5 @@ using namespace hbase::pb;
      namespace hbase {

      Client::Client(string quorum_spec)
    - : location_cache(quorum_spec, wangle::getCPUExecutor()) {}
    + : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/client.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
    index c2dc226..b583285 100644
    --- a/hbase-native-client/core/client.h
    +++ b/hbase-native-client/core/client.h
    @@ -19,8 +19,8 @@

      #pragma once

    -#include <folly/io/IOBuf.h>
      #include <folly/futures/Future.h>
    +#include <folly/io/IOBuf.h>

      #include <string>

    @@ -36,7 +36,7 @@ public:
        folly::Future<GetResult> get(const GetRequest &get_request);

      private:
    - LocationCache location_cache;
    + LocationCache location_cache_;
      };

      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
    index 70ca6f1..f3166fb 100644
    --- a/hbase-native-client/core/location-cache-test.cc
    +++ b/hbase-native-client/core/location-cache-test.cc
    @@ -16,8 +16,8 @@
       * limitations under the License.
       *
       */
    -#include <gtest/gtest.h>
      #include <folly/Memory.h>
    +#include <gtest/gtest.h>
      #include <wangle/concurrent/GlobalExecutor.h>

      #include "location-cache.h"

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 52e86e3..5925f4a 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -16,11 +16,13 @@
       * limitations under the License.
       *
       */
    -#include "location-cache.h"
    +#include "core/location-cache.h"

      #include <folly/Logging.h>
    +#include <folly/io/IOBuf.h>

      #include "if/ZooKeeper.pb.h"
    +#include "serde/zk-deserializer.h"

      using namespace std;
      using namespace folly;
    @@ -28,7 +30,7 @@ using namespace hbase::pb;
      using namespace hbase;

      // TODO(eclark): make this configurable on client creation
    -static const char META_LOCATION[] = "/hbase/meta-region-server";
    +static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";

      LocationCache::LocationCache(string quorum_spec,
                                   shared_ptr<folly::Executor> executor)
    @@ -66,45 +68,25 @@ void LocationCache::RefreshMetaLocation() {
      }

      ServerName LocationCache::ReadMetaLocation() {
    - char contents[4096];
    + auto buf = IOBuf::create(4096);
    + ZkDeserializer derser;
    +
        // This needs to be int rather than size_t as that's what ZK expects.
    - int len = sizeof(contents);
    + int len = buf->capacity();
        // TODO(elliott): handle disconnects/reconntion as needed.
    - int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
    + int zk_result =
    + zoo_get(this->zk_, META_ZNODE_NAME, 0,
    + reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
    + LOG(ERROR) << "len = " << len;
        if (zk_result != ZOK || len < 9) {
          LOG(ERROR) << "Error getting meta location.";
          throw runtime_error("Error getting meta location");
        }
    - // There should be a magic number for recoverable zk
    - if (static_cast<uint8_t>(contents[0]) != 255) {
    - LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
    - << unsigned(static_cast<uint8_t>(contents[0]));
    - throw runtime_error("Magic number not in znode data");
    - }
    - // pos will keep track of skipped bytes.
    - int pos = 1;
    - // How long is the id?
    - int id_len = 0;
    - for (int i = 0; i < 4; ++i) {
    - id_len = id_len << 8;
    - id_len = id_len | static_cast<uint8_t>(contents[pos]);
    - ++pos;
    - }
    - // Skip the id
    - pos += id_len;
    - // Then all protobuf's for HBase are prefixed with a magic string.
    - // PBUF, so we skip that.
    - // TODO(eclark): check to make sure that the magic string is correct
    - // though I am not sure that will get us much.
    - pos += 4;
    + buf->append(len);

        MetaRegionServer mrs;
    - // Try to decode the protobuf.
    - // If there's an error bail out.
    - if (mrs.ParseFromArray(contents + pos, len - pos) == false) {
    - LOG(ERROR) << "Error parsing Protobuf Message";
    - throw runtime_error("Error parsing protobuf");
    + if (derser.parse(buf.get(), &mrs) == false) {
    + LOG(ERROR) << "Unable to decode";
        }
    -
        return mrs.server();
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index 28a1ee1..ce97c43 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -18,10 +18,10 @@
       */
      #pragma once

    -#include <zookeeper/zookeeper.h>
    +#include <folly/Executor.h>
      #include <folly/futures/Future.h>
      #include <folly/futures/SharedPromise.h>
    -#include <folly/Executor.h>
    +#include <zookeeper/zookeeper.h>

      #include <memory>
      #include <mutex>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index 065f70b..5e6f82e 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -22,13 +22,13 @@
      #include <gflags/gflags.h>
      #include <wangle/concurrent/GlobalExecutor.h>

    -#include <iostream>
      #include <chrono>
    +#include <iostream>

    -#include "core/client.h"
      #include "connection/connection-factory.h"
    -#include "if/ZooKeeper.pb.h"
    +#include "core/client.h"
      #include "if/Client.pb.h"
    +#include "if/ZooKeeper.pb.h"

      using namespace folly;
      using namespace std;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
    index 207607f..0014c0b 100644
    --- a/hbase-native-client/serde/BUCK
    +++ b/hbase-native-client/serde/BUCK
    @@ -19,10 +19,12 @@ cxx_library(name="serde",
                  exported_headers=[
                      "client-serializer.h",
                      "client-deserializer.h",
    + "zk-deserializer.h",
                  ],
                  srcs=[
                      "client-serializer.cc",
                      "client-deserializer.cc",
    + "zk-deserializer.cc",
                  ],
                  deps=[
                      "//if:if",
    @@ -52,3 +54,11 @@ cxx_test(name="client-deserializer-test",
                   ":serde",
                   "//if:if",
               ], )
    +cxx_test(name="zk-deserializer-test",
    + srcs=[
    + "zk-deserializer-test.cc",
    + ],
    + deps=[
    + ":serde",
    + "//if:if",
    + ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc
    index bb57e50..9fef093 100644
    --- a/hbase-native-client/serde/client-deserializer-test.cc
    +++ b/hbase-native-client/serde/client-deserializer-test.cc
    @@ -17,12 +17,12 @@
       *
       */

    -#include <gtest/gtest.h>
      #include <folly/io/IOBuf.h>
    +#include <gtest/gtest.h>

    +#include "if/Client.pb.h"
      #include "serde/client-deserializer.h"
      #include "serde/client-serializer.h"
    -#include "if/Client.pb.h"

      using namespace hbase;
      using folly::IOBuf;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-deserializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc
    index 118b0d1..acca7ea 100644
    --- a/hbase-native-client/serde/client-deserializer.cc
    +++ b/hbase-native-client/serde/client-deserializer.cc
    @@ -19,10 +19,10 @@

      #include "serde/client-deserializer.h"

    -#include <google/protobuf/message.h>
    +#include <folly/Logging.h>
      #include <google/protobuf/io/coded_stream.h>
      #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
    -#include <folly/Logging.h>
    +#include <google/protobuf/message.h>

      using namespace hbase;


    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc
    index b32b55d..9bf38af 100644
    --- a/hbase-native-client/serde/client-serializer-test.cc
    +++ b/hbase-native-client/serde/client-serializer-test.cc
    @@ -22,9 +22,9 @@

      #include <string>

    -#include "serde/client-serializer.h"
      #include "if/HBase.pb.h"
      #include "if/RPC.pb.h"
    +#include "serde/client-serializer.h"

      using namespace hbase;
      using namespace hbase::pb;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc
    index 881b6e4..09b81c8 100644
    --- a/hbase-native-client/serde/client-serializer.cc
    +++ b/hbase-native-client/serde/client-serializer.cc
    @@ -18,8 +18,8 @@
       */
      #include "serde/client-serializer.h"

    -#include <folly/io/Cursor.h>
      #include <folly/Logging.h>
    +#include <folly/io/Cursor.h>
      #include <google/protobuf/io/coded_stream.h>
      #include <google/protobuf/io/zero_copy_stream_impl_lite.h>


    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/client-serializer.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h
    index 685095d..9c819fe 100644
    --- a/hbase-native-client/serde/client-serializer.h
    +++ b/hbase-native-client/serde/client-serializer.h
    @@ -18,9 +18,9 @@
       */
      #pragma once

    +#include <cstdint>
      #include <folly/io/IOBuf.h>
      #include <string>
    -#include <cstdint>

      // Forward
      namespace google {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer-test.cc b/hbase-native-client/serde/zk-deserializer-test.cc
    new file mode 100644
    index 0000000..92d85a0
    --- /dev/null
    +++ b/hbase-native-client/serde/zk-deserializer-test.cc
    @@ -0,0 +1,123 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/zk-deserializer.h"
    +
    +#include <folly/Logging.h>
    +#include <folly/io/Cursor.h>
    +#include <folly/io/IOBuf.h>
    +#include <gtest/gtest.h>
    +
    +#include "if/ZooKeeper.pb.h"
    +
    +using namespace hbase;
    +using namespace hbase::pb;
    +using namespace folly;
    +using namespace std;
    +using namespace folly::io;
    +
    +// Test that would test if there's nothing there.
    +TEST(TestZkDesializer, TestThrowNoMagicNum) {
    + ZkDeserializer deser;
    + MetaRegionServer mrs;
    +
    + auto buf = IOBuf::create(100);
    + buf->append(100);
    + RWPrivateCursor c{buf.get()};
    + c.write<uint8_t>(99);
    + ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
    +}
    +
    +// Test if the protobuf is in a format that we can't decode
    +TEST(TestZkDesializer, TestBadProtoThrow) {
    + ZkDeserializer deser;
    + MetaRegionServer mrs;
    + string magic{"PBUF"};
    +
    + // Set ServerName
    + mrs.mutable_server()->set_host_name("test");
    + mrs.mutable_server()->set_port(567);
    + mrs.mutable_server()->set_start_code(9567);
    +
    + // One byte magic number
    + // four bytes for id length
    + // four bytes for id
    + // four bytes for PBUF
    + uint32_t start_len = 1 + 4 + 4 + 4;
    + // How large the protobuf will be
    + uint32_t pbuf_size = mrs.ByteSize();
    +
    + auto buf = IOBuf::create(start_len + pbuf_size);
    + buf->append(start_len + pbuf_size);
    + RWPrivateCursor c{buf.get()};
    +
    + // Write the magic number
    + c.write<uint8_t>(255);
    + // Write the id len
    + c.writeBE<uint32_t>(4);
    + // Write the id
    + c.write<uint32_t>(13);
    + // Write the PBUF string
    + c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
    +
    + // Create the protobuf
    + MetaRegionServer out;
    + ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
    +}
    +
    +// Test to make sure the whole thing works.
    +TEST(TestZkDesializer, TestNoThrow) {
    + ZkDeserializer deser;
    + MetaRegionServer mrs;
    + string magic{"PBUF"};
    +
    + // Set ServerName
    + mrs.mutable_server()->set_host_name("test");
    + mrs.mutable_server()->set_port(567);
    + mrs.mutable_server()->set_start_code(9567);
    +
    + // One byte magic number
    + // four bytes for id length
    + // four bytes for id
    + // four bytes for PBUF
    + uint32_t start_len = 1 + 4 + 4 + 4;
    + // How large the protobuf will be
    + uint32_t pbuf_size = mrs.ByteSize();
    +
    + auto buf = IOBuf::create(start_len + pbuf_size);
    + buf->append(start_len + pbuf_size);
    + RWPrivateCursor c{buf.get()};
    +
    + // Write the magic number
    + c.write<uint8_t>(255);
    + // Write the id len
    + c.writeBE<uint32_t>(4);
    + // Write the id
    + c.write<uint32_t>(13);
    + // Write the PBUF string
    + c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
    +
    + // Now write the serialized protobuf
    + mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len);
    +
    + // Create the protobuf
    + MetaRegionServer out;
    + ASSERT_TRUE(deser.parse(buf.get(), &out));
    + ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer.cc b/hbase-native-client/serde/zk-deserializer.cc
    new file mode 100644
    index 0000000..33cf809
    --- /dev/null
    +++ b/hbase-native-client/serde/zk-deserializer.cc
    @@ -0,0 +1,78 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/zk-deserializer.h"
    +
    +#include <folly/io/Cursor.h>
    +#include <folly/io/IOBuf.h>
    +#include <google/protobuf/message.h>
    +
    +using hbase::ZkDeserializer;
    +using std::runtime_error;
    +using folly::IOBuf;
    +using folly::io::Cursor;
    +using google::protobuf::Message;
    +
    +static const std::string MAGIC_STRING = "PBUF";
    +
    +bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
    +
    + // The format is like this
    + // 1 byte of magic number. 255
    + // 4 bytes of id length.
    + // id_length number of bytes for the id of who put up the znode
    + // 4 bytes of a magic string PBUF
    + // Then the protobuf serialized without a varint header.
    +
    + Cursor c{buf};
    +
    + // There should be a magic number for recoverable zk
    + uint8_t magic_num = c.read<uint8_t>();
    + if (magic_num != 255) {
    + LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
    + << unsigned(magic_num);
    + throw runtime_error("Magic number not in znode data");
    + }
    + // How long is the id?
    + uint32_t id_len = c.readBE<uint32_t>();
    +
    + if (id_len >= c.length()) {
    + LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
    + "left to read anything else";
    + throw runtime_error("Not enough bytes to decode from zookeeper");
    + }
    +
    + // Skip the id
    + c.skip(id_len);
    +
    + // Make sure that the magic string is there.
    + if (MAGIC_STRING != c.readFixedString(4)) {
    + LOG(ERROR) << "There was no PBUF magic string.";
    + throw runtime_error("No PBUF magic string in the zookpeeper data.");
    + }
    +
    + // Try to decode the protobuf.
    + // If there's an error bail out.
    + if (out->ParseFromArray(c.data(), c.length()) == false) {
    + LOG(ERROR) << "Error parsing Protobuf Message";
    + throw runtime_error("Error parsing protobuf");
    + }
    +
    + return true;
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/serde/zk-deserializer.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer.h b/hbase-native-client/serde/zk-deserializer.h
    new file mode 100644
    index 0000000..aa91661
    --- /dev/null
    +++ b/hbase-native-client/serde/zk-deserializer.h
    @@ -0,0 +1,35 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +namespace google {
    +namespace protobuf {
    +class Message;
    +}
    +}
    +namespace folly {
    +class IOBuf;
    +}
    +
    +namespace hbase {
    +class ZkDeserializer {
    +public:
    + bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
    +};
    +} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/65cd2da5/hbase-native-client/utils/user-util-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
    index 2a7434f..1e72bcc 100644
    --- a/hbase-native-client/utils/user-util-test.cc
    +++ b/hbase-native-client/utils/user-util-test.cc
    @@ -30,5 +30,6 @@ TEST(TestUserUtil, TestGetSomething) {
        UserUtil u_util;
        string name = u_util.user_name();

    + // TODO shell out to whoami to check this.
        ASSERT_GT(name.length(), 0);
      }
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15766 Show working puts

    Summary:
    Add on showing how a set of puts to a single connection will work.
    This still needs retries and looking up what region each request is going to

    Test Plan:
         ./buck-out/gen/core/simple-client -columns 100
         ../bin/hbase shell
         count 't'
         100 row(s) in 0.2470 seconds

    Differential Revision: https://reviews.facebook.net/D57603


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/285b114c
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/285b114c
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/285b114c

    Branch: refs/heads/HBASE-14850
    Commit: 285b114ce8bbaf42d2d9d06c71fbe4674cbb33e5
    Parents: 5271987
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed May 4 01:54:21 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      .../connection/client-dispatcher.cc | 9 +--
      .../connection/client-dispatcher.h | 11 ++-
      .../connection/client-handler.cc | 13 ++--
      hbase-native-client/connection/client-handler.h | 4 +-
      .../connection/connection-factory.cc | 46 ++++++------
      .../connection/connection-factory.h | 11 ++-
      .../connection/connection-pool-test.cc | 54 +++++++++++---
      .../connection/connection-pool.cc | 33 +++++++--
      .../connection/connection-pool.h | 10 ++-
      hbase-native-client/core/client.cc | 6 +-
      hbase-native-client/core/location-cache.cc | 25 +++----
      hbase-native-client/core/location-cache.h | 6 +-
      hbase-native-client/core/meta-utils.cc | 6 +-
      hbase-native-client/core/meta-utils.h | 4 +-
      hbase-native-client/core/region-location.h | 8 ++-
      hbase-native-client/core/simple-client.cc | 76 +++++++++++++++++---
      .../serde/region-info-deserializer-test.cc | 1 -
      hbase-native-client/serde/region-info.h | 6 +-
      18 files changed, 231 insertions(+), 98 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/client-dispatcher.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
    index 6e2dc54..655d765 100644
    --- a/hbase-native-client/connection/client-dispatcher.cc
    +++ b/hbase-native-client/connection/client-dispatcher.cc
    @@ -22,10 +22,11 @@ using namespace folly;
      using namespace hbase;
      using namespace wangle;

    -ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
    +ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}

      void ClientDispatcher::read(Context *ctx, Response in) {
        auto call_id = in.call_id();
    +
        auto search = requests_.find(call_id);
        CHECK(search != requests_.end());
        auto p = std::move(search->second);
    @@ -38,10 +39,10 @@ void ClientDispatcher::read(Context *ctx, Response in) {
      }

      Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
    - auto call_id = ++current_call_id_;
    -
    + auto call_id = current_call_id_++;
        arg->set_call_id(call_id);
    - auto &p = requests_[call_id];
    + requests_.insert(call_id, Promise<Response>{});
    + auto &p = requests_.find(call_id)->second;
        auto f = p.getFuture();
        p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
          LOG(ERROR) << "e = " << call_id;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/client-dispatcher.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
    index 826fc6a..4435a1b 100644
    --- a/hbase-native-client/connection/client-dispatcher.h
    +++ b/hbase-native-client/connection/client-dispatcher.h
    @@ -19,8 +19,12 @@

      #pragma once

    +#include <folly/AtomicHashMap.h>
    +#include <folly/Logging.h>
      #include <wangle/service/ClientDispatcher.h>

    +#include <atomic>
    +
      #include "connection/pipeline.h"
      #include "connection/request.h"
      #include "connection/response.h"
    @@ -31,13 +35,16 @@ class ClientDispatcher
                                                std::unique_ptr<Request>, Response> {
      public:
        ClientDispatcher();
    + ~ClientDispatcher() {
    + LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
    + }
        void read(Context *ctx, Response in) override;
        folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
        folly::Future<folly::Unit> close(Context *ctx) override;
        folly::Future<folly::Unit> close() override;

      private:
    - std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
    + folly::AtomicHashMap<uint32_t, folly::Promise<Response>> requests_;
        // Start at some number way above what could
        // be there for un-initialized call id counters.
        //
    @@ -46,6 +53,6 @@ private:
        //
        // uint32_t has a max of 4Billion so 10 more or less is
        // not a big deal.
    - uint32_t current_call_id_;
    + std::atomic<uint32_t> current_call_id_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index 496e4f2..b92ad89 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -37,7 +37,10 @@ using hbase::pb::GetResponse;
      using google::protobuf::Message;

      ClientHandler::ClientHandler(std::string user_name)
    - : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
    + : user_name_(user_name), need_send_header_(true), serde_(),
    + resp_msgs_(
    + make_unique<folly::AtomicHashMap<
    + uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}

      void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
        if (LIKELY(buf != nullptr)) {
    @@ -51,14 +54,14 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
                    << " has_exception=" << header.has_exception();

          // Get the response protobuf from the map
    - auto search = resp_msgs_.find(header.call_id());
    + auto search = resp_msgs_->find(header.call_id());
          // It's an error if it's not there.
    - CHECK(search != resp_msgs_.end());
    + CHECK(search != resp_msgs_->end());
          auto resp_msg = search->second;
          CHECK(resp_msg != nullptr);

          // Make sure we don't leak the protobuf
    - resp_msgs_.erase(search);
    + resp_msgs_->erase(header.call_id());

          // set the call_id.
          // This will be used to by the dispatcher to match up
    @@ -96,7 +99,7 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
          ctx->fireWrite(std::move(pre));
        }

    - resp_msgs_[r->call_id()] = r->resp_msg();
    + resp_msgs_->insert(r->call_id(), r->resp_msg());
        return ctx->fireWrite(
            serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index ce99c9e..be5143c 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -18,6 +18,7 @@
       */
      #pragma once

    +#include <folly/AtomicHashMap.h>
      #include <wangle/channel/Handler.h>

      #include <string>
    @@ -51,7 +52,8 @@ private:
        RpcSerde serde_;

        // in flight requests
    - std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
    + std::unique_ptr<folly::AtomicHashMap<
    + uint32_t, std::shared_ptr<google::protobuf::Message>>>
            resp_msgs_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/connection-factory.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
    index 9102d60..635d12d 100644
    --- a/hbase-native-client/connection/connection-factory.cc
    +++ b/hbase-native-client/connection/connection-factory.cc
    @@ -19,40 +19,36 @@

      #include "connection/connection-factory.h"

    -#include <folly/futures/Future.h>
    -#include <wangle/bootstrap/ClientBootstrap.h>
    -#include <wangle/channel/AsyncSocketHandler.h>
    -#include <wangle/channel/EventBaseHandler.h>
    -#include <wangle/channel/OutputBufferingHandler.h>
    -#include <wangle/service/ClientDispatcher.h>
    -#include <wangle/service/CloseOnReleaseFilter.h>
    -#include <wangle/service/ExpiringFilter.h>
    -
    -#include <string>
    +#include <wangle/concurrent/GlobalExecutor.h>

      #include "connection/client-dispatcher.h"
      #include "connection/pipeline.h"
    -#include "connection/request.h"
    -#include "connection/response.h"
      #include "connection/service.h"

      using namespace folly;
      using namespace hbase;
    -using namespace wangle;

    -ConnectionFactory::ConnectionFactory() : bootstrap_() {
    - bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
    - bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
    -}
    +ConnectionFactory::ConnectionFactory()
    + : io_pool_(std::static_pointer_cast<wangle::IOThreadPoolExecutor>(
    + wangle::getIOExecutor())),
    + pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}

    -std::shared_ptr<HBaseService>
    -ConnectionFactory::make_connection(const std::string &host, int port) {
    - // Connect to a given server
    - // Then when connected create a ClientDispactcher.
    - auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
    +std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
    +ConnectionFactory::MakeBootstrap() {
    + auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
    + client->group(io_pool_);
    + client->pipelineFactory(pipeline_factory_);
    +
    + return client;
    +}
    +std::shared_ptr<HBaseService> ConnectionFactory::Connect(
    + std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
    + const std::string &hostname, int port) {
    + // Yes this will block however it makes dealing with connection pool soooooo
    + // much nicer.
    + // TODO see about using shared promise for this.
    + auto pipeline = client->connect(SocketAddress(hostname, port, true)).get();
        auto dispatcher = std::make_shared<ClientDispatcher>();
        dispatcher->setPipeline(pipeline);
    - auto service = std::make_shared<
    - CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
    - return service;
    + return dispatcher;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/connection-factory.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
    index fc4e161..2284a7c 100644
    --- a/hbase-native-client/connection/connection-factory.h
    +++ b/hbase-native-client/connection/connection-factory.h
    @@ -33,10 +33,15 @@ public:
        ConnectionFactory();
        virtual ~ConnectionFactory() = default;

    - virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
    - int port);
    + virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
    + MakeBootstrap();
    +
    + virtual std::shared_ptr<HBaseService>
    + Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
    + const std::string &hostname, int port);

      private:
    - wangle::ClientBootstrap<SerializePipeline> bootstrap_;
    + std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
    + std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/connection-pool-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
    index 975bc5e..b1a0ba0 100644
    --- a/hbase-native-client/connection/connection-pool-test.cc
    +++ b/hbase-native-client/connection/connection-pool-test.cc
    @@ -24,6 +24,7 @@

      #include "connection/connection-factory.h"
      #include "if/HBase.pb.h"
    +#include "serde/server-name.h"

      using namespace hbase;

    @@ -33,11 +34,16 @@ using ::testing::_;

      class MockConnectionFactory : public ConnectionFactory {
      public:
    - MOCK_METHOD2(make_connection,
    - std::shared_ptr<HBaseService>(const std::string &hostname,
    - int port));
    + MOCK_METHOD0(MakeBootstrap,
    + std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
    + MOCK_METHOD3(Connect,
    + std::shared_ptr<HBaseService>(
    + std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
    + const std::string &hostname, int port));
      };

    +class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
    +
      class MockServiceBase : public HBaseService {
      public:
        folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
    @@ -54,19 +60,20 @@ public:
      };

      TEST(TestConnectionPool, TestOnlyCreateOnce) {
    - std::string hostname{"hostname"};
    + auto hostname = std::string{"hostname"};
    + auto mock_boot = std::make_shared<MockBootstrap>();
        auto mock_service = std::make_shared<MockService>();
    + auto mock_cf = std::make_shared<MockConnectionFactory>();
        uint32_t port{999};

    - LOG(ERROR) << "About to make a MockConnectionFactory";
    - auto mock_cf = std::make_shared<MockConnectionFactory>();
    - EXPECT_CALL((*mock_cf), make_connection(_, _))
    + EXPECT_CALL((*mock_cf), Connect(_, _, _))
            .Times(1)
            .WillRepeatedly(Return(mock_service));
    + EXPECT_CALL((*mock_cf), MakeBootstrap())
    + .Times(1)
    + .WillRepeatedly(Return(mock_boot));
        ConnectionPool cp{mock_cf};

    - LOG(ERROR) << "Created ConnectionPool";
    -
        ServerName sn;
        sn.set_host_name(hostname);
        sn.set_port(port);
    @@ -75,3 +82,32 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
        ASSERT_TRUE(result != nullptr);
        result = cp.get(sn);
      }
    +
    +TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
    + std::string hostname_one{"hostname"};
    + std::string hostname_two{"hostname_two"};
    + uint32_t port{999};
    +
    + auto mock_boot = std::make_shared<MockBootstrap>();
    + auto mock_service = std::make_shared<MockService>();
    + auto mock_cf = std::make_shared<MockConnectionFactory>();
    +
    + EXPECT_CALL((*mock_cf), Connect(_, _, _))
    + .Times(2)
    + .WillRepeatedly(Return(mock_service));
    + EXPECT_CALL((*mock_cf), MakeBootstrap())
    + .Times(2)
    + .WillRepeatedly(Return(mock_boot));
    + ConnectionPool cp{mock_cf};
    +
    + {
    + auto result_one = cp.get(folly::to<ServerName>(
    + hostname_one + ":" + folly::to<std::string>(port)));
    + auto result_two = cp.get(folly::to<ServerName>(
    + hostname_two + ":" + folly::to<std::string>(port)));
    + }
    + auto result_one = cp.get(
    + folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
    + auto result_two = cp.get(
    + folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index eafe60a..6ed5ad9 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -19,6 +19,7 @@

      #include "connection/connection-pool.h"

    +#include <folly/SocketAddress.h>
      #include <wangle/service/Service.h>

      using std::mutex;
    @@ -26,28 +27,46 @@ using std::unique_ptr;
      using std::shared_ptr;
      using hbase::pb::ServerName;
      using folly::SharedMutexWritePriority;
    +using folly::SocketAddress;

      namespace hbase {

      ConnectionPool::ConnectionPool()
    - : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
    -}
    + : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
    + map_mutex_() {}
      ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
    - : cf_(cf), connections_(), map_mutex_() {}
    + : cf_(cf), clients_(), connections_(), map_mutex_() {}
    +
    +ConnectionPool::~ConnectionPool() {
    + SharedMutexWritePriority::WriteHolder holder(map_mutex_);
    + for (auto &item : connections_) {
    + auto &con = item.second;
    + con->close();
    + }
    + connections_.clear();
    + clients_.clear();
    +}

      std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
    + // Create a read lock.
        SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
    +
        auto found = connections_.find(sn);
        if (found == connections_.end() || found->second == nullptr) {
    + // Move the upgradable lock into the write lock if the connection
    + // hasn't been found.
          SharedMutexWritePriority::WriteHolder holder(std::move(holder));
    - auto new_con = cf_->make_connection(sn.host_name(), sn.port());
    - connections_[sn] = new_con;
    - return new_con;
    + auto client = cf_->MakeBootstrap();
    + auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
    + clients_.insert(std::make_pair(sn, client));
    + connections_.insert(std::make_pair(sn, dispatcher));
    + return dispatcher;
        }
        return found->second;
      }
    +
      void ConnectionPool::close(const ServerName &sn) {
    - SharedMutexWritePriority::WriteHolder holder(map_mutex_);
    + SharedMutexWritePriority::WriteHolder holder{map_mutex_};

        auto found = connections_.find(sn);
        if (found == connections_.end() || found->second == nullptr) {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index b8330e3..907afdb 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -37,23 +37,29 @@ struct ServerNameHash {
        std::size_t operator()(hbase::pb::ServerName const &s) const {
          std::size_t h1 = std::hash<std::string>()(s.host_name());
          std::size_t h2 = std::hash<uint32_t>()(s.port());
    - return h1 ^ (h2 << 1);
    + return h1 ^ (h2 << 2);
        }
      };

      class ConnectionPool {
      public:
        ConnectionPool();
    + ~ConnectionPool();
        explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
        std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
        void close(const hbase::pb::ServerName &sn);

      private:
    - std::shared_ptr<ConnectionFactory> cf_;
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
                           ServerNameHash, ServerNameEquals>
            connections_;
    + std::unordered_map<
    + hbase::pb::ServerName,
    + std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
    + ServerNameHash, ServerNameEquals>
    + clients_;
        folly::SharedMutexWritePriority map_mutex_;
    + std::shared_ptr<ConnectionFactory> cf_;
      };

      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
    index 266c239..4b9f844 100644
    --- a/hbase-native-client/core/client.cc
    +++ b/hbase-native-client/core/client.cc
    @@ -33,8 +33,4 @@ using namespace folly;
      using namespace std;
      using namespace hbase::pb;

    -namespace hbase {
    -
    -Client::Client(string quorum_spec)
    - : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
    -}
    +namespace hbase {}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 2667f11..e2a6251 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -25,8 +25,8 @@
      #include "connection/response.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
    -#include "serde/server-name.h"
      #include "serde/region-info.h"
    +#include "serde/server-name.h"
      #include "serde/zk.h"

      using namespace std;
    @@ -109,17 +109,22 @@ ServerName LocationCache::ReadMetaLocation() {

      Future<std::shared_ptr<RegionLocation>>
      LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
    - auto exc = wangle::getIOExecutor();
    + auto exec = wangle::getCPUExecutor();
        return this->LocateMeta()
    - .then([&](ServerName sn) { return this->cp_.get(sn); })
    - .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
    + .via(exec.get())
    + .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
            .then([&](std::shared_ptr<HBaseService> service) {
              return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
    - .then([&](Response resp) {
    + .then([this](Response resp) {
              // take the protobuf response and make it into
              // a region location.
              return this->CreateLocation(std::move(resp));
    + })
    + .then([ exec = exec, this ](std::shared_ptr<RegionLocation> rl) {
    + // Now fill out the connection.
    + rl->set_service(cp_.get(rl->server_name()));
    + return rl;
            });
      }

    @@ -162,16 +167,12 @@ private:
      };

      std::shared_ptr<RegionLocation>
    -LocationCache::CreateLocation(const Response &resp){
    +LocationCache::CreateLocation(const Response &resp) {
        auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
        auto &results = resp_msg->results().Get(0);
        auto &cells = results.cell();
    - LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
        auto ri = folly::to<RegionInfo>(cells.Get(0).value());
        auto sn = folly::to<ServerName>(cells.Get(1).value());
    -
    - LOG(ERROR) << "RegionInfo = " << ri.DebugString();
    - LOG(ERROR) << "ServerName = " << sn.DebugString();
    - auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
    - return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
    + return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
    + nullptr);
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index 99b5e5e..7f76428 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -48,9 +48,10 @@ public:
        // Meta Related Methods.
        // These are only public until testing is complete
        folly::Future<hbase::pb::ServerName> LocateMeta();
    - folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
    - const std::string &row);
    + folly::Future<std::shared_ptr<RegionLocation>>
    + LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
        void InvalidateMeta();
    + ConnectionPool cp_;

      private:
        void RefreshMetaLocation();
    @@ -61,7 +62,6 @@ private:
        std::shared_ptr<folly::Executor> executor_;
        std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
        std::mutex meta_lock_;
    - ConnectionPool cp_;
        MetaUtil meta_util_;

        // TODO: migrate this to a smart pointer with a deleter.

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/meta-utils.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
    index 1325d83..23d2041 100644
    --- a/hbase-native-client/core/meta-utils.cc
    +++ b/hbase-native-client/core/meta-utils.cc
    @@ -37,12 +37,12 @@ using hbase::pb::RegionSpecifier_RegionSpecifierType;
      static const std::string META_REGION = "1588230740";

      std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
    - const std::string &row) const {
    + const std::string &row) const {
        return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
      }

    -std::unique_ptr<Request>
    -MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
    +std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn,
    + const std::string &row) const {
        auto request = Request::scan();
        auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());


    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/meta-utils.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
    index 5a659f3..dfef065 100644
    --- a/hbase-native-client/core/meta-utils.h
    +++ b/hbase-native-client/core/meta-utils.h
    @@ -29,8 +29,8 @@ namespace hbase {
      class MetaUtil {
      public:
        std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
    - const std::string &row) const;
    + const std::string &row) const;
        std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
    - const std::string &row) const;
    + const std::string &row) const;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/region-location.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
    index 7922c95..7887526 100644
    --- a/hbase-native-client/core/region-location.h
    +++ b/hbase-native-client/core/region-location.h
    @@ -27,15 +27,19 @@ namespace hbase {

      class RegionLocation {
      public:
    - RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
    + RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
    + hbase::pb::ServerName sn,
                       std::shared_ptr<HBaseService> service)
    - : ri_(ri), sn_(sn), service_(service) {}
    + : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}

        const hbase::pb::RegionInfo &region_info() { return ri_; }
        const hbase::pb::ServerName &server_name() { return sn_; }
    + const std::string &region_name() { return region_name_; }
        std::shared_ptr<HBaseService> service() { return service_; }
    + void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }

      private:
    + std::string region_name_;
        hbase::pb::RegionInfo ri_;
        hbase::pb::ServerName sn_;
        std::shared_ptr<HBaseService> service_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index 00e3369..39c82c3 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -19,16 +19,21 @@

      #include <folly/Logging.h>
      #include <folly/Random.h>
    +#include <folly/futures/Future.h>
      #include <gflags/gflags.h>
    +#include <wangle/concurrent/CPUThreadPoolExecutor.h>
      #include <wangle/concurrent/GlobalExecutor.h>

    +#include <atomic>
      #include <chrono>
      #include <iostream>
    +#include <thread>

      #include "connection/connection-pool.h"
      #include "core/client.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
    +#include "serde/server-name.h"
      #include "serde/table-name.h"

      using namespace folly;
    @@ -39,16 +44,41 @@ using hbase::Request;
      using hbase::HBaseService;
      using hbase::LocationCache;
      using hbase::ConnectionPool;
    +using hbase::ConnectionFactory;
      using hbase::pb::TableName;
      using hbase::pb::ServerName;
      using hbase::pb::RegionSpecifier_RegionSpecifierType;
    -using hbase::pb::GetRequest;
    -using hbase::pb::GetResponse;
    +using hbase::pb::MutateRequest;
    +using hbase::pb::MutationProto_MutationType;

      // TODO(eclark): remove the need for this.
      DEFINE_string(table, "t", "What region to send a get");
      DEFINE_string(row, "test", "What row to get");
      DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
    +DEFINE_uint64(columns, 10000, "How many columns to write");
    +DEFINE_int32(threads, 6, "How many cpu threads");
    +
    +std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
    + auto req = Request::mutate();
    + auto msg = std::static_pointer_cast<MutateRequest>(req->req_msg());
    + auto region = msg->mutable_region();
    + auto suf = folly::to<std::string>(col);
    +
    + region->set_value(region_name);
    + region->set_type(RegionSpecifier_RegionSpecifierType::
    + RegionSpecifier_RegionSpecifierType_REGION_NAME);
    + auto mutation = msg->mutable_mutation();
    + mutation->set_row(FLAGS_row + suf);
    + mutation->set_mutate_type(
    + MutationProto_MutationType::MutationProto_MutationType_PUT);
    + auto column = mutation->add_column_value();
    + column->set_family("d");
    + auto qual = column->add_qualifier_value();
    + qual->set_qualifier(suf);
    + qual->set_value(".");
    +
    + return std::move(req);
    +}

      int main(int argc, char *argv[]) {
        google::SetUsageMessage(
    @@ -56,13 +86,41 @@ int main(int argc, char *argv[]) {
        google::ParseCommandLineFlags(&argc, &argv, true);
        google::InitGoogleLogging(argv[0]);

    - // Create a connection factory
    - ConnectionPool cp;
    - auto cpu_ex = wangle::getCPUExecutor();
    - LocationCache cache{FLAGS_zookeeper, cpu_ex};
    - auto result =
    - cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
    - .get(milliseconds(5000));
    + // Set up thread pools.
    + auto cpu_pool =
    + std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
    + wangle::setCPUExecutor(cpu_pool);
    + auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
    + wangle::setIOExecutor(io_pool);
    +
    + // Create the cache.
    + LocationCache cache{FLAGS_zookeeper, cpu_pool};
    +
    + auto row = FLAGS_row;
    + auto tn = folly::to<TableName>(FLAGS_table);
    +
    + auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(5000));
    + auto connection = loc->service();
    +
    + auto num_puts = FLAGS_columns;
    +
    + auto results = std::vector<Future<Response>>{};
    + uint64_t col{0};
    + for (; col < num_puts; col++) {
    + results.push_back(folly::makeFuture(col)
    + .via(cpu_pool.get())
    + .then([loc](uint64_t col) {
    + return MakeRequest(col, loc->region_name());
    + })
    + .then([connection](std::unique_ptr<Request> req) {
    + return (*connection)(std::move(req));
    + }));
    + }
    + auto allf = folly::collect(results).get();
    +
    + LOG(ERROR) << "Successfully sent " << allf.size() << " requests.";
    +
    + io_pool->stop();

        return 0;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/serde/region-info-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/region-info-deserializer-test.cc b/hbase-native-client/serde/region-info-deserializer-test.cc
    index ce8dedf..5cb8482 100644
    --- a/hbase-native-client/serde/region-info-deserializer-test.cc
    +++ b/hbase-native-client/serde/region-info-deserializer-test.cc
    @@ -44,7 +44,6 @@ TEST(TestRegionInfoDesializer, TestDeserialize) {
        ri_out.set_start_key(start_row);
        ri_out.set_end_key(stop_row);

    -
        string header{"PBUF"};
        string ser = header + ri_out.SerializeAsString();


    http://git-wip-us.apache.org/repos/asf/hbase/blob/285b114c/hbase-native-client/serde/region-info.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/region-info.h b/hbase-native-client/serde/region-info.h
    index 6af351c..e2ecfc9 100644
    --- a/hbase-native-client/serde/region-info.h
    +++ b/hbase-native-client/serde/region-info.h
    @@ -21,16 +21,16 @@

      #include "if/HBase.pb.h"

    -#include <folly/Conv.h>
      #include <boost/algorithm/string/predicate.hpp>
    +#include <folly/Conv.h>

      namespace hbase {
      namespace pb {
    -template <class String> void parseTo(String in, RegionInfo& out) {
    +template <class String> void parseTo(String in, RegionInfo &out) {
        // TODO(eclark): there has to be something better.
        std::string s = folly::to<std::string>(in);

    - if (!boost::starts_with(s, "PBUF") ) {
    + if (!boost::starts_with(s, "PBUF")) {
          throw std::runtime_error("Region Info field doesn't contain preamble");
        }
        if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15739 Add region location lookup from meta

    Summary:
    Add the ability for location-cache to read from meta.
    There's still no cache. Or any parsing.

    Test Plan: Needs tests.

    Differential Revision: https://reviews.facebook.net/D57489


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7225e42
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7225e42
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7225e42

    Branch: refs/heads/HBASE-14850
    Commit: a7225e4248d2e883143ebf8cc33a269101e3dba4
    Parents: 59f9612
    Author: Elliott Clark <eclark@apache.org>
    Authored: Sat Apr 30 17:12:11 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/connection/BUCK | 1 +
      .../connection/client-dispatcher.cc | 2 +
      .../connection/client-dispatcher.h | 3 +-
      .../connection/client-handler.cc | 5 +-
      hbase-native-client/connection/client-handler.h | 2 +-
      .../connection/connection-factory.cc | 2 +-
      .../connection/connection-factory.h | 1 +
      .../connection/connection-pool.cc | 14 ++---
      hbase-native-client/connection/pipeline.cc | 2 +
      hbase-native-client/connection/pipeline.h | 1 +
      hbase-native-client/connection/response.h | 6 ++-
      hbase-native-client/core/BUCK | 3 +-
      hbase-native-client/core/location-cache.cc | 37 ++++++++++++--
      hbase-native-client/core/location-cache.h | 19 ++++++-
      hbase-native-client/core/meta-utils.cc | 54 +++++++++++++++++++-
      hbase-native-client/core/meta-utils.h | 9 +++-
      hbase-native-client/core/region-location.h | 45 ++++++++++++++++
      hbase-native-client/core/simple-client.cc | 48 ++---------------
      hbase-native-client/core/table-name-test.cc | 28 +++++-----
      hbase-native-client/core/table-name.cc | 34 ------------
      hbase-native-client/core/table-name.h | 35 ++++++-------
      hbase-native-client/serde/BUCK | 1 +
      hbase-native-client/utils/BUCK | 7 +--
      hbase-native-client/utils/user-util.cc | 2 +
      hbase-native-client/utils/user-util.h | 3 +-
      25 files changed, 231 insertions(+), 133 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
    index 96f2136..39b8be5 100644
    --- a/hbase-native-client/connection/BUCK
    +++ b/hbase-native-client/connection/BUCK
    @@ -43,6 +43,7 @@ cxx_library(name="connection",
                      "//third-party:folly",
                      "//third-party:wangle",
                  ],
    + compiler_flags=['-Weffc++'],
                  visibility=['//core/...', ], )
      cxx_test(name="connection-pool-test",
               srcs=["connection-pool-test.cc", ],

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/client-dispatcher.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
    index eea0a17..817adc1 100644
    --- a/hbase-native-client/connection/client-dispatcher.cc
    +++ b/hbase-native-client/connection/client-dispatcher.cc
    @@ -22,6 +22,8 @@ using namespace folly;
      using namespace hbase;
      using namespace wangle;

    +ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
    +
      void ClientDispatcher::read(Context *ctx, Response in) {
        auto call_id = in.call_id();
        auto search = requests_.find(call_id);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/client-dispatcher.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
    index 877e877..826fc6a 100644
    --- a/hbase-native-client/connection/client-dispatcher.h
    +++ b/hbase-native-client/connection/client-dispatcher.h
    @@ -30,6 +30,7 @@ class ClientDispatcher
          : public wangle::ClientDispatcherBase<SerializePipeline,
                                                std::unique_ptr<Request>, Response> {
      public:
    + ClientDispatcher();
        void read(Context *ctx, Response in) override;
        folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
        folly::Future<folly::Unit> close(Context *ctx) override;
    @@ -45,6 +46,6 @@ private:
        //
        // uint32_t has a max of 4Billion so 10 more or less is
        // not a big deal.
    - uint32_t current_call_id_ = 10;
    + uint32_t current_call_id_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index abcf5c1..3180f4e 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -34,8 +34,11 @@ using namespace folly;
      using namespace wangle;
      using hbase::pb::ResponseHeader;
      using hbase::pb::GetResponse;
    +using google::protobuf::Message;

    -ClientHandler::ClientHandler(std::string user_name) : user_name_(user_name) {}
    +ClientHandler::ClientHandler(std::string user_name)
    + : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
    + resp_msgs_() {}

      void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
        if (LIKELY(buf != nullptr)) {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index 41bb883..68513de 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -47,7 +47,7 @@ public:
                                         std::unique_ptr<Request> r) override;

      private:
    - bool need_send_header_ = true;
    + bool need_send_header_;
        std::string user_name_;
        ClientSerializer ser_;
        ClientDeserializer deser_;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/connection-factory.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
    index b546269..9102d60 100644
    --- a/hbase-native-client/connection/connection-factory.cc
    +++ b/hbase-native-client/connection/connection-factory.cc
    @@ -40,7 +40,7 @@ using namespace folly;
      using namespace hbase;
      using namespace wangle;

    -ConnectionFactory::ConnectionFactory() {
    +ConnectionFactory::ConnectionFactory() : bootstrap_() {
        bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
        bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/connection-factory.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
    index 5a45316..fc4e161 100644
    --- a/hbase-native-client/connection/connection-factory.h
    +++ b/hbase-native-client/connection/connection-factory.h
    @@ -31,6 +31,7 @@ namespace hbase {
      class ConnectionFactory {
      public:
        ConnectionFactory();
    + virtual ~ConnectionFactory() = default;

        virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
                                                              int port);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index a967df2..72c1306 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -35,14 +35,14 @@ class RemoveServiceFilter

      public:
        RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    - ConnectionPool *cp)
    + ConnectionPool &cp)
            : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
              cp_(cp) {}

        folly::Future<folly::Unit> close() override {
          if (!released.exchange(true)) {
            return this->service_->close().then(
    - [this]() { this->cp_->close(this->sn_); });
    + [this]() { this->cp_.close(this->sn_); });
          } else {
            return folly::makeFuture();
          }
    @@ -57,12 +57,14 @@ public:
      private:
        std::atomic<bool> released{false};
        hbase::pb::ServerName sn_;
    - ConnectionPool *cp_;
    + ConnectionPool &cp_;
      };

    -ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
    +ConnectionPool::ConnectionPool()
    + : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
    +}
      ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
    - : cf_(cf) {}
    + : cf_(cf), connections_(), map_mutex_() {}

      std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
        SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
    @@ -70,7 +72,7 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
        if (found == connections_.end() || found->second == nullptr) {
          SharedMutexWritePriority::WriteHolder holder(std::move(holder));
          auto new_con = cf_->make_connection(sn.host_name(), sn.port());
    - auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
    + auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
          connections_[sn] = wrapped;
          return new_con;
        }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/pipeline.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/pipeline.cc b/hbase-native-client/connection/pipeline.cc
    index b9f5e0b..39cce8b 100644
    --- a/hbase-native-client/connection/pipeline.cc
    +++ b/hbase-native-client/connection/pipeline.cc
    @@ -30,6 +30,8 @@ using namespace folly;
      using namespace hbase;
      using namespace wangle;

    +RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
    +
      SerializePipeline::Ptr
      RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
        auto pipeline = SerializePipeline::create();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/pipeline.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
    index 6c4f4ff..e90cb93 100644
    --- a/hbase-native-client/connection/pipeline.h
    +++ b/hbase-native-client/connection/pipeline.h
    @@ -31,6 +31,7 @@ using SerializePipeline =

      class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
      public:
    + RpcPipelineFactory();
        SerializePipeline::Ptr
        newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;


    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/connection/response.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
    index a7f7939..d2eb19a 100644
    --- a/hbase-native-client/connection/response.h
    +++ b/hbase-native-client/connection/response.h
    @@ -31,10 +31,12 @@ namespace hbase {

      class Response {
      public:
    - Response() : call_id_(0) {}
    + Response() : call_id_(0), response_(nullptr) {}
        uint32_t call_id() { return call_id_; }
        void set_call_id(uint32_t call_id) { call_id_ = call_id; }
    - std::shared_ptr<google::protobuf::Message> response() { return response_; }
    + std::shared_ptr<google::protobuf::Message> response() const {
    + return response_;
    + }
        void set_response(std::shared_ptr<google::protobuf::Message> response) {
          response_ = std::move(response);
        }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index e555ba4..447248b 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -22,6 +22,7 @@ cxx_library(
              "client.h",
              "connection.h",
              "hbase_macros.h",
    + "region-location.h",
              "location-cache.h",
              "table-name.h",
              # TODO: move this out of exported
    @@ -32,7 +33,6 @@ cxx_library(
              "client.cc",
              "location-cache.cc",
              "meta-utils.cc",
    - "table-name.cc",
          ],
          deps=[
              "//connection:connection",
    @@ -42,6 +42,7 @@ cxx_library(
              "//third-party:wangle",
              "//third-party:zookeeper_mt",
          ],
    + compiler_flags=['-Weffc++'],
          visibility=[
              'PUBLIC',
          ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index c81deba..539051a 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -21,20 +21,31 @@
      #include <folly/Logging.h>
      #include <folly/io/IOBuf.h>

    +#include "connection/response.h"
    +#include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
      #include "serde/zk-deserializer.h"

      using namespace std;
      using namespace folly;
    -using namespace hbase::pb;
    -using namespace hbase;
    +
    +using hbase::Response;
    +using hbase::LocationCache;
    +using hbase::RegionLocation;
    +using hbase::HBaseService;
    +using hbase::pb::ScanResponse;
    +using hbase::pb::TableName;
    +using hbase::pb::ServerName;
    +using hbase::pb::MetaRegionServer;
    +using hbase::pb::RegionInfo;

      // TODO(eclark): make this configurable on client creation
      static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";

      LocationCache::LocationCache(string quorum_spec,
                                   shared_ptr<folly::Executor> executor)
    - : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
    + : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
    + meta_lock_(), cp_(), meta_util_() {
        zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
      }

    @@ -89,3 +100,23 @@ ServerName LocationCache::ReadMetaLocation() {
        }
        return mrs.server();
      }
    +
    +Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
    + const string &row) {
    + return this->LocateMeta()
    + .then([&](ServerName sn) { return this->cp_.get(sn); })
    + .then([&](std::shared_ptr<HBaseService> service) {
    + return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
    + })
    + .then([&](Response resp) {
    + // take the protobuf response and make it into
    + // a region location.
    + return this->parse_response(std::move(resp));
    + });
    +}
    +
    +RegionLocation LocationCache::parse_response(const Response &resp) {
    + auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
    + LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
    + return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index ce97c43..cfd6838 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -27,9 +27,19 @@
      #include <mutex>
      #include <string>

    -#include "if/HBase.pb.h"
    +#include "connection/connection-pool.h"
    +#include "core/meta-utils.h"
    +#include "core/table-name.h"
    +#include "core/region-location.h"

      namespace hbase {
    +
    +class Request;
    +class Response;
    +namespace pb {
    +class ServerName;
    +}
    +
      class LocationCache {
      public:
        explicit LocationCache(std::string quorum_spec,
    @@ -38,6 +48,9 @@ public:
        // Meta Related Methods.
        // These are only public until testing is complete
        folly::Future<hbase::pb::ServerName> LocateMeta();
    + folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
    + const std::string &row);
    + RegionLocation parse_response(const Response &resp);
        void InvalidateMeta();

      private:
    @@ -48,7 +61,11 @@ private:
        std::shared_ptr<folly::Executor> executor_;
        std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
        std::mutex meta_lock_;
    + ConnectionPool cp_;
    + MetaUtil meta_util_;
    +

    + // TODO: migrate this to a smart pointer with a deleter.
        zhandle_t *zk_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/meta-utils.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
    index 7c7d45f..d2fdd88 100644
    --- a/hbase-native-client/core/meta-utils.cc
    +++ b/hbase-native-client/core/meta-utils.cc
    @@ -21,11 +21,61 @@

      #include <folly/Conv.h>

    +#include "connection/request.h"
    +#include "connection/response.h"
      #include "core/table-name.h"
    +#include "if/Client.pb.h"

    -using namespace hbase;
    +using hbase::pb::TableName;
    +using hbase::MetaUtil;
    +using hbase::Request;
    +using hbase::Response;
    +using hbase::pb::ScanRequest;
    +using hbase::pb::RegionSpecifier_RegionSpecifierType;
    +
    +static const std::string META_REGION = "1588230740";

      std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
    - const std::string &row) {
    + const std::string &row) const {
        return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
      }
    +
    +std::unique_ptr<Request>
    +MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
    + auto request = Request::scan();
    + auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
    +
    + msg->set_number_of_rows(1);
    + msg->set_close_scanner(true);
    +
    + // Set the region this scan goes to
    + auto region = msg->mutable_region();
    + region->set_value(META_REGION);
    + region->set_type(RegionSpecifier_RegionSpecifierType::
    + RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
    +
    + auto scan = msg->mutable_scan();
    + // We don't care about before, just now.
    + scan->set_max_versions(1);
    + // Meta should be cached at all times.
    + scan->set_cache_blocks(true);
    + // We only want one row right now.
    + //
    + // TODO(eclark): Figure out if we should get more.
    + scan->set_caching(1);
    + // Close the scan after we have data.
    + scan->set_small(true);
    + // We know where to start but not where to end.
    + scan->set_reversed(true);
    + // Give me everything or nothing.
    + scan->set_allow_partial_results(false);
    +
    + // Set the columns that we need
    + auto info_col = scan->add_column();
    + info_col->set_family("info");
    + info_col->add_qualifier("server");
    + info_col->add_qualifier("regioninfo");
    +
    + scan->set_start_row(region_lookup_rowkey(tn, row));
    + return request;
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/meta-utils.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
    index 391d0ea..e007d02 100644
    --- a/hbase-native-client/core/meta-utils.h
    +++ b/hbase-native-client/core/meta-utils.h
    @@ -20,11 +20,16 @@

      #include <string>

    +#include "connection/Request.h"
    +#include "core/table-name.h"
    +
      namespace hbase {
    -class TableName;

      class MetaUtil {
      public:
    - std::string region_lookup_rowkey(const TableName &tn, const std::string &row);
    + std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
    + const std::string &row) const;
    + std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
    + const std::string &row) const;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/region-location.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
    new file mode 100644
    index 0000000..a46b8e2
    --- /dev/null
    +++ b/hbase-native-client/core/region-location.h
    @@ -0,0 +1,45 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +#include <memory>
    +
    +#include "connection/service.h"
    +#include "if/HBase.pb.h"
    +
    +
    +namespace hbase {
    +
    +class RegionLocation {
    +public:
    + RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
    + std::shared_ptr<HBaseService> service)
    + : ri_(ri), sn_(sn), service_(service) {}
    +
    + const hbase::pb::RegionInfo& region_info() { return ri_; }
    + const hbase::pb::ServerName& server_name() { return sn_; }
    + std::shared_ptr<HBaseService> service() { return service_; }
    +
    +private:
    + hbase::pb::RegionInfo ri_;
    + hbase::pb::ServerName sn_;
    + std::shared_ptr<HBaseService> service_;
    +};
    +
    +} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index 11dcd68..ab614e4 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -27,6 +27,7 @@

      #include "connection/connection-pool.h"
      #include "core/client.h"
    +#include "core/table-name.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"

    @@ -38,13 +39,14 @@ using hbase::Request;
      using hbase::HBaseService;
      using hbase::LocationCache;
      using hbase::ConnectionPool;
    +using hbase::TableNameUtil;
      using hbase::pb::ServerName;
      using hbase::pb::RegionSpecifier_RegionSpecifierType;
      using hbase::pb::GetRequest;
      using hbase::pb::GetResponse;

      // TODO(eclark): remove the need for this.
    -DEFINE_string(region, "1588230740", "What region to send a get to");
    +DEFINE_string(table, "t", "What region to send a get");
      DEFINE_string(row, "test", "What row to get");
      DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");

    @@ -59,48 +61,8 @@ int main(int argc, char *argv[]) {
        auto cpu_ex = wangle::getCPUExecutor();
        LocationCache cache{FLAGS_zookeeper, cpu_ex};
        auto result =
    - cache.LocateMeta()
    - .then([&cp = cp](ServerName sn) { return cp.get(sn); })
    - .then([](shared_ptr<HBaseService> con) {
    - // Send the request
    - auto r = Request::get();
    - // This is a get request so make that
    - auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
    - // Set what region
    - req_msg->mutable_region()->set_value(FLAGS_region);
    - // It's always this.
    - req_msg->mutable_region()->set_type(
    - RegionSpecifier_RegionSpecifierType::
    - RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
    -
    - // What row.
    - req_msg->mutable_get()->set_row(FLAGS_row);
    -
    - return (*con)(std::move(r));
    - })
    - .then([](Response resp) {
    - return static_pointer_cast<GetResponse>(resp.response());
    - })
    - .via(cpu_ex.get())
    - .then([](shared_ptr<GetResponse> get_resp) {
    - cout << "GetResponse has_result = " << get_resp->has_result()
    - << '\n';
    - if (get_resp->has_result()) {
    - auto &r = get_resp->result();
    - cout << "Result cell_size = " << r.cell_size() << endl;
    - for (auto &cell : r.cell()) {
    - cout << "\trow = " << cell.row()
    - << " family = " << cell.family()
    - << " qualifier = " << cell.qualifier()
    - << " timestamp = " << cell.timestamp()
    - << " value = " << cell.value() << endl;
    - }
    - return 0;
    - }
    -
    - return 1;
    - })
    + cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
                .get(milliseconds(5000));

    - return result;
    + return 0;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/table-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/table-name-test.cc b/hbase-native-client/core/table-name-test.cc
    index 8aff375..7bad3f1 100644
    --- a/hbase-native-client/core/table-name-test.cc
    +++ b/hbase-native-client/core/table-name-test.cc
    @@ -25,26 +25,30 @@
      #include "core/table-name.h"

      using namespace hbase;
    +using hbase::pb::TableName;

      TEST(TestTableName, TestToStringNoDefault) {
    - TableName tn{"TestTableName"};
    + TableName tn;
    + tn.set_qualifier("TestTableName");
        std::string result = folly::to<std::string>(tn);
        ASSERT_EQ(result.find("default"), std::string::npos);
        ASSERT_EQ("TestTableName", result);
      }
    +
    +TEST(TestTableName, TestToStringNoDefaltWhenSet) {
    + TableName tn;
    + tn.set_namespace_("default");
    + tn.set_qualifier("TestTableName");
    + std::string result = folly::to<std::string>(tn);
    + ASSERT_EQ(result.find("default"), std::string::npos);
    + ASSERT_EQ("TestTableName", result);
    +}
    +
      TEST(TestTableName, TestToStringIncludeNS) {
    - TableName tn{"hbase", "acl"};
    + TableName tn;
    + tn.set_namespace_("hbase");
    + tn.set_qualifier("acl");
        std::string result = folly::to<std::string>(tn);
        ASSERT_EQ(result.find("hbase"), 0);
        ASSERT_EQ("hbase:acl", result);
      }
    -TEST(TestTableName, TestIsDefault) {
    - TableName default_t1{"in_default"};
    - ASSERT_TRUE(default_t1.is_default_name_space());
    -
    - TableName default_t2{"default", "in_also"};
    - ASSERT_TRUE(default_t2.is_default_name_space());
    -
    - TableName non_default{"testing", "hmm"};
    - ASSERT_FALSE(non_default.is_default_name_space());
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/table-name.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/table-name.cc b/hbase-native-client/core/table-name.cc
    deleted file mode 100644
    index 33087fd..0000000
    --- a/hbase-native-client/core/table-name.cc
    +++ /dev/null
    @@ -1,34 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#include "core/table-name.h"
    -
    -#include <folly/Conv.h>
    -
    -using namespace hbase;
    -
    -TableName::TableName(std::string table_name)
    - : name_space_("default"), table_(table_name) {}
    -TableName::TableName(std::string name_space, std::string table_name)
    - : name_space_(name_space), table_(table_name) {}
    -bool TableName::is_default_name_space() const {
    - return name_space_.length() == 0 || name_space_ == "default";
    -}
    -bool TableName::operator==(const TableName &other) const {
    - return name_space_ == other.name_space_ && table_ == other.table_;
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/core/table-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
    index 816f7da..1612667 100644
    --- a/hbase-native-client/core/table-name.h
    +++ b/hbase-native-client/core/table-name.h
    @@ -21,33 +21,30 @@
      #include <memory>
      #include <string>

    +#include "if/HBase.pb.h"
      #include <folly/Conv.h>

      namespace hbase {
    -
    -// This is the core class of a HBase client.
    -class TableName {
    -public:
    - explicit TableName(std::string table_name);
    - explicit TableName(std::string name_space, std::string table_name);
    -
    - std::string name_space() const { return name_space_; };
    - std::string table() const { return table_; };
    - bool is_default_name_space() const;
    - bool operator==(const TableName &other) const;
    -
    -private:
    - std::string name_space_;
    - std::string table_;
    -};
    +namespace pb {

      // Provide folly::to<std::string>(TableName);
      template <class String> void toAppend(const TableName &in, String *result) {
    - if (in.is_default_name_space()) {
    - folly::toAppend(in.table(), result);
    + if (!in.has_namespace_() || in.namespace_() == "default") {
    + folly::toAppend(in.qualifier(), result);
        } else {
    - folly::toAppend(in.name_space(), ':', in.table(), result);
    + folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
        }
      }

    +} // namespace pb
    +
    +class TableNameUtil {
    +public:
    + static ::hbase::pb::TableName create(std::string table_name) {
    + ::hbase::pb::TableName tn;
    + tn.set_namespace_("default");
    + tn.set_qualifier(table_name);
    + return tn;
    + }
    +};
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/serde/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
    index 0014c0b..539a221 100644
    --- a/hbase-native-client/serde/BUCK
    +++ b/hbase-native-client/serde/BUCK
    @@ -34,6 +34,7 @@ cxx_library(name="serde",
                      ":client-serializer-test",
                      ":client-deserializer-test",
                  ],
    + compiler_flags=['-Weffc++'],
                  visibility=[
                      'PUBLIC',
                  ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/utils/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
    index ad12f7d..2ab6a89 100644
    --- a/hbase-native-client/utils/BUCK
    +++ b/hbase-native-client/utils/BUCK
    @@ -25,12 +25,13 @@ cxx_library(name="utils",
                  deps=[
                      '//third-party:folly',
                  ],
    + tests=[
    + ":user-util-test"
    + ],
                  visibility=[
                      'PUBLIC',
                  ],
    - tests=[
    - ":user-util-test"
    - ], )
    + compiler_flags=['-Weffc++'], )
      cxx_test(name="user-util-test",
               srcs=[
                   "user-util-test.cc",

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/utils/user-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
    index b801537..3d963b3 100644
    --- a/hbase-native-client/utils/user-util.cc
    +++ b/hbase-native-client/utils/user-util.cc
    @@ -27,6 +27,8 @@
      using namespace hbase;
      using namespace std;

    +UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
    +
      string UserUtil::user_name() {
        if (!init_) {
          compute_user_name();

    http://git-wip-us.apache.org/repos/asf/hbase/blob/a7225e42/hbase-native-client/utils/user-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
    index 9ca28e3..da95d78 100644
    --- a/hbase-native-client/utils/user-util.h
    +++ b/hbase-native-client/utils/user-util.h
    @@ -26,11 +26,12 @@
      namespace hbase {
      class UserUtil {
      public:
    + UserUtil();
        std::string user_name();

      private:
        void compute_user_name();
    - std::atomic<bool> init_{false};
    + std::atomic<bool> init_;
        std::string user_name_;
        std::mutex m_;
      };
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15690 Add utility to get current username

    Summary:
    Add a class to fine the username of the current process.
    It will only call out once and is multithread safe

    Test Plan: Unit Test

    Differential Revision: https://reviews.facebook.net/D57081


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de514926
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de514926
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de514926

    Branch: refs/heads/HBASE-14850
    Commit: de5149269d2ea4dceb4ad2d7d75fc9a09954d95b
    Parents: 0ec0a25
    Author: Elliott Clark <eclark@apache.org>
    Authored: Thu Apr 21 21:17:11 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/utils/BUCK | 40 +++++++++++++++++
      hbase-native-client/utils/user-util-test.cc | 34 +++++++++++++++
      hbase-native-client/utils/user-util.cc | 55 ++++++++++++++++++++++++
      hbase-native-client/utils/user-util.h | 37 ++++++++++++++++
      4 files changed, 166 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/de514926/hbase-native-client/utils/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
    new file mode 100644
    index 0000000..2b65b12
    --- /dev/null
    +++ b/hbase-native-client/utils/BUCK
    @@ -0,0 +1,40 @@
    +##
    +# Licensed to the Apache Software Foundation (ASF) under one
    +# or more contributor license agreements. See the NOTICE file
    +# distributed with this work for additional information
    +# regarding copyright ownership. The ASF licenses this file
    +# to you under the Apache License, Version 2.0 (the
    +# "License"); you may not use this file except in compliance
    +# with the License. You may obtain a copy of the License at
    +#
    +# http://www.apache.org/licenses/LICENSE-2.0
    +#
    +# Unless required by applicable law or agreed to in writing, software
    +# distributed under the License is distributed on an "AS IS" BASIS,
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +# See the License for the specific language governing permissions and
    +# limitations under the License.
    +
    +cxx_library(name="utils",
    + exported_headers=[
    + "user-util.h",
    + ],
    + srcs=[
    + "user-util.cc",
    + ],
    + deps=[
    + '//third-party:folly',
    + ],
    + visibility=[
    + 'PUBLIC',
    + ],
    + tests=[
    + ":user-util-test"
    + ],)
    +cxx_test(name="user-util-test",
    + srcs=[
    + "user-util-test.cc",
    + ],
    + deps=[
    + ":utils",
    + ],)

    http://git-wip-us.apache.org/repos/asf/hbase/blob/de514926/hbase-native-client/utils/user-util-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
    new file mode 100644
    index 0000000..2a7434f
    --- /dev/null
    +++ b/hbase-native-client/utils/user-util-test.cc
    @@ -0,0 +1,34 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include <gtest/gtest.h>
    +#include <string>
    +#include <folly/Logging.h>
    +
    +#include "utils/user-util.h"
    +
    +using namespace std;
    +using namespace hbase;
    +
    +TEST(TestUserUtil, TestGetSomething) {
    + UserUtil u_util;
    + string name = u_util.user_name();
    +
    + ASSERT_GT(name.length(), 0);
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/de514926/hbase-native-client/utils/user-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
    new file mode 100644
    index 0000000..c4427e3
    --- /dev/null
    +++ b/hbase-native-client/utils/user-util.cc
    @@ -0,0 +1,55 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "utils/user-util.h"
    +
    +#include <pwd.h>
    +#include <unistd.h>
    +#include <sys/types.h>
    +#include <folly/Logging.h>
    +
    +using namespace hbase;
    +using namespace std;
    +
    +string UserUtil::user_name() {
    + if (!init_) {
    + compute_user_name();
    + }
    + return user_name_;
    +}
    +
    +void UserUtil::compute_user_name() {
    + lock_guard<mutex> lock(m_);
    +
    + if (init_) {
    + return;
    + }
    +
    + // According to the man page of getpwuid
    + // this should never be free'd
    + //
    + // So yeah a raw pointer with no ownership....
    + struct passwd *passwd = getpwuid(getuid());
    +
    + // make sure that we got something.
    + if (passwd && passwd->pw_name) {
    + user_name_ = string{passwd->pw_name};
    + init_ = true;
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/de514926/hbase-native-client/utils/user-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
    new file mode 100644
    index 0000000..4f74948
    --- /dev/null
    +++ b/hbase-native-client/utils/user-util.h
    @@ -0,0 +1,37 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#pragma once
    +
    +#include <memory>
    +#include <atomic>
    +#include <mutex>
    +
    +namespace hbase {
    +class UserUtil {
    +public:
    + std::string user_name();
    +
    +private:
    + void compute_user_name();
    + std::atomic<bool> init_{false};
    + std::string user_name_;
    + std::mutex m_;
    +};
    +} // namespace hbase
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15687 Allow decoding more than GetResponse from the server

    Summary:
    We'll need more than get's for the client to be usable. So now the
    Request class contains the protobufs needed to encode and decode
    rpc's.

    I also added some helper methods to create initial requests.

    Test Plan: It compiles and still gets data from HBase meta

    Differential Revision: https://reviews.facebook.net/D57327


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a98ebe7
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a98ebe7
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a98ebe7

    Branch: refs/heads/HBASE-14850
    Commit: 3a98ebe7d8073c8eb2e2835b3ec8dc97e8d692ad
    Parents: fef51e2
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed Apr 27 15:27:09 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/connection/BUCK | 1 +
      .../connection/client-dispatcher.cc | 6 +--
      .../connection/client-dispatcher.h | 6 +--
      .../connection/client-handler.cc | 30 ++++++++++---
      hbase-native-client/connection/client-handler.h | 18 ++++++--
      .../connection/connection-factory.cc | 6 +--
      .../connection/connection-factory.h | 2 +-
      hbase-native-client/connection/pipeline.h | 3 +-
      hbase-native-client/connection/request.cc | 45 ++++++++++++++++++++
      hbase-native-client/connection/request.h | 25 +++++++++--
      hbase-native-client/core/simple-client.cc | 13 +++---
      11 files changed, 123 insertions(+), 32 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
    index 5067708..d393885 100644
    --- a/hbase-native-client/connection/BUCK
    +++ b/hbase-native-client/connection/BUCK
    @@ -32,6 +32,7 @@ cxx_library(name="connection",
                      "client-handler.cc",
                      "connection-factory.cc",
                      "pipeline.cc",
    + "request.cc",
                  ],
                  deps=[
                      "//if:if",

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/client-dispatcher.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
    index 25cff7d..eea0a17 100644
    --- a/hbase-native-client/connection/client-dispatcher.cc
    +++ b/hbase-native-client/connection/client-dispatcher.cc
    @@ -35,16 +35,16 @@ void ClientDispatcher::read(Context *ctx, Response in) {
        p.setValue(in);
      }

    -Future<Response> ClientDispatcher::operator()(Request arg) {
    +Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
        auto call_id = ++current_call_id_;

    - arg.set_call_id(call_id);
    + arg->set_call_id(call_id);
        auto &p = requests_[call_id];
        auto f = p.getFuture();
        p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
          this->requests_.erase(call_id);
        });
    - this->pipeline_->write(arg);
    + this->pipeline_->write(std::move(arg));

        return f;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/client-dispatcher.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
    index 89c7119..877e877 100644
    --- a/hbase-native-client/connection/client-dispatcher.h
    +++ b/hbase-native-client/connection/client-dispatcher.h
    @@ -27,11 +27,11 @@

      namespace hbase {
      class ClientDispatcher
    - : public wangle::ClientDispatcherBase<SerializePipeline, Request,
    - Response> {
    + : public wangle::ClientDispatcherBase<SerializePipeline,
    + std::unique_ptr<Request>, Response> {
      public:
        void read(Context *ctx, Response in) override;
    - folly::Future<Response> operator()(Request arg) override;
    + folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
        folly::Future<folly::Unit> close(Context *ctx) override;
        folly::Future<folly::Unit> close() override;


    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index 205993a7..abcf5c1 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -47,14 +47,30 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
          LOG(INFO) << "Read ResponseHeader size=" << used_bytes
                    << " call_id=" << header.call_id()
                    << " has_exception=" << header.has_exception();
    +
    + // Get the response protobuf from the map
    + auto search = resp_msgs_.find(header.call_id());
    + // It's an error if it's not there.
    + CHECK(search != resp_msgs_.end());
    + auto resp_msg = search->second;
    + CHECK(resp_msg != nullptr);
    +
    + // Make sure we don't leak the protobuf
    + resp_msgs_.erase(search);
    +
    + // set the call_id.
    + // This will be used to by the dispatcher to match up
    + // the promise with the response.
          received.set_call_id(header.call_id());

    + // If there was an exception then there's no
    + // data left on the wire.
          if (header.has_exception() == false) {
            buf->trimStart(used_bytes);
    - // For now assume that everything was a get.
    - // We'll need to set this up later.
    - received.set_response(std::make_shared<GetResponse>());
    - used_bytes = deser_.parse_delimited(buf.get(), received.response().get());
    + used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
    + // Make sure that bytes were parsed.
    + CHECK(used_bytes == buf->length());
    + received.set_response(resp_msg);
          }
          ctx->fireRead(std::move(received));
        }
    @@ -62,7 +78,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {

      // TODO(eclark): Figure out how to handle the
      // network errors that are going to come.
    -Future<Unit> ClientHandler::write(Context *ctx, Request r) {
    +Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
        // Keep track of if we have sent the header.
        if (UNLIKELY(need_send_header_)) {
          need_send_header_ = false;
    @@ -78,5 +94,7 @@ Future<Unit> ClientHandler::write(Context *ctx, Request r) {
          ctx->fireWrite(std::move(pre));
        }

    - return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
    + resp_msgs_[r->call_id()] = r->resp_msg();
    + return ctx->fireWrite(
    + ser_.request(r->call_id(), r->method(), r->req_msg().get()));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index dbaf5a0..41bb883 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -30,20 +30,30 @@ namespace hbase {
      class Request;
      class Response;
      }
    +namespace google {
    +namespace protobuf {
    +class Message;
    +}
    +}

      namespace hbase {
    -class ClientHandler
    - : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
    - std::unique_ptr<folly::IOBuf>> {
    +class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
    + Response, std::unique_ptr<Request>,
    + std::unique_ptr<folly::IOBuf>> {
      public:
        ClientHandler(std::string user_name);
        void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
    - folly::Future<folly::Unit> write(Context *ctx, Request r) override;
    + folly::Future<folly::Unit> write(Context *ctx,
    + std::unique_ptr<Request> r) override;

      private:
        bool need_send_header_ = true;
        std::string user_name_;
        ClientSerializer ser_;
        ClientDeserializer deser_;
    +
    + // in flight requests
    + std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
    + resp_msgs_;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/connection-factory.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
    index 5d1b0da..7073f9d 100644
    --- a/hbase-native-client/connection/connection-factory.cc
    +++ b/hbase-native-client/connection/connection-factory.cc
    @@ -45,14 +45,14 @@ ConnectionFactory::ConnectionFactory() {
        bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
      }

    -std::shared_ptr<Service<Request, Response>>
    +std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
      ConnectionFactory::make_connection(std::string host, int port) {
        // Connect to a given server
        // Then when connected create a ClientDispactcher.
        auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
        auto dispatcher = std::make_shared<ClientDispatcher>();
        dispatcher->setPipeline(pipeline);
    - auto service =
    - std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
    + auto service = std::make_shared<
    + CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
        return service;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/connection-factory.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
    index 73ac032..8d1d2f0 100644
    --- a/hbase-native-client/connection/connection-factory.h
    +++ b/hbase-native-client/connection/connection-factory.h
    @@ -31,7 +31,7 @@ namespace hbase {
      class ConnectionFactory {
      public:
        ConnectionFactory();
    - std::shared_ptr<wangle::Service<Request, Response>>
    + std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
        make_connection(std::string host, int port);

      private:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/pipeline.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
    index 8114fab..6c4f4ff 100644
    --- a/hbase-native-client/connection/pipeline.h
    +++ b/hbase-native-client/connection/pipeline.h
    @@ -26,7 +26,8 @@
      #include "utils/user-util.h"

      namespace hbase {
    -using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
    +using SerializePipeline =
    + wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;

      class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
      public:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/request.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/request.cc b/hbase-native-client/connection/request.cc
    new file mode 100644
    index 0000000..50ea029
    --- /dev/null
    +++ b/hbase-native-client/connection/request.cc
    @@ -0,0 +1,45 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "connection/request.h"
    +
    +#include "if/Client.pb.h"
    +
    +using namespace hbase;
    +
    +Request::Request(std::shared_ptr<google::protobuf::Message> req,
    + std::shared_ptr<google::protobuf::Message> resp,
    + std::string method)
    + : req_msg_(req), resp_msg_(resp), method_(method), call_id_(0) {}
    +
    +std::unique_ptr<Request> Request::get() {
    + return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
    + std::make_shared<hbase::pb::GetResponse>(),
    + "Get");
    +}
    +std::unique_ptr<Request> Request::mutate() {
    + return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
    + std::make_shared<hbase::pb::MutateResponse>(),
    + "Mutate");
    +}
    +std::unique_ptr<Request> Request::scan() {
    + return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
    + std::make_shared<hbase::pb::ScanResponse>(),
    + "Scan");
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/connection/request.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
    index e9e3e88..743c469 100644
    --- a/hbase-native-client/connection/request.h
    +++ b/hbase-native-client/connection/request.h
    @@ -21,22 +21,39 @@
      #include <google/protobuf/message.h>

      #include <cstdint>
    +#include <memory>
      #include <string>

      namespace hbase {
      class Request {
      public:
    - Request() : call_id_(0) {}
    + static std::unique_ptr<Request> get();
    + static std::unique_ptr<Request> mutate();
    + static std::unique_ptr<Request> scan();
    +
    + Request(std::shared_ptr<google::protobuf::Message> req,
    + std::shared_ptr<google::protobuf::Message> resp, std::string method);
    +
        uint32_t call_id() { return call_id_; }
        void set_call_id(uint32_t call_id) { call_id_ = call_id; }
    - google::protobuf::Message *msg() { return msg_.get(); }
    - void set_msg(std::shared_ptr<google::protobuf::Message> msg) { msg_ = msg; }
    +
    + std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
    + std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
    +
    + void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
    + req_msg_ = msg;
    + }
    + void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
    + resp_msg_ = msg;
    + }
    +
        std::string method() { return method_; }
        void set_method(std::string method) { method_ = method; }

      private:
        uint32_t call_id_;
    - std::shared_ptr<google::protobuf::Message> msg_ = nullptr;
    + std::shared_ptr<google::protobuf::Message> req_msg_ = nullptr;
    + std::shared_ptr<google::protobuf::Message> resp_msg_ = nullptr;
        std::string method_ = "Get";
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/3a98ebe7/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index 8b2fae5..2cb6200 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -58,24 +58,23 @@ int main(int argc, char *argv[]) {
        auto conn = cf.make_connection(result.host_name(), result.port());

        // Send the request
    - Request r;
    + auto r = Request::get();

        // This is a get request so make that
    - auto msg = make_shared<hbase::pb::GetRequest>();
    + auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());

        // Set what region
    - msg->mutable_region()->set_value(FLAGS_region);
    + req_msg->mutable_region()->set_value(FLAGS_region);
        // It's always this.
    - msg->mutable_region()->set_type(
    + req_msg->mutable_region()->set_type(
            RegionSpecifier_RegionSpecifierType::
                RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);

        // What row.
    - msg->mutable_get()->set_row(FLAGS_row);
    + req_msg->mutable_get()->set_row(FLAGS_row);

        // Send it.
    - r.set_msg(msg);
    - auto resp = (*conn)(r).get(milliseconds(5000));
    + auto resp = (*conn)(std::move(r)).get(milliseconds(5000));

        auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
        cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15655 Set up doxygen for documentation

    Summary:
    Add on doxygen config.
    It outputs to the docs folder.
    Added on a makefile

    Test Plan:
         doxygen hbase.doxygen

    Differential Revision: https://reviews.facebook.net/D57663


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52719872
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52719872
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52719872

    Branch: refs/heads/HBASE-14850
    Commit: 52719872e25108111cc8c7a33a13ba5f316e34ce
    Parents: 6b4d759
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed May 4 18:19:33 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      .gitignore | 1 +
      hbase-native-client/Dockerfile | 2 +-
      hbase-native-client/Makefile | 20 +++
      hbase-native-client/hbase.doxygen | 319 +++++++++++++++++++++++++++++++++
      4 files changed, 341 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/52719872/.gitignore
    ----------------------------------------------------------------------
    diff --git a/.gitignore b/.gitignore
    index 626a4d0..1b1e9d5 100644
    --- a/.gitignore
    +++ b/.gitignore
    @@ -16,3 +16,4 @@ hbase-*/test
      *.ipr
      patchprocess/
      dependency-reduced-pom.xml
    +docs/

    http://git-wip-us.apache.org/repos/asf/hbase/blob/52719872/hbase-native-client/Dockerfile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
    index 8e33e74..be0cbc6 100644
    --- a/hbase-native-client/Dockerfile
    +++ b/hbase-native-client/Dockerfile
    @@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -p

      ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"

    -RUN apt-get install -y vim maven inetutils-ping python-pip && \
    +RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && \
            pip install yapf && \
            apt-get -qq clean && \
            apt-get -y -qq autoremove && \

    http://git-wip-us.apache.org/repos/asf/hbase/blob/52719872/hbase-native-client/Makefile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
    new file mode 100644
    index 0000000..90ec6ea
    --- /dev/null
    +++ b/hbase-native-client/Makefile
    @@ -0,0 +1,20 @@
    +build:
    + $(shell buck build core/... )
    +
    +check:
    + $(shell buck test --all --no-results-cache )
    +
    +doc:
    + $(shell doxygen hbase.doxygen > /dev/null )
    +
    +clean:
    + $(shell rm -rf docs )
    +
    +help:
    + @echo "Available targets:"
    + @echo ""
    + @echo " build : will build everything."
    + @echo " clean : will remove the docs folder"
    + @echo " check : will test everything."
    +
    +all: build doc check

    http://git-wip-us.apache.org/repos/asf/hbase/blob/52719872/hbase-native-client/hbase.doxygen
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/hbase.doxygen b/hbase-native-client/hbase.doxygen
    new file mode 100644
    index 0000000..22e2d21
    --- /dev/null
    +++ b/hbase-native-client/hbase.doxygen
    @@ -0,0 +1,319 @@
    +# Doxyfile 1.8.11
    +
    +#---------------------------------------------------------------------------
    +# Project related configuration options
    +#---------------------------------------------------------------------------
    +DOXYFILE_ENCODING = UTF-8
    +PROJECT_NAME = "HBase Native Client"
    +PROJECT_NUMBER =
    +PROJECT_BRIEF =
    +PROJECT_LOGO = ../hbase-server/src/main/resources/hbase-webapps/static/hbase_logo_small.png
    +OUTPUT_DIRECTORY = docs
    +CREATE_SUBDIRS = YES
    +ALLOW_UNICODE_NAMES = NO
    +OUTPUT_LANGUAGE = English
    +BRIEF_MEMBER_DESC = YES
    +REPEAT_BRIEF = YES
    +ABBREVIATE_BRIEF =
    +ALWAYS_DETAILED_SEC = NO
    +INLINE_INHERITED_MEMB = NO
    +FULL_PATH_NAMES = NO
    +STRIP_FROM_PATH =
    +STRIP_FROM_INC_PATH =
    +SHORT_NAMES = NO
    +JAVADOC_AUTOBRIEF = NO
    +QT_AUTOBRIEF = NO
    +MULTILINE_CPP_IS_BRIEF = NO
    +INHERIT_DOCS = YES
    +SEPARATE_MEMBER_PAGES = NO
    +TAB_SIZE = 4
    +ALIASES =
    +TCL_SUBST =
    +OPTIMIZE_OUTPUT_FOR_C = NO
    +OPTIMIZE_OUTPUT_JAVA = NO
    +OPTIMIZE_FOR_FORTRAN = NO
    +OPTIMIZE_OUTPUT_VHDL = NO
    +EXTENSION_MAPPING =
    +MARKDOWN_SUPPORT = YES
    +AUTOLINK_SUPPORT = YES
    +BUILTIN_STL_SUPPORT = NO
    +CPP_CLI_SUPPORT = NO
    +SIP_SUPPORT = NO
    +IDL_PROPERTY_SUPPORT = YES
    +DISTRIBUTE_GROUP_DOC = NO
    +GROUP_NESTED_COMPOUNDS = NO
    +SUBGROUPING = YES
    +INLINE_GROUPED_CLASSES = NO
    +INLINE_SIMPLE_STRUCTS = NO
    +TYPEDEF_HIDES_STRUCT = NO
    +LOOKUP_CACHE_SIZE = 0
    +#---------------------------------------------------------------------------
    +# Build related configuration options
    +#---------------------------------------------------------------------------
    +EXTRACT_ALL = NO
    +EXTRACT_PRIVATE = NO
    +EXTRACT_PACKAGE = NO
    +EXTRACT_STATIC = NO
    +EXTRACT_LOCAL_CLASSES = YES
    +EXTRACT_LOCAL_METHODS = NO
    +EXTRACT_ANON_NSPACES = NO
    +HIDE_UNDOC_MEMBERS = NO
    +HIDE_UNDOC_CLASSES = NO
    +HIDE_FRIEND_COMPOUNDS = NO
    +HIDE_IN_BODY_DOCS = NO
    +INTERNAL_DOCS = NO
    +CASE_SENSE_NAMES = YES
    +HIDE_SCOPE_NAMES = NO
    +HIDE_COMPOUND_REFERENCE= NO
    +SHOW_INCLUDE_FILES = YES
    +SHOW_GROUPED_MEMB_INC = NO
    +FORCE_LOCAL_INCLUDES = NO
    +INLINE_INFO = YES
    +SORT_MEMBER_DOCS = YES
    +SORT_BRIEF_DOCS = NO
    +SORT_MEMBERS_CTORS_1ST = NO
    +SORT_GROUP_NAMES = NO
    +SORT_BY_SCOPE_NAME = NO
    +STRICT_PROTO_MATCHING = NO
    +GENERATE_TODOLIST = YES
    +GENERATE_TESTLIST = YES
    +GENERATE_BUGLIST = YES
    +GENERATE_DEPRECATEDLIST= YES
    +ENABLED_SECTIONS =
    +MAX_INITIALIZER_LINES = 30
    +SHOW_USED_FILES = YES
    +SHOW_FILES = YES
    +SHOW_NAMESPACES = YES
    +FILE_VERSION_FILTER =
    +LAYOUT_FILE =
    +CITE_BIB_FILES =
    +#---------------------------------------------------------------------------
    +# Configuration options related to warning and progress messages
    +#---------------------------------------------------------------------------
    +QUIET = NO
    +WARNINGS = YES
    +WARN_IF_UNDOCUMENTED = YES
    +WARN_IF_DOC_ERROR = YES
    +WARN_NO_PARAMDOC = NO
    +WARN_AS_ERROR = NO
    +WARN_FORMAT = "$file:$line: $text"
    +WARN_LOGFILE =
    +#---------------------------------------------------------------------------
    +# Configuration options related to the input files
    +#---------------------------------------------------------------------------
    +INPUT =
    +INPUT_ENCODING = UTF-8
    +FILE_PATTERNS =
    +RECURSIVE = YES
    +EXCLUDE = third-party/ buck-out/
    +EXCLUDE_SYMLINKS = NO
    +EXCLUDE_PATTERNS = /*-test.cc
    +EXCLUDE_SYMBOLS =
    +EXAMPLE_PATH =
    +EXAMPLE_PATTERNS =
    +EXAMPLE_RECURSIVE = NO
    +IMAGE_PATH =
    +INPUT_FILTER =
    +FILTER_PATTERNS =
    +FILTER_SOURCE_FILES = NO
    +FILTER_SOURCE_PATTERNS =
    +USE_MDFILE_AS_MAINPAGE =
    +#---------------------------------------------------------------------------
    +# Configuration options related to source browsing
    +#---------------------------------------------------------------------------
    +SOURCE_BROWSER = NO
    +INLINE_SOURCES = NO
    +STRIP_CODE_COMMENTS = YES
    +REFERENCED_BY_RELATION = NO
    +REFERENCES_RELATION = NO
    +REFERENCES_LINK_SOURCE = YES
    +SOURCE_TOOLTIPS = YES
    +USE_HTAGS = NO
    +VERBATIM_HEADERS = YES
    +CLANG_ASSISTED_PARSING = NO
    +CLANG_OPTIONS =
    +#---------------------------------------------------------------------------
    +# Configuration options related to the alphabetical class index
    +#---------------------------------------------------------------------------
    +ALPHABETICAL_INDEX = YES
    +COLS_IN_ALPHA_INDEX = 5
    +IGNORE_PREFIX =
    +#---------------------------------------------------------------------------
    +# Configuration options related to the HTML output
    +#---------------------------------------------------------------------------
    +GENERATE_HTML = YES
    +HTML_OUTPUT = html
    +HTML_FILE_EXTENSION = .html
    +HTML_HEADER =
    +HTML_FOOTER =
    +HTML_STYLESHEET =
    +HTML_EXTRA_STYLESHEET =
    +HTML_EXTRA_FILES =
    +HTML_COLORSTYLE_HUE = 220
    +HTML_COLORSTYLE_SAT = 100
    +HTML_COLORSTYLE_GAMMA = 80
    +HTML_TIMESTAMP = NO
    +HTML_DYNAMIC_SECTIONS = NO
    +HTML_INDEX_NUM_ENTRIES = 100
    +GENERATE_DOCSET = NO
    +DOCSET_FEEDNAME = "Doxygen generated docs"
    +DOCSET_BUNDLE_ID = org.doxygen.Project
    +DOCSET_PUBLISHER_ID = org.doxygen.Publisher
    +DOCSET_PUBLISHER_NAME = Publisher
    +GENERATE_HTMLHELP = NO
    +CHM_FILE =
    +HHC_LOCATION =
    +GENERATE_CHI = NO
    +CHM_INDEX_ENCODING =
    +BINARY_TOC = NO
    +TOC_EXPAND = NO
    +GENERATE_QHP = NO
    +QCH_FILE =
    +QHP_NAMESPACE = org.doxygen.Project
    +QHP_VIRTUAL_FOLDER = doc
    +QHP_CUST_FILTER_NAME =
    +QHP_CUST_FILTER_ATTRS =
    +QHP_SECT_FILTER_ATTRS =
    +QHG_LOCATION =
    +GENERATE_ECLIPSEHELP = NO
    +ECLIPSE_DOC_ID = org.doxygen.Project
    +DISABLE_INDEX = NO
    +GENERATE_TREEVIEW = NO
    +ENUM_VALUES_PER_LINE = 4
    +TREEVIEW_WIDTH = 250
    +EXT_LINKS_IN_WINDOW = NO
    +FORMULA_FONTSIZE = 10
    +FORMULA_TRANSPARENT = YES
    +USE_MATHJAX = NO
    +MATHJAX_FORMAT = HTML-CSS
    +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
    +MATHJAX_EXTENSIONS =
    +MATHJAX_CODEFILE =
    +SEARCHENGINE = YES
    +SERVER_BASED_SEARCH = NO
    +EXTERNAL_SEARCH = NO
    +SEARCHENGINE_URL =
    +SEARCHDATA_FILE = searchdata.xml
    +EXTERNAL_SEARCH_ID =
    +EXTRA_SEARCH_MAPPINGS =
    +#---------------------------------------------------------------------------
    +# Configuration options related to the LaTeX output
    +#---------------------------------------------------------------------------
    +GENERATE_LATEX = YES
    +LATEX_OUTPUT = latex
    +LATEX_CMD_NAME = latex
    +MAKEINDEX_CMD_NAME = makeindex
    +COMPACT_LATEX = NO
    +PAPER_TYPE = a4
    +EXTRA_PACKAGES =
    +LATEX_HEADER =
    +LATEX_FOOTER =
    +LATEX_EXTRA_STYLESHEET =
    +LATEX_EXTRA_FILES =
    +PDF_HYPERLINKS = YES
    +USE_PDFLATEX = YES
    +LATEX_BATCHMODE = NO
    +LATEX_HIDE_INDICES = NO
    +LATEX_SOURCE_CODE = NO
    +LATEX_BIB_STYLE = plain
    +LATEX_TIMESTAMP = NO
    +#---------------------------------------------------------------------------
    +# Configuration options related to the RTF output
    +#---------------------------------------------------------------------------
    +GENERATE_RTF = NO
    +RTF_OUTPUT = rtf
    +COMPACT_RTF = NO
    +RTF_HYPERLINKS = NO
    +RTF_STYLESHEET_FILE =
    +RTF_EXTENSIONS_FILE =
    +RTF_SOURCE_CODE = NO
    +#---------------------------------------------------------------------------
    +# Configuration options related to the man page output
    +#---------------------------------------------------------------------------
    +GENERATE_MAN = NO
    +MAN_OUTPUT = man
    +MAN_EXTENSION = .3
    +MAN_SUBDIR =
    +MAN_LINKS = NO
    +#---------------------------------------------------------------------------
    +# Configuration options related to the XML output
    +#---------------------------------------------------------------------------
    +GENERATE_XML = NO
    +XML_OUTPUT = xml
    +XML_PROGRAMLISTING = YES
    +#---------------------------------------------------------------------------
    +# Configuration options related to the DOCBOOK output
    +#---------------------------------------------------------------------------
    +GENERATE_DOCBOOK = NO
    +DOCBOOK_OUTPUT = docbook
    +DOCBOOK_PROGRAMLISTING = NO
    +#---------------------------------------------------------------------------
    +# Configuration options for the AutoGen Definitions output
    +#---------------------------------------------------------------------------
    +GENERATE_AUTOGEN_DEF = NO
    +#---------------------------------------------------------------------------
    +# Configuration options related to the Perl module output
    +#---------------------------------------------------------------------------
    +GENERATE_PERLMOD = NO
    +PERLMOD_LATEX = NO
    +PERLMOD_PRETTY = YES
    +PERLMOD_MAKEVAR_PREFIX =
    +#---------------------------------------------------------------------------
    +# Configuration options related to the preprocessor
    +#---------------------------------------------------------------------------
    +ENABLE_PREPROCESSING = YES
    +MACRO_EXPANSION = NO
    +EXPAND_ONLY_PREDEF = NO
    +SEARCH_INCLUDES = YES
    +INCLUDE_PATH =
    +INCLUDE_FILE_PATTERNS =
    +PREDEFINED =
    +EXPAND_AS_DEFINED =
    +SKIP_FUNCTION_MACROS = YES
    +#---------------------------------------------------------------------------
    +# Configuration options related to external references
    +#---------------------------------------------------------------------------
    +TAGFILES =
    +GENERATE_TAGFILE =
    +ALLEXTERNALS = NO
    +EXTERNAL_GROUPS = YES
    +EXTERNAL_PAGES = YES
    +PERL_PATH = /usr/bin/perl
    +#---------------------------------------------------------------------------
    +# Configuration options related to the dot tool
    +#---------------------------------------------------------------------------
    +CLASS_DIAGRAMS = YES
    +MSCGEN_PATH =
    +DIA_PATH =
    +HIDE_UNDOC_RELATIONS = YES
    +HAVE_DOT = YES
    +DOT_NUM_THREADS = 0
    +DOT_FONTNAME = Helvetica
    +DOT_FONTSIZE = 10
    +DOT_FONTPATH =
    +CLASS_GRAPH = YES
    +COLLABORATION_GRAPH = YES
    +GROUP_GRAPHS = YES
    +UML_LOOK = NO
    +UML_LIMIT_NUM_FIELDS = 10
    +TEMPLATE_RELATIONS = NO
    +INCLUDE_GRAPH = YES
    +INCLUDED_BY_GRAPH = YES
    +CALL_GRAPH = NO
    +CALLER_GRAPH = NO
    +GRAPHICAL_HIERARCHY = YES
    +DIRECTORY_GRAPH = YES
    +DOT_IMAGE_FORMAT = png
    +INTERACTIVE_SVG = NO
    +DOT_PATH =
    +DOTFILE_DIRS =
    +MSCFILE_DIRS =
    +DIAFILE_DIRS =
    +PLANTUML_JAR_PATH =
    +PLANTUML_INCLUDE_PATH =
    +DOT_GRAPH_MAX_NODES = 50
    +MAX_DOT_GRAPH_DEPTH = 0
    +DOT_TRANSPARENT = NO
    +DOT_MULTI_TARGETS = NO
    +GENERATE_LEGEND = YES
    +DOT_CLEANUP = YES
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15823 Use call once for user util


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/616405a6
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/616405a6
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/616405a6

    Branch: refs/heads/HBASE-14850
    Commit: 616405a69a3ad9c4c9ac0d7baa0f065a7a2efb33
    Parents: 6bc1ab3
    Author: Elliott Clark <eclark@apache.org>
    Authored: Fri May 13 13:07:03 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/utils/user-util.cc | 13 ++-----------
      hbase-native-client/utils/user-util.h | 6 ++----
      2 files changed, 4 insertions(+), 15 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/616405a6/hbase-native-client/utils/user-util.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
    index 3d963b3..9e170e0 100644
    --- a/hbase-native-client/utils/user-util.cc
    +++ b/hbase-native-client/utils/user-util.cc
    @@ -27,22 +27,14 @@
      using namespace hbase;
      using namespace std;

    -UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
    +UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}

      string UserUtil::user_name() {
    - if (!init_) {
    - compute_user_name();
    - }
    + std::call_once(once_flag_, [this]() { compute_user_name(); });
        return user_name_;
      }

      void UserUtil::compute_user_name() {
    - lock_guard<mutex> lock(m_);
    -
    - if (init_) {
    - return;
    - }
    -
        // According to the man page of getpwuid
        // this should never be free'd
        //
    @@ -52,6 +44,5 @@ void UserUtil::compute_user_name() {
        // make sure that we got something.
        if (passwd && passwd->pw_name) {
          user_name_ = string{passwd->pw_name};
    - init_ = true;
        }
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/616405a6/hbase-native-client/utils/user-util.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
    index 0b4cc73..fdfc0c8 100644
    --- a/hbase-native-client/utils/user-util.h
    +++ b/hbase-native-client/utils/user-util.h
    @@ -19,8 +19,7 @@

      #pragma once

    -#include <atomic>
    -#include <memory>
    +#include <string>
      #include <mutex>

      namespace hbase {
    @@ -49,8 +48,7 @@ private:
         * Compute the username. This will block.
         */
        void compute_user_name();
    - std::atomic<bool> init_;
    + std::once_flag once_flag_;
        std::string user_name_;
    - std::mutex m_;
      };
      } // namespace hbase
  • Eclark at May 18, 2016 at 10:55 pm
    HBASE-15750 Add on meta deserialization

    Summary: Add on meta region info deserialization

    Test Plan:
    Unit tests.
    Simple client connects.

    Differential Revision: https://reviews.facebook.net/D57555


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6b4d7599
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6b4d7599
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6b4d7599

    Branch: refs/heads/HBASE-14850
    Commit: 6b4d75995a52f079a87334951f8cfe20418db25c
    Parents: 3915478
    Author: Elliott Clark <eclark@apache.org>
    Authored: Tue May 3 12:17:07 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      .../connection/client-dispatcher.cc | 1 +
      .../connection/client-handler.cc | 13 +-
      hbase-native-client/connection/client-handler.h | 6 +-
      .../connection/connection-pool.cc | 36 +---
      .../connection/connection-pool.h | 8 +-
      hbase-native-client/core/BUCK | 4 -
      hbase-native-client/core/location-cache-test.cc | 1 +
      hbase-native-client/core/location-cache.cc | 73 +++++++-
      hbase-native-client/core/location-cache.h | 7 +-
      hbase-native-client/core/meta-utils.cc | 9 +-
      hbase-native-client/core/meta-utils.h | 7 +-
      hbase-native-client/core/region-location.h | 5 +-
      hbase-native-client/core/simple-client.cc | 6 +-
      hbase-native-client/core/table-name-test.cc | 54 ------
      hbase-native-client/core/table-name.h | 50 -----
      hbase-native-client/serde/BUCK | 56 +++---
      .../serde/client-deserializer-test.cc | 25 ++-
      .../serde/client-deserializer.cc | 68 -------
      hbase-native-client/serde/client-deserializer.h | 36 ----
      .../serde/client-serializer-test.cc | 26 +--
      hbase-native-client/serde/client-serializer.cc | 139 --------------
      hbase-native-client/serde/client-serializer.h | 55 ------
      .../serde/region-info-deserializer-test.cc | 54 ++++++
      hbase-native-client/serde/region-info.h | 41 +++++
      hbase-native-client/serde/rpc.cc | 181 +++++++++++++++++++
      hbase-native-client/serde/rpc.h | 58 ++++++
      hbase-native-client/serde/server-name-test.cc | 32 ++++
      hbase-native-client/serde/server-name.h | 21 +++
      hbase-native-client/serde/table-name-test.cc | 54 ++++++
      hbase-native-client/serde/table-name.h | 54 ++++++
      .../serde/zk-deserializer-test.cc | 8 +-
      hbase-native-client/serde/zk-deserializer.cc | 78 --------
      hbase-native-client/serde/zk-deserializer.h | 35 ----
      hbase-native-client/serde/zk.cc | 78 ++++++++
      hbase-native-client/serde/zk.h | 35 ++++
      35 files changed, 761 insertions(+), 653 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/connection/client-dispatcher.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
    index 817adc1..6e2dc54 100644
    --- a/hbase-native-client/connection/client-dispatcher.cc
    +++ b/hbase-native-client/connection/client-dispatcher.cc
    @@ -44,6 +44,7 @@ Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
        auto &p = requests_[call_id];
        auto f = p.getFuture();
        p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
    + LOG(ERROR) << "e = " << call_id;
          this->requests_.erase(call_id);
        });
        this->pipeline_->write(std::move(arg));

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/connection/client-handler.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
    index 3180f4e..496e4f2 100644
    --- a/hbase-native-client/connection/client-handler.cc
    +++ b/hbase-native-client/connection/client-handler.cc
    @@ -37,8 +37,7 @@ using hbase::pb::GetResponse;
      using google::protobuf::Message;

      ClientHandler::ClientHandler(std::string user_name)
    - : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
    - resp_msgs_() {}
    + : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}

      void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
        if (LIKELY(buf != nullptr)) {
    @@ -46,7 +45,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
          Response received;
          ResponseHeader header;

    - int used_bytes = deser_.parse_delimited(buf.get(), &header);
    + int used_bytes = serde_.ParseDelimited(buf.get(), &header);
          LOG(INFO) << "Read ResponseHeader size=" << used_bytes
                    << " call_id=" << header.call_id()
                    << " has_exception=" << header.has_exception();
    @@ -70,7 +69,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
          // data left on the wire.
          if (header.has_exception() == false) {
            buf->trimStart(used_bytes);
    - used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
    + used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
            // Make sure that bytes were parsed.
            CHECK(used_bytes == buf->length());
            received.set_response(resp_msg);
    @@ -91,13 +90,13 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
          // and one for the request.
          //
          // That doesn't seem like too bad, but who knows.
    - auto pre = ser_.preamble();
    - auto header = ser_.header(user_name_);
    + auto pre = serde_.Preamble();
    + auto header = serde_.Header(user_name_);
          pre->appendChain(std::move(header));
          ctx->fireWrite(std::move(pre));
        }

        resp_msgs_[r->call_id()] = r->resp_msg();
        return ctx->fireWrite(
    - ser_.request(r->call_id(), r->method(), r->req_msg().get()));
    + serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/connection/client-handler.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
    index 68513de..ce99c9e 100644
    --- a/hbase-native-client/connection/client-handler.h
    +++ b/hbase-native-client/connection/client-handler.h
    @@ -22,8 +22,7 @@

      #include <string>

    -#include "serde/client-deserializer.h"
    -#include "serde/client-serializer.h"
    +#include "serde/rpc.h"

      // Forward decs.
      namespace hbase {
    @@ -49,8 +48,7 @@ public:
      private:
        bool need_send_header_;
        std::string user_name_;
    - ClientSerializer ser_;
    - ClientDeserializer deser_;
    + RpcSerde serde_;

        // in flight requests
        std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/connection/connection-pool.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
    index 72c1306..eafe60a 100644
    --- a/hbase-native-client/connection/connection-pool.cc
    +++ b/hbase-native-client/connection/connection-pool.cc
    @@ -25,41 +25,10 @@ using std::mutex;
      using std::unique_ptr;
      using std::shared_ptr;
      using hbase::pb::ServerName;
    -using wangle::ServiceFilter;
      using folly::SharedMutexWritePriority;

      namespace hbase {

    -class RemoveServiceFilter
    - : public ServiceFilter<unique_ptr<Request>, Response> {
    -
    -public:
    - RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    - ConnectionPool &cp)
    - : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
    - cp_(cp) {}
    -
    - folly::Future<folly::Unit> close() override {
    - if (!released.exchange(true)) {
    - return this->service_->close().then(
    - [this]() { this->cp_.close(this->sn_); });
    - } else {
    - return folly::makeFuture();
    - }
    - }
    -
    - virtual bool isAvailable() override { return service_->isAvailable(); }
    -
    - folly::Future<Response> operator()(unique_ptr<Request> req) override {
    - return (*this->service_)(std::move(req));
    - }
    -
    -private:
    - std::atomic<bool> released{false};
    - hbase::pb::ServerName sn_;
    - ConnectionPool &cp_;
    -};
    -
      ConnectionPool::ConnectionPool()
          : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
      }
    @@ -72,13 +41,12 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
        if (found == connections_.end() || found->second == nullptr) {
          SharedMutexWritePriority::WriteHolder holder(std::move(holder));
          auto new_con = cf_->make_connection(sn.host_name(), sn.port());
    - auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
    - connections_[sn] = wrapped;
    + connections_[sn] = new_con;
          return new_con;
        }
        return found->second;
      }
    -void ConnectionPool::close(ServerName sn) {
    +void ConnectionPool::close(const ServerName &sn) {
        SharedMutexWritePriority::WriteHolder holder(map_mutex_);

        auto found = connections_.find(sn);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/connection/connection-pool.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
    index 394cd71..b8330e3 100644
    --- a/hbase-native-client/connection/connection-pool.h
    +++ b/hbase-native-client/connection/connection-pool.h
    @@ -27,13 +27,13 @@
      #include "if/HBase.pb.h"

      namespace hbase {
    -struct MyServerNameEquals {
    +struct ServerNameEquals {
        bool operator()(const hbase::pb::ServerName &lhs,
                        const hbase::pb::ServerName &rhs) const {
          return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
        }
      };
    -struct MyServerNameHash {
    +struct ServerNameHash {
        std::size_t operator()(hbase::pb::ServerName const &s) const {
          std::size_t h1 = std::hash<std::string>()(s.host_name());
          std::size_t h2 = std::hash<uint32_t>()(s.port());
    @@ -46,12 +46,12 @@ public:
        ConnectionPool();
        explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
        std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
    - void close(hbase::pb::ServerName sn);
    + void close(const hbase::pb::ServerName &sn);

      private:
        std::shared_ptr<ConnectionFactory> cf_;
        std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
    - MyServerNameHash, MyServerNameEquals>
    + ServerNameHash, ServerNameEquals>
            connections_;
        folly::SharedMutexWritePriority map_mutex_;
      };

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index 447248b..ef8c2f8 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -24,7 +24,6 @@ cxx_library(
              "hbase_macros.h",
              "region-location.h",
              "location-cache.h",
    - "table-name.h",
              # TODO: move this out of exported
              # Once meta lookup works
              "meta-utils.h",
    @@ -53,9 +52,6 @@ cxx_test(name="location-cache-test",
               ],
               deps=[":core", ],
               run_test_separately=True, )
    -cxx_test(name="table-name-test",
    - srcs=["table-name-test.cc", ],
    - deps=[":core", ], )
      cxx_binary(name="simple-client",
                 srcs=["simple-client.cc", ],
                 deps=[":core", "//connection:connection"], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/location-cache-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
    index f3166fb..172799d 100644
    --- a/hbase-native-client/core/location-cache-test.cc
    +++ b/hbase-native-client/core/location-cache-test.cc
    @@ -30,4 +30,5 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
        auto result = f.get();
        ASSERT_FALSE(f.hasException());
        ASSERT_TRUE(result.has_port());
    + ASSERT_TRUE(result.has_host_name());
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    index 539051a..2667f11 100644
    --- a/hbase-native-client/core/location-cache.cc
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -20,19 +20,25 @@

      #include <folly/Logging.h>
      #include <folly/io/IOBuf.h>
    +#include <wangle/concurrent/GlobalExecutor.h>

      #include "connection/response.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
    -#include "serde/zk-deserializer.h"
    +#include "serde/server-name.h"
    +#include "serde/region-info.h"
    +#include "serde/zk.h"

      using namespace std;
      using namespace folly;

    +using wangle::ServiceFilter;
    +using hbase::Request;
      using hbase::Response;
      using hbase::LocationCache;
      using hbase::RegionLocation;
      using hbase::HBaseService;
    +using hbase::ConnectionPool;
      using hbase::pb::ScanResponse;
      using hbase::pb::TableName;
      using hbase::pb::ServerName;
    @@ -45,7 +51,7 @@ static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
      LocationCache::LocationCache(string quorum_spec,
                                   shared_ptr<folly::Executor> executor)
          : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
    - meta_lock_(), cp_(), meta_util_() {
    + meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
        zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
      }

    @@ -95,28 +101,77 @@ ServerName LocationCache::ReadMetaLocation() {
        buf->append(len);

        MetaRegionServer mrs;
    - if (derser.parse(buf.get(), &mrs) == false) {
    + if (derser.Parse(buf.get(), &mrs) == false) {
          LOG(ERROR) << "Unable to decode";
        }
        return mrs.server();
      }

    -Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
    - const string &row) {
    +Future<std::shared_ptr<RegionLocation>>
    +LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
    + auto exc = wangle::getIOExecutor();
        return this->LocateMeta()
            .then([&](ServerName sn) { return this->cp_.get(sn); })
    + .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
            .then([&](std::shared_ptr<HBaseService> service) {
    - return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
    + return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
            })
            .then([&](Response resp) {
              // take the protobuf response and make it into
              // a region location.
    - return this->parse_response(std::move(resp));
    + return this->CreateLocation(std::move(resp));
            });
      }

    -RegionLocation LocationCache::parse_response(const Response &resp) {
    +class RemoveServiceFilter
    + : public ServiceFilter<std::unique_ptr<Request>, Response> {
    +
    +public:
    + RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
    + ConnectionPool &cp)
    + : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
    + cp_(cp) {}
    +
    + folly::Future<folly::Unit> close() override {
    + if (!released.exchange(true)) {
    + return this->service_->close().then([this]() {
    + // TODO(eclark): remove the service from the meta cache.
    + this->cp_.close(this->sn_);
    + });
    + } else {
    + return folly::makeFuture();
    + }
    + }
    +
    + virtual bool isAvailable() override {
    + return !released && service_->isAvailable();
    + }
    +
    + folly::Future<Response> operator()(unique_ptr<Request> req) override {
    + // TODO(eclark): add in an on error handler that will
    + // remove the region location from the cache if needed.
    + // Also close the connection if this is likely to be an error
    + // that needs to get a new connection.
    + return (*this->service_)(std::move(req));
    + }
    +
    +private:
    + std::atomic<bool> released{false};
    + hbase::pb::ServerName sn_;
    + ConnectionPool &cp_;
    +};
    +
    +std::shared_ptr<RegionLocation>
    +LocationCache::CreateLocation(const Response &resp){
        auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
    + auto &results = resp_msg->results().Get(0);
    + auto &cells = results.cell();
        LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
    - return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
    + auto ri = folly::to<RegionInfo>(cells.Get(0).value());
    + auto sn = folly::to<ServerName>(cells.Get(1).value());
    +
    + LOG(ERROR) << "RegionInfo = " << ri.DebugString();
    + LOG(ERROR) << "ServerName = " << sn.DebugString();
    + auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
    + return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    index cfd6838..99b5e5e 100644
    --- a/hbase-native-client/core/location-cache.h
    +++ b/hbase-native-client/core/location-cache.h
    @@ -29,8 +29,8 @@

      #include "connection/connection-pool.h"
      #include "core/meta-utils.h"
    -#include "core/table-name.h"
      #include "core/region-location.h"
    +#include "serde/table-name.h"

      namespace hbase {

    @@ -48,14 +48,14 @@ public:
        // Meta Related Methods.
        // These are only public until testing is complete
        folly::Future<hbase::pb::ServerName> LocateMeta();
    - folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
    + folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
                                                     const std::string &row);
    - RegionLocation parse_response(const Response &resp);
        void InvalidateMeta();

      private:
        void RefreshMetaLocation();
        hbase::pb::ServerName ReadMetaLocation();
    + std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);

        std::string quorum_spec_;
        std::shared_ptr<folly::Executor> executor_;
    @@ -64,7 +64,6 @@ private:
        ConnectionPool cp_;
        MetaUtil meta_util_;

    -
        // TODO: migrate this to a smart pointer with a deleter.
        zhandle_t *zk_;
      };

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/meta-utils.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
    index d2fdd88..1325d83 100644
    --- a/hbase-native-client/core/meta-utils.cc
    +++ b/hbase-native-client/core/meta-utils.cc
    @@ -23,25 +23,26 @@

      #include "connection/request.h"
      #include "connection/response.h"
    -#include "core/table-name.h"
      #include "if/Client.pb.h"
    +#include "serde/table-name.h"

      using hbase::pb::TableName;
      using hbase::MetaUtil;
      using hbase::Request;
      using hbase::Response;
      using hbase::pb::ScanRequest;
    +using hbase::pb::ServerName;
      using hbase::pb::RegionSpecifier_RegionSpecifierType;

      static const std::string META_REGION = "1588230740";

    -std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
    +std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
                                                 const std::string &row) const {
        return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
      }

      std::unique_ptr<Request>
    -MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
    +MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
        auto request = Request::scan();
        auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());

    @@ -76,6 +77,6 @@ MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
        info_col->add_qualifier("server");
        info_col->add_qualifier("regioninfo");

    - scan->set_start_row(region_lookup_rowkey(tn, row));
    + scan->set_start_row(RegionLookupRowkey(tn, row));
        return request;
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/meta-utils.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
    index e007d02..5a659f3 100644
    --- a/hbase-native-client/core/meta-utils.h
    +++ b/hbase-native-client/core/meta-utils.h
    @@ -21,15 +21,16 @@
      #include <string>

      #include "connection/Request.h"
    -#include "core/table-name.h"
    +#include "if/HBase.pb.h"
    +#include "serde/table-name.h"

      namespace hbase {

      class MetaUtil {
      public:
    - std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
    + std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
                                         const std::string &row) const;
    - std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
    + std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
                                                   const std::string &row) const;
      };
      } // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/region-location.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
    index a46b8e2..7922c95 100644
    --- a/hbase-native-client/core/region-location.h
    +++ b/hbase-native-client/core/region-location.h
    @@ -23,7 +23,6 @@
      #include "connection/service.h"
      #include "if/HBase.pb.h"

    -
      namespace hbase {

      class RegionLocation {
    @@ -32,8 +31,8 @@ public:
                       std::shared_ptr<HBaseService> service)
            : ri_(ri), sn_(sn), service_(service) {}

    - const hbase::pb::RegionInfo& region_info() { return ri_; }
    - const hbase::pb::ServerName& server_name() { return sn_; }
    + const hbase::pb::RegionInfo &region_info() { return ri_; }
    + const hbase::pb::ServerName &server_name() { return sn_; }
        std::shared_ptr<HBaseService> service() { return service_; }

      private:

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/simple-client.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
    index ab614e4..00e3369 100644
    --- a/hbase-native-client/core/simple-client.cc
    +++ b/hbase-native-client/core/simple-client.cc
    @@ -27,9 +27,9 @@

      #include "connection/connection-pool.h"
      #include "core/client.h"
    -#include "core/table-name.h"
      #include "if/Client.pb.h"
      #include "if/ZooKeeper.pb.h"
    +#include "serde/table-name.h"

      using namespace folly;
      using namespace std;
    @@ -39,7 +39,7 @@ using hbase::Request;
      using hbase::HBaseService;
      using hbase::LocationCache;
      using hbase::ConnectionPool;
    -using hbase::TableNameUtil;
    +using hbase::pb::TableName;
      using hbase::pb::ServerName;
      using hbase::pb::RegionSpecifier_RegionSpecifierType;
      using hbase::pb::GetRequest;
    @@ -61,7 +61,7 @@ int main(int argc, char *argv[]) {
        auto cpu_ex = wangle::getCPUExecutor();
        LocationCache cache{FLAGS_zookeeper, cpu_ex};
        auto result =
    - cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
    + cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
                .get(milliseconds(5000));

        return 0;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/table-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/table-name-test.cc b/hbase-native-client/core/table-name-test.cc
    deleted file mode 100644
    index 7bad3f1..0000000
    --- a/hbase-native-client/core/table-name-test.cc
    +++ /dev/null
    @@ -1,54 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#include <folly/Conv.h>
    -#include <gtest/gtest.h>
    -
    -#include <string>
    -
    -#include "core/table-name.h"
    -
    -using namespace hbase;
    -using hbase::pb::TableName;
    -
    -TEST(TestTableName, TestToStringNoDefault) {
    - TableName tn;
    - tn.set_qualifier("TestTableName");
    - std::string result = folly::to<std::string>(tn);
    - ASSERT_EQ(result.find("default"), std::string::npos);
    - ASSERT_EQ("TestTableName", result);
    -}
    -
    -TEST(TestTableName, TestToStringNoDefaltWhenSet) {
    - TableName tn;
    - tn.set_namespace_("default");
    - tn.set_qualifier("TestTableName");
    - std::string result = folly::to<std::string>(tn);
    - ASSERT_EQ(result.find("default"), std::string::npos);
    - ASSERT_EQ("TestTableName", result);
    -}
    -
    -TEST(TestTableName, TestToStringIncludeNS) {
    - TableName tn;
    - tn.set_namespace_("hbase");
    - tn.set_qualifier("acl");
    - std::string result = folly::to<std::string>(tn);
    - ASSERT_EQ(result.find("hbase"), 0);
    - ASSERT_EQ("hbase:acl", result);
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/core/table-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
    deleted file mode 100644
    index 1612667..0000000
    --- a/hbase-native-client/core/table-name.h
    +++ /dev/null
    @@ -1,50 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -#include <memory>
    -#include <string>
    -
    -#include "if/HBase.pb.h"
    -#include <folly/Conv.h>
    -
    -namespace hbase {
    -namespace pb {
    -
    -// Provide folly::to<std::string>(TableName);
    -template <class String> void toAppend(const TableName &in, String *result) {
    - if (!in.has_namespace_() || in.namespace_() == "default") {
    - folly::toAppend(in.qualifier(), result);
    - } else {
    - folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
    - }
    -}
    -
    -} // namespace pb
    -
    -class TableNameUtil {
    -public:
    - static ::hbase::pb::TableName create(std::string table_name) {
    - ::hbase::pb::TableName tn;
    - tn.set_namespace_("default");
    - tn.set_qualifier(table_name);
    - return tn;
    - }
    -};
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
    index 539a221..db15026 100644
    --- a/hbase-native-client/serde/BUCK
    +++ b/hbase-native-client/serde/BUCK
    @@ -17,49 +17,47 @@

      cxx_library(name="serde",
                  exported_headers=[
    - "client-serializer.h",
    - "client-deserializer.h",
    - "zk-deserializer.h",
    + "region-info.h",
    + "rpc.h",
    + "server-name.h",
    + "table-name.h",
    + "zk.h",
                  ],
                  srcs=[
    - "client-serializer.cc",
    - "client-deserializer.cc",
    - "zk-deserializer.cc",
    + "rpc.cc",
    + "zk.cc",
                  ],
                  deps=[
                      "//if:if",
                      "//third-party:folly",
                  ],
                  tests=[
    - ":client-serializer-test",
                      ":client-deserializer-test",
    + ":client-serializer-test",
    + ":server-name-test",
    + ":table-name-test",
    + ":zk-deserializer-test",
    + ":region-info-deserializer-test",
                  ],
                  compiler_flags=['-Weffc++'],
                  visibility=[
                      'PUBLIC',
                  ], )
    -
    +cxx_test(name="table-name-test",
    + srcs=["table-name-test.cc", ],
    + deps=[":serde", ], )
    +cxx_test(name="server-name-test",
    + srcs=["server-name-test.cc", ],
    + deps=[":serde", ], )
      cxx_test(name="client-serializer-test",
    - srcs=[
    - "client-serializer-test.cc",
    - ],
    - deps=[
    - ":serde",
    - "//if:if",
    - ], )
    + srcs=["client-serializer-test.cc", ],
    + deps=[":serde", ], )
      cxx_test(name="client-deserializer-test",
    - srcs=[
    - "client-deserializer-test.cc",
    - ],
    - deps=[
    - ":serde",
    - "//if:if",
    - ], )
    + srcs=["client-deserializer-test.cc", ],
    + deps=[":serde", ], )
      cxx_test(name="zk-deserializer-test",
    - srcs=[
    - "zk-deserializer-test.cc",
    - ],
    - deps=[
    - ":serde",
    - "//if:if",
    - ], )
    + srcs=["zk-deserializer-test.cc", ],
    + deps=[":serde", ], )
    +cxx_test(name="region-info-deserializer-test",
    + srcs=["region-info-deserializer-test.cc", ],
    + deps=[":serde", ], )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc
    index 9fef093..8c571b1 100644
    --- a/hbase-native-client/serde/client-deserializer-test.cc
    +++ b/hbase-native-client/serde/client-deserializer-test.cc
    @@ -16,13 +16,12 @@
       * limitations under the License.
       *
       */
    +#include "serde/rpc.h"

      #include <folly/io/IOBuf.h>
      #include <gtest/gtest.h>

      #include "if/Client.pb.h"
    -#include "serde/client-deserializer.h"
    -#include "serde/client-serializer.h"

      using namespace hbase;
      using folly::IOBuf;
    @@ -30,23 +29,23 @@ using hbase::pb::GetRequest;
      using hbase::pb::RegionSpecifier;
      using hbase::pb::RegionSpecifier_RegionSpecifierType;

    -TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
    - ClientDeserializer deser;
    - ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
    +TEST(TestRpcSerde, TestReturnFalseOnNullPtr) {
    + RpcSerde deser;
    + ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0);
      }

    -TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
    - ClientDeserializer deser;
    +TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
    + RpcSerde deser;
        auto buf = IOBuf::copyBuffer("test");
        GetRequest gr;

    - ASSERT_LT(deser.parse_delimited(buf.get(), &gr), 0);
    + ASSERT_LT(deser.ParseDelimited(buf.get(), &gr), 0);
      }

    -TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
    +TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
        GetRequest in;
    - ClientSerializer ser;
    - ClientDeserializer deser;
    + RpcSerde ser;
    + RpcSerde deser;

        // fill up the GetRequest.
        in.mutable_region()->set_value("test_region_id");
    @@ -56,11 +55,11 @@ TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
        in.mutable_get()->set_row("test_row");

        // Create the buffer
    - auto buf = ser.serialize_delimited(in);
    + auto buf = ser.SerializeDelimited(in);

        GetRequest out;

    - int used_bytes = deser.parse_delimited(buf.get(), &out);
    + int used_bytes = deser.ParseDelimited(buf.get(), &out);

        ASSERT_GT(used_bytes, 0);
        ASSERT_EQ(used_bytes, buf->length());

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-deserializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc
    deleted file mode 100644
    index acca7ea..0000000
    --- a/hbase-native-client/serde/client-deserializer.cc
    +++ /dev/null
    @@ -1,68 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#include "serde/client-deserializer.h"
    -
    -#include <folly/Logging.h>
    -#include <google/protobuf/io/coded_stream.h>
    -#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
    -#include <google/protobuf/message.h>
    -
    -using namespace hbase;
    -
    -using folly::IOBuf;
    -using google::protobuf::Message;
    -using google::protobuf::io::ArrayInputStream;
    -using google::protobuf::io::CodedInputStream;
    -
    -int ClientDeserializer::parse_delimited(const IOBuf *buf, Message *msg) {
    - if (buf == nullptr || msg == nullptr) {
    - return -2;
    - }
    -
    - DCHECK(!buf->isChained());
    -
    - ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
    - CodedInputStream coded_stream{&ais};
    -
    - uint32_t msg_size;
    -
    - // Try and read the varint.
    - if (coded_stream.ReadVarint32(&msg_size) == false) {
    - FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
    - return -3;
    - }
    -
    - coded_stream.PushLimit(msg_size);
    - // Parse the message.
    - if (msg->MergeFromCodedStream(&coded_stream) == false) {
    - FB_LOG_EVERY_MS(ERROR, 1000)
    - << "Unable to read a protobuf message from data.";
    - return -4;
    - }
    -
    - // Make sure all the data was consumed.
    - if (coded_stream.ConsumedEntireMessage() == false) {
    - FB_LOG_EVERY_MS(ERROR, 1000)
    - << "Orphaned data left after reading protobuf message";
    - return -5;
    - }
    -
    - return coded_stream.CurrentPosition();
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-deserializer.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-deserializer.h b/hbase-native-client/serde/client-deserializer.h
    deleted file mode 100644
    index b9664b0..0000000
    --- a/hbase-native-client/serde/client-deserializer.h
    +++ /dev/null
    @@ -1,36 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -#include <folly/io/IOBuf.h>
    -
    -// Forward
    -namespace google {
    -namespace protobuf {
    -class Message;
    -}
    -}
    -
    -namespace hbase {
    -class ClientDeserializer {
    -public:
    - int parse_delimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
    -};
    -
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-serializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc
    index 9bf38af..2bd17fb 100644
    --- a/hbase-native-client/serde/client-serializer-test.cc
    +++ b/hbase-native-client/serde/client-serializer-test.cc
    @@ -24,16 +24,16 @@

      #include "if/HBase.pb.h"
      #include "if/RPC.pb.h"
    -#include "serde/client-serializer.h"
    +#include "serde/rpc.h"

      using namespace hbase;
      using namespace hbase::pb;
      using namespace folly;
      using namespace folly::io;

    -TEST(ClientSerializerTest, PreambleIncludesHBas) {
    - ClientSerializer ser;
    - auto buf = ser.preamble();
    +TEST(RpcSerdeTest, PreambleIncludesHBas) {
    + RpcSerde ser;
    + auto buf = ser.Preamble();
        const char *p = reinterpret_cast<const char *>(buf->data());
        // Take the first for chars and make sure they are the
        // magic string
    @@ -42,16 +42,16 @@ TEST(ClientSerializerTest, PreambleIncludesHBas) {
        EXPECT_EQ(6, buf->computeChainDataLength());
      }

    -TEST(ClientSerializerTest, PreambleIncludesVersion) {
    - ClientSerializer ser;
    - auto buf = ser.preamble();
    +TEST(RpcSerdeTest, PreambleIncludesVersion) {
    + RpcSerde ser;
    + auto buf = ser.Preamble();
        EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
        EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
      }

    -TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
    - ClientSerializer ser;
    - auto header = ser.header("elliott");
    +TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
    + RpcSerde ser;
    + auto header = ser.Header("elliott");

        // The header should be prefixed by 4 bytes of length.
        EXPECT_EQ(4, header->length());
    @@ -64,9 +64,9 @@ TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
        EXPECT_EQ(prefixed_len, header->next()->length());
      }

    -TEST(ClientSerializerTest, TestHeaderDecode) {
    - ClientSerializer ser;
    - auto buf = ser.header("elliott");
    +TEST(RpcSerdeTest, TestHeaderDecode) {
    + RpcSerde ser;
    + auto buf = ser.Header("elliott");
        auto header_buf = buf->next();
        ConnectionHeader h;


    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-serializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc
    deleted file mode 100644
    index 09b81c8..0000000
    --- a/hbase-native-client/serde/client-serializer.cc
    +++ /dev/null
    @@ -1,139 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#include "serde/client-serializer.h"
    -
    -#include <folly/Logging.h>
    -#include <folly/io/Cursor.h>
    -#include <google/protobuf/io/coded_stream.h>
    -#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
    -
    -#include "if/HBase.pb.h"
    -#include "if/RPC.pb.h"
    -
    -using namespace hbase;
    -
    -using folly::IOBuf;
    -using folly::io::RWPrivateCursor;
    -using google::protobuf::Message;
    -using google::protobuf::io::ArrayOutputStream;
    -using google::protobuf::io::CodedOutputStream;
    -using google::protobuf::io::ZeroCopyOutputStream;
    -using std::string;
    -using std::unique_ptr;
    -
    -static const std::string PREAMBLE = "HBas";
    -static const std::string INTERFACE = "ClientService";
    -static const uint8_t RPC_VERSION = 0;
    -static const uint8_t DEFAULT_AUTH_TYPE = 80;
    -
    -ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
    -
    -unique_ptr<IOBuf> ClientSerializer::preamble() {
    - auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
    - magic->append(2);
    - RWPrivateCursor c(magic.get());
    - c.skip(4);
    - // Version
    - c.write(RPC_VERSION);
    - // Standard security aka Please don't lie to me.
    - c.write(auth_type_);
    - return magic;
    -}
    -
    -unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
    - pb::ConnectionHeader h;
    -
    - // TODO(eclark): Make this not a total lie.
    - h.mutable_user_info()->set_effective_user(user);
    - // The service name that we want to talk to.
    - //
    - // Right now we're completely ignoring the service interface.
    - // That may or may not be the correct thing to do.
    - // It worked for a while with the java client; until it
    - // didn't.
    - h.set_service_name(INTERFACE);
    - return prepend_length(serialize_message(h));
    -}
    -
    -unique_ptr<IOBuf> ClientSerializer::request(const uint32_t call_id,
    - const string &method,
    - const Message *msg) {
    - pb::RequestHeader rq;
    - rq.set_method_name(method);
    - rq.set_call_id(call_id);
    - rq.set_request_param(msg != nullptr);
    - auto ser_header = serialize_delimited(rq);
    - if (msg != nullptr) {
    - auto ser_req = serialize_delimited(*msg);
    - ser_header->appendChain(std::move(ser_req));
    - }
    -
    - return prepend_length(std::move(ser_header));
    -}
    -
    -unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
    - // Java ints are 4 long. So create a buffer that large
    - auto len_buf = IOBuf::create(4);
    - // Then make those bytes visible.
    - len_buf->append(4);
    -
    - RWPrivateCursor c(len_buf.get());
    - // Get the size of the data to be pushed out the network.
    - auto size = msg->computeChainDataLength();
    -
    - // Write the length to this IOBuf.
    - c.writeBE(static_cast<uint32_t>(size));
    -
    - // Then attach the origional to the back of len_buf
    - len_buf->appendChain(std::move(msg));
    - return len_buf;
    -}
    -
    -unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
    - // Get the buffer size needed for just the message.
    - int msg_size = msg.ByteSize();
    - int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
    -
    - // Create a buffer big enough to hold the varint and the object.
    - auto buf = IOBuf::create(buf_size);
    - buf->append(buf_size);
    -
    - // Create the array output stream.
    - ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
    - // Wrap the ArrayOuputStream in the coded output stream to allow writing
    - // Varint32
    - CodedOutputStream cos{&aos};
    -
    - // Write out the size.
    - cos.WriteVarint32(msg_size);
    -
    - // Now write the rest out.
    - // We're using the protobuf output streams here to keep track
    - // of where in the output array we are rather than IOBuf.
    - msg.SerializeWithCachedSizesToArray(
    - cos.GetDirectBufferForNBytesAndAdvance(msg_size));
    -
    - // Return the buffer.
    - return buf;
    -}
    -// TODO(eclark): Make this 1 copy.
    -unique_ptr<IOBuf> ClientSerializer::serialize_message(const Message &msg) {
    - auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
    - return buf;
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/client-serializer.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h
    deleted file mode 100644
    index 9c819fe..0000000
    --- a/hbase-native-client/serde/client-serializer.h
    +++ /dev/null
    @@ -1,55 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -#include <cstdint>
    -#include <folly/io/IOBuf.h>
    -#include <string>
    -
    -// Forward
    -namespace google {
    -namespace protobuf {
    -class Message;
    -}
    -}
    -namespace hbase {
    -class Request;
    -}
    -
    -namespace hbase {
    -class ClientSerializer {
    -public:
    - ClientSerializer();
    - std::unique_ptr<folly::IOBuf> preamble();
    - std::unique_ptr<folly::IOBuf> header(const std::string &user);
    - std::unique_ptr<folly::IOBuf> request(const uint32_t call_id,
    - const std::string &method,
    - const google::protobuf::Message *msg);
    - std::unique_ptr<folly::IOBuf>
    - serialize_delimited(const google::protobuf::Message &msg);
    -
    - std::unique_ptr<folly::IOBuf>
    - serialize_message(const google::protobuf::Message &msg);
    -
    - std::unique_ptr<folly::IOBuf>
    - prepend_length(std::unique_ptr<folly::IOBuf> msg);
    -
    - uint8_t auth_type_;
    -};
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/region-info-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/region-info-deserializer-test.cc b/hbase-native-client/serde/region-info-deserializer-test.cc
    new file mode 100644
    index 0000000..ce8dedf
    --- /dev/null
    +++ b/hbase-native-client/serde/region-info-deserializer-test.cc
    @@ -0,0 +1,54 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/region-info.h"
    +
    +#include <gtest/gtest.h>
    +
    +#include <string>
    +
    +#include "if/HBase.pb.h"
    +#include "serde/table-name.h"
    +
    +using std::string;
    +using hbase::pb::RegionInfo;
    +using hbase::pb::TableName;
    +
    +TEST(TestRegionInfoDesializer, TestDeserialize) {
    + string ns{"test_ns"};
    + string tn{"table_name"};
    + string start_row{"AAAAAA"};
    + string stop_row{"BBBBBBBBBBBB"};
    + uint64_t region_id = 2345678;
    +
    + RegionInfo ri_out;
    + ri_out.set_region_id(region_id);
    + ri_out.mutable_table_name()->set_namespace_(ns);
    + ri_out.mutable_table_name()->set_qualifier(tn);
    + ri_out.set_start_key(start_row);
    + ri_out.set_end_key(stop_row);
    +
    +
    + string header{"PBUF"};
    + string ser = header + ri_out.SerializeAsString();
    +
    + auto out = folly::to<RegionInfo>(ser);
    +
    + EXPECT_EQ(region_id, out.region_id());
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/region-info.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/region-info.h b/hbase-native-client/serde/region-info.h
    new file mode 100644
    index 0000000..6af351c
    --- /dev/null
    +++ b/hbase-native-client/serde/region-info.h
    @@ -0,0 +1,41 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#pragma once
    +
    +#include "if/HBase.pb.h"
    +
    +#include <folly/Conv.h>
    +#include <boost/algorithm/string/predicate.hpp>
    +
    +namespace hbase {
    +namespace pb {
    +template <class String> void parseTo(String in, RegionInfo& out) {
    + // TODO(eclark): there has to be something better.
    + std::string s = folly::to<std::string>(in);
    +
    + if (!boost::starts_with(s, "PBUF") ) {
    + throw std::runtime_error("Region Info field doesn't contain preamble");
    + }
    + if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
    + throw std::runtime_error("Bad protobuf for RegionInfo");
    + }
    +}
    +} // namespace pb
    +} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/rpc.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/rpc.cc b/hbase-native-client/serde/rpc.cc
    new file mode 100644
    index 0000000..4c3c999
    --- /dev/null
    +++ b/hbase-native-client/serde/rpc.cc
    @@ -0,0 +1,181 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/rpc.h"
    +
    +#include <folly/Logging.h>
    +#include <folly/Logging.h>
    +#include <folly/io/Cursor.h>
    +#include <google/protobuf/io/coded_stream.h>
    +#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
    +#include <google/protobuf/message.h>
    +
    +#include "if/HBase.pb.h"
    +#include "if/RPC.pb.h"
    +
    +using namespace hbase;
    +
    +using folly::IOBuf;
    +using folly::io::RWPrivateCursor;
    +using google::protobuf::Message;
    +using google::protobuf::Message;
    +using google::protobuf::io::ArrayInputStream;
    +using google::protobuf::io::ArrayOutputStream;
    +using google::protobuf::io::CodedInputStream;
    +using google::protobuf::io::CodedOutputStream;
    +using google::protobuf::io::ZeroCopyOutputStream;
    +using std::string;
    +using std::unique_ptr;
    +
    +static const std::string PREAMBLE = "HBas";
    +static const std::string INTERFACE = "ClientService";
    +static const uint8_t RPC_VERSION = 0;
    +static const uint8_t DEFAULT_AUTH_TYPE = 80;
    +
    +int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
    + if (buf == nullptr || msg == nullptr) {
    + return -2;
    + }
    +
    + DCHECK(!buf->isChained());
    +
    + ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
    + CodedInputStream coded_stream{&ais};
    +
    + uint32_t msg_size;
    +
    + // Try and read the varint.
    + if (coded_stream.ReadVarint32(&msg_size) == false) {
    + FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
    + return -3;
    + }
    +
    + coded_stream.PushLimit(msg_size);
    + // Parse the message.
    + if (msg->MergeFromCodedStream(&coded_stream) == false) {
    + FB_LOG_EVERY_MS(ERROR, 1000)
    + << "Unable to read a protobuf message from data.";
    + return -4;
    + }
    +
    + // Make sure all the data was consumed.
    + if (coded_stream.ConsumedEntireMessage() == false) {
    + FB_LOG_EVERY_MS(ERROR, 1000)
    + << "Orphaned data left after reading protobuf message";
    + return -5;
    + }
    +
    + return coded_stream.CurrentPosition();
    +}
    +
    +RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
    +RpcSerde::~RpcSerde() {}
    +
    +unique_ptr<IOBuf> RpcSerde::Preamble() {
    + auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
    + magic->append(2);
    + RWPrivateCursor c(magic.get());
    + c.skip(4);
    + // Version
    + c.write(RPC_VERSION);
    + // Standard security aka Please don't lie to me.
    + c.write(auth_type_);
    + return magic;
    +}
    +
    +unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
    + pb::ConnectionHeader h;
    +
    + // TODO(eclark): Make this not a total lie.
    + h.mutable_user_info()->set_effective_user(user);
    + // The service name that we want to talk to.
    + //
    + // Right now we're completely ignoring the service interface.
    + // That may or may not be the correct thing to do.
    + // It worked for a while with the java client; until it
    + // didn't.
    + h.set_service_name(INTERFACE);
    + return PrependLength(SerializeMessage(h));
    +}
    +
    +unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id,
    + const string &method, const Message *msg) {
    + pb::RequestHeader rq;
    + rq.set_method_name(method);
    + rq.set_call_id(call_id);
    + rq.set_request_param(msg != nullptr);
    + auto ser_header = SerializeDelimited(rq);
    + if (msg != nullptr) {
    + auto ser_req = SerializeDelimited(*msg);
    + ser_header->appendChain(std::move(ser_req));
    + }
    +
    + return PrependLength(std::move(ser_header));
    +}
    +
    +unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
    + // Java ints are 4 long. So create a buffer that large
    + auto len_buf = IOBuf::create(4);
    + // Then make those bytes visible.
    + len_buf->append(4);
    +
    + RWPrivateCursor c(len_buf.get());
    + // Get the size of the data to be pushed out the network.
    + auto size = msg->computeChainDataLength();
    +
    + // Write the length to this IOBuf.
    + c.writeBE(static_cast<uint32_t>(size));
    +
    + // Then attach the origional to the back of len_buf
    + len_buf->appendChain(std::move(msg));
    + return len_buf;
    +}
    +
    +unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
    + // Get the buffer size needed for just the message.
    + int msg_size = msg.ByteSize();
    + int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
    +
    + // Create a buffer big enough to hold the varint and the object.
    + auto buf = IOBuf::create(buf_size);
    + buf->append(buf_size);
    +
    + // Create the array output stream.
    + ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
    + // Wrap the ArrayOuputStream in the coded output stream to allow writing
    + // Varint32
    + CodedOutputStream cos{&aos};
    +
    + // Write out the size.
    + cos.WriteVarint32(msg_size);
    +
    + // Now write the rest out.
    + // We're using the protobuf output streams here to keep track
    + // of where in the output array we are rather than IOBuf.
    + msg.SerializeWithCachedSizesToArray(
    + cos.GetDirectBufferForNBytesAndAdvance(msg_size));
    +
    + // Return the buffer.
    + return buf;
    +}
    +// TODO(eclark): Make this 1 copy.
    +unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
    + auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
    + return buf;
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/rpc.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/rpc.h b/hbase-native-client/serde/rpc.h
    new file mode 100644
    index 0000000..cefb583
    --- /dev/null
    +++ b/hbase-native-client/serde/rpc.h
    @@ -0,0 +1,58 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +#include <memory>
    +#include <string>
    +
    +// Forward
    +namespace folly {
    +class IOBuf;
    +}
    +namespace google {
    +namespace protobuf {
    +class Message;
    +}
    +}
    +
    +namespace hbase {
    +class RpcSerde {
    +public:
    + RpcSerde();
    + virtual ~RpcSerde();
    + int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
    + std::unique_ptr<folly::IOBuf> Preamble();
    + std::unique_ptr<folly::IOBuf> Header(const std::string &user);
    + std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
    + const std::string &method,
    + const google::protobuf::Message *msg);
    + std::unique_ptr<folly::IOBuf>
    + SerializeDelimited(const google::protobuf::Message &msg);
    +
    + std::unique_ptr<folly::IOBuf>
    + SerializeMessage(const google::protobuf::Message &msg);
    +
    + std::unique_ptr<folly::IOBuf>
    + PrependLength(std::unique_ptr<folly::IOBuf> msg);
    +
    +private:
    + /* data */
    + uint8_t auth_type_;
    +};
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/server-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
    new file mode 100644
    index 0000000..35dcbc1
    --- /dev/null
    +++ b/hbase-native-client/serde/server-name-test.cc
    @@ -0,0 +1,32 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/server-name.h"
    +
    +#include <gtest/gtest.h>
    +#include <string>
    +
    +using hbase::pb::ServerName;
    +
    +TEST(TestServerName, TestMakeServerName) {
    + auto sn = folly::to<ServerName>("test:123");
    +
    + ASSERT_EQ("test", sn.host_name());
    + ASSERT_EQ(123, sn.port());
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/server-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
    new file mode 100644
    index 0000000..bdba087
    --- /dev/null
    +++ b/hbase-native-client/serde/server-name.h
    @@ -0,0 +1,21 @@
    +#pragma once
    +
    +#include "if/HBase.pb.h"
    +#include <folly/Conv.h>
    +#include <folly/String.h>
    +
    +namespace hbase {
    +namespace pb {
    +
    +template <class String> void parseTo(String in, ServerName &out) {
    + // TODO see about getting rsplit into folly.
    + std::string s = folly::to<std::string>(in);
    +
    + auto delim = s.rfind(":");
    + DCHECK(delim != std::string::npos);
    + out.set_host_name(s.substr(0, delim));
    + // Now keep everything after the : (delim + 1) to the end.
    + out.set_port(folly::to<int>(s.substr(delim + 1)));
    +}
    +}
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/table-name-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/table-name-test.cc b/hbase-native-client/serde/table-name-test.cc
    new file mode 100644
    index 0000000..877d522
    --- /dev/null
    +++ b/hbase-native-client/serde/table-name-test.cc
    @@ -0,0 +1,54 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include <folly/Conv.h>
    +#include <gtest/gtest.h>
    +
    +#include <string>
    +
    +#include "serde/table-name.h"
    +
    +using namespace hbase;
    +using hbase::pb::TableName;
    +
    +TEST(TestTableName, TestToStringNoDefault) {
    + TableName tn;
    + tn.set_qualifier("TestTableName");
    + std::string result = folly::to<std::string>(tn);
    + ASSERT_EQ(result.find("default"), std::string::npos);
    + ASSERT_EQ("TestTableName", result);
    +}
    +
    +TEST(TestTableName, TestToStringNoDefaltWhenSet) {
    + TableName tn;
    + tn.set_namespace_("default");
    + tn.set_qualifier("TestTableName");
    + std::string result = folly::to<std::string>(tn);
    + ASSERT_EQ(result.find("default"), std::string::npos);
    + ASSERT_EQ("TestTableName", result);
    +}
    +
    +TEST(TestTableName, TestToStringIncludeNS) {
    + TableName tn;
    + tn.set_namespace_("hbase");
    + tn.set_qualifier("acl");
    + std::string result = folly::to<std::string>(tn);
    + ASSERT_EQ(result.find("hbase"), 0);
    + ASSERT_EQ("hbase:acl", result);
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/table-name.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/table-name.h b/hbase-native-client/serde/table-name.h
    new file mode 100644
    index 0000000..c81e166
    --- /dev/null
    +++ b/hbase-native-client/serde/table-name.h
    @@ -0,0 +1,54 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +#include <memory>
    +#include <string>
    +
    +#include "if/HBase.pb.h"
    +#include <folly/Conv.h>
    +#include <folly/String.h>
    +
    +namespace hbase {
    +namespace pb {
    +
    +// Provide folly::to<std::string>(TableName);
    +template <class String> void toAppend(const TableName &in, String *result) {
    + if (!in.has_namespace_() || in.namespace_() == "default") {
    + folly::toAppend(in.qualifier(), result);
    + } else {
    + folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
    + }
    +}
    +
    +template <class String> void parseTo(String in, TableName &out) {
    + std::vector<std::string> v;
    + folly::split(":", in, v);
    +
    + if (v.size() == 1) {
    + out.set_namespace_("default");
    + out.set_qualifier(v[0]);
    + } else {
    + out.set_namespace_(v[0]);
    + out.set_qualifier(v[1]);
    + }
    +}
    +
    +} // namespace pb
    +} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/zk-deserializer-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer-test.cc b/hbase-native-client/serde/zk-deserializer-test.cc
    index 92d85a0..f07eecf 100644
    --- a/hbase-native-client/serde/zk-deserializer-test.cc
    +++ b/hbase-native-client/serde/zk-deserializer-test.cc
    @@ -17,7 +17,7 @@
       *
       */

    -#include "serde/zk-deserializer.h"
    +#include "serde/zk.h"

      #include <folly/Logging.h>
      #include <folly/io/Cursor.h>
    @@ -41,7 +41,7 @@ TEST(TestZkDesializer, TestThrowNoMagicNum) {
        buf->append(100);
        RWPrivateCursor c{buf.get()};
        c.write<uint8_t>(99);
    - ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
    + ASSERT_THROW(deser.Parse(buf.get(), &mrs), runtime_error);
      }

      // Test if the protobuf is in a format that we can't decode
    @@ -78,7 +78,7 @@ TEST(TestZkDesializer, TestBadProtoThrow) {

        // Create the protobuf
        MetaRegionServer out;
    - ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
    + ASSERT_THROW(deser.Parse(buf.get(), &out), runtime_error);
      }

      // Test to make sure the whole thing works.
    @@ -118,6 +118,6 @@ TEST(TestZkDesializer, TestNoThrow) {

        // Create the protobuf
        MetaRegionServer out;
    - ASSERT_TRUE(deser.parse(buf.get(), &out));
    + ASSERT_TRUE(deser.Parse(buf.get(), &out));
        ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
      }

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/zk-deserializer.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer.cc b/hbase-native-client/serde/zk-deserializer.cc
    deleted file mode 100644
    index 33cf809..0000000
    --- a/hbase-native-client/serde/zk-deserializer.cc
    +++ /dev/null
    @@ -1,78 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#include "serde/zk-deserializer.h"
    -
    -#include <folly/io/Cursor.h>
    -#include <folly/io/IOBuf.h>
    -#include <google/protobuf/message.h>
    -
    -using hbase::ZkDeserializer;
    -using std::runtime_error;
    -using folly::IOBuf;
    -using folly::io::Cursor;
    -using google::protobuf::Message;
    -
    -static const std::string MAGIC_STRING = "PBUF";
    -
    -bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
    -
    - // The format is like this
    - // 1 byte of magic number. 255
    - // 4 bytes of id length.
    - // id_length number of bytes for the id of who put up the znode
    - // 4 bytes of a magic string PBUF
    - // Then the protobuf serialized without a varint header.
    -
    - Cursor c{buf};
    -
    - // There should be a magic number for recoverable zk
    - uint8_t magic_num = c.read<uint8_t>();
    - if (magic_num != 255) {
    - LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
    - << unsigned(magic_num);
    - throw runtime_error("Magic number not in znode data");
    - }
    - // How long is the id?
    - uint32_t id_len = c.readBE<uint32_t>();
    -
    - if (id_len >= c.length()) {
    - LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
    - "left to read anything else";
    - throw runtime_error("Not enough bytes to decode from zookeeper");
    - }
    -
    - // Skip the id
    - c.skip(id_len);
    -
    - // Make sure that the magic string is there.
    - if (MAGIC_STRING != c.readFixedString(4)) {
    - LOG(ERROR) << "There was no PBUF magic string.";
    - throw runtime_error("No PBUF magic string in the zookpeeper data.");
    - }
    -
    - // Try to decode the protobuf.
    - // If there's an error bail out.
    - if (out->ParseFromArray(c.data(), c.length()) == false) {
    - LOG(ERROR) << "Error parsing Protobuf Message";
    - throw runtime_error("Error parsing protobuf");
    - }
    -
    - return true;
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/zk-deserializer.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk-deserializer.h b/hbase-native-client/serde/zk-deserializer.h
    deleted file mode 100644
    index aa91661..0000000
    --- a/hbase-native-client/serde/zk-deserializer.h
    +++ /dev/null
    @@ -1,35 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -#pragma once
    -
    -namespace google {
    -namespace protobuf {
    -class Message;
    -}
    -}
    -namespace folly {
    -class IOBuf;
    -}
    -
    -namespace hbase {
    -class ZkDeserializer {
    -public:
    - bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
    -};
    -} // namespace hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/zk.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk.cc b/hbase-native-client/serde/zk.cc
    new file mode 100644
    index 0000000..59871a5
    --- /dev/null
    +++ b/hbase-native-client/serde/zk.cc
    @@ -0,0 +1,78 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "serde/zk.h"
    +
    +#include <folly/io/Cursor.h>
    +#include <folly/io/IOBuf.h>
    +#include <google/protobuf/message.h>
    +
    +using hbase::ZkDeserializer;
    +using std::runtime_error;
    +using folly::IOBuf;
    +using folly::io::Cursor;
    +using google::protobuf::Message;
    +
    +static const std::string MAGIC_STRING = "PBUF";
    +
    +bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
    +
    + // The format is like this
    + // 1 byte of magic number. 255
    + // 4 bytes of id length.
    + // id_length number of bytes for the id of who put up the znode
    + // 4 bytes of a magic string PBUF
    + // Then the protobuf serialized without a varint header.
    +
    + Cursor c{buf};
    +
    + // There should be a magic number for recoverable zk
    + uint8_t magic_num = c.read<uint8_t>();
    + if (magic_num != 255) {
    + LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
    + << unsigned(magic_num);
    + throw runtime_error("Magic number not in znode data");
    + }
    + // How long is the id?
    + uint32_t id_len = c.readBE<uint32_t>();
    +
    + if (id_len >= c.length()) {
    + LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
    + "left to read anything else";
    + throw runtime_error("Not enough bytes to decode from zookeeper");
    + }
    +
    + // Skip the id
    + c.skip(id_len);
    +
    + // Make sure that the magic string is there.
    + if (MAGIC_STRING != c.readFixedString(4)) {
    + LOG(ERROR) << "There was no PBUF magic string.";
    + throw runtime_error("No PBUF magic string in the zookpeeper data.");
    + }
    +
    + // Try to decode the protobuf.
    + // If there's an error bail out.
    + if (out->ParseFromArray(c.data(), c.length()) == false) {
    + LOG(ERROR) << "Error parsing Protobuf Message";
    + throw runtime_error("Error parsing protobuf");
    + }
    +
    + return true;
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/6b4d7599/hbase-native-client/serde/zk.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/serde/zk.h b/hbase-native-client/serde/zk.h
    new file mode 100644
    index 0000000..b672bf4
    --- /dev/null
    +++ b/hbase-native-client/serde/zk.h
    @@ -0,0 +1,35 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +#pragma once
    +
    +namespace google {
    +namespace protobuf {
    +class Message;
    +}
    +}
    +namespace folly {
    +class IOBuf;
    +}
    +
    +namespace hbase {
    +class ZkDeserializer {
    +public:
    + bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
    +};
    +} // namespace hbase
  • Eclark at May 18, 2016 at 10:56 pm
    HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.

    Summary: On every start of docker make sure that protos are up to date.

    Test Plan: Added the script and it updated the protos, but didn't delete BUCK.

    Differential Revision: https://reviews.facebook.net/D56763


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f3442379
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f3442379
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f3442379

    Branch: refs/heads/HBASE-14850
    Commit: f3442379616c5874504695d5702e2d701ab3c512
    Parents: 17185fe
    Author: Elliott Clark <eclark@apache.org>
    Authored: Wed Apr 13 15:59:13 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/bin/copy-protobuf.sh | 8 ++
      hbase-native-client/bin/start-docker.sh | 22 +++-
      hbase-native-client/if/Admin.proto | 1 +
      hbase-native-client/if/Client.proto | 8 +-
      hbase-native-client/if/ClusterStatus.proto | 3 +
      hbase-native-client/if/Filter.proto | 1 +
      hbase-native-client/if/Master.proto | 47 +++++++-
      hbase-native-client/if/MasterProcedure.proto | 40 +++++++
      hbase-native-client/if/RSGroup.proto | 34 ++++++
      hbase-native-client/if/RSGroupAdmin.proto | 136 ++++++++++++++++++++++
      hbase-native-client/if/ZooKeeper.proto | 13 +++
      11 files changed, 304 insertions(+), 9 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/bin/copy-protobuf.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/copy-protobuf.sh b/hbase-native-client/bin/copy-protobuf.sh
    new file mode 100755
    index 0000000..c9e70f2
    --- /dev/null
    +++ b/hbase-native-client/bin/copy-protobuf.sh
    @@ -0,0 +1,8 @@
    +#!/usr/bin/env bash
    +set -euo pipefail
    +IFS=$'\n\t'
    +
    +BIN_DIR=$(dirname "$0")
    +PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
    +PB_DEST_DIR="${BIN_DIR}/../if/"
    +rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/bin/start-docker.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
    index 725ed6a..9cbd8b7 100755
    --- a/hbase-native-client/bin/start-docker.sh
    +++ b/hbase-native-client/bin/start-docker.sh
    @@ -23,22 +23,38 @@ set -x
      eval "$(docker-machine env docker-vm)"
      eval "$(docker-machine env dinghy)"

    -# Build the image
    -docker build -t hbase_native .
    +BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
    +BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)

    +${BIN_DIR}/copy-protobuf.sh

    +# Go into the base dir. This just makes things cleaner.
    +pushd ${BASE_DIR}
    +
    +# Make sure that there is a thrid-party dir.
      mkdir third-party || true
    +
    +# Get gtest
    +# TODO(eclark): Remove this ( see HBASE-15427 )
      if [[ ! -d third-party/googletest ]]; then
              git clone https://github.com/google/googletest.git third-party/googletest
      fi

    +# We don't want to have to re-download all the jars in docker if we can help it
      if [[ ! -d ~/.m2 ]]; then
          echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
          exit 1
      fi;

    +# Build the image
    +#
    +# This shouldn't be needed after the development environment is a little more stable.
    +docker build -t hbase_native .
    +
    +# After the image is built run the thing
      docker run -p 16010:16010/tcp \
                 -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
    - -v ${PWD}/..:/usr/src/hbase \
    + -v ${BASE_DIR}/..:/usr/src/hbase \
                 -v ~/.m2:/root/.m2 \
                 -it hbase_native /bin/bash
    +popd

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Admin.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/Admin.proto b/hbase-native-client/if/Admin.proto
    index e905340..a1905a4 100644
    --- a/hbase-native-client/if/Admin.proto
    +++ b/hbase-native-client/if/Admin.proto
    @@ -25,6 +25,7 @@ option java_generic_services = true;
      option java_generate_equals_and_hash = true;
      option optimize_for = SPEED;

    +import "Client.proto";
      import "HBase.proto";
      import "WAL.proto";


    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Client.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/Client.proto b/hbase-native-client/if/Client.proto
    index ca9abdc..8a4d459 100644
    --- a/hbase-native-client/if/Client.proto
    +++ b/hbase-native-client/if/Client.proto
    @@ -402,6 +402,11 @@ message RegionLoadStats {
        optional int32 compactionPressure = 3 [default = 0];
      }

    +message MultiRegionLoadStats{
    + repeated RegionSpecifier region = 1;
    + repeated RegionLoadStats stat = 2;
    +}
    +
      /**
       * Either a Result or an Exception NameBytesPair (keyed by
       * exception name whose value is the exception stringified)
    @@ -416,7 +421,7 @@ message ResultOrException {
        // result if this was a coprocessor service call
        optional CoprocessorServiceResult service_result = 4;
        // current load on the region
    - optional RegionLoadStats loadStats = 5;
    + optional RegionLoadStats loadStats = 5 [deprecated=true];
      }

      /**
    @@ -445,6 +450,7 @@ message MultiResponse {
        repeated RegionActionResult regionActionResult = 1;
        // used for mutate to indicate processed only
        optional bool processed = 2;
    + optional MultiRegionLoadStats regionStatistics = 3;
      }



    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/ClusterStatus.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/ClusterStatus.proto b/hbase-native-client/if/ClusterStatus.proto
    index 228be7e..54bc0c3 100644
    --- a/hbase-native-client/if/ClusterStatus.proto
    +++ b/hbase-native-client/if/ClusterStatus.proto
    @@ -136,6 +136,9 @@ message RegionLoad {

        /** the most recent sequence Id of store from cache flush */
        repeated StoreSequenceId store_complete_sequence_id = 18;
    +
    + /** the current total filtered read requests made to region */
    + optional uint64 filtered_read_requests_count = 19;
      }

      /* Server-level protobufs */

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Filter.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/Filter.proto b/hbase-native-client/if/Filter.proto
    index 67d5717..1fa6697 100644
    --- a/hbase-native-client/if/Filter.proto
    +++ b/hbase-native-client/if/Filter.proto
    @@ -146,6 +146,7 @@ message SkipFilter {

      message TimestampsFilter {
        repeated int64 timestamps = 1 [packed=true];
    + optional bool can_hint = 2;
      }

      message ValueFilter {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/Master.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto
    index 4d3a2e1..1f7a3b7 100644
    --- a/hbase-native-client/if/Master.proto
    +++ b/hbase-native-client/if/Master.proto
    @@ -186,6 +186,7 @@ message CreateNamespaceRequest {
      }

      message CreateNamespaceResponse {
    + optional uint64 proc_id = 1;
      }

      message DeleteNamespaceRequest {
    @@ -195,6 +196,7 @@ message DeleteNamespaceRequest {
      }

      message DeleteNamespaceResponse {
    + optional uint64 proc_id = 1;
      }

      message ModifyNamespaceRequest {
    @@ -204,6 +206,7 @@ message ModifyNamespaceRequest {
      }

      message ModifyNamespaceResponse {
    + optional uint64 proc_id = 1;
      }

      message GetNamespaceDescriptorRequest {
    @@ -276,6 +279,29 @@ message IsBalancerEnabledResponse {
        required bool enabled = 1;
      }

    +enum MasterSwitchType {
    + SPLIT = 0;
    + MERGE = 1;
    +}
    +
    +message SetSplitOrMergeEnabledRequest {
    + required bool enabled = 1;
    + optional bool synchronous = 2;
    + repeated MasterSwitchType switch_types = 3;
    +}
    +
    +message SetSplitOrMergeEnabledResponse {
    + repeated bool prev_value = 1;
    +}
    +
    +message IsSplitOrMergeEnabledRequest {
    + required MasterSwitchType switch_type = 1;
    +}
    +
    +message IsSplitOrMergeEnabledResponse {
    + required bool enabled = 1;
    +}
    +
      message NormalizeRequest {
      }

    @@ -344,9 +370,12 @@ message DeleteSnapshotResponse {

      message RestoreSnapshotRequest {
        required SnapshotDescription snapshot = 1;
    + optional uint64 nonce_group = 2 [default = 0];
    + optional uint64 nonce = 3 [default = 0];
      }

      message RestoreSnapshotResponse {
    + required uint64 proc_id = 1;
      }

      /* if you don't send the snapshot, then you will get it back
    @@ -630,6 +659,19 @@ service MasterService {
          returns(IsBalancerEnabledResponse);

        /**
    + * Turn the split or merge switch on or off.
    + * If synchronous is true, it waits until current operation call, if outstanding, to return.
    + */
    + rpc SetSplitOrMergeEnabled(SetSplitOrMergeEnabledRequest)
    + returns(SetSplitOrMergeEnabledResponse);
    +
    + /**
    + * Query whether the split or merge switch is on/off.
    + */
    + rpc IsSplitOrMergeEnabled(IsSplitOrMergeEnabledRequest)
    + returns(IsSplitOrMergeEnabledResponse);
    +
    + /**
         * Run region normalizer. Can NOT run for various reasons. Check logs.
         */
        rpc Normalize(NormalizeRequest)
    @@ -696,11 +738,6 @@ service MasterService {
        rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);

        /**
    - * Determine if the snapshot restore is done yet.
    - */
    - rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse);
    -
    - /**
         * Execute a distributed procedure.
         */
        rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/MasterProcedure.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/MasterProcedure.proto b/hbase-native-client/if/MasterProcedure.proto
    index 2d2aff4..87aae6a 100644
    --- a/hbase-native-client/if/MasterProcedure.proto
    +++ b/hbase-native-client/if/MasterProcedure.proto
    @@ -222,6 +222,46 @@ message DisableTableStateData {
        required bool skip_table_state_check = 3;
      }

    +message RestoreParentToChildRegionsPair {
    + required string parent_region_name = 1;
    + required string child1_region_name = 2;
    + required string child2_region_name = 3;
    +}
    +
    +enum CloneSnapshotState {
    + CLONE_SNAPSHOT_PRE_OPERATION = 1;
    + CLONE_SNAPSHOT_WRITE_FS_LAYOUT = 2;
    + CLONE_SNAPSHOT_ADD_TO_META = 3;
    + CLONE_SNAPSHOT_ASSIGN_REGIONS = 4;
    + CLONE_SNAPSHOT_UPDATE_DESC_CACHE = 5;
    + CLONE_SNAPSHOT_POST_OPERATION = 6;
    +}
    +
    +message CloneSnapshotStateData {
    + required UserInformation user_info = 1;
    + required SnapshotDescription snapshot = 2;
    + required TableSchema table_schema = 3;
    + repeated RegionInfo region_info = 4;
    + repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5;
    +}
    +
    +enum RestoreSnapshotState {
    + RESTORE_SNAPSHOT_PRE_OPERATION = 1;
    + RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR = 2;
    + RESTORE_SNAPSHOT_WRITE_FS_LAYOUT = 3;
    + RESTORE_SNAPSHOT_UPDATE_META = 4;
    +}
    +
    +message RestoreSnapshotStateData {
    + required UserInformation user_info = 1;
    + required SnapshotDescription snapshot = 2;
    + required TableSchema modified_table_schema = 3;
    + repeated RegionInfo region_info_for_restore = 4;
    + repeated RegionInfo region_info_for_remove = 5;
    + repeated RegionInfo region_info_for_add = 6;
    + repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 7;
    +}
    +
      message ServerCrashStateData {
        required ServerName server_name = 1;
        optional bool distributed_log_replay = 2;

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/RSGroup.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/RSGroup.proto b/hbase-native-client/if/RSGroup.proto
    new file mode 100644
    index 0000000..7358941
    --- /dev/null
    +++ b/hbase-native-client/if/RSGroup.proto
    @@ -0,0 +1,34 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package hbase.pb;
    +
    +option java_package = "org.apache.hadoop.hbase.protobuf.generated";
    +option java_outer_classname = "RSGroupProtos";
    +option java_generic_services = true;
    +option java_generate_equals_and_hash = true;
    +option optimize_for = SPEED;
    +
    +import "HBase.proto";
    +
    +message RSGroupInfo {
    + required string name = 1;
    + repeated ServerName servers = 4;
    + repeated TableName tables = 3;
    +}
    +

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/RSGroupAdmin.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/RSGroupAdmin.proto b/hbase-native-client/if/RSGroupAdmin.proto
    new file mode 100644
    index 0000000..fda9b09
    --- /dev/null
    +++ b/hbase-native-client/if/RSGroupAdmin.proto
    @@ -0,0 +1,136 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package hbase.pb;
    +
    +option java_package = "org.apache.hadoop.hbase.protobuf.generated";
    +option java_outer_classname = "RSGroupAdminProtos";
    +option java_generic_services = true;
    +option java_generate_equals_and_hash = true;
    +option optimize_for = SPEED;
    +
    +import "HBase.proto";
    +import "RSGroup.proto";
    +
    +/** Group level protobufs */
    +
    +message ListTablesOfRSGroupRequest {
    + required string r_s_group_name = 1;
    +}
    +
    +message ListTablesOfRSGroupResponse {
    + repeated TableName table_name = 1;
    +}
    +
    +message GetRSGroupInfoRequest {
    + required string r_s_group_name = 1;
    +}
    +
    +message GetRSGroupInfoResponse {
    + optional RSGroupInfo r_s_group_info = 1;
    +}
    +
    +message GetRSGroupInfoOfTableRequest {
    + required TableName table_name = 1;
    +}
    +
    +message GetRSGroupInfoOfTableResponse {
    + optional RSGroupInfo r_s_group_info = 1;
    +}
    +
    +message MoveServersRequest {
    + required string target_group = 1;
    + repeated ServerName servers = 3;
    +}
    +
    +message MoveServersResponse {
    +}
    +
    +message MoveTablesRequest {
    + required string target_group = 1;
    + repeated TableName table_name = 2;
    +}
    +
    +message MoveTablesResponse {
    +}
    +
    +message AddRSGroupRequest {
    + required string r_s_group_name = 1;
    +}
    +
    +message AddRSGroupResponse {
    +}
    +
    +message RemoveRSGroupRequest {
    + required string r_s_group_name = 1;
    +}
    +
    +message RemoveRSGroupResponse {
    +}
    +
    +message BalanceRSGroupRequest {
    + required string r_s_group_name = 1;
    +}
    +
    +message BalanceRSGroupResponse {
    + required bool balanceRan = 1;
    +}
    +
    +message ListRSGroupInfosRequest {
    +}
    +
    +message ListRSGroupInfosResponse {
    + repeated RSGroupInfo r_s_group_info = 1;
    +}
    +
    +message GetRSGroupInfoOfServerRequest {
    + required ServerName server = 2;
    +}
    +
    +message GetRSGroupInfoOfServerResponse {
    + optional RSGroupInfo r_s_group_info = 1;
    +}
    +
    +service RSGroupAdminService {
    + rpc GetRSGroupInfo(GetRSGroupInfoRequest)
    + returns (GetRSGroupInfoResponse);
    +
    + rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest)
    + returns (GetRSGroupInfoOfTableResponse);
    +
    + rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest)
    + returns (GetRSGroupInfoOfServerResponse);
    +
    + rpc MoveServers(MoveServersRequest)
    + returns (MoveServersResponse);
    +
    + rpc MoveTables(MoveTablesRequest)
    + returns (MoveTablesResponse);
    +
    + rpc AddRSGroup(AddRSGroupRequest)
    + returns (AddRSGroupResponse);
    +
    + rpc RemoveRSGroup(RemoveRSGroupRequest)
    + returns (RemoveRSGroupResponse);
    +
    + rpc BalanceRSGroup(BalanceRSGroupRequest)
    + returns (BalanceRSGroupResponse);
    +
    + rpc ListRSGroupInfos(ListRSGroupInfosRequest)
    + returns (ListRSGroupInfosResponse);
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/f3442379/hbase-native-client/if/ZooKeeper.proto
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto
    index 54652af..186d183 100644
    --- a/hbase-native-client/if/ZooKeeper.proto
    +++ b/hbase-native-client/if/ZooKeeper.proto
    @@ -105,6 +105,11 @@ message DeprecatedTableState {
        required State state = 1 [default = ENABLED];
      }

    +message TableCF {
    + optional TableName table_name = 1;
    + repeated bytes families = 2;
    +}
    +
      /**
       * Used by replication. Holds a replication peer key.
       */
    @@ -115,6 +120,7 @@ message ReplicationPeer {
        optional string replicationEndpointImpl = 2;
        repeated BytesBytesPair data = 3;
        repeated NameStringPair configuration = 4;
    + repeated TableCF table_cfs = 5;
      }

      /**
    @@ -153,3 +159,10 @@ message TableLock {
        optional string purpose = 5;
        optional int64 create_time = 6;
      }
    +
    +/**
    + * State of the switch.
    + */
    +message SwitchState {
    + optional bool enabled = 1;
    +}
    \ No newline at end of file
  • Eclark at May 18, 2016 at 10:56 pm
    HBASE-14854 Read meta location from zk


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b9881d21
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b9881d21
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b9881d21

    Branch: refs/heads/HBASE-14850
    Commit: b9881d21beedc1354094d558eb6458ecfdd51cc1
    Parents: dd8bb5c
    Author: Elliott Clark <eclark@apache.org>
    Authored: Sat Mar 5 00:09:08 2016 -0800
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/Dockerfile | 20 ++-
      hbase-native-client/bin/start-docker.sh | 5 +-
      hbase-native-client/core/BUCK | 106 +++++++-------
      .../core/HBaseNativeClientTestEnv.cc | 42 ------
      .../core/SampleNativeClientTest.cc | 28 ----
      hbase-native-client/core/location-cache-test.cc | 14 ++
      hbase-native-client/core/location-cache.cc | 67 +++++++++
      hbase-native-client/core/location-cache.h | 35 +++++
      .../core/native-client-test-env.cc | 42 ++++++
      .../core/simple-native-client-test.cc | 28 ++++
      hbase-native-client/core/test_env.h | 2 +
      hbase-native-client/if/BUCK | 1 +
      hbase-native-client/third-party/BUCK | 138 ++++++++++---------
      13 files changed, 339 insertions(+), 189 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/Dockerfile
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
    index 1364d22..36959a5 100644
    --- a/hbase-native-client/Dockerfile
    +++ b/hbase-native-client/Dockerfile
    @@ -15,7 +15,7 @@
      # See the License for the specific language governing permissions and
      # limitations under the License.

    -FROM cpp_update
    +FROM pjameson/buck-folly-watchman

      ARG CC=/usr/bin/gcc-5
      ARG CXX=/usr/bin/g++-5
    @@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -p
      RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
      RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
        cd /usr/src/protobuf/ && \
    + ldconfig && \
        ./autogen.sh && \
    - ./configure --disable-shared && \
    + ./configure && \
        make && \
    - make check && \
    - make install
    + make install && \
    + make clean && \
    + rm -rf .git
    +
      RUN cd /usr/src && \
        wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
        tar zxf zookeeper-3.4.8.tar.gz && \
        rm -rf zookeeper-3.4.8.tar.gz && \
        cd zookeeper-3.4.8 && \
        cd src/c && \
    - ./configure --disable-shared && \
    + ldconfig && \
    + ./configure && \
        make && \
        make install && \
    - make clean
    + make clean
    +
    +RUN ldconfig

    -WORKDIR /usr/local/src/hbase/hbase-native-client
    +WORKDIR /usr/src/hbase/hbase-native-client

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/bin/start-docker.sh
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
    index 4426705..725ed6a 100755
    --- a/hbase-native-client/bin/start-docker.sh
    +++ b/hbase-native-client/bin/start-docker.sh
    @@ -19,8 +19,11 @@
      set -e
      set -x

    +# Try out some standard docker machine names that could work
      eval "$(docker-machine env docker-vm)"
      eval "$(docker-machine env dinghy)"
    +
    +# Build the image
      docker build -t hbase_native .


    @@ -36,6 +39,6 @@ fi;

      docker run -p 16010:16010/tcp \
                 -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
    - -v ${PWD}/..:/usr/local/src/hbase \
    + -v ${PWD}/..:/usr/src/hbase \
                 -v ~/.m2:/root/.m2 \
                 -it hbase_native /bin/bash

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index ef027a1..817b5a0 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -15,52 +15,62 @@
      # See the License for the specific language governing permissions and
      # limitations under the License.

    +cxx_library(name="core",
    + headers=[
    + "admin.h",
    + "client.h",
    + "connection.h",
    + "connection_attr.h",
    + "delete.h",
    + "get.h",
    + "hbase_macros.h",
    + "mutation.h",
    + "put.h",
    + "scanner.h",
    + "location-cache.h",
    + ],
    + srcs=[
    + "admin.cc",
    + "client.cc",
    + "connection.cc",
    + "get.cc",
    + "mutation.cc",
    + "put.cc",
    + "delete.cc",
    + "scanner.cc",
    + "location-cache.cc",
    + ],
    + deps=[
    + "//if:if",
    + "//third-party:zookeeper_mt",
    + "//third-party:folly",
    + "//third-party:wangle",
    + ],
    + visibility=[
    + 'PUBLIC',
    + ], )

    -cxx_binary(
    - name = "core",
    - headers = [
    - "admin.h",
    - "client.h",
    - "connection.h",
    - "connection_attr.h",
    - "delete.h",
    - "get.h",
    - "hbase_macros.h",
    - "mutation.h",
    - "put.h",
    - "scanner.h",
    - ],
    - srcs = [
    - "admin.cc",
    - "client.cc",
    - "connection.cc",
    - "get.cc",
    - "mutation.cc",
    - "put.cc",
    - "delete.cc",
    - "scanner.cc",
    - ],
    - deps = [
    - "//if:if",
    - "//third-party:folly",
    - "//third-party:wangle",
    - ],
    - visibility = [
    - 'PUBLIC',
    - ],
    -)
    -
    -cxx_test(
    - name = "core_test",
    - headers = [
    - "test_env.h",
    - ],
    - srcs = [
    - "HBaseNativeClientTestEnv.cc",
    - "SampleNativeClientTest.cc",
    - ],
    - deps = [
    - ":core",
    - ],
    - run_test_separately = True,
    -)
    +cxx_test(name="simple-test",
    + headers=[
    + "test_env.h",
    + ],
    + srcs=[
    + "native-client-test-env.cc",
    + "simple-native-client-test.cc",
    + ],
    + deps=[
    + ":core",
    + ],
    + run_test_separately=True, )
    +cxx_test(name="location-cache-test",
    + headers=[
    + "test_env.h",
    + ],
    + srcs=[
    + "native-client-test-env.cc",
    + "location-cache-test.cc",
    + ],
    + deps=[
    + ":core",
    + ],
    + run_test_separately=True, )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/HBaseNativeClientTestEnv.cc b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    deleted file mode 100644
    index b8cb8db..0000000
    --- a/hbase-native-client/core/HBaseNativeClientTestEnv.cc
    +++ /dev/null
    @@ -1,42 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#include <gtest/gtest.h>
    -#include <core/test_env.h>
    -
    -namespace {
    -
    -class HBaseNativeClientTestEnv : public ::testing::Environment {
    - public:
    - void SetUp() override {
    - init_test_env();
    - }
    -
    - void TearDown() override {
    - clean_test_env();
    - }
    -};
    -
    -} // anonymous
    -
    -int main(int argc, char** argv) {
    - testing::InitGoogleTest(&argc, argv);
    - ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
    - return RUN_ALL_TESTS();
    -}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/SampleNativeClientTest.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/SampleNativeClientTest.cc b/hbase-native-client/core/SampleNativeClientTest.cc
    deleted file mode 100644
    index ef564f7..0000000
    --- a/hbase-native-client/core/SampleNativeClientTest.cc
    +++ /dev/null
    @@ -1,28 +0,0 @@
    -/*
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - *
    - */
    -
    -#include "gtest/gtest.h"
    -
    -/**
    - * Sample test.
    - */
    -TEST(SampleTest, sample) {
    - EXPECT_TRUE(true);
    -}
    -

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/location-cache-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
    new file mode 100644
    index 0000000..3106e36
    --- /dev/null
    +++ b/hbase-native-client/core/location-cache-test.cc
    @@ -0,0 +1,14 @@
    +#include <gtest/gtest.h>
    +#include <folly/Memory.h>
    +#include <wangle/concurrent/GlobalExecutor.h>
    +
    +#include "location-cache.h"
    +using namespace hbase;
    +
    +TEST(LocationCacheTest, TestGetMetaNodeContents) {
    + // TODO(elliott): need to make a test utility for this.
    + LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
    + auto result = cache.LocateMeta();
    + result.wait();
    + ASSERT_FALSE(result.hasException());
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/location-cache.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
    new file mode 100644
    index 0000000..cf61e24
    --- /dev/null
    +++ b/hbase-native-client/core/location-cache.cc
    @@ -0,0 +1,67 @@
    +#include "location-cache.h"
    +
    +#include <folly/Logging.h>
    +
    +#include "if/ZooKeeper.pb.h"
    +
    +using namespace std;
    +using namespace folly;
    +using namespace hbase::pb;
    +
    +namespace hbase {
    +
    +// TODO(elliott): make this configurable on client creation
    +const static string META_LOCATION = "/hbase/meta-region-server";
    +
    +LocationCache::LocationCache(string quorum_spec,
    + shared_ptr<folly::Executor> executor)
    + : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
    + zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
    +}
    +
    +LocationCache::~LocationCache() {
    + zookeeper_close(zk_);
    + zk_ = nullptr;
    + LOG(INFO) << "Closed connection to ZooKeeper.";
    +}
    +
    +Future<ServerName> LocationCache::LocateMeta() {
    + lock_guard<mutex> g(meta_lock_);
    + if (meta_promise_ == nullptr) {
    + this->RefreshMetaLocation();
    + }
    + return meta_promise_->getFuture();
    +}
    +
    +void LocationCache::InvalidateMeta() {
    + if (meta_promise_ != nullptr) {
    + lock_guard<mutex> g(meta_lock_);
    + meta_promise_ = nullptr;
    + }
    +}
    +
    +/// MUST hold the meta_lock_
    +void LocationCache::RefreshMetaLocation() {
    + meta_promise_ = make_unique<SharedPromise<ServerName>>();
    + executor_->add([&] {
    + meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
    + });
    +}
    +
    +ServerName LocationCache::ReadMetaLocation() {
    + char contents[4096];
    + int len = sizeof(contents);
    + // TODO(elliott): handle disconnects/reconntion as needed.
    + int zk_result =
    + zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr);
    +
    + if (zk_result != ZOK) {
    + LOG(ERROR) << "Error getting meta location.";
    + throw runtime_error("Error getting meta location");
    + }
    +
    + MetaRegionServer mrs;
    + mrs.ParseFromArray(contents, len);
    + return mrs.server();
    +}
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/location-cache.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
    new file mode 100644
    index 0000000..8dc2760
    --- /dev/null
    +++ b/hbase-native-client/core/location-cache.h
    @@ -0,0 +1,35 @@
    +#pragma once
    +
    +#include <memory>
    +#include <mutex>
    +
    +#include <zookeeper/zookeeper.h>
    +#include <folly/futures/Future.h>
    +#include <folly/futures/SharedPromise.h>
    +
    +#include <folly/Executor.h>
    +#include "if/HBase.pb.h"
    +
    +namespace hbase {
    +class LocationCache {
    +public:
    + explicit LocationCache(std::string quorum_spec,
    + std::shared_ptr<folly::Executor> executor);
    + ~LocationCache();
    + // Meta Related Methods.
    + // These are only public until testing is complete
    + folly::Future<hbase::pb::ServerName> LocateMeta();
    + void InvalidateMeta();
    +
    +private:
    + void RefreshMetaLocation();
    + hbase::pb::ServerName ReadMetaLocation();
    +
    + std::string quorum_spec_;
    + std::shared_ptr<folly::Executor> executor_;
    + std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
    + std::mutex meta_lock_;
    +
    + zhandle_t *zk_;
    +};
    +} // hbase

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/native-client-test-env.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc
    new file mode 100644
    index 0000000..a86961f
    --- /dev/null
    +++ b/hbase-native-client/core/native-client-test-env.cc
    @@ -0,0 +1,42 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include <gtest/gtest.h>
    +#include <core/test_env.h>
    +
    +namespace {
    +
    +class NativeClientTestEnv : public ::testing::Environment {
    + public:
    + void SetUp() override {
    + init_test_env();
    + }
    +
    + void TearDown() override {
    + clean_test_env();
    + }
    +};
    +
    +} // anonymous
    +
    +int main(int argc, char** argv) {
    + testing::InitGoogleTest(&argc, argv);
    + ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
    + return RUN_ALL_TESTS();
    +}

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/simple-native-client-test.cc
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/simple-native-client-test.cc b/hbase-native-client/core/simple-native-client-test.cc
    new file mode 100644
    index 0000000..ef564f7
    --- /dev/null
    +++ b/hbase-native-client/core/simple-native-client-test.cc
    @@ -0,0 +1,28 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + *
    + */
    +
    +#include "gtest/gtest.h"
    +
    +/**
    + * Sample test.
    + */
    +TEST(SampleTest, sample) {
    + EXPECT_TRUE(true);
    +}
    +

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/core/test_env.h
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/test_env.h b/hbase-native-client/core/test_env.h
    index 5796ae1..79bdbec 100644
    --- a/hbase-native-client/core/test_env.h
    +++ b/hbase-native-client/core/test_env.h
    @@ -17,6 +17,8 @@
       *
       */

    +#pragma once
    +
      #include <cstdlib>

      inline void init_test_env() {

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/if/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK
    index 3490a05..9b989b5 100644
    --- a/hbase-native-client/if/BUCK
    +++ b/hbase-native-client/if/BUCK
    @@ -33,4 +33,5 @@ cxx_library(
              + [':' + x for x in CC_FILENAMES]
              + [ ':' + x for x in HEADER_FILENAMES ],
        visibility = [ 'PUBLIC', ],
    + exported_deps = ['//third-party:protobuf']
      )

    http://git-wip-us.apache.org/repos/asf/hbase/blob/b9881d21/hbase-native-client/third-party/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
    index e577a5f..6548695 100644
    --- a/hbase-native-client/third-party/BUCK
    +++ b/hbase-native-client/third-party/BUCK
    @@ -15,85 +15,97 @@
      # See the License for the specific language governing permissions and
      # limitations under the License.

    -def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = [], exported_linker_flags = []):
    - rules = []
    - for name in names:
    - gen_rule_name = "gen_lib{}".format(name)
    - genrule(
    - name = gen_rule_name,
    - out = gen_rule_name,
    - bash = "mkdir -p $OUT && cp {}/lib{}.* $OUT".format(lib_dir,name),
    - )
    - prebuilt_cxx_library(
    - name = name,
    - lib_name = name,
    - lib_dir = '$(location :{})'.format(gen_rule_name),
    - force_static = True,
    - deps = deps,
    - visibility = [ 'PUBLIC' ],
    - exported_linker_flags = exported_linker_flags,
    - )
    - rules.append(":" + name)
    - return rules

    -system_libs = [
    - "unwind",
    - "lzma",
    -]
    +def add_system_libs(names=[],
    + lib_dir="/usr/lib/x86_64-linux-gnu",
    + deps=[],
    + exported_deps=[],
    + exported_linker_flags=[]):
    + rules = []
    + for name in names:
    + gen_rule_name = "gen_lib{}".format(name)
    + genrule(name=gen_rule_name,
    + out=gen_rule_name,
    + bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir,
    + name), )
    + prebuilt_cxx_library(name=name,
    + lib_name=name,
    + lib_dir='$(location :{})'.format(gen_rule_name),
    + deps=deps,
    + force_static = True,
    + exported_deps=exported_deps,
    + visibility=['PUBLIC'],
    + exported_linker_flags=exported_linker_flags, )
    + rules.append(":" + name)
    + return rules
    +
    +
    +system_libs = ["unwind", "lzma", "event", ]
      local_libs = [
    - "double-conversion",
    - "glog",
    - "gflags",
    - "protobuf",
    - "zookeeper_mt",
    - "boost_regex",
    + "double-conversion",
    + "boost_regex",
    + "boost_context",
    + "boost_thread",
    + "boost_system",
    + "boost_filesystem",
    + "boost_program_options",
    + "boost_chrono",
    + "gflags",
    + "glog",
    + "protobuf",
      ]

    +
    +
      tp_dep_rules = add_system_libs(system_libs) \
    - + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
    -folly = add_system_libs(
    - ['folly'],
    - lib_dir = '/usr/local/lib',
    - deps = tp_dep_rules,
    - exported_linker_flags = [ "-pthread", "-lstdc++",]
    - )
    -folly_bench = add_system_libs(
    - ['follybenchmark'],
    - lib_dir = '/usr/local/lib',
    - deps = tp_dep_rules + folly,
    - exported_linker_flags = [ "-pthread", "-lstdc++",]
    - )
    -wangle = add_system_libs(
    - ['wangle'],
    - lib_dir = '/usr/local/lib',
    - deps = tp_dep_rules + folly,
    - exported_linker_flags = [ "-pthread", "-lstdc++",]
    - )
    + + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
    +
    +zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib")
    +folly = add_system_libs(['folly'],
    + lib_dir='/usr/local/lib',
    + exported_deps=tp_dep_rules,
    + exported_linker_flags=["-pthread",
    + "-lstdc++", ])
    +folly_bench = add_system_libs(['follybenchmark'],
    + lib_dir='/usr/local/lib',
    + exported_deps=tp_dep_rules + folly,
    + exported_linker_flags=["-pthread",
    + "-lstdc++", ])
    +wangle = add_system_libs(['wangle'],
    + lib_dir='/usr/local/lib',
    + exported_deps=tp_dep_rules + folly,
    + exported_linker_flags=["-pthread",
    + "-lstdc++", ])
    +
    +
    +genrule(
    +name = "gen_zk",
    +out = "gen_zk",
    +bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
    +)
      cxx_library(
    - name = 'google-test',
    - srcs = [
    + name = 'google-test',
    + srcs = [
          'googletest/googletest/src/gtest-all.cc',
          'googletest/googlemock/src/gmock-all.cc',
          'googletest/googlemock/src/gmock_main.cc',
    - ],
    - header_namespace = '',
    - exported_headers = subdir_glob([
    + ],
    + header_namespace = '',
    + exported_headers = subdir_glob([
          ('googletest/googletest/include', '**/*.h'),
          ('googletest/googlemock/include', '**/*.h'),
    - ]),
    - headers = subdir_glob([
    + ]),
    + headers = subdir_glob([
          ('googletest/googletest', 'src/*.h'),
          ('googletest/googletest', 'src/*.cc'),
          ('googletest/googlemock', 'src/*.h'),
          ('googletest/googlemock', 'src/*.cc'),
    - ]),
    - exported_linker_flags = [
    + ]),
    + exported_linker_flags = [
          "-pthread",
          "-lstdc++",
    - ],
    - visibility = [
    + ],
    + visibility = [
          'PUBLIC',
    - ],
    - deps = [
    - ]
    + ],
      )
  • Eclark at May 18, 2016 at 10:56 pm
    HBASE-15850 Localize the configuration change in testCheckTableLocks to reduce flakiness of TestHBaseFsck test suite (Stephen Yuan Jiang)


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8aa8abfc
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8aa8abfc
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8aa8abfc

    Branch: refs/heads/HBASE-14850
    Commit: 8aa8abfcb672e4b905c487a10f1511db431f7006
    Parents: 5ac54e6
    Author: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Authored: Wed May 18 09:41:13 2016 -0700
    Committer: Stephen Yuan Jiang <syuanjiangdev@gmail.com>
    Committed: Wed May 18 13:22:41 2016 -0700

    ----------------------------------------------------------------------
      .../hadoop/hbase/util/TestHBaseFsckOneRS.java | 16 ++++++++++------
      1 file changed, 10 insertions(+), 6 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/8aa8abfc/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    ----------------------------------------------------------------------
    diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    index 165fea6..57bc77e 100644
    --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckOneRS.java
    @@ -1472,7 +1472,8 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
              TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});
    + assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
    + HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});

          final CountDownLatch latch = new CountDownLatch(1);
          new Thread() {
    @@ -1496,24 +1497,27 @@ public class TestHBaseFsckOneRS extends BaseTestHBaseFsck {
          Threads.sleep(300); // wait some more to ensure writeLock.acquire() is called

          hbck = doFsck(conf, false);
    + // still one expired, one not-expired
          assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
    - HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK}); // still one expired, one not-expired
    + HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK});

          edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
              TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS)); // let table lock expire

          hbck = doFsck(conf, false);
    - assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK,
    + assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] {
    + HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK,
              HBaseFsck.ErrorReporter.ERROR_CODE.EXPIRED_TABLE_LOCK}); // both are expired

    - conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
    + Configuration localConf = new Configuration(conf);
          // reaping from ZKInterProcessWriteLock uses znode cTime,
          // which is not injectable through EnvironmentEdge
    + localConf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);

          Threads.sleep(10);
    - hbck = doFsck(conf, true); // now fix both cases
    + hbck = doFsck(localConf, true); // now fix both cases

    - hbck = doFsck(conf, false);
    + hbck = doFsck(localConf, false);
          assertNoErrors(hbck);

          // ensure that locks are deleted
  • Eclark at May 18, 2016 at 10:56 pm
    HBASE-14855 Connect to regionserver

    Summary:
    * Client dispatcher to put request and response together with call id. Though right now this is a pretty empty version of that.
    * pipeline. The Serialization pipeline.
    * client-serilization-handler.h This does the work of sending the preamable, the connection header, and length prepending data. This will obviously need to be split up into h and cc files

    Test Plan: Use simple-client to connect to a standalone server

    Differential Revision: https://reviews.facebook.net/D56385


    Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0ec0a25c
    Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0ec0a25c
    Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0ec0a25c

    Branch: refs/heads/HBASE-14850
    Commit: 0ec0a25cc08c3166f70032e6385819024567488b
    Parents: f344237
    Author: Elliott Clark <eclark@apache.org>
    Authored: Mon Apr 4 13:52:05 2016 -0700
    Committer: Elliott Clark <elliott@fb.com>
    Committed: Wed May 18 15:48:52 2016 -0700

    ----------------------------------------------------------------------
      hbase-native-client/core/BUCK | 30 +++++-
      hbase-native-client/core/client-dispatcher.cc | 54 ++++++++++
      hbase-native-client/core/client-dispatcher.h | 42 ++++++++
      .../core/client-serialize-handler.cc | 104 ++++++++++++++++++
      .../core/client-serialize-handler.h | 44 ++++++++
      hbase-native-client/core/client.cc | 16 ++-
      hbase-native-client/core/client.h | 18 +++-
      hbase-native-client/core/connection-factory.cc | 57 ++++++++++
      hbase-native-client/core/connection-factory.h | 41 ++++++++
      hbase-native-client/core/get-request.cc | 19 ++++
      hbase-native-client/core/get-request.h | 35 +++++++
      hbase-native-client/core/get-result.cc | 19 ++++
      hbase-native-client/core/get-result.h | 32 ++++++
      hbase-native-client/core/location-cache-test.cc | 25 ++++-
      hbase-native-client/core/location-cache.cc | 62 +++++++++--
      hbase-native-client/core/location-cache.h | 29 ++++-
      .../core/native-client-test-env.cc | 6 +-
      hbase-native-client/core/pipeline.cc | 42 ++++++++
      hbase-native-client/core/pipeline.h | 34 ++++++
      hbase-native-client/core/request.h | 33 ++++++
      hbase-native-client/core/response.h | 34 ++++++
      hbase-native-client/core/service.h | 26 +++++
      hbase-native-client/core/simple-client.cc | 57 ++++++++++
      .../core/simple-native-client-test.cc | 5 +-
      hbase-native-client/core/table-name.cc | 19 ++++
      hbase-native-client/core/table-name.h | 32 ++++++
      hbase-native-client/if/BUCK | 24 ++++-
      hbase-native-client/third-party/BUCK | 105 +++++++++----------
      28 files changed, 948 insertions(+), 96 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hbase/blob/0ec0a25c/hbase-native-client/core/BUCK
    ----------------------------------------------------------------------
    diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
    index d1e89d1..2b00d66 100644
    --- a/hbase-native-client/core/BUCK
    +++ b/hbase-native-client/core/BUCK
    @@ -16,35 +16,52 @@
      # limitations under the License.

      cxx_library(name="core",
    - headers=[
    + exported_headers=[
                      "admin.h",
    + "client-dispatcher.h",
    + "client-serialize-handler.h",
                      "client.h",
    + "connection-factory.h",
                      "connection.h",
                      "connection_attr.h",
                      "delete.h",
    + "get-request.h",
    + "get-result.h",
                      "get.h",
                      "hbase_macros.h",
    + "location-cache.h",
                      "mutation.h",
    + "pipeline.h",
                      "put.h",
    + "request.h",
    + "response.h",
                      "scanner.h",
    - "location-cache.h",
    + "service.h",
    + "table-name.h",
                  ],
                  srcs=[
                      "admin.cc",
    + "client-dispatcher.cc",
    + "client-serialize-handler.cc",
                      "client.cc",
    + "connection-factory.cc",
                      "connection.cc",
    + "delete.cc",
    + "get-request.cc",
    + "get-result.cc",
                      "get.cc",
    + "location-cache.cc",
                      "mutation.cc",
    + "pipeline.cc",
              &nb