client.h

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.


#ifndef KUDU_CLIENT_CLIENT_H
#define KUDU_CLIENT_CLIENT_H

#include <stdint.h>
#include <string>
#include <vector>

#include "kudu/client/resource_metrics.h"
#include "kudu/client/row_result.h"
#include "kudu/client/scan_batch.h"
#include "kudu/client/scan_predicate.h"
#include "kudu/client/schema.h"
#include "kudu/client/shared_ptr.h"
#ifdef KUDU_HEADERS_NO_STUBS
#include <gtest/gtest_prod.h>
#include "kudu/gutil/macros.h"
#include "kudu/gutil/port.h"
#else
#include "kudu/client/stubs.h"
#endif
#include "kudu/client/write_op.h"
#include "kudu/util/kudu_export.h"
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"

namespace kudu {

class ClientStressTest_TestUniqueClientIds_Test;
class LinkedListTester;
class PartitionSchema;

namespace client {

class KuduLoggingCallback;
class KuduScanToken;
class KuduSession;
class KuduStatusCallback;
class KuduTable;
class KuduTableAlterer;
class KuduTableCreator;
class KuduTablet;
class KuduTabletServer;
class KuduValue;
class KuduWriteOperation;

namespace internal {
class Batcher;
class GetTableSchemaRpc;
class LookupRpc;
class MetaCache;
class RemoteTablet;
class RemoteTabletServer;
class WriteRpc;
} // namespace internal

void KUDU_EXPORT InstallLoggingCallback(KuduLoggingCallback* cb);

void KUDU_EXPORT UninstallLoggingCallback();

void KUDU_EXPORT SetVerboseLogLevel(int level);

Status KUDU_EXPORT SetInternalSignalNumber(int signum);

std::string KUDU_EXPORT GetShortVersionString();

std::string KUDU_EXPORT GetAllVersionInfo();

class KUDU_EXPORT KuduClientBuilder {
 public:
  KuduClientBuilder();
  ~KuduClientBuilder();

  KuduClientBuilder& clear_master_server_addrs();

  KuduClientBuilder& master_server_addrs(const std::vector<std::string>& addrs);

  KuduClientBuilder& add_master_server_addr(const std::string& addr);

  KuduClientBuilder& default_admin_operation_timeout(const MonoDelta& timeout);

  KuduClientBuilder& default_rpc_timeout(const MonoDelta& timeout);

  Status Build(sp::shared_ptr<KuduClient>* client);

 private:
  class KUDU_NO_EXPORT Data;

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduClientBuilder);
};

class KUDU_EXPORT KuduClient : public sp::enable_shared_from_this<KuduClient> {
 public:
  ~KuduClient();

  KuduTableCreator* NewTableCreator();

  Status IsCreateTableInProgress(const std::string& table_name,
                                 bool *create_in_progress);

  Status DeleteTable(const std::string& table_name);

  KuduTableAlterer* NewTableAlterer(const std::string& table_name);

  Status IsAlterTableInProgress(const std::string& table_name,
                                bool *alter_in_progress);
  Status GetTableSchema(const std::string& table_name,
                        KuduSchema* schema);

  Status ListTabletServers(std::vector<KuduTabletServer*>* tablet_servers);

  Status ListTables(std::vector<std::string>* tables,
                    const std::string& filter = "");

  Status TableExists(const std::string& table_name, bool* exists);

  Status OpenTable(const std::string& table_name,
                   sp::shared_ptr<KuduTable>* table);

  sp::shared_ptr<KuduSession> NewSession();


  Status KUDU_NO_EXPORT GetTablet(const std::string& tablet_id,
                                  KuduTablet** tablet);


  enum ReplicaSelection {
    LEADER_ONLY,      

    CLOSEST_REPLICA,  

    FIRST_REPLICA     
  };

  bool IsMultiMaster() const;

  const MonoDelta& default_admin_operation_timeout() const;

  const MonoDelta& default_rpc_timeout() const;

  static const uint64_t kNoTimestamp;

  uint64_t GetLatestObservedTimestamp() const;

  void SetLatestObservedTimestamp(uint64_t ht_timestamp);

 private:
  class KUDU_NO_EXPORT Data;

  friend class internal::Batcher;
  friend class internal::GetTableSchemaRpc;
  friend class internal::LookupRpc;
  friend class internal::MetaCache;
  friend class internal::RemoteTablet;
  friend class internal::RemoteTabletServer;
  friend class internal::WriteRpc;
  friend class ClientTest;
  friend class KuduClientBuilder;
  friend class KuduScanner;
  friend class KuduScanTokenBuilder;
  friend class KuduSession;
  friend class KuduTable;
  friend class KuduTableAlterer;
  friend class KuduTableCreator;

  FRIEND_TEST(kudu::ClientStressTest, TestUniqueClientIds);
  FRIEND_TEST(ClientTest, TestGetTabletServerBlacklist);
  FRIEND_TEST(ClientTest, TestMasterDown);
  FRIEND_TEST(ClientTest, TestMasterLookupPermits);
  FRIEND_TEST(ClientTest, TestMetaCacheExpiry);
  FRIEND_TEST(ClientTest, TestNonCoveringRangePartitions);
  FRIEND_TEST(ClientTest, TestReplicatedTabletWritesWithLeaderElection);
  FRIEND_TEST(ClientTest, TestScanFaultTolerance);
  FRIEND_TEST(ClientTest, TestScanTimeout);
  FRIEND_TEST(ClientTest, TestWriteWithDeadMaster);
  FRIEND_TEST(MasterFailoverTest, TestPauseAfterCreateTableIssued);

  KuduClient();

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduClient);
};

class KUDU_EXPORT KuduTabletServer {
 public:
  ~KuduTabletServer();

  const std::string& uuid() const;

  const std::string& hostname() const;

  uint16_t port() const;

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduClient;
  friend class KuduScanner;
  friend class KuduScanTokenBuilder;

  KuduTabletServer();

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduTabletServer);
};

class KUDU_EXPORT KuduReplica {
 public:
  ~KuduReplica();

  bool is_leader() const;

  const KuduTabletServer& ts() const;

 private:
  friend class KuduClient;
  friend class KuduScanTokenBuilder;

  class KUDU_NO_EXPORT Data;

  KuduReplica();

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduReplica);
};

class KUDU_EXPORT KuduTablet {
 public:
  ~KuduTablet();

  const std::string& id() const;

  const std::vector<const KuduReplica*>& replicas() const;

 private:
  friend class KuduClient;
  friend class KuduScanTokenBuilder;

  class KUDU_NO_EXPORT Data;

  KuduTablet();

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduTablet);
};

class KUDU_EXPORT KuduTableCreator {
 public:
  ~KuduTableCreator();

  KuduTableCreator& table_name(const std::string& name);

  KuduTableCreator& schema(const KuduSchema* schema);

  KuduTableCreator& add_hash_partitions(const std::vector<std::string>& columns,
                                        int32_t num_buckets);

  KuduTableCreator& add_hash_partitions(const std::vector<std::string>& columns,
                                        int32_t num_buckets, int32_t seed);

  KuduTableCreator& set_range_partition_columns(const std::vector<std::string>& columns);

  enum RangePartitionBound {
    EXCLUSIVE_BOUND, 
    INCLUSIVE_BOUND, 
  };

  KuduTableCreator& add_range_partition(KuduPartialRow* lower_bound,
                                        KuduPartialRow* upper_bound,
                                        RangePartitionBound lower_bound_type = INCLUSIVE_BOUND,
                                        RangePartitionBound upper_bound_type = EXCLUSIVE_BOUND);

  KuduTableCreator& add_range_partition_split(KuduPartialRow* split_row);

  KuduTableCreator& split_rows(const std::vector<const KuduPartialRow*>& split_rows)
      ATTRIBUTE_DEPRECATED("use add_range_partition_split() instead");

  KuduTableCreator& num_replicas(int n_replicas);

  KuduTableCreator& timeout(const MonoDelta& timeout);

  KuduTableCreator& wait(bool wait);

  Status Create();

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduClient;

  explicit KuduTableCreator(KuduClient* client);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduTableCreator);
};

class KUDU_EXPORT KuduTable : public sp::enable_shared_from_this<KuduTable> {
 public:
  ~KuduTable();

  const std::string& name() const;

  const std::string& id() const;

  const KuduSchema& schema() const;

  int num_replicas() const;

  KuduInsert* NewInsert();

  KuduUpsert* NewUpsert();

  KuduUpdate* NewUpdate();

  KuduDelete* NewDelete();

  KuduPredicate* NewComparisonPredicate(const Slice& col_name,
                                        KuduPredicate::ComparisonOp op,
                                        KuduValue* value);

  KuduPredicate* NewInListPredicate(const Slice& col_name,
                                    std::vector<KuduValue*>* values);

  KuduClient* client() const;

  const PartitionSchema& partition_schema() const;

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduClient;

  KuduTable(const sp::shared_ptr<KuduClient>& client,
            const std::string& name,
            const std::string& id,
            int num_replicas,
            const KuduSchema& schema,
            const PartitionSchema& partition_schema);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduTable);
};

class KUDU_EXPORT KuduTableAlterer {
 public:
  ~KuduTableAlterer();

  KuduTableAlterer* RenameTo(const std::string& new_name);

  KuduColumnSpec* AddColumn(const std::string& name);

  KuduColumnSpec* AlterColumn(const std::string& name);

  KuduTableAlterer* DropColumn(const std::string& name);

  KuduTableAlterer* AddRangePartition(
      KuduPartialRow* lower_bound,
      KuduPartialRow* upper_bound,
      KuduTableCreator::RangePartitionBound lower_bound_type = KuduTableCreator::INCLUSIVE_BOUND,
      KuduTableCreator::RangePartitionBound upper_bound_type = KuduTableCreator::EXCLUSIVE_BOUND);

  KuduTableAlterer* DropRangePartition(
      KuduPartialRow* lower_bound,
      KuduPartialRow* upper_bound,
      KuduTableCreator::RangePartitionBound lower_bound_type = KuduTableCreator::INCLUSIVE_BOUND,
      KuduTableCreator::RangePartitionBound upper_bound_type = KuduTableCreator::EXCLUSIVE_BOUND);

  KuduTableAlterer* timeout(const MonoDelta& timeout);

  KuduTableAlterer* wait(bool wait);

  Status Alter();

 private:
  class KUDU_NO_EXPORT Data;
  friend class KuduClient;

  KuduTableAlterer(KuduClient* client,
                   const std::string& name);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduTableAlterer);
};

class KUDU_EXPORT KuduError {
 public:
  ~KuduError();

  const Status& status() const;

  const KuduWriteOperation& failed_op() const;

  KuduWriteOperation* release_failed_op();

  bool was_possibly_successful() const;

 private:
  class KUDU_NO_EXPORT Data;

  friend class internal::Batcher;
  friend class KuduSession;

  KuduError(KuduWriteOperation* failed_op, const Status& error);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduError);
};


class KUDU_EXPORT KuduSession : public sp::enable_shared_from_this<KuduSession> {
 public:
  ~KuduSession();

  enum FlushMode {
    AUTO_FLUSH_SYNC,

    AUTO_FLUSH_BACKGROUND,

    MANUAL_FLUSH
  };

  Status SetFlushMode(FlushMode m) WARN_UNUSED_RESULT;

  enum ExternalConsistencyMode {
    CLIENT_PROPAGATED,

    COMMIT_WAIT
  };

  Status SetExternalConsistencyMode(ExternalConsistencyMode m)
    WARN_UNUSED_RESULT;

  Status SetMutationBufferSpace(size_t size_bytes) WARN_UNUSED_RESULT;

  Status SetMutationBufferFlushWatermark(double watermark_pct)
      WARN_UNUSED_RESULT;

  Status SetMutationBufferFlushInterval(unsigned int millis) WARN_UNUSED_RESULT;

  Status SetMutationBufferMaxNum(unsigned int max_num) WARN_UNUSED_RESULT;

  void SetTimeoutMillis(int millis);


  Status Apply(KuduWriteOperation* write_op) WARN_UNUSED_RESULT;

  Status Flush() WARN_UNUSED_RESULT;

  void FlushAsync(KuduStatusCallback* cb);

  Status Close() WARN_UNUSED_RESULT;

  bool HasPendingOperations() const;

  int CountBufferedOperations() const
      ATTRIBUTE_DEPRECATED("this method is experimental and will disappear "
                           "in a future release");

  int CountPendingErrors() const;

  void GetPendingErrors(std::vector<KuduError*>* errors, bool* overflowed);

  KuduClient* client() const;

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduClient;
  friend class internal::Batcher;
  friend class ClientTest;
  FRIEND_TEST(ClientTest, TestAutoFlushBackgroundApplyBlocks);
  FRIEND_TEST(ClientTest, TestAutoFlushBackgroundAndErrorCollector);

  explicit KuduSession(const sp::shared_ptr<KuduClient>& client);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduSession);
};


class KUDU_EXPORT KuduScanner {
 public:
  enum ReadMode {
    READ_LATEST,

    READ_AT_SNAPSHOT
  };

  enum OrderMode {
    UNORDERED,

    ORDERED
  };

  enum { kScanTimeoutMillis = 30000 };

  explicit KuduScanner(KuduTable* table);
  ~KuduScanner();

  Status SetProjectedColumnNames(const std::vector<std::string>& col_names)
    WARN_UNUSED_RESULT;

  Status SetProjectedColumnIndexes(const std::vector<int>& col_indexes)
    WARN_UNUSED_RESULT;

  Status SetProjectedColumns(const std::vector<std::string>& col_names)
      WARN_UNUSED_RESULT
      ATTRIBUTE_DEPRECATED("use SetProjectedColumnNames() instead");

  Status AddConjunctPredicate(KuduPredicate* pred) WARN_UNUSED_RESULT;

  Status AddLowerBound(const KuduPartialRow& key);

  Status AddLowerBoundRaw(const Slice& key)
      ATTRIBUTE_DEPRECATED("use AddLowerBound() instead");

  Status AddExclusiveUpperBound(const KuduPartialRow& key);

  Status AddExclusiveUpperBoundRaw(const Slice& key)
      ATTRIBUTE_DEPRECATED("use AddExclusiveUpperBound() instead");

  Status AddLowerBoundPartitionKeyRaw(const Slice& partition_key);

  Status AddExclusiveUpperBoundPartitionKeyRaw(const Slice& partition_key);

  Status SetCacheBlocks(bool cache_blocks);

  Status Open();

  Status KeepAlive();

  void Close();

  bool HasMoreRows() const;

  Status NextBatch(std::vector<KuduRowResult>* rows)
      ATTRIBUTE_DEPRECATED("use NextBatch(KuduScanBatch*) instead");

  Status NextBatch(KuduScanBatch* batch);

  Status GetCurrentServer(KuduTabletServer** server);

  const ResourceMetrics& GetResourceMetrics() const;

  Status SetBatchSizeBytes(uint32_t batch_size);

  Status SetSelection(KuduClient::ReplicaSelection selection)
    WARN_UNUSED_RESULT;

  Status SetReadMode(ReadMode read_mode) WARN_UNUSED_RESULT;

  Status SetOrderMode(OrderMode order_mode) WARN_UNUSED_RESULT
      ATTRIBUTE_DEPRECATED("use SetFaultTolerant() instead");

  Status SetFaultTolerant() WARN_UNUSED_RESULT;

  Status SetSnapshotMicros(uint64_t snapshot_timestamp_micros) WARN_UNUSED_RESULT;

  Status SetSnapshotRaw(uint64_t snapshot_timestamp) WARN_UNUSED_RESULT;

  Status SetTimeoutMillis(int millis);

  KuduSchema GetProjectionSchema() const;

  std::string ToString() const;

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduScanToken;
  FRIEND_TEST(ClientTest, TestScanCloseProxy);
  FRIEND_TEST(ClientTest, TestScanFaultTolerance);
  FRIEND_TEST(ClientTest, TestScanNoBlockCaching);
  FRIEND_TEST(ClientTest, TestScanTimeout);

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduScanner);
};

class KUDU_EXPORT KuduScanToken {
 public:

  ~KuduScanToken();

  Status IntoKuduScanner(KuduScanner** scanner) const WARN_UNUSED_RESULT;

  const KuduTablet& tablet() const;

  Status Serialize(std::string* buf) const WARN_UNUSED_RESULT;

  static Status DeserializeIntoScanner(KuduClient* client,
                                       const std::string& serialized_token,
                                       KuduScanner** scanner) WARN_UNUSED_RESULT;

 private:
  class KUDU_NO_EXPORT Data;

  friend class KuduScanTokenBuilder;

  KuduScanToken();

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduScanToken);
};

class KUDU_EXPORT KuduScanTokenBuilder {
 public:

  explicit KuduScanTokenBuilder(KuduTable* table);
  ~KuduScanTokenBuilder();

  Status SetProjectedColumnNames(const std::vector<std::string>& col_names)
    WARN_UNUSED_RESULT;

  Status SetProjectedColumnIndexes(const std::vector<int>& col_indexes)
    WARN_UNUSED_RESULT;

  Status AddConjunctPredicate(KuduPredicate* pred) WARN_UNUSED_RESULT;

  Status AddLowerBound(const KuduPartialRow& key) WARN_UNUSED_RESULT;

  Status AddUpperBound(const KuduPartialRow& key) WARN_UNUSED_RESULT;

  Status SetCacheBlocks(bool cache_blocks) WARN_UNUSED_RESULT;

  Status SetBatchSizeBytes(uint32_t batch_size) WARN_UNUSED_RESULT;

  Status SetSelection(KuduClient::ReplicaSelection selection)
    WARN_UNUSED_RESULT;

  Status SetReadMode(KuduScanner::ReadMode read_mode) WARN_UNUSED_RESULT;

  Status SetFaultTolerant() WARN_UNUSED_RESULT;

  Status SetSnapshotMicros(uint64_t snapshot_timestamp_micros)
    WARN_UNUSED_RESULT;

  Status SetSnapshotRaw(uint64_t snapshot_timestamp) WARN_UNUSED_RESULT;

  Status SetTimeoutMillis(int millis) WARN_UNUSED_RESULT;

  Status Build(std::vector<KuduScanToken*>* tokens) WARN_UNUSED_RESULT;

  std::string ToString() const;

 private:
  class KUDU_NO_EXPORT Data;

  // Owned.
  Data* data_;

  DISALLOW_COPY_AND_ASSIGN(KuduScanTokenBuilder);
};

} // namespace client
} // namespace kudu
#endif