From 6fe39d15373766ac3d0cf525fc3f005546a81c21 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 24 Dec 2020 17:38:26 +0800 Subject: [PATCH 01/19] add WFGovernancePolicy.h --- src/factory/WFGovernancePolicy.h | 189 +++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 src/factory/WFGovernancePolicy.h diff --git a/src/factory/WFGovernancePolicy.h b/src/factory/WFGovernancePolicy.h new file mode 100644 index 00000000..ce0241e1 --- /dev/null +++ b/src/factory/WFGovernancePolicy.h @@ -0,0 +1,189 @@ +class EndpointAddress +{ +public: + AddressParams params; + std::string address; + std::string host; + std::string port; + unsigned short port_value; + //... + +public: + EndpointAddress(const std::string& address, + const struct AddressParams *address_params); +}; + +//using ENDPOINT_TABLE = std::map>; +using ENDPOINT_TABLE = std::vector; + +class WFGovernancePolicy : public WFDNSResolver +{ +public: + WFGovernancePolicy() + { + } + + virtual WFRouterTask *create_router_task(const struct WFNSParams *params, + router_callback_t callback) + { + EndpointAddress *addr; + + if (this->select(parmas->uri, this->endpoint_table, addr)) + { + if (addr.is_sockaddr) + { + this->result.cookie = const_cast(addr); + } + auto *task = WFDNSResolver::create_router_task(params, callback); + task->set_cookie(addr); + return task; + } + + return new WFRouterTask(std::move(callback)); + } + + virtual void success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) + { + // ... + WFDNSResolver::success(result, cookie, target); + } + + virtual void failed(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) + { + // ... + WFDNSResolver::failed(result, cookie, target); + } + +private: + // breaker_list; + +public: + WFGovernancePolicy() + { + this->try_another = false; + } + +private: + virtual bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr); + ENDPOINT_TABLE endpoint_table; + bool try_another; +}; + +class WFWeightedRandomPolicy : public WFGovernancePolicy +{ +public: + bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) + { + uri.host = addr->host; +/* + int x = 0; + int s = 0; + int temp_weight = total_weight_; + + if (temp_weight > 0) + x = rand() % temp_weight; + + if (idx == n) + idx = n - 1; +*/ + } +}; + +using select_t = std::function; + +class WFConsistentHashPolicy : public WFGovernancePolicy +{ +public: + WFConsistentHashPolicy() + { + this->consistent_hash = this->default_consistent_hash; + } + + WFConsistentHashSelector(select_t select) + { + this->consistent_hash = std::move(select); + } + + bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) + { + unsigned int hash_value = this->consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + EndpointAddress *addr = consistent_hash_select(hash_value); + // if not available: try another + } + +private: + select_t consistent_hash; + +private: + const EndpointAddress *consistent_hash_select(unsigned int hash) const + { + } + + static unsigned int default_consistent_hash(const char *path, + const char *query, + const char *fragment) + { + static std::hash std_hash; + std::string str(path); + + str += query; + str += fragment; + return std_hash(str); + } +}; + +class WFManualSelector : public WFSelector +{ +public: + WFManualSelector(bool try_another, select_t select, select_t try_another_select) + { + this->try_another = try_another; + this->select = select; + this->try_another_select = try_another_select; + } + + bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) + { + int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + if (idx >= n) + idx %= n; + + EndpointAddress *addr = endpoint_table[idx]; + // if not available: try another + } + +private: + select_t manual_select; + select_t try_another_select; +}; + +/* +class WFSelectorTask : public WFRouterTask +{ +private: + virtual void dispatch() + { + found = this->policy->selector->select(this->result); + + if (!found) + { + //for upstream: create_dns_task... + //for sosclient: create_cs_route_task(selector_callback) + series_of(this)->push_front(new_task); + } + + return this->subtask_done; + } + + +}; +*/ + From 13c168dc00975725188152871601f0c541769233 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 6 Jan 2021 19:45:49 +0800 Subject: [PATCH 02/19] add GovernancePolicy for demo version --- CMakeLists_Headers.txt | 1 + src/factory/CMakeLists.txt | 1 + src/factory/GovernancePolicy.cc | 402 ++++++++++++++++++++++++++++++++ src/factory/GovernancePolicy.h | 353 ++++++++++++++++++++++++++++ 4 files changed, 757 insertions(+) create mode 100644 src/factory/GovernancePolicy.cc create mode 100644 src/factory/GovernancePolicy.h diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index cd9eb530..22465031 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -84,6 +84,7 @@ set(INCLUDE_HEADERS src/factory/WFOperator.h src/factory/WFNameService.h src/factory/WFDNSResolver.h + src/factory/GovernancePolicy.h ) if(KAFKA STREQUAL "y") diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 74153a68..18202644 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -10,6 +10,7 @@ set(SRC Workflow.cc WFNameService.cc WFDNSResolver.cc + GovernancePolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/factory/GovernancePolicy.cc b/src/factory/GovernancePolicy.cc new file mode 100644 index 00000000..19a478bc --- /dev/null +++ b/src/factory/GovernancePolicy.cc @@ -0,0 +1,402 @@ +#include "GovernancePolicy.h" +#include "StringUtil.h" + +EndpointAddress::EndpointAddress(const std::string& address, + const struct AddressParams *address_params) +{ + std::vector arr = StringUtil::split(address, ':'); + this->params = *address_params; + this->address = address; + + if (this->params.weight == 0) + this->params.weight = 1; + + if (this->params.max_fails == 0) + this->params.max_fails = 1; + + if (this->params.group_id < 0) + this->params.group_id = -1; + + if (arr.size() == 0) + this->host = ""; + else + this->host = arr[0]; + + if (arr.size() <= 1) + { + this->port = ""; + this->port_value = 0; + } + else + { + this->port = arr[1]; + this->port_value = atoi(arr[1].c_str()); + } +} + +void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) +{ + EndpointAddress *server = (EndpointAddress *)cookie; + + pthread_rwlock_rdlock(&this->rwlock); + server->fail_count = 0; + this->breaker_lock.lock(); + if (server->list.next) + { + list_del(&server->list); + server->list.next = NULL; + this->recover_server(server); + //this->server_list_change(); + } + this->breaker_lock.unlock(); + pthread_rwlock_unlock(&this->rwlock); + + WFDNSResolver::success(result, NULL, target); +} + +void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) +{ + EndpointAddress *server = (EndpointAddress *)cookie; + + pthread_rwlock_rdlock(&this->rwlock); + int fail_count = ++server->fail_count; + if (fail_count == server->params.max_fails) + { + this->breaker_lock.lock(); + if (!server->list.next) + { + server->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; + list_add_tail(&server->list, &this->breaker_list); + this->fuse_server(server); + //this->server_list_change(); + } + this->breaker_lock.unlock(); + } + pthread_rwlock_unlock(&this->rwlock); + + WFDNSResolver::failed(result, NULL, target); +} + +void GovernancePolicy::recover_server(const EndpointAddress *addr) +{ + this->nalive++; +} + +void GovernancePolicy::fuse_server(const EndpointAddress *addr) +{ + this->nalive--; +} + +void GovernancePolicy::recover_breaker() +{ + this->breaker_lock.lock(); + if (!list_empty(&this->breaker_list)) + { + int64_t cur_time = GET_CURRENT_SECOND; + struct list_head *pos, *tmp; + EndpointAddress *addr; + + list_for_each_safe(pos, tmp, &this->breaker_list) + { + addr = list_entry(pos, EndpointAddress, list); + if (cur_time >= addr->broken_timeout) + { + if (addr->fail_count >= addr->params.max_fails) + { + addr->fail_count = addr->params.max_fails - 1; + this->nalive++; + } + list_del(pos); + addr->list.next = NULL; + } + } + } + this->breaker_lock.unlock(); + + //this->server_list_change(); +} + +const EndpointAddress *GovernancePolicy::first_stradegy(const ParsedURI& uri) +{ + unsigned int idx = rand() % this->servers.size(); + return this->servers[idx]; +} + +const EndpointAddress *GovernancePolicy::another_stradegy(const ParsedURI& uri) +{ + return this->first_stradegy(uri); +} + +bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + this->recover_breaker(); + if (this->nalive == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->first_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + if (this->try_another) + select_addr = this->another_stradegy(uri); + } + + pthread_rwlock_unlock(&this->rwlock); + + if (select_addr) + { + *addr = (EndpointAddress *)select_addr; + return true; + } + + return false; +} + +void GovernancePolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + auto *addr = new EndpointAddress(address, address_params); + + pthread_rwlock_wrlock(&this->rwlock); + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + this->servers.push_back(addr); + this->nalive++; + pthread_rwlock_unlock(&this->rwlock); +} + +int GovernancePolicy::remove_server(const std::string& address) +{ + pthread_rwlock_wrlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (auto addr : map_it->second) + { + if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalive + this->nalive--; + } + + this->server_map.erase(map_it); + } + + int n = this->servers.size(); + int new_n = 0; + + for (int i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + int ret = 0; + if (new_n < n) + { + this->servers.resize(new_n); + ret = n - new_n; + } + + pthread_rwlock_unlock(&this->rwlock); + return ret; +} + +void GovernancePolicy::enable_server(const std::string& address) +{ +} + +void GovernancePolicy::disable_server(const std::string& address) +{ +} + +/* +void GroupPolicy::recover_breaker() +{ + // recover every group in this policy +} + +bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + return false; + + if (n == 1) + { + addr = this->servers[0]; + return true; + } + + this->recover_breaker(); + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->select_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + select_addr = this->get_one(); // check_one_strong/weak() + if (!select_addr && this->try_another) + select_addr = this->another_stradegy(); + } + addr = select_addr; + + pthread_rwlock_unlock(&this->rwlock); + return addr != NULL; +} +void GroupPolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + auto *addr = new EndpointAddress(address, address_params); + int group_id = addr->group_id; + rb_node **p = &this->group_map.rb_node; + rb_node *parent = NULL; + EndpointGroup *group; + + pthread_rwlock_wrlock(&this->rwlock); + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + + while (*p) + { + parent = *p; + group = rb_entry(*p, EndpointGroup, rb); + + if (group_id < group->group_id) + p = &(*p)->rb_left; + else if (group_id > group->group_id) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + group = new EndpointGroup(group_id, this); + rb_link_node(&group->rb, parent, p); + rb_insert_color(&group->rb, &this->group_map); + } + + if (addr->params.server_type == 0) + { + this->total_weight += addr->params.weight; + this->main.push_back(addr); + } + + group->mutex.lock(); + this->gain_one_server(addr); + addr->group = group; + if (addr->params.server_type == 0) + { + group->weight += addr->params.weight; + group->mains.push_back(addr); + } + else + group->backups.push_back(addr); + group->mutex.unlock(); + + pthread_rwlock_unlock(&this->rwlock); + return; +} + +int GroupPolicy::remove_server(const std::string& address) +{ + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + + if (map_it != this->server_map.cend()) + { + for (auto addr : map_it->second) + { + auto *group = addr->group; + std::vector *vec; + + if (addr->params->server_type == 0) + { + this->total_weight -= addr->params.weight; // this is for WeightedRandom + vec = &group->mains; + } + else + vec = &group->backups; + + std::lock_guard lock(group->mutex); + if (addr->fail_count < addr->params.max_fails) + group->lose_one_server(addr); + if (addr->params.server_type == 0) + group->weight -= addr->params.weight; + for (auto it = vec->begin(); it != vec->end(); ++it) + { + if (*it == addr) + { + vec->erase(it); + break; + } + } + } + + this->server_map.erase(map_it); + } + + int n = this->servers.size(); + int new_n = 0; + + for (int i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + if (new_n < n) + { + this->servers.resize(new_n); + pthread_rwlock_unlock(&this->rwlock); + return n - new_n; + } + + pthread_rwlock_unlock(&this->rwlock); + return 0; +} + +void GroupPolicy::gain_one_server(const EndpointAddress *addr) +{ + //this->navlive++; + this->group->gain_one_server(); +} + +void GroupPolicy::lose_one_server(const EndpointAddress *addr) +{ + this->group->lose_one_server(); +} + +void WeightedRandomPolicy::gain_one_server(const EndpointAddress *addr) +{ + // TODO: + this->available_weight += addr->params.weight; +} +*/ + diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h new file mode 100644 index 00000000..2cafd0f0 --- /dev/null +++ b/src/factory/GovernancePolicy.h @@ -0,0 +1,353 @@ +#include "EndpointParams.h" +#include "WFNameService.h" +#include "WFDNSResolver.h" +#include "WFGlobal.h" +#include "WFTaskError.h" + +#include +#include + +#ifndef _GOVERNANCE_POLICY_H_ +#define _GOVERNANCE_POLICY_H_ + +#define MTTR_SECOND 30 +#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() +//#define VIRTUAL_GROUP_SIZE 16 + +// or #include "WFTaskFactory.h" +#define DNS_CACHE_LEVEL_1 1 +#define DNS_CACHE_LEVEL_2 2 + +struct AddressParams +{ + struct EndpointParams endpoint_params; + unsigned int dns_ttl_default; + unsigned int dns_ttl_min; + unsigned int max_fails; + unsigned short weight; + int server_type; + int group_id; +}; + +class EndpointGroup; +class GovernancePolicy; +class GroupPolicy; + +class EndpointAddress +{ +public: + EndpointGroup *group; + AddressParams params; + std::string address; + std::string host; + std::string port; + short port_value; + struct list_head list; + std::atomic fail_count; + int64_t broken_timeout; + // unsigned int consistent_hash[VIRTUAL_GROUP_SIZE]; + +public: + EndpointAddress(const std::string& address, + const struct AddressParams *address_params); +}; + +class EndpointGroup +{ +public: + int group_id; + GroupPolicy *policy; + struct rb_node rb; + std::mutex mutex; + std::vector mains; + std::vector backups; + + EndpointGroup(int group_id, GroupPolicy *policy) + { + this->group_id = group_id; + this->policy = policy; + } + +public: + const EndpointAddress *get_one(); + const EndpointAddress *get_one_backup(); +}; + +class WFSelectorFailTask : public WFRouterTask +{ +public: + WFSelectorFailTask(router_callback_t&& cb) + : WFRouterTask(std::move(cb)) + { + } + + virtual void dispatch() + { + this->state = WFT_STATE_TASK_ERROR; + this->error = WFT_ERR_UPSTREAM_UNAVAILABLE; + + return this->subtask_done(); + } +}; + +class GovernancePolicy : public WFDNSResolver +{ +public: + virtual WFRouterTask *create_router_task(const struct WFNSParams *params, + router_callback_t callback) + { + EndpointAddress *addr = NULL; + WFRouterTask *task = NULL; + + if (this->select(params->uri, &addr)) + { + char *host = NULL; + char *port = NULL; + if (!addr->host.empty()) + { + host = strdup(addr->host.c_str()); + if (host) + { + if (addr->port_value > 0) + { + port = strdup(addr->port.c_str()); + if (port) + { + free(params->uri.port); + params->uri.port = port; + free(params->uri.host); + params->uri.host = host; + const auto *settings = WFGlobal::get_global_settings(); + unsigned int dns_ttl_default = settings->dns_ttl_default; + unsigned int dns_ttl_min = settings->dns_ttl_min; + const struct EndpointParams *endpoint_params = &settings->endpoint_params; + int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : + DNS_CACHE_LEVEL_1; + task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, + endpoint_params, std::move(callback)); + } + else + free(host); + } + } + } + } + + if (!task) + task = new WFSelectorFailTask(std::move(callback)); + + task->set_cookie(addr); + + return task; + } + + virtual void success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target); + virtual void failed(RouteManager::RouteResult *result, void *cookie, + CommTarget *target); + + // dynamic_cast<> + virtual void add_server(const std::string& address, const AddressParams *address_params); + virtual int remove_server(const std::string& address); + + virtual void enable_server(const std::string& address); + virtual void disable_server(const std::string& address); + // virtual void server_list_change(/* std::vector status */) {} + +public: + GovernancePolicy() + { + this->nalive = 0; + this->try_another = false; + pthread_rwlock_init(&this->rwlock, NULL); + INIT_LIST_HEAD(&this->breaker_list); + } + + ~GovernancePolicy() + { + pthread_rwlock_destroy(&this->rwlock); + for (EndpointAddress *addr : this->addresses) + delete addr; + } + +private: + virtual bool select(const ParsedURI& uri, EndpointAddress **addr); + virtual const EndpointAddress *first_stradegy(const ParsedURI& uri); + virtual const EndpointAddress *another_stradegy(const ParsedURI& uri); + virtual void fuse_server(const EndpointAddress *addr); + virtual void recover_server(const EndpointAddress *addr); + void recover_breaker(); // check_all_breaker() + + bool try_another; + std::atomic nalive; + std::vector servers; // current servers + std::vector addresses; // memory management + std::unordered_map> server_map; + pthread_rwlock_t rwlock; + struct list_head breaker_list; + std::mutex breaker_lock; +}; + +class GroupPolicy : public GovernancePolicy +{ +public: + struct rb_root group_map; + std::vector mains; + std::vector backups; + EndpointGroup *default_group; + + GroupPolicy() + { + this->group_map.rb_node = NULL; + this->default_group = new EndpointGroup(-1, this); + rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); + rb_insert_color(&this->default_group->rb, &this->group_map); + } + +private: + virtual void gain_one_server(EndpointGroup *group, const EndpointAddress *addr); + virtual void lose_one_server(EndpointGroup *group, const EndpointAddress *addr); +}; + +/* +class WeightedRandomPolicy : public GroupPolicy +{ +public: + EndpointAddrress *first_stradegy(const ParsedURI& uri) + { + int x = 0; + int s = 0; + int idx, temp_weight; + + temp_weight = this->total_weight; + + if (temp_weight > 0) + x = rand() % temp_weight; + + for (idx = 0; idx < this->servers.size(); idx++) + { + s += this->servers[idx].params.weight; + if (s > x) + break; + } + if (idx == this->servers.size()) + idx--; + + return &this->servers[idx]; + } + + EndpointAddrress *another_stradegy(const ParsedURI& uri) + { + // weighted_random_try_another(); + } + + WeightedRandomPolicy() + { + this->total_weight = 0; + this->available_weight = 0; + } + +protected: + int total_weight; + int available_weight; +}; + +using select_t = std::function; + +class ConsistentHashPolicy : public GroupPolicy +{ +public: + ConsistentHashPolicy() + { + this->consistent_hash = this->default_consistent_hash; + } + + ConsistentHashPolicy(select_t select) + { + this->consistent_hash = std::move(select); + } + + EndpointAddrress *another_stradegy(const ParsedURI& uri) + { + unsigned int hash_value = this->consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + return consistent_hash_select(hash_value); + } + + EndpointAddrress *first_stradegy(const ParsedURI& uri) + { + return NULL; + } + +private: + select_t consistent_hash; + +private: + const EndpointAddress *consistent_hash_select(unsigned int hash) const + { + } + + static unsigned int default_consistent_hash(const char *path, + const char *query, + const char *fragment) + { + static std::hash std_hash; + std::string str(path); + + str += query; + str += fragment; + return std_hash(str); + } +}; + +class ManualPolicy : public GroupPolicy +{ +public: + ManualPolicy(bool try_another, select_t select, select_t try_another_select) + { + this->try_another = try_another; + this->select = select; + this->try_another_select = try_another_select; + } + + EndpointAddrress *first_stradegy(const ParsedURI& uri) + { + int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + if (idx >= n) + idx %= n; + + return this->servers[idx]; + } + + EndpointAddrress *another_stradegy(const ParsedURI& uri) + { + if (this->try_another_select) + { + int idx = this->try_another_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + if (idx >= n) + idx %= n; + return this->servers[idx]; + } + + // get address after hash_value + return this->default_consistent_hash_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + } + +private: + select_t manual_select; + select_t try_another_select; +}; +*/ + +#endif + From 37231fdcdbcc92d73fddfafe064339b983b04450 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 7 Jan 2021 20:46:32 +0800 Subject: [PATCH 03/19] update disable/enable_server() --- src/factory/GovernancePolicy.cc | 80 ++++++++++++++++++++++----------- src/factory/GovernancePolicy.h | 11 +++-- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/factory/GovernancePolicy.cc b/src/factory/GovernancePolicy.cc index 19a478bc..3657c5c0 100644 --- a/src/factory/GovernancePolicy.cc +++ b/src/factory/GovernancePolicy.cc @@ -34,22 +34,38 @@ EndpointAddress::EndpointAddress(const std::string& address, } } -void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, - CommTarget *target) +inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) { - EndpointAddress *server = (EndpointAddress *)cookie; - - pthread_rwlock_rdlock(&this->rwlock); - server->fail_count = 0; + addr->fail_count = 0; this->breaker_lock.lock(); - if (server->list.next) + if (addr->list.next) { - list_del(&server->list); - server->list.next = NULL; - this->recover_server(server); + list_del(&addr->list); + addr->list.next = NULL; + this->recover_one_server(addr); //this->server_list_change(); } this->breaker_lock.unlock(); +} + +inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) +{ + this->breaker_lock.lock(); + if (!addr->list.next) + { + addr->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; + list_add_tail(&addr->list, &this->breaker_list); + this->fuse_one_server(addr); + //this->server_list_change(); + } + this->breaker_lock.unlock(); +} + +void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) +{ + pthread_rwlock_rdlock(&this->rwlock); + this->recover_server_from_breaker((EndpointAddress *)cookie); pthread_rwlock_unlock(&this->rwlock); WFDNSResolver::success(result, NULL, target); @@ -63,33 +79,24 @@ void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, pthread_rwlock_rdlock(&this->rwlock); int fail_count = ++server->fail_count; if (fail_count == server->params.max_fails) - { - this->breaker_lock.lock(); - if (!server->list.next) - { - server->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; - list_add_tail(&server->list, &this->breaker_list); - this->fuse_server(server); - //this->server_list_change(); - } - this->breaker_lock.unlock(); - } + this->fuse_server_to_breaker(server); + pthread_rwlock_unlock(&this->rwlock); WFDNSResolver::failed(result, NULL, target); } -void GovernancePolicy::recover_server(const EndpointAddress *addr) +inline void GovernancePolicy::recover_one_server(const EndpointAddress *addr) { this->nalive++; } -void GovernancePolicy::fuse_server(const EndpointAddress *addr) +inline void GovernancePolicy::fuse_one_server(const EndpointAddress *addr) { this->nalive--; } -void GovernancePolicy::recover_breaker() +void GovernancePolicy::check_breaker() { this->breaker_lock.lock(); if (!list_empty(&this->breaker_list)) @@ -140,7 +147,7 @@ bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) return false; } - this->recover_breaker(); + this->check_breaker(); if (this->nalive == 0) { pthread_rwlock_unlock(&this->rwlock); @@ -222,14 +229,33 @@ int GovernancePolicy::remove_server(const std::string& address) void GovernancePolicy::enable_server(const std::string& address) { + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + this->recover_server_from_breaker(addr); + } + pthread_rwlock_unlock(&this->rwlock); } void GovernancePolicy::disable_server(const std::string& address) { + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + addr->fail_count = addr->params.max_fails; + this->fuse_server_to_breaker(addr); + } + } + pthread_rwlock_unlock(&this->rwlock); } /* -void GroupPolicy::recover_breaker() +void GroupPolicy::check_breaker() { // recover every group in this policy } @@ -248,7 +274,7 @@ bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) return true; } - this->recover_breaker(); + this->check_breaker(); // select_addr == NULL will only happened in consistent_hash const EndpointAddress *select_addr = this->select_stradegy(uri); diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h index 2cafd0f0..419f25e7 100644 --- a/src/factory/GovernancePolicy.h +++ b/src/factory/GovernancePolicy.h @@ -154,6 +154,8 @@ public: virtual void disable_server(const std::string& address); // virtual void server_list_change(/* std::vector status */) {} +private: + public: GovernancePolicy() { @@ -174,9 +176,12 @@ private: virtual bool select(const ParsedURI& uri, EndpointAddress **addr); virtual const EndpointAddress *first_stradegy(const ParsedURI& uri); virtual const EndpointAddress *another_stradegy(const ParsedURI& uri); - virtual void fuse_server(const EndpointAddress *addr); - virtual void recover_server(const EndpointAddress *addr); - void recover_breaker(); // check_all_breaker() + virtual void fuse_one_server(const EndpointAddress *addr); + virtual void recover_one_server(const EndpointAddress *addr); + + void fuse_server_to_breaker(EndpointAddress *addr); + void recover_server_from_breaker(EndpointAddress *addr); + void check_breaker(); bool try_another; std::atomic nalive; From b4551978be307368bc13cb3b5e6b765d9fd45f6e Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 8 Jan 2021 19:16:17 +0800 Subject: [PATCH 04/19] add strdup_uri() --- src/factory/GovernancePolicy.h | 73 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h index 419f25e7..f5320e17 100644 --- a/src/factory/GovernancePolicy.h +++ b/src/factory/GovernancePolicy.h @@ -73,6 +73,35 @@ public: const EndpointAddress *get_one_backup(); }; +static inline bool strdup_uri(ParsedURI& uri, EndpointAddress *addr) +{ + char *host = NULL; + char *port = NULL; + + if (!addr->host.empty()) + { + host = strdup(addr->host.c_str()); + if (!host) + return false; + } + + if (addr->port_value > 0) + { + port = strdup(addr->port.c_str()); + if (!port) + { + free(host); + return false; + } + free(uri.port); + uri.port = port; + } + + free(uri.host); + uri.host = host; + return true; +} + class WFSelectorFailTask : public WFRouterTask { public: @@ -99,41 +128,19 @@ public: EndpointAddress *addr = NULL; WFRouterTask *task = NULL; - if (this->select(params->uri, &addr)) + if (this->select(params->uri, &addr) && + strdup_uri(params->uri, addr)) { - char *host = NULL; - char *port = NULL; - if (!addr->host.empty()) - { - host = strdup(addr->host.c_str()); - if (host) - { - if (addr->port_value > 0) - { - port = strdup(addr->port.c_str()); - if (port) - { - free(params->uri.port); - params->uri.port = port; - free(params->uri.host); - params->uri.host = host; - const auto *settings = WFGlobal::get_global_settings(); - unsigned int dns_ttl_default = settings->dns_ttl_default; - unsigned int dns_ttl_min = settings->dns_ttl_min; - const struct EndpointParams *endpoint_params = &settings->endpoint_params; - int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : - DNS_CACHE_LEVEL_1; - task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, - endpoint_params, std::move(callback)); - } - else - free(host); - } - } - } + const auto *settings = WFGlobal::get_global_settings(); + unsigned int dns_ttl_default = settings->dns_ttl_default; + unsigned int dns_ttl_min = settings->dns_ttl_min; + const struct EndpointParams *endpoint_params = &settings->endpoint_params; + int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : + DNS_CACHE_LEVEL_1; + task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, + endpoint_params, std::move(callback)); } - - if (!task) + else task = new WFSelectorFailTask(std::move(callback)); task->set_cookie(addr); From d90d92063e5e15010ebfceacac6ae044b21c2e7d Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 12 Jan 2021 15:47:14 +0800 Subject: [PATCH 05/19] add GroupPolicy --- src/factory/GovernancePolicy.cc | 250 ++++++++----------------------- src/factory/GovernancePolicy.h | 93 ++++-------- src/factory/GroupPolicy.cc | 200 +++++++++++++++++++++++++ src/factory/WFGovernancePolicy.h | 189 ----------------------- 4 files changed, 290 insertions(+), 442 deletions(-) create mode 100644 src/factory/GroupPolicy.cc delete mode 100644 src/factory/WFGovernancePolicy.h diff --git a/src/factory/GovernancePolicy.cc b/src/factory/GovernancePolicy.cc index 3657c5c0..d3d5e94c 100644 --- a/src/factory/GovernancePolicy.cc +++ b/src/factory/GovernancePolicy.cc @@ -1,6 +1,35 @@ #include "GovernancePolicy.h" #include "StringUtil.h" +static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) +{ + char *host = NULL; + char *port = NULL; + + if (!addr->host.empty()) + { + host = strdup(addr->host.c_str()); + if (!host) + return false; + } + + if (addr->port_value > 0) + { + port = strdup(addr->port.c_str()); + if (!port) + { + free(host); + return false; + } + free(uri.port); + uri.port = port; + } + + free(uri.host); + uri.host = host; + return true; +} + EndpointAddress::EndpointAddress(const std::string& address, const struct AddressParams *address_params) { @@ -34,6 +63,31 @@ EndpointAddress::EndpointAddress(const std::string& address, } } +WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *params, + router_callback_t callback) +{ + EndpointAddress *addr = NULL; + WFRouterTask *task = NULL; + + if (this->select(params->uri, &addr) && copy_host_port(params->uri, addr)) + { + const auto *settings = WFGlobal::get_global_settings(); + unsigned int dns_ttl_default = settings->dns_ttl_default; + unsigned int dns_ttl_min = settings->dns_ttl_min; + const struct EndpointParams *endpoint_params = &settings->endpoint_params; + int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : + DNS_CACHE_LEVEL_1; + task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, + endpoint_params, std::move(callback)); + } + else + task = new WFSelectorFailTask(std::move(callback)); + + task->set_cookie(addr); + + return task; +} + inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) { addr->fail_count = 0; @@ -86,16 +140,6 @@ void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, WFDNSResolver::failed(result, NULL, target); } -inline void GovernancePolicy::recover_one_server(const EndpointAddress *addr) -{ - this->nalive++; -} - -inline void GovernancePolicy::fuse_one_server(const EndpointAddress *addr) -{ - this->nalive--; -} - void GovernancePolicy::check_breaker() { this->breaker_lock.lock(); @@ -113,7 +157,7 @@ void GovernancePolicy::check_breaker() if (addr->fail_count >= addr->params.max_fails) { addr->fail_count = addr->params.max_fails - 1; - this->nalive++; + this->recover_one_server(addr); } list_del(pos); addr->list.next = NULL; @@ -183,7 +227,7 @@ void GovernancePolicy::add_server(const std::string& address, this->addresses.push_back(addr); this->server_map[addr->address].push_back(addr); this->servers.push_back(addr); - this->nalive++; + this->recover_one_server(addr); pthread_rwlock_unlock(&this->rwlock); } @@ -193,19 +237,19 @@ int GovernancePolicy::remove_server(const std::string& address) const auto map_it = this->server_map.find(address); if (map_it != this->server_map.cend()) { - for (auto addr : map_it->second) + for (EndpointAddress *addr : map_it->second) { if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalive - this->nalive--; + this->fuse_one_server(addr); } this->server_map.erase(map_it); } - int n = this->servers.size(); - int new_n = 0; + size_t n = this->servers.size(); + size_t new_n = 0; - for (int i = 0; i < n; i++) + for (size_t i = 0; i < n; i++) { if (this->servers[i]->address != address) { @@ -254,175 +298,3 @@ void GovernancePolicy::disable_server(const std::string& address) pthread_rwlock_unlock(&this->rwlock); } -/* -void GroupPolicy::check_breaker() -{ - // recover every group in this policy -} - -bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) -{ - pthread_rwlock_rdlock(&this->rwlock); - unsigned int n = (unsigned int)this->servers.size(); - - if (n == 0) - return false; - - if (n == 1) - { - addr = this->servers[0]; - return true; - } - - this->check_breaker(); - - // select_addr == NULL will only happened in consistent_hash - const EndpointAddress *select_addr = this->select_stradegy(uri); - - if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) - { - select_addr = this->get_one(); // check_one_strong/weak() - if (!select_addr && this->try_another) - select_addr = this->another_stradegy(); - } - addr = select_addr; - - pthread_rwlock_unlock(&this->rwlock); - return addr != NULL; -} -void GroupPolicy::add_server(const std::string& address, - const AddressParams *address_params) -{ - auto *addr = new EndpointAddress(address, address_params); - int group_id = addr->group_id; - rb_node **p = &this->group_map.rb_node; - rb_node *parent = NULL; - EndpointGroup *group; - - pthread_rwlock_wrlock(&this->rwlock); - this->addresses.push_back(addr); - this->server_map[addr->address].push_back(addr); - - while (*p) - { - parent = *p; - group = rb_entry(*p, EndpointGroup, rb); - - if (group_id < group->group_id) - p = &(*p)->rb_left; - else if (group_id > group->group_id) - p = &(*p)->rb_right; - else - break; - } - - if (*p == NULL) - { - group = new EndpointGroup(group_id, this); - rb_link_node(&group->rb, parent, p); - rb_insert_color(&group->rb, &this->group_map); - } - - if (addr->params.server_type == 0) - { - this->total_weight += addr->params.weight; - this->main.push_back(addr); - } - - group->mutex.lock(); - this->gain_one_server(addr); - addr->group = group; - if (addr->params.server_type == 0) - { - group->weight += addr->params.weight; - group->mains.push_back(addr); - } - else - group->backups.push_back(addr); - group->mutex.unlock(); - - pthread_rwlock_unlock(&this->rwlock); - return; -} - -int GroupPolicy::remove_server(const std::string& address) -{ - pthread_rwlock_rdlock(&this->rwlock); - const auto map_it = this->server_map.find(address); - - if (map_it != this->server_map.cend()) - { - for (auto addr : map_it->second) - { - auto *group = addr->group; - std::vector *vec; - - if (addr->params->server_type == 0) - { - this->total_weight -= addr->params.weight; // this is for WeightedRandom - vec = &group->mains; - } - else - vec = &group->backups; - - std::lock_guard lock(group->mutex); - if (addr->fail_count < addr->params.max_fails) - group->lose_one_server(addr); - if (addr->params.server_type == 0) - group->weight -= addr->params.weight; - for (auto it = vec->begin(); it != vec->end(); ++it) - { - if (*it == addr) - { - vec->erase(it); - break; - } - } - } - - this->server_map.erase(map_it); - } - - int n = this->servers.size(); - int new_n = 0; - - for (int i = 0; i < n; i++) - { - if (this->servers[i]->address != address) - { - if (new_n != i) - this->servers[new_n++] = this->servers[i]; - else - new_n++; - } - } - - if (new_n < n) - { - this->servers.resize(new_n); - pthread_rwlock_unlock(&this->rwlock); - return n - new_n; - } - - pthread_rwlock_unlock(&this->rwlock); - return 0; -} - -void GroupPolicy::gain_one_server(const EndpointAddress *addr) -{ - //this->navlive++; - this->group->gain_one_server(); -} - -void GroupPolicy::lose_one_server(const EndpointAddress *addr) -{ - this->group->lose_one_server(); -} - -void WeightedRandomPolicy::gain_one_server(const EndpointAddress *addr) -{ - // TODO: - this->available_weight += addr->params.weight; -} -*/ - diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h index f5320e17..6f4f3659 100644 --- a/src/factory/GovernancePolicy.h +++ b/src/factory/GovernancePolicy.h @@ -61,11 +61,13 @@ public: std::mutex mutex; std::vector mains; std::vector backups; + std::atomic nalive; EndpointGroup(int group_id, GroupPolicy *policy) { this->group_id = group_id; this->policy = policy; + this->nalive = 0; } public: @@ -73,35 +75,6 @@ public: const EndpointAddress *get_one_backup(); }; -static inline bool strdup_uri(ParsedURI& uri, EndpointAddress *addr) -{ - char *host = NULL; - char *port = NULL; - - if (!addr->host.empty()) - { - host = strdup(addr->host.c_str()); - if (!host) - return false; - } - - if (addr->port_value > 0) - { - port = strdup(addr->port.c_str()); - if (!port) - { - free(host); - return false; - } - free(uri.port); - uri.port = port; - } - - free(uri.host); - uri.host = host; - return true; -} - class WFSelectorFailTask : public WFRouterTask { public: @@ -123,44 +96,19 @@ class GovernancePolicy : public WFDNSResolver { public: virtual WFRouterTask *create_router_task(const struct WFNSParams *params, - router_callback_t callback) - { - EndpointAddress *addr = NULL; - WFRouterTask *task = NULL; - - if (this->select(params->uri, &addr) && - strdup_uri(params->uri, addr)) - { - const auto *settings = WFGlobal::get_global_settings(); - unsigned int dns_ttl_default = settings->dns_ttl_default; - unsigned int dns_ttl_min = settings->dns_ttl_min; - const struct EndpointParams *endpoint_params = &settings->endpoint_params; - int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : - DNS_CACHE_LEVEL_1; - task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, - endpoint_params, std::move(callback)); - } - else - task = new WFSelectorFailTask(std::move(callback)); - - task->set_cookie(addr); - - return task; - } - + router_callback_t callback); virtual void success(RouteManager::RouteResult *result, void *cookie, CommTarget *target); virtual void failed(RouteManager::RouteResult *result, void *cookie, CommTarget *target); - // dynamic_cast<> + // GovernancePolicy *gp = dynamic_cast(policy); gp->add_server(...); virtual void add_server(const std::string& address, const AddressParams *address_params); virtual int remove_server(const std::string& address); virtual void enable_server(const std::string& address); virtual void disable_server(const std::string& address); // virtual void server_list_change(/* std::vector status */) {} - private: public: @@ -183,15 +131,22 @@ private: virtual bool select(const ParsedURI& uri, EndpointAddress **addr); virtual const EndpointAddress *first_stradegy(const ParsedURI& uri); virtual const EndpointAddress *another_stradegy(const ParsedURI& uri); - virtual void fuse_one_server(const EndpointAddress *addr); - virtual void recover_one_server(const EndpointAddress *addr); - void fuse_server_to_breaker(EndpointAddress *addr); + virtual void recover_one_server(const EndpointAddress *addr) + { + this->nalive++; + } + + virtual void fuse_one_server(const EndpointAddress *addr) + { + this->nalive--; + } + void recover_server_from_breaker(EndpointAddress *addr); + void fuse_server_to_breaker(EndpointAddress *addr); void check_breaker(); bool try_another; - std::atomic nalive; std::vector servers; // current servers std::vector addresses; // memory management std::unordered_map nalive; }; class GroupPolicy : public GovernancePolicy { public: struct rb_root group_map; - std::vector mains; - std::vector backups; EndpointGroup *default_group; GroupPolicy() @@ -218,8 +174,17 @@ public: } private: - virtual void gain_one_server(EndpointGroup *group, const EndpointAddress *addr); - virtual void lose_one_server(EndpointGroup *group, const EndpointAddress *addr); + virtual void recover_one_server(const EndpointAddress *addr) + { + this->nalive++; + addr->group->nalive++; + } + + virtual void fuse_one_server(const EndpointAddress *addr) + { + this->nalive--; + addr->group->nalive--; + } }; /* diff --git a/src/factory/GroupPolicy.cc b/src/factory/GroupPolicy.cc new file mode 100644 index 00000000..774a6412 --- /dev/null +++ b/src/factory/GroupPolicy.cc @@ -0,0 +1,200 @@ +#include "GovernancePolicy.h" + +// no need override for success(), failed(), check_breaker() + +bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + this->check_breaker(); + if (this->nalive == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->select_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + if (select_addr) + select_addr = addr->group->get_one(); + + if (!select_addr && this->try_another) + select_addr = this->another_stradegy(); + } + + if (!select_addr) + this->default_group->get_one_backup(); + + pthread_rwlock_unlock(&this->rwlock); + + if (select_addr) + { + *addr = (EndpointAddress *)select_addr; + return true; + } + + return false; +} + +const EndpointAddress *EndpointGroup::get_one() +{ + if (this->nalive == 0) + return NULL; + + std::lock_guard lock(this->mutex); + + std::random_shuffle(this->mains.begin(), this->mains.end()); + for (const EndpointAddress *main : this->mains) + { + if (main->fail_count < main->params.max_fails) + return main; + } + + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (const EndpointAddress *backup : this->backups) + { + if (backup->fail_count < backup->params.max_fails) + return backup; + } + + return NULL; +} + +const EndpointAddress *EndpointGroup::get_one_backup() +{ + if (this->nalive == 0) + return NULL; + + std::lock_guard lock(this->mutex); + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (const EndpointAddress *backup : this->backups) + { + if (backup->fail_count < backup->params.max_fails) + return backup; + } + return NULL; +} + +void GroupPolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + auto *addr = new EndpointAddress(address, address_params); + int group_id = addr->group_id; + rb_node **p = &this->group_map.rb_node; + rb_node *parent = NULL; + EndpointGroup *group; + + pthread_rwlock_wrlock(&this->rwlock); + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + + while (*p) + { + parent = *p; + group = rb_entry(*p, EndpointGroup, rb); + + if (group_id < group->group_id) + p = &(*p)->rb_left; + else if (group_id > group->group_id) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + group = new EndpointGroup(group_id, this); + rb_link_node(&group->rb, parent, p); + rb_insert_color(&group->rb, &this->group_map); + } + + group->mutex.lock(); + this->recover_one_server(); + addr->group = group; + if (addr->params.server_type == 0) + group->mains.push_back(addr); + else + group->backups.push_back(addr); + group->mutex.unlock(); + + pthread_rwlock_unlock(&this->rwlock); + return; +} + +int GroupPolicy::remove_server(const std::string& address) +{ + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + EndpointGroup *group = addr->group; + std::vector *vec; + + if (addr->params->server_type == 0) + vec = &group->mains; + else + vec = &group->backups; + + std::lock_guard lock(group->mutex); + if (addr->fail_count < addr->params.max_fails) + this->fuse_one_server(addr); + + for (auto it = vec->begin(); it != vec->end(); ++it) + { + if (*it == addr) + { + vec->erase(it); + break; + } + } + } + + this->server_map.erase(map_it); + } + + size_t n = this->servers.size(); + size_t new_n = 0; + + for (size_t i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + int ret = 0; + if (new_n < n) + { + this->servers.resize(new_n); + ret = n - new_n; + } + + pthread_rwlock_unlock(&this->rwlock); + return ret; +} + +/* +void WeightedRandomPolicy::gain_one_server(const EndpointAddress *addr) +{ + // TODO: + this->available_weight += addr->params.weight; +} +*/ + diff --git a/src/factory/WFGovernancePolicy.h b/src/factory/WFGovernancePolicy.h deleted file mode 100644 index ce0241e1..00000000 --- a/src/factory/WFGovernancePolicy.h +++ /dev/null @@ -1,189 +0,0 @@ -class EndpointAddress -{ -public: - AddressParams params; - std::string address; - std::string host; - std::string port; - unsigned short port_value; - //... - -public: - EndpointAddress(const std::string& address, - const struct AddressParams *address_params); -}; - -//using ENDPOINT_TABLE = std::map>; -using ENDPOINT_TABLE = std::vector; - -class WFGovernancePolicy : public WFDNSResolver -{ -public: - WFGovernancePolicy() - { - } - - virtual WFRouterTask *create_router_task(const struct WFNSParams *params, - router_callback_t callback) - { - EndpointAddress *addr; - - if (this->select(parmas->uri, this->endpoint_table, addr)) - { - if (addr.is_sockaddr) - { - this->result.cookie = const_cast(addr); - } - auto *task = WFDNSResolver::create_router_task(params, callback); - task->set_cookie(addr); - return task; - } - - return new WFRouterTask(std::move(callback)); - } - - virtual void success(RouteManager::RouteResult *result, void *cookie, - CommTarget *target) - { - // ... - WFDNSResolver::success(result, cookie, target); - } - - virtual void failed(RouteManager::RouteResult *result, void *cookie, - CommTarget *target) - { - // ... - WFDNSResolver::failed(result, cookie, target); - } - -private: - // breaker_list; - -public: - WFGovernancePolicy() - { - this->try_another = false; - } - -private: - virtual bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr); - ENDPOINT_TABLE endpoint_table; - bool try_another; -}; - -class WFWeightedRandomPolicy : public WFGovernancePolicy -{ -public: - bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) - { - uri.host = addr->host; -/* - int x = 0; - int s = 0; - int temp_weight = total_weight_; - - if (temp_weight > 0) - x = rand() % temp_weight; - - if (idx == n) - idx = n - 1; -*/ - } -}; - -using select_t = std::function; - -class WFConsistentHashPolicy : public WFGovernancePolicy -{ -public: - WFConsistentHashPolicy() - { - this->consistent_hash = this->default_consistent_hash; - } - - WFConsistentHashSelector(select_t select) - { - this->consistent_hash = std::move(select); - } - - bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) - { - unsigned int hash_value = this->consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - EndpointAddress *addr = consistent_hash_select(hash_value); - // if not available: try another - } - -private: - select_t consistent_hash; - -private: - const EndpointAddress *consistent_hash_select(unsigned int hash) const - { - } - - static unsigned int default_consistent_hash(const char *path, - const char *query, - const char *fragment) - { - static std::hash std_hash; - std::string str(path); - - str += query; - str += fragment; - return std_hash(str); - } -}; - -class WFManualSelector : public WFSelector -{ -public: - WFManualSelector(bool try_another, select_t select, select_t try_another_select) - { - this->try_another = try_another; - this->select = select; - this->try_another_select = try_another_select; - } - - bool select(const ParsedURI& uri, const ENDPOINT_TABLE& endpoint_table, EndpointAddr *addr) - { - int idx = this->manual_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - if (idx >= n) - idx %= n; - - EndpointAddress *addr = endpoint_table[idx]; - // if not available: try another - } - -private: - select_t manual_select; - select_t try_another_select; -}; - -/* -class WFSelectorTask : public WFRouterTask -{ -private: - virtual void dispatch() - { - found = this->policy->selector->select(this->result); - - if (!found) - { - //for upstream: create_dns_task... - //for sosclient: create_cs_route_task(selector_callback) - series_of(this)->push_front(new_task); - } - - return this->subtask_done; - } - - -}; -*/ - From 39551c5dc4ab9d58d895fc0b7288c33c973572b5 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 12 Jan 2021 16:00:47 +0800 Subject: [PATCH 06/19] update GroupPolicy --- src/factory/GovernancePolicy.h | 9 +-------- src/factory/GroupPolicy.cc | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h index 6f4f3659..ac94b343 100644 --- a/src/factory/GovernancePolicy.h +++ b/src/factory/GovernancePolicy.h @@ -165,14 +165,6 @@ public: struct rb_root group_map; EndpointGroup *default_group; - GroupPolicy() - { - this->group_map.rb_node = NULL; - this->default_group = new EndpointGroup(-1, this); - rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); - rb_insert_color(&this->default_group->rb, &this->group_map); - } - private: virtual void recover_one_server(const EndpointAddress *addr) { @@ -185,6 +177,7 @@ private: this->nalive--; addr->group->nalive--; } + // override: select() add_server() remove_server() }; /* diff --git a/src/factory/GroupPolicy.cc b/src/factory/GroupPolicy.cc index 774a6412..dcb94150 100644 --- a/src/factory/GroupPolicy.cc +++ b/src/factory/GroupPolicy.cc @@ -1,6 +1,24 @@ #include "GovernancePolicy.h" -// no need override for success(), failed(), check_breaker() +GroupPolicy::GroupPolicy() +{ + this->group_map.rb_node = NULL; + this->default_group = new EndpointGroup(-1, this); + rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); + rb_insert_color(&this->default_group->rb, &this->group_map); +} + +GroupPolicy::~GroupPolicy() +{ + EndpointGroup *group; + + while (this->group_map.rb_node) + { + group = rb_entry(this->group_map.rb_node, EndpointGroup, rb); + rb_erase(this->group.rb_node, &this->group_map); + delete group; + } +} bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) { From 807cc87dac1f3f004772ddf67ab808602e94b650 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 14 Jan 2021 23:41:38 +0800 Subject: [PATCH 07/19] add SelectPolicy --- src/factory/GovernancePolicy.cc | 36 +++++-- src/factory/GovernancePolicy.h | 181 +++++++++++--------------------- src/factory/GroupPolicy.cc | 95 ++++++++++++----- src/factory/SelectPolicy.cc | 111 ++++++++++++++++++++ 4 files changed, 272 insertions(+), 151 deletions(-) create mode 100644 src/factory/SelectPolicy.cc diff --git a/src/factory/GovernancePolicy.cc b/src/factory/GovernancePolicy.cc index d3d5e94c..6ad2ecb8 100644 --- a/src/factory/GovernancePolicy.cc +++ b/src/factory/GovernancePolicy.cc @@ -1,6 +1,26 @@ #include "GovernancePolicy.h" #include "StringUtil.h" +#define DNS_CACHE_LEVEL_1 1 +#define DNS_CACHE_LEVEL_2 2 + +class WFSelectorFailTask : public WFRouterTask +{ +public: + WFSelectorFailTask(router_callback_t&& cb) + : WFRouterTask(std::move(cb)) + { + } + + virtual void dispatch() + { + this->state = WFT_STATE_TASK_ERROR; + this->error = WFT_ERR_UPSTREAM_UNAVAILABLE; + + return this->subtask_done(); + } +}; + static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) { char *host = NULL; @@ -91,7 +111,7 @@ WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *para inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) { addr->fail_count = 0; - this->breaker_lock.lock(); + pthread_mutex_lock(&this->breaker_lock); if (addr->list.next) { list_del(&addr->list); @@ -99,12 +119,12 @@ inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) this->recover_one_server(addr); //this->server_list_change(); } - this->breaker_lock.unlock(); + pthread_mutex_unlock(&this->breaker_lock); } inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) { - this->breaker_lock.lock(); + pthread_mutex_lock(&this->breaker_lock); if (!addr->list.next) { addr->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; @@ -112,7 +132,7 @@ inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) this->fuse_one_server(addr); //this->server_list_change(); } - this->breaker_lock.unlock(); + pthread_mutex_unlock(&this->breaker_lock); } void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, @@ -142,7 +162,7 @@ void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, void GovernancePolicy::check_breaker() { - this->breaker_lock.lock(); + pthread_mutex_lock(&this->breaker_lock); if (!list_empty(&this->breaker_list)) { int64_t cur_time = GET_CURRENT_SECOND; @@ -164,7 +184,7 @@ void GovernancePolicy::check_breaker() } } } - this->breaker_lock.unlock(); + pthread_mutex_unlock(&this->breaker_lock); //this->server_list_change(); } @@ -192,7 +212,7 @@ bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) } this->check_breaker(); - if (this->nalive == 0) + if (this->nalives == 0) { pthread_rwlock_unlock(&this->rwlock); return false; @@ -239,7 +259,7 @@ int GovernancePolicy::remove_server(const std::string& address) { for (EndpointAddress *addr : map_it->second) { - if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalive + if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalives this->fuse_one_server(addr); } diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h index ac94b343..7ffe12f4 100644 --- a/src/factory/GovernancePolicy.h +++ b/src/factory/GovernancePolicy.h @@ -14,10 +14,6 @@ #define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() //#define VIRTUAL_GROUP_SIZE 16 -// or #include "WFTaskFactory.h" -#define DNS_CACHE_LEVEL_1 1 -#define DNS_CACHE_LEVEL_2 2 - struct AddressParams { struct EndpointParams endpoint_params; @@ -55,19 +51,27 @@ public: class EndpointGroup { public: - int group_id; + int id; GroupPolicy *policy; struct rb_node rb; - std::mutex mutex; + pthread_mutex_t mutex; std::vector mains; std::vector backups; - std::atomic nalive; + std::atomic nalives; + int weight; EndpointGroup(int group_id, GroupPolicy *policy) { - this->group_id = group_id; + this->id = group_id; this->policy = policy; - this->nalive = 0; + this->nalives = 0; + this->weight = 0; + pthread_mutex_init(&this->mutex, NULL); + } + + ~EndpointGroup() + { + pthread_mutex_destroy(&this->mutex); } public: @@ -75,23 +79,6 @@ public: const EndpointAddress *get_one_backup(); }; -class WFSelectorFailTask : public WFRouterTask -{ -public: - WFSelectorFailTask(router_callback_t&& cb) - : WFRouterTask(std::move(cb)) - { - } - - virtual void dispatch() - { - this->state = WFT_STATE_TASK_ERROR; - this->error = WFT_ERR_UPSTREAM_UNAVAILABLE; - - return this->subtask_done(); - } -}; - class GovernancePolicy : public WFDNSResolver { public: @@ -114,14 +101,16 @@ private: public: GovernancePolicy() { - this->nalive = 0; + this->nalives = 0; this->try_another = false; pthread_rwlock_init(&this->rwlock, NULL); + pthread_mutex_init(&this->breaker_lock, NULL); INIT_LIST_HEAD(&this->breaker_list); } ~GovernancePolicy() { + pthread_mutex_destroy(&this->breaker_lock); pthread_rwlock_destroy(&this->rwlock); for (EndpointAddress *addr : this->addresses) delete addr; @@ -134,29 +123,29 @@ private: virtual void recover_one_server(const EndpointAddress *addr) { - this->nalive++; + this->nalives++; } virtual void fuse_one_server(const EndpointAddress *addr) { - this->nalive--; + this->nalives--; } void recover_server_from_breaker(EndpointAddress *addr); void fuse_server_to_breaker(EndpointAddress *addr); void check_breaker(); - bool try_another; - std::vector servers; // current servers std::vector addresses; // memory management std::unordered_map> server_map; pthread_rwlock_t rwlock; struct list_head breaker_list; - std::mutex breaker_lock; + pthread_mutex_t breaker_lock; protected: - std::atomic nalive; + std::vector servers; // current servers + std::atomic nalives; + bool try_another; }; class GroupPolicy : public GovernancePolicy @@ -168,59 +157,58 @@ public: private: virtual void recover_one_server(const EndpointAddress *addr) { - this->nalive++; - addr->group->nalive++; + this->nalives++; + addr->group->nalives++; } virtual void fuse_one_server(const EndpointAddress *addr) { - this->nalive--; - addr->group->nalive--; + this->nalives--; + addr->group->nalives--; } // override: select() add_server() remove_server() + +protected: + const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; + + inline const EndpointAddress *check_and_get(const EndpointAddress *addr) // check_get_weak + { + if (addr && addr->fail_count >= addr->params.max_fails && addr->params.group_id >= 0) + { + const auto *ret = addr->group->get_one(); + + if (ret) + addr = ret; + } + return addr; + } + + inline bool is_alive_or_group_alive(const EndpointAddress *addr) + { + return ((addr->params.group_id < 0 && addr->fail_count < addr->params.max_fails) || + (addr->params.group_id >= 0 && addr->group->nalives > 0)); + } }; -/* class WeightedRandomPolicy : public GroupPolicy { public: - EndpointAddrress *first_stradegy(const ParsedURI& uri) - { - int x = 0; - int s = 0; - int idx, temp_weight; - - temp_weight = this->total_weight; - - if (temp_weight > 0) - x = rand() % temp_weight; - - for (idx = 0; idx < this->servers.size(); idx++) - { - s += this->servers[idx].params.weight; - if (s > x) - break; - } - if (idx == this->servers.size()) - idx--; - - return &this->servers[idx]; - } - - EndpointAddrress *another_stradegy(const ParsedURI& uri) - { - // weighted_random_try_another(); - } - - WeightedRandomPolicy() + WeightedRandomPolicy(bool try_another) { this->total_weight = 0; this->available_weight = 0; + this->try_another = try_another; } + const EndpointAddress *first_stradegy(const ParsedURI& uri); + const EndpointAddress *another_stradegy(const ParsedURI& uri); protected: int total_weight; int available_weight; + +private: + virtual void recover_one_server(const EndpointAddress *addr); + virtual void fuse_one_server(const EndpointAddress *addr); }; using select_t = std::function; @@ -233,33 +221,18 @@ public: this->consistent_hash = this->default_consistent_hash; } - ConsistentHashPolicy(select_t select) + ConsistentHashPolicy(select_t consistent_hash) { - this->consistent_hash = std::move(select); + this->consistent_hash = std::move(consistent_hash); } - EndpointAddrress *another_stradegy(const ParsedURI& uri) - { - unsigned int hash_value = this->consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - return consistent_hash_select(hash_value); - } - - EndpointAddrress *first_stradegy(const ParsedURI& uri) - { - return NULL; - } +protected: + const EndpointAddress *first_stradegy(const ParsedURI& uri); private: select_t consistent_hash; -private: - const EndpointAddress *consistent_hash_select(unsigned int hash) const - { - } - +public: static unsigned int default_consistent_hash(const char *path, const char *query, const char *fragment) @@ -279,45 +252,17 @@ public: ManualPolicy(bool try_another, select_t select, select_t try_another_select) { this->try_another = try_another; - this->select = select; + this->manual_select = select; this->try_another_select = try_another_select; } - EndpointAddrress *first_stradegy(const ParsedURI& uri) - { - int idx = this->manual_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - if (idx >= n) - idx %= n; - - return this->servers[idx]; - } - - EndpointAddrress *another_stradegy(const ParsedURI& uri) - { - if (this->try_another_select) - { - int idx = this->try_another_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - if (idx >= n) - idx %= n; - return this->servers[idx]; - } - - // get address after hash_value - return this->default_consistent_hash_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - } + const EndpointAddress *first_stradegy(const ParsedURI& uri); + const EndpointAddress *another_stradegy(const ParsedURI& uri); private: select_t manual_select; select_t try_another_select; }; -*/ #endif diff --git a/src/factory/GroupPolicy.cc b/src/factory/GroupPolicy.cc index dcb94150..8c6dc0ac 100644 --- a/src/factory/GroupPolicy.cc +++ b/src/factory/GroupPolicy.cc @@ -1,5 +1,7 @@ #include "GovernancePolicy.h" +#define VIRTUAL_GROUP_SIZE 16 + GroupPolicy::GroupPolicy() { this->group_map.rb_node = NULL; @@ -32,14 +34,14 @@ bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) } this->check_breaker(); - if (this->nalive == 0) + if (this->nalives == 0) { pthread_rwlock_unlock(&this->rwlock); return false; } // select_addr == NULL will only happened in consistent_hash - const EndpointAddress *select_addr = this->select_stradegy(uri); + const EndpointAddress *select_addr = this->first_stradegy(uri); if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) { @@ -66,41 +68,59 @@ bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) const EndpointAddress *EndpointGroup::get_one() { - if (this->nalive == 0) + if (this->nalives == 0) return NULL; - std::lock_guard lock(this->mutex); + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); std::random_shuffle(this->mains.begin(), this->mains.end()); - for (const EndpointAddress *main : this->mains) + for (size_t i = 0; i < this->mains.size(); i++) { - if (main->fail_count < main->params.max_fails) - return main; + if (this->mains[i]->fail_count < this->mains[i]->params.max_fails) + { + addr = this->mains[i]; + break; + } } - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (const EndpointAddress *backup : this->backups) + if (!addr) { - if (backup->fail_count < backup->params.max_fails) - return backup; + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (size_t i = 0; i < this->backups.size(); i++) + { + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } + } } - return NULL; + pthread_mutex_unlock(&this->mutex); + return addr; } const EndpointAddress *EndpointGroup::get_one_backup() { - if (this->nalive == 0) + if (this->nalives == 0) return NULL; - std::lock_guard lock(this->mutex); + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); + std::random_shuffle(this->backups.begin(), this->backups.end()); - for (const EndpointAddress *backup : this->backups) + for (size_t i = 0; i < this->backups.size(); i++) { - if (backup->fail_count < backup->params.max_fails) - return backup; + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } } - return NULL; + + pthread_mutex_unlock(&this->mutex); + return addr; } void GroupPolicy::add_server(const std::string& address, @@ -121,9 +141,9 @@ void GroupPolicy::add_server(const std::string& address, parent = *p; group = rb_entry(*p, EndpointGroup, rb); - if (group_id < group->group_id) + if (group_id < group->id) p = &(*p)->rb_left; - else if (group_id > group->group_id) + else if (group_id > group->id) p = &(*p)->rb_right; else break; @@ -140,7 +160,10 @@ void GroupPolicy::add_server(const std::string& address, this->recover_one_server(); addr->group = group; if (addr->params.server_type == 0) + { group->mains.push_back(addr); + group->weight += addr->params.weight; // TODO + } else group->backups.push_back(addr); group->mutex.unlock(); @@ -170,6 +193,9 @@ int GroupPolicy::remove_server(const std::string& address) if (addr->fail_count < addr->params.max_fails) this->fuse_one_server(addr); + if (addr->params->server_type == 0) + group->weight -= ua->params.weight; // TODO + for (auto it = vec->begin(); it != vec->end(); ++it) { if (*it == addr) @@ -208,11 +234,30 @@ int GroupPolicy::remove_server(const std::string& address) return ret; } -/* -void WeightedRandomPolicy::gain_one_server(const EndpointAddress *addr) +const EndpointAddress *GroupPolicy::consistent_hash_with_group(unsigned int hash) const { - // TODO: - this->available_weight += addr->params.weight; + const EndpointAddress *addr = NULL; + unsigned int min_dis = (unsigned int)-1; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) + { + unsigned int dis = std::min + (hash - main->consistent_hash[i], + main->consistent_hash[i] - hash); + + if (dis < min_dis) + { + min_dis = dis; + addr = server; + } + } + } + } + + return this->check_and_get(addr); } -*/ diff --git a/src/factory/SelectPolicy.cc b/src/factory/SelectPolicy.cc new file mode 100644 index 00000000..1ee25060 --- /dev/null +++ b/src/factory/SelectPolicy.cc @@ -0,0 +1,111 @@ +#include "GovernancePolicy.h" + +const EndpointAddress *WeightedRandomPolicy::first_stradegy(const ParsedURI& uri) +{ + int x = 0; + int s = 0; + int idx, temp_weight; + + temp_weight = this->total_weight; + + if (temp_weight > 0) + x = rand() % temp_weight; + + for (idx = 0; idx < this->servers.size(); idx++) + { + s += this->servers[idx]->params.weight; + if (s > x) + break; + } + if (idx == this->servers.size()) + idx--; + + return this->servers[idx]; +} + +const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& uri) +{ + int temp_weight = this->available_weight; + if (temp_weight == 0) + return NULL; + + const EndpointAddress *addr = NULL; + int x = rand() % temp_weight; + int s = 0; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + addr = server; + s += server->params.weight; + if (s > x) + break; + } + } + return this->check_and_get(addr); +} + + +virtual void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) +{ + this->nalives++; + if (addr->group->nalives++ == 0 && addr->group->id > 0) + this->available_weight += addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight += addr->params.weight; +} + +virtual void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) +{ + this->nalives--; + if (--addr->group->nalives == 0 && addr->group->id > 0) + this->available_weight -= addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight -= addr->params.weight; +} + +const EndpointAddress *ConsistentHashPolicy::first_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->consistent_hash) + hash_value = this->consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = this->default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + +const EndpointAddress *ManualPolicy::first_stradegy(const ParsedURI& uri) +{ + int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + if (idx >= this->servers.size()) + idx %= this->servers.size(); + + return this->servers[idx]; +} + +const EndpointAddress *ManualPolicy::another_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->try_another_select) + hash_value = this->try_another_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = ConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + From 1a5c3a2d5f674b6b502ebdfeccc8c0074834a098 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 14 Jan 2021 23:49:52 +0800 Subject: [PATCH 08/19] update SelectPolicy --- src/factory/SelectPolicy.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/factory/SelectPolicy.cc b/src/factory/SelectPolicy.cc index 1ee25060..e89bba42 100644 --- a/src/factory/SelectPolicy.cc +++ b/src/factory/SelectPolicy.cc @@ -47,7 +47,7 @@ const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& u } -virtual void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) +void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) { this->nalives++; if (addr->group->nalives++ == 0 && addr->group->id > 0) @@ -57,7 +57,7 @@ virtual void WeightedRandomPolicy::recover_one_server(const EndpointAddress *add this->available_weight += addr->params.weight; } -virtual void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) +void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) { this->nalives--; if (--addr->group->nalives == 0 && addr->group->id > 0) From 3a14072832733d43bfec2b0be4d6704d44dd5673 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 19 Jan 2021 19:57:10 +0800 Subject: [PATCH 09/19] udpate GovernancePolicy GroupPolicy and SelectPolicy --- src/manager/GovernancePolicy.cc | 324 ++++++++++++++++++++++++++++++++ src/manager/GovernancePolicy.h | 274 +++++++++++++++++++++++++++ src/manager/GroupPolicy.cc | 265 ++++++++++++++++++++++++++ src/manager/SelectPolicy.cc | 111 +++++++++++ 4 files changed, 974 insertions(+) create mode 100644 src/manager/GovernancePolicy.cc create mode 100644 src/manager/GovernancePolicy.h create mode 100644 src/manager/GroupPolicy.cc create mode 100644 src/manager/SelectPolicy.cc diff --git a/src/manager/GovernancePolicy.cc b/src/manager/GovernancePolicy.cc new file mode 100644 index 00000000..d2ac8097 --- /dev/null +++ b/src/manager/GovernancePolicy.cc @@ -0,0 +1,324 @@ +#include "GovernancePolicy.h" +#include "StringUtil.h" + +#define DNS_CACHE_LEVEL_1 1 +#define DNS_CACHE_LEVEL_2 2 + +class WFSelectorFailTask : public WFRouterTask +{ +public: + WFSelectorFailTask(router_callback_t&& cb) + : WFRouterTask(std::move(cb)) + { + } + + virtual void dispatch() + { + this->state = WFT_STATE_TASK_ERROR; + this->error = WFT_ERR_UPSTREAM_UNAVAILABLE; + + return this->subtask_done(); + } +}; + +static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) +{ + char *host = NULL; + char *port = NULL; + + if (!addr->host.empty()) + { + host = strdup(addr->host.c_str()); + if (!host) + return false; + } + + if (addr->port_value > 0) + { + port = strdup(addr->port.c_str()); + if (!port) + { + free(host); + return false; + } + free(uri.port); + uri.port = port; + } + + free(uri.host); + uri.host = host; + return true; +} + +EndpointAddress::EndpointAddress(const std::string& address, + const struct AddressParams *address_params) +{ + std::vector arr = StringUtil::split(address, ':'); + this->params = *address_params; + this->address = address; + + static std::hash std_hash; + for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) + this->consistent_hash[i] = std_hash(address + "|v" + std::to_string(i)); + + if (this->params.weight == 0) + this->params.weight = 1; + + if (this->params.max_fails == 0) + this->params.max_fails = 1; + + if (this->params.group_id < 0) + this->params.group_id = -1; + + if (arr.size() == 0) + this->host = ""; + else + this->host = arr[0]; + + if (arr.size() <= 1) + { + this->port = ""; + this->port_value = 0; + } + else + { + this->port = arr[1]; + this->port_value = atoi(arr[1].c_str()); + } +} + +WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *params, + router_callback_t callback) +{ + EndpointAddress *addr = NULL; + WFRouterTask *task = NULL; + + if (this->select(params->uri, &addr) && copy_host_port(params->uri, addr)) + { + const auto *settings = WFGlobal::get_global_settings(); + unsigned int dns_ttl_default = settings->dns_ttl_default; + unsigned int dns_ttl_min = settings->dns_ttl_min; + const struct EndpointParams *endpoint_params = &settings->endpoint_params; + int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : + DNS_CACHE_LEVEL_1; + task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, + endpoint_params, std::move(callback)); + } + else + task = new WFSelectorFailTask(std::move(callback)); + + task->set_cookie(addr); + + return task; +} + +inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) +{ + addr->fail_count = 0; + pthread_mutex_lock(&this->breaker_lock); + if (addr->list.next) + { + list_del(&addr->list); + addr->list.next = NULL; + this->recover_one_server(addr); + //this->server_list_change(); + } + pthread_mutex_unlock(&this->breaker_lock); +} + +inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) +{ + pthread_mutex_lock(&this->breaker_lock); + if (!addr->list.next) + { + addr->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; + list_add_tail(&addr->list, &this->breaker_list); + this->fuse_one_server(addr); + //this->server_list_change(); + } + pthread_mutex_unlock(&this->breaker_lock); +} + +void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) +{ + pthread_rwlock_rdlock(&this->rwlock); + this->recover_server_from_breaker((EndpointAddress *)cookie); + pthread_rwlock_unlock(&this->rwlock); + + WFDNSResolver::success(result, NULL, target); +} + +void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, + CommTarget *target) +{ + EndpointAddress *server = (EndpointAddress *)cookie; + + pthread_rwlock_rdlock(&this->rwlock); + int fail_count = ++server->fail_count; + if (fail_count == server->params.max_fails) + this->fuse_server_to_breaker(server); + + pthread_rwlock_unlock(&this->rwlock); + + WFDNSResolver::failed(result, NULL, target); +} + +void GovernancePolicy::check_breaker() +{ + pthread_mutex_lock(&this->breaker_lock); + if (!list_empty(&this->breaker_list)) + { + int64_t cur_time = GET_CURRENT_SECOND; + struct list_head *pos, *tmp; + EndpointAddress *addr; + + list_for_each_safe(pos, tmp, &this->breaker_list) + { + addr = list_entry(pos, EndpointAddress, list); + if (cur_time >= addr->broken_timeout) + { + if (addr->fail_count >= addr->params.max_fails) + { + addr->fail_count = addr->params.max_fails - 1; + this->recover_one_server(addr); + } + list_del(pos); + addr->list.next = NULL; + } + } + } + pthread_mutex_unlock(&this->breaker_lock); + + //this->server_list_change(); +} + +const EndpointAddress *GovernancePolicy::first_stradegy(const ParsedURI& uri) +{ + unsigned int idx = rand() % this->servers.size(); + return this->servers[idx]; +} + +const EndpointAddress *GovernancePolicy::another_stradegy(const ParsedURI& uri) +{ + return this->first_stradegy(uri); +} + +bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + this->check_breaker(); + if (this->nalives == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->first_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + if (this->try_another) + select_addr = this->another_stradegy(uri); + } + + pthread_rwlock_unlock(&this->rwlock); + + if (select_addr) + { + *addr = (EndpointAddress *)select_addr; + return true; + } + + return false; +} + +void GovernancePolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + auto *addr = new EndpointAddress(address, address_params); + + pthread_rwlock_wrlock(&this->rwlock); + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + this->servers.push_back(addr); + this->recover_one_server(addr); + pthread_rwlock_unlock(&this->rwlock); +} + +int GovernancePolicy::remove_server(const std::string& address) +{ + pthread_rwlock_wrlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalives + this->fuse_one_server(addr); + } + + this->server_map.erase(map_it); + } + + size_t n = this->servers.size(); + size_t new_n = 0; + + for (size_t i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + int ret = 0; + if (new_n < n) + { + this->servers.resize(new_n); + ret = n - new_n; + } + + pthread_rwlock_unlock(&this->rwlock); + return ret; +} + +void GovernancePolicy::enable_server(const std::string& address) +{ + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + this->recover_server_from_breaker(addr); + } + pthread_rwlock_unlock(&this->rwlock); +} + +void GovernancePolicy::disable_server(const std::string& address) +{ + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + addr->fail_count = addr->params.max_fails; + this->fuse_server_to_breaker(addr); + } + } + pthread_rwlock_unlock(&this->rwlock); +} + diff --git a/src/manager/GovernancePolicy.h b/src/manager/GovernancePolicy.h new file mode 100644 index 00000000..bd10faf4 --- /dev/null +++ b/src/manager/GovernancePolicy.h @@ -0,0 +1,274 @@ +#include "EndpointParams.h" +#include "WFNameService.h" +#include "WFDNSResolver.h" +#include "WFGlobal.h" +#include "WFTaskError.h" + +#include +#include + +#ifndef _GOVERNANCE_POLICY_H_ +#define _GOVERNANCE_POLICY_H_ + +#define MTTR_SECOND 30 +#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() +#define VIRTUAL_GROUP_SIZE 16 + +struct AddressParams +{ + struct EndpointParams endpoint_params; + unsigned int dns_ttl_default; + unsigned int dns_ttl_min; + unsigned int max_fails; + unsigned short weight; + int server_type; + int group_id; +}; + +class EndpointGroup; +class GovernancePolicy; +class GroupPolicy; + +class EndpointAddress +{ +public: + EndpointGroup *group; + AddressParams params; + std::string address; + std::string host; + std::string port; + short port_value; + struct list_head list; + std::atomic fail_count; + int64_t broken_timeout; + unsigned int consistent_hash[VIRTUAL_GROUP_SIZE]; + +public: + EndpointAddress(const std::string& address, + const struct AddressParams *address_params); +}; + +class EndpointGroup +{ +public: + int id; + GroupPolicy *policy; + struct rb_node rb; + pthread_mutex_t mutex; + std::vector mains; + std::vector backups; + std::atomic nalives; + int weight; + + EndpointGroup(int group_id, GroupPolicy *policy) + { + this->id = group_id; + this->policy = policy; + this->nalives = 0; + this->weight = 0; + pthread_mutex_init(&this->mutex, NULL); + } + + ~EndpointGroup() + { + pthread_mutex_destroy(&this->mutex); + } + +public: + const EndpointAddress *get_one(); + const EndpointAddress *get_one_backup(); +}; + +class GovernancePolicy : public WFDNSResolver +{ +public: + virtual WFRouterTask *create_router_task(const struct WFNSParams *params, + router_callback_t callback); + virtual void success(RouteManager::RouteResult *result, void *cookie, + CommTarget *target); + virtual void failed(RouteManager::RouteResult *result, void *cookie, + CommTarget *target); + + // GovernancePolicy *gp = dynamic_cast(policy); gp->add_server(...); + virtual void add_server(const std::string& address, const AddressParams *address_params); + virtual int remove_server(const std::string& address); + + virtual void enable_server(const std::string& address); + virtual void disable_server(const std::string& address); + // virtual void server_list_change(/* std::vector status */) {} +private: + +public: + GovernancePolicy() + { + this->nalives = 0; + this->try_another = false; + pthread_rwlock_init(&this->rwlock, NULL); + pthread_mutex_init(&this->breaker_lock, NULL); + INIT_LIST_HEAD(&this->breaker_list); + } + + ~GovernancePolicy() + { + pthread_mutex_destroy(&this->breaker_lock); + pthread_rwlock_destroy(&this->rwlock); + for (EndpointAddress *addr : this->addresses) + delete addr; + } + +private: + virtual bool select(const ParsedURI& uri, EndpointAddress **addr); + + virtual void recover_one_server(const EndpointAddress *addr) + { + this->nalives++; + } + + virtual void fuse_one_server(const EndpointAddress *addr) + { + this->nalives--; + } + + void recover_server_from_breaker(EndpointAddress *addr); + void fuse_server_to_breaker(EndpointAddress *addr); + + struct list_head breaker_list; + pthread_mutex_t breaker_lock; + +protected: + virtual const EndpointAddress *first_stradegy(const ParsedURI& uri); + virtual const EndpointAddress *another_stradegy(const ParsedURI& uri); + void check_breaker(); + + std::vector servers; // current servers + std::vector addresses; // memory management + std::unordered_map> server_map; + pthread_rwlock_t rwlock; + std::atomic nalives; + bool try_another; +}; + +class GroupPolicy : public GovernancePolicy +{ +public: + GroupPolicy(); + ~GroupPolicy(); + struct rb_root group_map; + EndpointGroup *default_group; + virtual void add_server(const std::string& address, const AddressParams *address_params); + virtual int remove_server(const std::string& address); + +private: + virtual void recover_one_server(const EndpointAddress *addr) + { + this->nalives++; + addr->group->nalives++; + } + + virtual void fuse_one_server(const EndpointAddress *addr) + { + this->nalives--; + addr->group->nalives--; + } + // override: select() add_server() remove_server() + virtual bool select(const ParsedURI& uri, EndpointAddress **addr); + +protected: + const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; + + inline const EndpointAddress *check_and_get(const EndpointAddress *addr) const // check_get_weak + { + if (addr && addr->fail_count >= addr->params.max_fails && addr->params.group_id >= 0) + { + const auto *ret = addr->group->get_one(); + + if (ret) + addr = ret; + } + return addr; + } + + inline bool is_alive_or_group_alive(const EndpointAddress *addr) const + { + return ((addr->params.group_id < 0 && addr->fail_count < addr->params.max_fails) || + (addr->params.group_id >= 0 && addr->group->nalives > 0)); + } +}; + +class WeightedRandomPolicy : public GroupPolicy +{ +public: + WeightedRandomPolicy(bool try_another) + { + this->total_weight = 0; + this->available_weight = 0; + this->try_another = try_another; + } + const EndpointAddress *first_stradegy(const ParsedURI& uri); + const EndpointAddress *another_stradegy(const ParsedURI& uri); + +protected: + int total_weight; + int available_weight; + +private: + virtual void recover_one_server(const EndpointAddress *addr); + virtual void fuse_one_server(const EndpointAddress *addr); +}; + +using select_t = std::function; + +class ConsistentHashPolicy : public GroupPolicy +{ +public: + ConsistentHashPolicy() + { + this->consistent_hash = this->default_consistent_hash; + } + + ConsistentHashPolicy(select_t consistent_hash) + { + this->consistent_hash = std::move(consistent_hash); + } + +protected: + const EndpointAddress *first_stradegy(const ParsedURI& uri); + +private: + select_t consistent_hash; + +public: + static unsigned int default_consistent_hash(const char *path, + const char *query, + const char *fragment) + { + static std::hash std_hash; + std::string str(path); + + str += query; + str += fragment; + return std_hash(str); + } +}; + +class ManualPolicy : public GroupPolicy +{ +public: + ManualPolicy(bool try_another, select_t select, select_t try_another_select) + { + this->try_another = try_another; + this->manual_select = select; + this->try_another_select = try_another_select; + } + + const EndpointAddress *first_stradegy(const ParsedURI& uri); + const EndpointAddress *another_stradegy(const ParsedURI& uri); + +private: + select_t manual_select; + select_t try_another_select; +}; + +#endif + diff --git a/src/manager/GroupPolicy.cc b/src/manager/GroupPolicy.cc new file mode 100644 index 00000000..550a474b --- /dev/null +++ b/src/manager/GroupPolicy.cc @@ -0,0 +1,265 @@ +#include "GovernancePolicy.h" + +#define VIRTUAL_GROUP_SIZE 16 + +GroupPolicy::GroupPolicy() +{ + this->group_map.rb_node = NULL; + this->default_group = new EndpointGroup(-1, this); + rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); + rb_insert_color(&this->default_group->rb, &this->group_map); +} + +GroupPolicy::~GroupPolicy() +{ + EndpointGroup *group; + + while (this->group_map.rb_node) + { + group = rb_entry(this->group_map.rb_node, EndpointGroup, rb); + rb_erase(this->group_map.rb_node, &this->group_map); + delete group; + } +} + +bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + this->check_breaker(); + if (this->nalives == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->first_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + if (select_addr) + select_addr = select_addr->group->get_one(); + + if (!select_addr && this->try_another) + select_addr = this->another_stradegy(uri); + } + + if (!select_addr) + this->default_group->get_one_backup(); + + pthread_rwlock_unlock(&this->rwlock); + + if (select_addr) + { + *addr = (EndpointAddress *)select_addr; + return true; + } + + return false; +} + +const EndpointAddress *EndpointGroup::get_one() +{ + if (this->nalives == 0) + return NULL; + + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); + + std::random_shuffle(this->mains.begin(), this->mains.end()); + for (size_t i = 0; i < this->mains.size(); i++) + { + if (this->mains[i]->fail_count < this->mains[i]->params.max_fails) + { + addr = this->mains[i]; + break; + } + } + + if (!addr) + { + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (size_t i = 0; i < this->backups.size(); i++) + { + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } + } + } + + pthread_mutex_unlock(&this->mutex); + return addr; +} + +const EndpointAddress *EndpointGroup::get_one_backup() +{ + if (this->nalives == 0) + return NULL; + + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); + + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (size_t i = 0; i < this->backups.size(); i++) + { + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } + } + + pthread_mutex_unlock(&this->mutex); + return addr; +} + +void GroupPolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + EndpointAddress *addr = new EndpointAddress(address, address_params); + int group_id = addr->params.group_id; + rb_node **p = &this->group_map.rb_node; + rb_node *parent = NULL; + EndpointGroup *group; + + pthread_rwlock_wrlock(&this->rwlock); + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + + while (*p) + { + parent = *p; + group = rb_entry(*p, EndpointGroup, rb); + + if (group_id < group->id) + p = &(*p)->rb_left; + else if (group_id > group->id) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + group = new EndpointGroup(group_id, this); + rb_link_node(&group->rb, parent, p); + rb_insert_color(&group->rb, &this->group_map); + } + + pthread_mutex_lock(&group->mutex); + this->recover_one_server(addr); + addr->group = group; + if (addr->params.server_type == 0) + { + group->mains.push_back(addr); + group->weight += addr->params.weight; // TODO + } + else + group->backups.push_back(addr); + pthread_mutex_unlock(&group->mutex); + + pthread_rwlock_unlock(&this->rwlock); + return; +} + +int GroupPolicy::remove_server(const std::string& address) +{ + pthread_rwlock_rdlock(&this->rwlock); + const auto map_it = this->server_map.find(address); + + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + EndpointGroup *group = addr->group; + std::vector *vec; + + if (addr->params.server_type == 0) + vec = &group->mains; + else + vec = &group->backups; + + //std::lock_guard lock(group->mutex); + pthread_mutex_lock(&group->mutex); + if (addr->fail_count < addr->params.max_fails) + this->fuse_one_server(addr); + + if (addr->params.server_type == 0) + group->weight -= addr->params.weight; // TODO + + for (auto it = vec->begin(); it != vec->end(); ++it) + { + if (*it == addr) + { + vec->erase(it); + break; + } + } + pthread_mutex_unlock(&group->mutex); + } + + this->server_map.erase(map_it); + } + + size_t n = this->servers.size(); + size_t new_n = 0; + + for (size_t i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + int ret = 0; + if (new_n < n) + { + this->servers.resize(new_n); + ret = n - new_n; + } + + pthread_rwlock_unlock(&this->rwlock); + return ret; +} + +const EndpointAddress *GroupPolicy::consistent_hash_with_group(unsigned int hash) const +{ + const EndpointAddress *addr = NULL; + unsigned int min_dis = (unsigned int)-1; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) + { + unsigned int dis = std::min + (hash - server->consistent_hash[i], + server->consistent_hash[i] - hash); + + if (dis < min_dis) + { + min_dis = dis; + addr = server; + } + } + } + } + + return this->check_and_get(addr); +} + diff --git a/src/manager/SelectPolicy.cc b/src/manager/SelectPolicy.cc new file mode 100644 index 00000000..e89bba42 --- /dev/null +++ b/src/manager/SelectPolicy.cc @@ -0,0 +1,111 @@ +#include "GovernancePolicy.h" + +const EndpointAddress *WeightedRandomPolicy::first_stradegy(const ParsedURI& uri) +{ + int x = 0; + int s = 0; + int idx, temp_weight; + + temp_weight = this->total_weight; + + if (temp_weight > 0) + x = rand() % temp_weight; + + for (idx = 0; idx < this->servers.size(); idx++) + { + s += this->servers[idx]->params.weight; + if (s > x) + break; + } + if (idx == this->servers.size()) + idx--; + + return this->servers[idx]; +} + +const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& uri) +{ + int temp_weight = this->available_weight; + if (temp_weight == 0) + return NULL; + + const EndpointAddress *addr = NULL; + int x = rand() % temp_weight; + int s = 0; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + addr = server; + s += server->params.weight; + if (s > x) + break; + } + } + return this->check_and_get(addr); +} + + +void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) +{ + this->nalives++; + if (addr->group->nalives++ == 0 && addr->group->id > 0) + this->available_weight += addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight += addr->params.weight; +} + +void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) +{ + this->nalives--; + if (--addr->group->nalives == 0 && addr->group->id > 0) + this->available_weight -= addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight -= addr->params.weight; +} + +const EndpointAddress *ConsistentHashPolicy::first_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->consistent_hash) + hash_value = this->consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = this->default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + +const EndpointAddress *ManualPolicy::first_stradegy(const ParsedURI& uri) +{ + int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + if (idx >= this->servers.size()) + idx %= this->servers.size(); + + return this->servers[idx]; +} + +const EndpointAddress *ManualPolicy::another_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->try_another_select) + hash_value = this->try_another_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = ConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + From 44122c2f1864c38177b5d60f94443a1c592b8635 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Tue, 19 Jan 2021 20:03:29 +0800 Subject: [PATCH 10/19] update GovernancePolicy GroupPolicy and SelectPolicy 3a14072 --- CMakeLists_Headers.txt | 2 +- src/factory/CMakeLists.txt | 1 - src/factory/GovernancePolicy.cc | 320 -------------------------------- src/factory/GovernancePolicy.h | 268 -------------------------- src/factory/GroupPolicy.cc | 263 -------------------------- src/factory/SelectPolicy.cc | 111 ----------- src/manager/CMakeLists.txt | 3 + 7 files changed, 4 insertions(+), 964 deletions(-) delete mode 100644 src/factory/GovernancePolicy.cc delete mode 100644 src/factory/GovernancePolicy.h delete mode 100644 src/factory/GroupPolicy.cc delete mode 100644 src/factory/SelectPolicy.cc diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 22465031..63585d21 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -84,7 +84,7 @@ set(INCLUDE_HEADERS src/factory/WFOperator.h src/factory/WFNameService.h src/factory/WFDNSResolver.h - src/factory/GovernancePolicy.h + src/manager/GovernancePolicy.h ) if(KAFKA STREQUAL "y") diff --git a/src/factory/CMakeLists.txt b/src/factory/CMakeLists.txt index 18202644..74153a68 100644 --- a/src/factory/CMakeLists.txt +++ b/src/factory/CMakeLists.txt @@ -10,7 +10,6 @@ set(SRC Workflow.cc WFNameService.cc WFDNSResolver.cc - GovernancePolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/factory/GovernancePolicy.cc b/src/factory/GovernancePolicy.cc deleted file mode 100644 index 6ad2ecb8..00000000 --- a/src/factory/GovernancePolicy.cc +++ /dev/null @@ -1,320 +0,0 @@ -#include "GovernancePolicy.h" -#include "StringUtil.h" - -#define DNS_CACHE_LEVEL_1 1 -#define DNS_CACHE_LEVEL_2 2 - -class WFSelectorFailTask : public WFRouterTask -{ -public: - WFSelectorFailTask(router_callback_t&& cb) - : WFRouterTask(std::move(cb)) - { - } - - virtual void dispatch() - { - this->state = WFT_STATE_TASK_ERROR; - this->error = WFT_ERR_UPSTREAM_UNAVAILABLE; - - return this->subtask_done(); - } -}; - -static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) -{ - char *host = NULL; - char *port = NULL; - - if (!addr->host.empty()) - { - host = strdup(addr->host.c_str()); - if (!host) - return false; - } - - if (addr->port_value > 0) - { - port = strdup(addr->port.c_str()); - if (!port) - { - free(host); - return false; - } - free(uri.port); - uri.port = port; - } - - free(uri.host); - uri.host = host; - return true; -} - -EndpointAddress::EndpointAddress(const std::string& address, - const struct AddressParams *address_params) -{ - std::vector arr = StringUtil::split(address, ':'); - this->params = *address_params; - this->address = address; - - if (this->params.weight == 0) - this->params.weight = 1; - - if (this->params.max_fails == 0) - this->params.max_fails = 1; - - if (this->params.group_id < 0) - this->params.group_id = -1; - - if (arr.size() == 0) - this->host = ""; - else - this->host = arr[0]; - - if (arr.size() <= 1) - { - this->port = ""; - this->port_value = 0; - } - else - { - this->port = arr[1]; - this->port_value = atoi(arr[1].c_str()); - } -} - -WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *params, - router_callback_t callback) -{ - EndpointAddress *addr = NULL; - WFRouterTask *task = NULL; - - if (this->select(params->uri, &addr) && copy_host_port(params->uri, addr)) - { - const auto *settings = WFGlobal::get_global_settings(); - unsigned int dns_ttl_default = settings->dns_ttl_default; - unsigned int dns_ttl_min = settings->dns_ttl_min; - const struct EndpointParams *endpoint_params = &settings->endpoint_params; - int dns_cache_level = params->retry_times == 0 ? DNS_CACHE_LEVEL_2 : - DNS_CACHE_LEVEL_1; - task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, - endpoint_params, std::move(callback)); - } - else - task = new WFSelectorFailTask(std::move(callback)); - - task->set_cookie(addr); - - return task; -} - -inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) -{ - addr->fail_count = 0; - pthread_mutex_lock(&this->breaker_lock); - if (addr->list.next) - { - list_del(&addr->list); - addr->list.next = NULL; - this->recover_one_server(addr); - //this->server_list_change(); - } - pthread_mutex_unlock(&this->breaker_lock); -} - -inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) -{ - pthread_mutex_lock(&this->breaker_lock); - if (!addr->list.next) - { - addr->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; - list_add_tail(&addr->list, &this->breaker_list); - this->fuse_one_server(addr); - //this->server_list_change(); - } - pthread_mutex_unlock(&this->breaker_lock); -} - -void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, - CommTarget *target) -{ - pthread_rwlock_rdlock(&this->rwlock); - this->recover_server_from_breaker((EndpointAddress *)cookie); - pthread_rwlock_unlock(&this->rwlock); - - WFDNSResolver::success(result, NULL, target); -} - -void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, - CommTarget *target) -{ - EndpointAddress *server = (EndpointAddress *)cookie; - - pthread_rwlock_rdlock(&this->rwlock); - int fail_count = ++server->fail_count; - if (fail_count == server->params.max_fails) - this->fuse_server_to_breaker(server); - - pthread_rwlock_unlock(&this->rwlock); - - WFDNSResolver::failed(result, NULL, target); -} - -void GovernancePolicy::check_breaker() -{ - pthread_mutex_lock(&this->breaker_lock); - if (!list_empty(&this->breaker_list)) - { - int64_t cur_time = GET_CURRENT_SECOND; - struct list_head *pos, *tmp; - EndpointAddress *addr; - - list_for_each_safe(pos, tmp, &this->breaker_list) - { - addr = list_entry(pos, EndpointAddress, list); - if (cur_time >= addr->broken_timeout) - { - if (addr->fail_count >= addr->params.max_fails) - { - addr->fail_count = addr->params.max_fails - 1; - this->recover_one_server(addr); - } - list_del(pos); - addr->list.next = NULL; - } - } - } - pthread_mutex_unlock(&this->breaker_lock); - - //this->server_list_change(); -} - -const EndpointAddress *GovernancePolicy::first_stradegy(const ParsedURI& uri) -{ - unsigned int idx = rand() % this->servers.size(); - return this->servers[idx]; -} - -const EndpointAddress *GovernancePolicy::another_stradegy(const ParsedURI& uri) -{ - return this->first_stradegy(uri); -} - -bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) -{ - pthread_rwlock_rdlock(&this->rwlock); - unsigned int n = (unsigned int)this->servers.size(); - - if (n == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - this->check_breaker(); - if (this->nalives == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - // select_addr == NULL will only happened in consistent_hash - const EndpointAddress *select_addr = this->first_stradegy(uri); - - if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) - { - if (this->try_another) - select_addr = this->another_stradegy(uri); - } - - pthread_rwlock_unlock(&this->rwlock); - - if (select_addr) - { - *addr = (EndpointAddress *)select_addr; - return true; - } - - return false; -} - -void GovernancePolicy::add_server(const std::string& address, - const AddressParams *address_params) -{ - auto *addr = new EndpointAddress(address, address_params); - - pthread_rwlock_wrlock(&this->rwlock); - this->addresses.push_back(addr); - this->server_map[addr->address].push_back(addr); - this->servers.push_back(addr); - this->recover_one_server(addr); - pthread_rwlock_unlock(&this->rwlock); -} - -int GovernancePolicy::remove_server(const std::string& address) -{ - pthread_rwlock_wrlock(&this->rwlock); - const auto map_it = this->server_map.find(address); - if (map_it != this->server_map.cend()) - { - for (EndpointAddress *addr : map_it->second) - { - if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalives - this->fuse_one_server(addr); - } - - this->server_map.erase(map_it); - } - - size_t n = this->servers.size(); - size_t new_n = 0; - - for (size_t i = 0; i < n; i++) - { - if (this->servers[i]->address != address) - { - if (new_n != i) - this->servers[new_n++] = this->servers[i]; - else - new_n++; - } - } - - int ret = 0; - if (new_n < n) - { - this->servers.resize(new_n); - ret = n - new_n; - } - - pthread_rwlock_unlock(&this->rwlock); - return ret; -} - -void GovernancePolicy::enable_server(const std::string& address) -{ - pthread_rwlock_rdlock(&this->rwlock); - const auto map_it = this->server_map.find(address); - if (map_it != this->server_map.cend()) - { - for (EndpointAddress *addr : map_it->second) - this->recover_server_from_breaker(addr); - } - pthread_rwlock_unlock(&this->rwlock); -} - -void GovernancePolicy::disable_server(const std::string& address) -{ - pthread_rwlock_rdlock(&this->rwlock); - const auto map_it = this->server_map.find(address); - if (map_it != this->server_map.cend()) - { - for (EndpointAddress *addr : map_it->second) - { - addr->fail_count = addr->params.max_fails; - this->fuse_server_to_breaker(addr); - } - } - pthread_rwlock_unlock(&this->rwlock); -} - diff --git a/src/factory/GovernancePolicy.h b/src/factory/GovernancePolicy.h deleted file mode 100644 index 7ffe12f4..00000000 --- a/src/factory/GovernancePolicy.h +++ /dev/null @@ -1,268 +0,0 @@ -#include "EndpointParams.h" -#include "WFNameService.h" -#include "WFDNSResolver.h" -#include "WFGlobal.h" -#include "WFTaskError.h" - -#include -#include - -#ifndef _GOVERNANCE_POLICY_H_ -#define _GOVERNANCE_POLICY_H_ - -#define MTTR_SECOND 30 -#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() -//#define VIRTUAL_GROUP_SIZE 16 - -struct AddressParams -{ - struct EndpointParams endpoint_params; - unsigned int dns_ttl_default; - unsigned int dns_ttl_min; - unsigned int max_fails; - unsigned short weight; - int server_type; - int group_id; -}; - -class EndpointGroup; -class GovernancePolicy; -class GroupPolicy; - -class EndpointAddress -{ -public: - EndpointGroup *group; - AddressParams params; - std::string address; - std::string host; - std::string port; - short port_value; - struct list_head list; - std::atomic fail_count; - int64_t broken_timeout; - // unsigned int consistent_hash[VIRTUAL_GROUP_SIZE]; - -public: - EndpointAddress(const std::string& address, - const struct AddressParams *address_params); -}; - -class EndpointGroup -{ -public: - int id; - GroupPolicy *policy; - struct rb_node rb; - pthread_mutex_t mutex; - std::vector mains; - std::vector backups; - std::atomic nalives; - int weight; - - EndpointGroup(int group_id, GroupPolicy *policy) - { - this->id = group_id; - this->policy = policy; - this->nalives = 0; - this->weight = 0; - pthread_mutex_init(&this->mutex, NULL); - } - - ~EndpointGroup() - { - pthread_mutex_destroy(&this->mutex); - } - -public: - const EndpointAddress *get_one(); - const EndpointAddress *get_one_backup(); -}; - -class GovernancePolicy : public WFDNSResolver -{ -public: - virtual WFRouterTask *create_router_task(const struct WFNSParams *params, - router_callback_t callback); - virtual void success(RouteManager::RouteResult *result, void *cookie, - CommTarget *target); - virtual void failed(RouteManager::RouteResult *result, void *cookie, - CommTarget *target); - - // GovernancePolicy *gp = dynamic_cast(policy); gp->add_server(...); - virtual void add_server(const std::string& address, const AddressParams *address_params); - virtual int remove_server(const std::string& address); - - virtual void enable_server(const std::string& address); - virtual void disable_server(const std::string& address); - // virtual void server_list_change(/* std::vector status */) {} -private: - -public: - GovernancePolicy() - { - this->nalives = 0; - this->try_another = false; - pthread_rwlock_init(&this->rwlock, NULL); - pthread_mutex_init(&this->breaker_lock, NULL); - INIT_LIST_HEAD(&this->breaker_list); - } - - ~GovernancePolicy() - { - pthread_mutex_destroy(&this->breaker_lock); - pthread_rwlock_destroy(&this->rwlock); - for (EndpointAddress *addr : this->addresses) - delete addr; - } - -private: - virtual bool select(const ParsedURI& uri, EndpointAddress **addr); - virtual const EndpointAddress *first_stradegy(const ParsedURI& uri); - virtual const EndpointAddress *another_stradegy(const ParsedURI& uri); - - virtual void recover_one_server(const EndpointAddress *addr) - { - this->nalives++; - } - - virtual void fuse_one_server(const EndpointAddress *addr) - { - this->nalives--; - } - - void recover_server_from_breaker(EndpointAddress *addr); - void fuse_server_to_breaker(EndpointAddress *addr); - void check_breaker(); - - std::vector addresses; // memory management - std::unordered_map> server_map; - pthread_rwlock_t rwlock; - struct list_head breaker_list; - pthread_mutex_t breaker_lock; - -protected: - std::vector servers; // current servers - std::atomic nalives; - bool try_another; -}; - -class GroupPolicy : public GovernancePolicy -{ -public: - struct rb_root group_map; - EndpointGroup *default_group; - -private: - virtual void recover_one_server(const EndpointAddress *addr) - { - this->nalives++; - addr->group->nalives++; - } - - virtual void fuse_one_server(const EndpointAddress *addr) - { - this->nalives--; - addr->group->nalives--; - } - // override: select() add_server() remove_server() - -protected: - const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; - - inline const EndpointAddress *check_and_get(const EndpointAddress *addr) // check_get_weak - { - if (addr && addr->fail_count >= addr->params.max_fails && addr->params.group_id >= 0) - { - const auto *ret = addr->group->get_one(); - - if (ret) - addr = ret; - } - return addr; - } - - inline bool is_alive_or_group_alive(const EndpointAddress *addr) - { - return ((addr->params.group_id < 0 && addr->fail_count < addr->params.max_fails) || - (addr->params.group_id >= 0 && addr->group->nalives > 0)); - } -}; - -class WeightedRandomPolicy : public GroupPolicy -{ -public: - WeightedRandomPolicy(bool try_another) - { - this->total_weight = 0; - this->available_weight = 0; - this->try_another = try_another; - } - const EndpointAddress *first_stradegy(const ParsedURI& uri); - const EndpointAddress *another_stradegy(const ParsedURI& uri); - -protected: - int total_weight; - int available_weight; - -private: - virtual void recover_one_server(const EndpointAddress *addr); - virtual void fuse_one_server(const EndpointAddress *addr); -}; - -using select_t = std::function; - -class ConsistentHashPolicy : public GroupPolicy -{ -public: - ConsistentHashPolicy() - { - this->consistent_hash = this->default_consistent_hash; - } - - ConsistentHashPolicy(select_t consistent_hash) - { - this->consistent_hash = std::move(consistent_hash); - } - -protected: - const EndpointAddress *first_stradegy(const ParsedURI& uri); - -private: - select_t consistent_hash; - -public: - static unsigned int default_consistent_hash(const char *path, - const char *query, - const char *fragment) - { - static std::hash std_hash; - std::string str(path); - - str += query; - str += fragment; - return std_hash(str); - } -}; - -class ManualPolicy : public GroupPolicy -{ -public: - ManualPolicy(bool try_another, select_t select, select_t try_another_select) - { - this->try_another = try_another; - this->manual_select = select; - this->try_another_select = try_another_select; - } - - const EndpointAddress *first_stradegy(const ParsedURI& uri); - const EndpointAddress *another_stradegy(const ParsedURI& uri); - -private: - select_t manual_select; - select_t try_another_select; -}; - -#endif - diff --git a/src/factory/GroupPolicy.cc b/src/factory/GroupPolicy.cc deleted file mode 100644 index 8c6dc0ac..00000000 --- a/src/factory/GroupPolicy.cc +++ /dev/null @@ -1,263 +0,0 @@ -#include "GovernancePolicy.h" - -#define VIRTUAL_GROUP_SIZE 16 - -GroupPolicy::GroupPolicy() -{ - this->group_map.rb_node = NULL; - this->default_group = new EndpointGroup(-1, this); - rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); - rb_insert_color(&this->default_group->rb, &this->group_map); -} - -GroupPolicy::~GroupPolicy() -{ - EndpointGroup *group; - - while (this->group_map.rb_node) - { - group = rb_entry(this->group_map.rb_node, EndpointGroup, rb); - rb_erase(this->group.rb_node, &this->group_map); - delete group; - } -} - -bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress *addr) -{ - pthread_rwlock_rdlock(&this->rwlock); - unsigned int n = (unsigned int)this->servers.size(); - - if (n == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - this->check_breaker(); - if (this->nalives == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - // select_addr == NULL will only happened in consistent_hash - const EndpointAddress *select_addr = this->first_stradegy(uri); - - if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) - { - if (select_addr) - select_addr = addr->group->get_one(); - - if (!select_addr && this->try_another) - select_addr = this->another_stradegy(); - } - - if (!select_addr) - this->default_group->get_one_backup(); - - pthread_rwlock_unlock(&this->rwlock); - - if (select_addr) - { - *addr = (EndpointAddress *)select_addr; - return true; - } - - return false; -} - -const EndpointAddress *EndpointGroup::get_one() -{ - if (this->nalives == 0) - return NULL; - - const EndpointAddress *addr = NULL; - pthread_mutex_lock(&this->mutex); - - std::random_shuffle(this->mains.begin(), this->mains.end()); - for (size_t i = 0; i < this->mains.size(); i++) - { - if (this->mains[i]->fail_count < this->mains[i]->params.max_fails) - { - addr = this->mains[i]; - break; - } - } - - if (!addr) - { - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (size_t i = 0; i < this->backups.size(); i++) - { - if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) - { - addr = this->backups[i]; - break; - } - } - } - - pthread_mutex_unlock(&this->mutex); - return addr; -} - -const EndpointAddress *EndpointGroup::get_one_backup() -{ - if (this->nalives == 0) - return NULL; - - const EndpointAddress *addr = NULL; - pthread_mutex_lock(&this->mutex); - - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (size_t i = 0; i < this->backups.size(); i++) - { - if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) - { - addr = this->backups[i]; - break; - } - } - - pthread_mutex_unlock(&this->mutex); - return addr; -} - -void GroupPolicy::add_server(const std::string& address, - const AddressParams *address_params) -{ - auto *addr = new EndpointAddress(address, address_params); - int group_id = addr->group_id; - rb_node **p = &this->group_map.rb_node; - rb_node *parent = NULL; - EndpointGroup *group; - - pthread_rwlock_wrlock(&this->rwlock); - this->addresses.push_back(addr); - this->server_map[addr->address].push_back(addr); - - while (*p) - { - parent = *p; - group = rb_entry(*p, EndpointGroup, rb); - - if (group_id < group->id) - p = &(*p)->rb_left; - else if (group_id > group->id) - p = &(*p)->rb_right; - else - break; - } - - if (*p == NULL) - { - group = new EndpointGroup(group_id, this); - rb_link_node(&group->rb, parent, p); - rb_insert_color(&group->rb, &this->group_map); - } - - group->mutex.lock(); - this->recover_one_server(); - addr->group = group; - if (addr->params.server_type == 0) - { - group->mains.push_back(addr); - group->weight += addr->params.weight; // TODO - } - else - group->backups.push_back(addr); - group->mutex.unlock(); - - pthread_rwlock_unlock(&this->rwlock); - return; -} - -int GroupPolicy::remove_server(const std::string& address) -{ - pthread_rwlock_rdlock(&this->rwlock); - const auto map_it = this->server_map.find(address); - - if (map_it != this->server_map.cend()) - { - for (EndpointAddress *addr : map_it->second) - { - EndpointGroup *group = addr->group; - std::vector *vec; - - if (addr->params->server_type == 0) - vec = &group->mains; - else - vec = &group->backups; - - std::lock_guard lock(group->mutex); - if (addr->fail_count < addr->params.max_fails) - this->fuse_one_server(addr); - - if (addr->params->server_type == 0) - group->weight -= ua->params.weight; // TODO - - for (auto it = vec->begin(); it != vec->end(); ++it) - { - if (*it == addr) - { - vec->erase(it); - break; - } - } - } - - this->server_map.erase(map_it); - } - - size_t n = this->servers.size(); - size_t new_n = 0; - - for (size_t i = 0; i < n; i++) - { - if (this->servers[i]->address != address) - { - if (new_n != i) - this->servers[new_n++] = this->servers[i]; - else - new_n++; - } - } - - int ret = 0; - if (new_n < n) - { - this->servers.resize(new_n); - ret = n - new_n; - } - - pthread_rwlock_unlock(&this->rwlock); - return ret; -} - -const EndpointAddress *GroupPolicy::consistent_hash_with_group(unsigned int hash) const -{ - const EndpointAddress *addr = NULL; - unsigned int min_dis = (unsigned int)-1; - - for (const EndpointAddress *server : this->servers) - { - if (this->is_alive_or_group_alive(server)) - { - for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) - { - unsigned int dis = std::min - (hash - main->consistent_hash[i], - main->consistent_hash[i] - hash); - - if (dis < min_dis) - { - min_dis = dis; - addr = server; - } - } - } - } - - return this->check_and_get(addr); -} - diff --git a/src/factory/SelectPolicy.cc b/src/factory/SelectPolicy.cc deleted file mode 100644 index e89bba42..00000000 --- a/src/factory/SelectPolicy.cc +++ /dev/null @@ -1,111 +0,0 @@ -#include "GovernancePolicy.h" - -const EndpointAddress *WeightedRandomPolicy::first_stradegy(const ParsedURI& uri) -{ - int x = 0; - int s = 0; - int idx, temp_weight; - - temp_weight = this->total_weight; - - if (temp_weight > 0) - x = rand() % temp_weight; - - for (idx = 0; idx < this->servers.size(); idx++) - { - s += this->servers[idx]->params.weight; - if (s > x) - break; - } - if (idx == this->servers.size()) - idx--; - - return this->servers[idx]; -} - -const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& uri) -{ - int temp_weight = this->available_weight; - if (temp_weight == 0) - return NULL; - - const EndpointAddress *addr = NULL; - int x = rand() % temp_weight; - int s = 0; - - for (const EndpointAddress *server : this->servers) - { - if (this->is_alive_or_group_alive(server)) - { - addr = server; - s += server->params.weight; - if (s > x) - break; - } - } - return this->check_and_get(addr); -} - - -void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) -{ - this->nalives++; - if (addr->group->nalives++ == 0 && addr->group->id > 0) - this->available_weight += addr->group->weight; - - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO - this->available_weight += addr->params.weight; -} - -void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) -{ - this->nalives--; - if (--addr->group->nalives == 0 && addr->group->id > 0) - this->available_weight -= addr->group->weight; - - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO - this->available_weight -= addr->params.weight; -} - -const EndpointAddress *ConsistentHashPolicy::first_stradegy(const ParsedURI& uri) -{ - unsigned int hash_value; - - if (this->consistent_hash) - hash_value = this->consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = this->default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - return this->consistent_hash_with_group(hash_value); -} - -const EndpointAddress *ManualPolicy::first_stradegy(const ParsedURI& uri) -{ - int idx = this->manual_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - if (idx >= this->servers.size()) - idx %= this->servers.size(); - - return this->servers[idx]; -} - -const EndpointAddress *ManualPolicy::another_stradegy(const ParsedURI& uri) -{ - unsigned int hash_value; - - if (this->try_another_select) - hash_value = this->try_another_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = ConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - return this->consistent_hash_with_group(hash_value); -} - diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 6a282556..2596d755 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -6,6 +6,9 @@ set(SRC UpstreamManager.cc RouteManager.cc WFGlobal.cc + GovernancePolicy.cc + GroupPolicy.cc + SelectPolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) From d64395ba08d9ef583e783d034d3a65668f37d0df Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 21 Jan 2021 15:31:35 +0800 Subject: [PATCH 11/19] rename UPSPolicy and add origin upstream api --- CMakeLists_Headers.txt | 2 +- src/manager/CMakeLists.txt | 8 +- .../{GroupPolicy.cc => UPSGroupPolicy.cc} | 20 +- .../{GovernancePolicy.cc => UPSPolicy.cc} | 67 +- .../{GovernancePolicy.h => UPSPolicy.h} | 49 +- .../{SelectPolicy.cc => UPSSelectPolicy.cc} | 18 +- src/manager/UpstreamManager.cc | 1147 ++--------------- src/manager/UpstreamManager.h | 11 + 8 files changed, 178 insertions(+), 1144 deletions(-) rename src/manager/{GroupPolicy.cc => UPSGroupPolicy.cc} (88%) rename src/manager/{GovernancePolicy.cc => UPSPolicy.cc} (80%) rename src/manager/{GovernancePolicy.h => UPSPolicy.h} (83%) rename src/manager/{SelectPolicy.cc => UPSSelectPolicy.cc} (77%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 63585d21..254260fc 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -84,7 +84,7 @@ set(INCLUDE_HEADERS src/factory/WFOperator.h src/factory/WFNameService.h src/factory/WFDNSResolver.h - src/manager/GovernancePolicy.h + src/manager/UPSPolicy.h ) if(KAFKA STREQUAL "y") diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 2596d755..c1663af4 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -3,12 +3,12 @@ project(manager) set(SRC DNSCache.cc - UpstreamManager.cc +# UpstreamManager.cc RouteManager.cc WFGlobal.cc - GovernancePolicy.cc - GroupPolicy.cc - SelectPolicy.cc + UPSPolicy.cc + UPSGroupPolicy.cc + UPSSelectPolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/manager/GroupPolicy.cc b/src/manager/UPSGroupPolicy.cc similarity index 88% rename from src/manager/GroupPolicy.cc rename to src/manager/UPSGroupPolicy.cc index 550a474b..6fa851cc 100644 --- a/src/manager/GroupPolicy.cc +++ b/src/manager/UPSGroupPolicy.cc @@ -1,8 +1,8 @@ -#include "GovernancePolicy.h" +#include "UPSPolicy.h" #define VIRTUAL_GROUP_SIZE 16 -GroupPolicy::GroupPolicy() +UPSGroupPolicy::UPSGroupPolicy() { this->group_map.rb_node = NULL; this->default_group = new EndpointGroup(-1, this); @@ -10,7 +10,7 @@ GroupPolicy::GroupPolicy() rb_insert_color(&this->default_group->rb, &this->group_map); } -GroupPolicy::~GroupPolicy() +UPSGroupPolicy::~UPSGroupPolicy() { EndpointGroup *group; @@ -22,7 +22,7 @@ GroupPolicy::~GroupPolicy() } } -bool GroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) +bool UPSGroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) { pthread_rwlock_rdlock(&this->rwlock); unsigned int n = (unsigned int)this->servers.size(); @@ -123,16 +123,13 @@ const EndpointAddress *EndpointGroup::get_one_backup() return addr; } -void GroupPolicy::add_server(const std::string& address, - const AddressParams *address_params) +void UPSGroupPolicy::__add_server(EndpointAddress *addr) { - EndpointAddress *addr = new EndpointAddress(address, address_params); int group_id = addr->params.group_id; rb_node **p = &this->group_map.rb_node; rb_node *parent = NULL; EndpointGroup *group; - pthread_rwlock_wrlock(&this->rwlock); this->addresses.push_back(addr); this->server_map[addr->address].push_back(addr); @@ -168,13 +165,11 @@ void GroupPolicy::add_server(const std::string& address, group->backups.push_back(addr); pthread_mutex_unlock(&group->mutex); - pthread_rwlock_unlock(&this->rwlock); return; } -int GroupPolicy::remove_server(const std::string& address) +int UPSGroupPolicy::__remove_server(const std::string& address) { - pthread_rwlock_rdlock(&this->rwlock); const auto map_it = this->server_map.find(address); if (map_it != this->server_map.cend()) @@ -232,11 +227,10 @@ int GroupPolicy::remove_server(const std::string& address) ret = n - new_n; } - pthread_rwlock_unlock(&this->rwlock); return ret; } -const EndpointAddress *GroupPolicy::consistent_hash_with_group(unsigned int hash) const +const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash) const { const EndpointAddress *addr = NULL; unsigned int min_dis = (unsigned int)-1; diff --git a/src/manager/GovernancePolicy.cc b/src/manager/UPSPolicy.cc similarity index 80% rename from src/manager/GovernancePolicy.cc rename to src/manager/UPSPolicy.cc index d2ac8097..913eb7ee 100644 --- a/src/manager/GovernancePolicy.cc +++ b/src/manager/UPSPolicy.cc @@ -1,4 +1,4 @@ -#include "GovernancePolicy.h" +#include "UPSPolicy.h" #include "StringUtil.h" #define DNS_CACHE_LEVEL_1 1 @@ -21,7 +21,7 @@ public: } }; -static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) +static bool copy_host_port(ParsedURI& uri, const EndpointAddress *addr) { char *host = NULL; char *port = NULL; @@ -87,7 +87,7 @@ EndpointAddress::EndpointAddress(const std::string& address, } } -WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *params, +WFRouterTask *UPSPolicy::create_router_task(const struct WFNSParams *params, router_callback_t callback) { EndpointAddress *addr = NULL; @@ -112,7 +112,7 @@ WFRouterTask *GovernancePolicy::create_router_task(const struct WFNSParams *para return task; } -inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) +inline void UPSPolicy::recover_server_from_breaker(EndpointAddress *addr) { addr->fail_count = 0; pthread_mutex_lock(&this->breaker_lock); @@ -126,7 +126,7 @@ inline void GovernancePolicy::recover_server_from_breaker(EndpointAddress *addr) pthread_mutex_unlock(&this->breaker_lock); } -inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) +inline void UPSPolicy::fuse_server_to_breaker(EndpointAddress *addr) { pthread_mutex_lock(&this->breaker_lock); if (!addr->list.next) @@ -139,7 +139,7 @@ inline void GovernancePolicy::fuse_server_to_breaker(EndpointAddress *addr) pthread_mutex_unlock(&this->breaker_lock); } -void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, +void UPSPolicy::success(RouteManager::RouteResult *result, void *cookie, CommTarget *target) { pthread_rwlock_rdlock(&this->rwlock); @@ -149,7 +149,7 @@ void GovernancePolicy::success(RouteManager::RouteResult *result, void *cookie, WFDNSResolver::success(result, NULL, target); } -void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, +void UPSPolicy::failed(RouteManager::RouteResult *result, void *cookie, CommTarget *target) { EndpointAddress *server = (EndpointAddress *)cookie; @@ -164,7 +164,7 @@ void GovernancePolicy::failed(RouteManager::RouteResult *result, void *cookie, WFDNSResolver::failed(result, NULL, target); } -void GovernancePolicy::check_breaker() +void UPSPolicy::check_breaker() { pthread_mutex_lock(&this->breaker_lock); if (!list_empty(&this->breaker_list)) @@ -193,18 +193,18 @@ void GovernancePolicy::check_breaker() //this->server_list_change(); } -const EndpointAddress *GovernancePolicy::first_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSPolicy::first_stradegy(const ParsedURI& uri) { unsigned int idx = rand() % this->servers.size(); return this->servers[idx]; } -const EndpointAddress *GovernancePolicy::another_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSPolicy::another_stradegy(const ParsedURI& uri) { return this->first_stradegy(uri); } -bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) +bool UPSPolicy::select(const ParsedURI& uri, EndpointAddress **addr) { pthread_rwlock_rdlock(&this->rwlock); unsigned int n = (unsigned int)this->servers.size(); @@ -242,22 +242,16 @@ bool GovernancePolicy::select(const ParsedURI& uri, EndpointAddress **addr) return false; } -void GovernancePolicy::add_server(const std::string& address, - const AddressParams *address_params) +void UPSPolicy::__add_server(EndpointAddress *addr) { - auto *addr = new EndpointAddress(address, address_params); - - pthread_rwlock_wrlock(&this->rwlock); this->addresses.push_back(addr); this->server_map[addr->address].push_back(addr); this->servers.push_back(addr); this->recover_one_server(addr); - pthread_rwlock_unlock(&this->rwlock); } -int GovernancePolicy::remove_server(const std::string& address) +int UPSPolicy::__remove_server(const std::string& address) { - pthread_rwlock_wrlock(&this->rwlock); const auto map_it = this->server_map.find(address); if (map_it != this->server_map.cend()) { @@ -291,11 +285,42 @@ int GovernancePolicy::remove_server(const std::string& address) ret = n - new_n; } + return ret; +} + +void UPSPolicy::add_server(const std::string& address, + const AddressParams *address_params) +{ + EndpointAddress *addr = new EndpointAddress(address, address_params); + + pthread_rwlock_wrlock(&this->rwlock); + this->__add_server(addr); + pthread_rwlock_unlock(&this->rwlock); +} + +int UPSPolicy::remove_server(const std::string& address) +{ + int ret; + pthread_rwlock_wrlock(&this->rwlock); + ret = this->__remove_server(address); pthread_rwlock_unlock(&this->rwlock); return ret; } -void GovernancePolicy::enable_server(const std::string& address) +int UPSPolicy::replace_server(const std::string& address, + const AddressParams *address_params) +{ + int ret; + EndpointAddress *addr = new EndpointAddress(address, address_params); + + pthread_rwlock_wrlock(&this->rwlock); + this->__add_server(addr); + ret = this->__remove_server(address); + pthread_rwlock_unlock(&this->rwlock); + return ret; +} + +void UPSPolicy::enable_server(const std::string& address) { pthread_rwlock_rdlock(&this->rwlock); const auto map_it = this->server_map.find(address); @@ -307,7 +332,7 @@ void GovernancePolicy::enable_server(const std::string& address) pthread_rwlock_unlock(&this->rwlock); } -void GovernancePolicy::disable_server(const std::string& address) +void UPSPolicy::disable_server(const std::string& address) { pthread_rwlock_rdlock(&this->rwlock); const auto map_it = this->server_map.find(address); diff --git a/src/manager/GovernancePolicy.h b/src/manager/UPSPolicy.h similarity index 83% rename from src/manager/GovernancePolicy.h rename to src/manager/UPSPolicy.h index bd10faf4..9da9e543 100644 --- a/src/manager/GovernancePolicy.h +++ b/src/manager/UPSPolicy.h @@ -26,8 +26,8 @@ struct AddressParams }; class EndpointGroup; -class GovernancePolicy; -class GroupPolicy; +class UPSPolicy; +class UPSGroupPolicy; class EndpointAddress { @@ -52,7 +52,7 @@ class EndpointGroup { public: int id; - GroupPolicy *policy; + UPSGroupPolicy *policy; struct rb_node rb; pthread_mutex_t mutex; std::vector mains; @@ -60,7 +60,7 @@ public: std::atomic nalives; int weight; - EndpointGroup(int group_id, GroupPolicy *policy) + EndpointGroup(int group_id, UPSGroupPolicy *policy) { this->id = group_id; this->policy = policy; @@ -79,7 +79,7 @@ public: const EndpointAddress *get_one_backup(); }; -class GovernancePolicy : public WFDNSResolver +class UPSPolicy : public WFDNSResolver { public: virtual WFRouterTask *create_router_task(const struct WFNSParams *params, @@ -89,17 +89,17 @@ public: virtual void failed(RouteManager::RouteResult *result, void *cookie, CommTarget *target); - // GovernancePolicy *gp = dynamic_cast(policy); gp->add_server(...); - virtual void add_server(const std::string& address, const AddressParams *address_params); - virtual int remove_server(const std::string& address); + // UPSPolicy *gp = dynamic_cast(policy); gp->add_server(...); + void add_server(const std::string& address, const AddressParams *address_params); + int remove_server(const std::string& address); + int replace_server(const std::string& address, const AddressParams *address_params); virtual void enable_server(const std::string& address); virtual void disable_server(const std::string& address); // virtual void server_list_change(/* std::vector status */) {} -private: public: - GovernancePolicy() + UPSPolicy() { this->nalives = 0; this->try_another = false; @@ -108,7 +108,7 @@ public: INIT_LIST_HEAD(&this->breaker_list); } - ~GovernancePolicy() + ~UPSPolicy() { pthread_mutex_destroy(&this->breaker_lock); pthread_rwlock_destroy(&this->rwlock); @@ -129,6 +129,9 @@ private: this->nalives--; } + virtual void __add_server(EndpointAddress *addr); + virtual int __remove_server(const std::string& address); + void recover_server_from_breaker(EndpointAddress *addr); void fuse_server_to_breaker(EndpointAddress *addr); @@ -149,15 +152,13 @@ protected: bool try_another; }; -class GroupPolicy : public GovernancePolicy +class UPSGroupPolicy : public UPSPolicy { public: - GroupPolicy(); - ~GroupPolicy(); + UPSGroupPolicy(); + ~UPSGroupPolicy(); struct rb_root group_map; EndpointGroup *default_group; - virtual void add_server(const std::string& address, const AddressParams *address_params); - virtual int remove_server(const std::string& address); private: virtual void recover_one_server(const EndpointAddress *addr) @@ -173,6 +174,8 @@ private: } // override: select() add_server() remove_server() virtual bool select(const ParsedURI& uri, EndpointAddress **addr); + virtual void __add_server(EndpointAddress *addr); + virtual int __remove_server(const std::string& address); protected: const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; @@ -196,10 +199,10 @@ protected: } }; -class WeightedRandomPolicy : public GroupPolicy +class UPSWeightedRandomPolicy : public UPSGroupPolicy { public: - WeightedRandomPolicy(bool try_another) + UPSWeightedRandomPolicy(bool try_another) { this->total_weight = 0; this->available_weight = 0; @@ -219,15 +222,15 @@ private: using select_t = std::function; -class ConsistentHashPolicy : public GroupPolicy +class UPSConsistentHashPolicy : public UPSGroupPolicy { public: - ConsistentHashPolicy() + UPSConsistentHashPolicy() { this->consistent_hash = this->default_consistent_hash; } - ConsistentHashPolicy(select_t consistent_hash) + UPSConsistentHashPolicy(select_t consistent_hash) { this->consistent_hash = std::move(consistent_hash); } @@ -252,10 +255,10 @@ public: } }; -class ManualPolicy : public GroupPolicy +class UPSManualPolicy : public UPSGroupPolicy { public: - ManualPolicy(bool try_another, select_t select, select_t try_another_select) + UPSManualPolicy(bool try_another, select_t select, select_t try_another_select) { this->try_another = try_another; this->manual_select = select; diff --git a/src/manager/SelectPolicy.cc b/src/manager/UPSSelectPolicy.cc similarity index 77% rename from src/manager/SelectPolicy.cc rename to src/manager/UPSSelectPolicy.cc index e89bba42..5d76976d 100644 --- a/src/manager/SelectPolicy.cc +++ b/src/manager/UPSSelectPolicy.cc @@ -1,6 +1,6 @@ -#include "GovernancePolicy.h" +#include "UPSPolicy.h" -const EndpointAddress *WeightedRandomPolicy::first_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& uri) { int x = 0; int s = 0; @@ -23,7 +23,7 @@ const EndpointAddress *WeightedRandomPolicy::first_stradegy(const ParsedURI& uri return this->servers[idx]; } -const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSWeightedRandomPolicy::another_stradegy(const ParsedURI& uri) { int temp_weight = this->available_weight; if (temp_weight == 0) @@ -47,7 +47,7 @@ const EndpointAddress *WeightedRandomPolicy::another_stradegy(const ParsedURI& u } -void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) +void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) { this->nalives++; if (addr->group->nalives++ == 0 && addr->group->id > 0) @@ -57,7 +57,7 @@ void WeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) this->available_weight += addr->params.weight; } -void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) +void UPSWeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) { this->nalives--; if (--addr->group->nalives == 0 && addr->group->id > 0) @@ -67,7 +67,7 @@ void WeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) this->available_weight -= addr->params.weight; } -const EndpointAddress *ConsistentHashPolicy::first_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSConsistentHashPolicy::first_stradegy(const ParsedURI& uri) { unsigned int hash_value; @@ -82,7 +82,7 @@ const EndpointAddress *ConsistentHashPolicy::first_stradegy(const ParsedURI& uri return this->consistent_hash_with_group(hash_value); } -const EndpointAddress *ManualPolicy::first_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSManualPolicy::first_stradegy(const ParsedURI& uri) { int idx = this->manual_select(uri.path ? uri.path : "", uri.query ? uri.query : "", @@ -94,7 +94,7 @@ const EndpointAddress *ManualPolicy::first_stradegy(const ParsedURI& uri) return this->servers[idx]; } -const EndpointAddress *ManualPolicy::another_stradegy(const ParsedURI& uri) +const EndpointAddress *UPSManualPolicy::another_stradegy(const ParsedURI& uri) { unsigned int hash_value; @@ -103,7 +103,7 @@ const EndpointAddress *ManualPolicy::another_stradegy(const ParsedURI& uri) uri.query ? uri.query : "", uri.fragment ? uri.fragment : ""); else - hash_value = ConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", + hash_value = UPSConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", uri.query ? uri.query : "", uri.fragment ? uri.fragment : ""); return this->consistent_hash_with_group(hash_value); diff --git a/src/manager/UpstreamManager.cc b/src/manager/UpstreamManager.cc index b5adad77..09cbac1e 100644 --- a/src/manager/UpstreamManager.cc +++ b/src/manager/UpstreamManager.cc @@ -16,1036 +16,36 @@ Authors: Wu Jiaxu (wujiaxu@sogou-inc.com) */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "list.h" -#include "rbtree.h" -#include "URIParser.h" -#include "StringUtil.h" -#include "EndpointParams.h" #include "UpstreamManager.h" +#include "workflow/WFNameService.h" +#include "UPSPolicy.h" -#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() -#define MTTR_SECOND 30 -#define VIRTUAL_GROUP_SIZE 16 - -namespace //anoymous namespace, for safe, avoid conflict +// existed: -1; success: 0; +UpstreamManager::~UpstreamManager() { -// RAII: YES -class ReadLock -{ -public: - ReadLock(pthread_rwlock_t& rwlock): rwlock_(&rwlock) { pthread_rwlock_rdlock(rwlock_); } - ReadLock(pthread_rwlock_t *rwlock): rwlock_(rwlock) { pthread_rwlock_rdlock(rwlock_); } - ~ReadLock() { pthread_rwlock_unlock(rwlock_); } - -private: - pthread_rwlock_t *rwlock_; -}; - -// RAII: YES -class WriteLock -{ -public: - WriteLock(pthread_rwlock_t& rwlock): rwlock_(&rwlock) { pthread_rwlock_wrlock(rwlock_); } - WriteLock(pthread_rwlock_t *rwlock): rwlock_(rwlock) { pthread_rwlock_wrlock(rwlock_); } - ~WriteLock() { pthread_rwlock_unlock(rwlock_); } - -private: - pthread_rwlock_t *rwlock_; -}; - + for (const NSPolicy *policy : this->policies) + delete policy; + pthread_mutex_destroy(&this->lock); } -class UpstreamAddress; -class UpstreamGroup; -class Upstream; - -class UpstreamAddress -{ -public: - UpstreamGroup *group; - AddressParams params; - struct list_head list; - int64_t broken_timeout; - unsigned int consistent_hash[VIRTUAL_GROUP_SIZE]; - std::string address; - std::string host; - std::string port; - std::atomic fail_count; - unsigned short port_value; - -public: - UpstreamAddress(const std::string& address, - const struct AddressParams *address_params); -}; - -class UpstreamGroup -{ -public: - Upstream *upstream; - struct rb_node rb; - std::mutex mutex; - std::vector mains; - std::vector backups; - struct list_head breaker_list; - std::atomic nbreak; - std::atomic nalive; - int weight; - int group_id; - -public: - UpstreamGroup(int group_id, Upstream *us): - upstream(us), - nbreak(0), - nalive(0), - weight(0), - group_id(group_id) - { - INIT_LIST_HEAD(&this->breaker_list); - } - - const UpstreamAddress *get_one(); - const UpstreamAddress *get_one_backup(); -}; - -class Upstream -{ -public: - Upstream(); - ~Upstream(); - - int add(UpstreamAddress *ua); - int del(const std::string& address); - void disable_server(const std::string& address); - void enable_server(const std::string& address); - const UpstreamAddress *get(const ParsedURI& uri); - int set_select_callback(upstream_route_t&& select_callback); - int set_consistent_mode(upstream_route_t&& consistent_callback); - int set_attr(bool try_another, upstream_route_t rehash_callback); - void check_one_breaker(UpstreamGroup *group, int64_t cur_time); - void check_all_breaker(); - void get_all_main(std::vector& addr_list); - - static void notify_unavailable(UpstreamAddress *ua); - static void notify_available(UpstreamAddress *ua); - -protected: - pthread_rwlock_t rwlock_; - int total_weight_; - int available_weight_; - std::vector mains_; - std::unordered_map> server_map_; - struct rb_root group_map_; - upstream_route_t select_callback_; - upstream_route_t consistent_callback_; - UpstreamGroup *default_group_; - bool try_another_; - bool is_consistent_; - -private: - void lose_one_server(UpstreamGroup *group, const UpstreamAddress *ua); - void gain_one_server(UpstreamGroup *group, const UpstreamAddress *ua); - const UpstreamAddress *weighted_random_try_another() const; - const UpstreamAddress *consistent_hash_select(unsigned int hash) const; -}; - -const UpstreamAddress *UpstreamGroup::get_one() -{ - if (this->nalive == 0) - return NULL; - - std::lock_guard lock(this->mutex); - - std::random_shuffle(this->mains.begin(), this->mains.end()); - for (const auto *main : this->mains) - { - if (main->fail_count < main->params.max_fails) - return main; - } - - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (const auto *backup : this->backups) - { - if (backup->fail_count < backup->params.max_fails) - return backup; - } - - return NULL; -} - -const UpstreamAddress *UpstreamGroup::get_one_backup() -{ - if (this->nalive == 0) - return NULL; - - std::lock_guard lock(this->mutex); - - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (const auto *backup : this->backups) - { - if (backup->fail_count < backup->params.max_fails) - return backup; - } - - return NULL; -} - -static unsigned int __default_consistent_hash(const char *path, - const char *query, - const char *fragment) -{ - static std::hash std_hash; - std::string str(path); - - str += query; - str += fragment; - return std_hash(str); -} - -UpstreamAddress::UpstreamAddress(const std::string& address, - const struct AddressParams *address_params) -{ - static std::hash std_hash; - std::vector arr = StringUtil::split(address, ':'); - - this->list.next = NULL; - this->fail_count = 0; - this->params = *address_params; - this->address = address; - for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) - this->consistent_hash[i] = std_hash(address + "|v" + std::to_string(i)); - - if (this->params.weight == 0) - this->params.weight = 1; - - if (this->params.max_fails == 0) - this->params.max_fails = 1; - - if (this->params.group_id < 0) - this->params.group_id = -1; - - if (arr.size() == 0) - this->host = ""; - else - this->host = arr[0]; - - if (arr.size() <= 1) - { - this->port = ""; - this->port_value = 0; - } - else - { - this->port = arr[1]; - this->port_value = atoi(arr[1].c_str()); - } -} - -Upstream::Upstream(): - rwlock_(PTHREAD_RWLOCK_INITIALIZER), - total_weight_(0), - available_weight_(0), - select_callback_(nullptr), - consistent_callback_(nullptr), - try_another_(false), - is_consistent_(false) -{ - group_map_.rb_node = NULL; - default_group_ = new UpstreamGroup(-1, this); - rb_link_node(&default_group_->rb, NULL, &group_map_.rb_node); - rb_insert_color(&default_group_->rb, &group_map_); -} - -Upstream::~Upstream() -{ - UpstreamGroup *group; - - while (group_map_.rb_node) - { - group = rb_entry(group_map_.rb_node, UpstreamGroup, rb); - rb_erase(group_map_.rb_node, &group_map_); - delete group; - } -} - -void Upstream::lose_one_server(UpstreamGroup *group, const UpstreamAddress *ua) -{ - if (--group->nalive == 0 && ua->params.group_id >= 0) - available_weight_ -= group->weight; - - if (ua->params.group_id < 0 && ua->params.server_type == 0) - available_weight_ -= ua->params.weight; -} - -void Upstream::gain_one_server(UpstreamGroup *group, const UpstreamAddress *ua) -{ - if (group->nalive++ == 0 && ua->params.group_id >= 0) - available_weight_ += group->weight; - - if (ua->params.group_id < 0 && ua->params.server_type == 0) - available_weight_ += ua->params.weight; -} - -int Upstream::add(UpstreamAddress *ua) -{ - int group_id = ua->params.group_id; - rb_node **p = &group_map_.rb_node; - rb_node *parent = NULL; - UpstreamGroup *group; - WriteLock lock(rwlock_); - - server_map_[ua->address].push_back(ua); - while (*p) - { - parent = *p; - group = rb_entry(*p, UpstreamGroup, rb); - - if (group_id < group->group_id) - p = &(*p)->rb_left; - else if (group_id > group->group_id) - p = &(*p)->rb_right; - else - break; - } - - if (*p == NULL) - { - group = new UpstreamGroup(group_id, this); - rb_link_node(&group->rb, parent, p); - rb_insert_color(&group->rb, &group_map_); - } - - if (ua->params.server_type == 0) - { - total_weight_ += ua->params.weight; - mains_.push_back(ua); - } - - group->mutex.lock(); - gain_one_server(group, ua); - ua->group = group; - if (ua->params.server_type == 0) - { - group->weight += ua->params.weight; - group->mains.push_back(ua); - } - else - group->backups.push_back(ua); - - group->mutex.unlock(); - - return 0; -} - -int Upstream::del(const std::string& address) -{ - WriteLock lock(rwlock_); - const auto map_it = server_map_.find(address); - - if (map_it != server_map_.cend()) - { - for (auto ua : map_it->second) - { - auto *group = ua->group; - std::vector *vec; - - if (ua->params.server_type == 0) - { - total_weight_ -= ua->params.weight; - vec = &group->mains; - } - else - vec = &group->backups; - - std::lock_guard lock(group->mutex); - - ua->group = NULL; - if (ua->fail_count < ua->params.max_fails) - lose_one_server(group, ua); - - if (ua->params.server_type == 0) - group->weight -= ua->params.weight; - - for (auto it = vec->begin(); it != vec->end(); ++it) - { - if (*it == ua) - { - vec->erase(it); - break; - } - } - } - - server_map_.erase(map_it); - } - - int n = (int)mains_.size(); - int new_n = 0; - - for (int i = 0; i < n; i++) - { - if (mains_[i]->address != address) - { - if (new_n != i) - mains_[new_n++] = mains_[i]; - else - new_n++; - } - } - - if (new_n < n) - { - mains_.resize(new_n); - return n - new_n; - } - - return 0; -} - -void Upstream::disable_server(const std::string& address) -{ - ReadLock lock(rwlock_); - const auto map_it = server_map_.find(address); - - if (map_it != server_map_.cend()) - { - for (auto ua : map_it->second) - { - auto *group = ua->group; - - if (group) - { - std::lock_guard lock(group->mutex); - - ua->fail_count = ua->params.max_fails; - if (ua->group == group && !ua->list.next) - { - ua->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; - list_add_tail(&ua->list, &group->breaker_list); - group->nbreak++; - group->upstream->lose_one_server(group, ua); - } - } - else - ua->fail_count = ua->params.max_fails; - } - } -} - -void Upstream::enable_server(const std::string& address) -{ - ReadLock lock(rwlock_); - const auto map_it = server_map_.find(address); - - if (map_it != server_map_.cend()) - { - for (auto ua : map_it->second) - UpstreamManager::notify_available(ua); - } -} - -void Upstream::get_all_main(std::vector& addr_list) -{ - ReadLock lock(rwlock_); - - for (const auto *main : mains_) - addr_list.push_back(main->address); -} - -static inline const UpstreamAddress *__check_get_strong(const UpstreamAddress *ua) -{ - if (ua->fail_count >= ua->params.max_fails) - { - if (ua->params.group_id < 0) - ua = NULL; - else - ua = ua->group->get_one(); - } - - return ua; -} - -static inline const UpstreamAddress *__check_get_weak(const UpstreamAddress *ua) -{ - if (ua && ua->fail_count >= ua->params.max_fails && ua->params.group_id >= 0) - { - const auto *ret = ua->group->get_one(); - - if (ret) - ua = ret; - } - - return ua; -} - -static inline bool __is_alive_or_group_alive(const UpstreamAddress *ua) -{ - return (ua->params.group_id >= 0 && ua->group->nalive > 0) - || (ua->params.group_id < 0 && ua->fail_count < ua->params.max_fails); -} - -const UpstreamAddress *Upstream::weighted_random_try_another() const -{ - int temp_weight = available_weight_; - - if (temp_weight == 0) - return NULL; - - const UpstreamAddress *ua = NULL; - int x = rand() % temp_weight; - int s = 0; - - for (const auto *main : mains_) - { - if (__is_alive_or_group_alive(main)) - { - ua = main; - s += main->params.weight; - if (s > x) - break; - } - } - - return __check_get_weak(ua); -} - -const UpstreamAddress *Upstream::consistent_hash_select(unsigned int hash) const -{ - const UpstreamAddress *ua = NULL; - unsigned int min_dis = (unsigned int)-1; - - for (const auto *main : mains_) - { - if (__is_alive_or_group_alive(main)) - { - for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) - { - unsigned int dis = std::min - (hash - main->consistent_hash[i], - main->consistent_hash[i] - hash); - - if (dis < min_dis) - { - min_dis = dis; - ua = main; - } - } - } - } - - return __check_get_weak(ua); -} - -void Upstream::check_one_breaker(UpstreamGroup *group, int64_t cur_time) -{ - struct list_head *pos, *tmp; - UpstreamAddress *ua; - - if (group->nbreak == 0) - return; - - std::lock_guard lock(group->mutex); - - list_for_each_safe(pos, tmp, &group->breaker_list) - { - ua = list_entry(pos, UpstreamAddress, list); - if (cur_time >= ua->broken_timeout) - { - if (ua->fail_count >= ua->params.max_fails) - { - ua->fail_count = ua->params.max_fails - 1; - if (ua->group == group) - gain_one_server(group, ua); - } - - list_del(pos); - ua->list.next = NULL; - group->nbreak--; - } - } -} - -void Upstream::check_all_breaker() -{ - if (!group_map_.rb_node) - return; - - struct rb_node *cur = rb_first(&group_map_); - UpstreamGroup *group; - int64_t cur_time = GET_CURRENT_SECOND; - - while (cur) - { - group = rb_entry(cur, UpstreamGroup, rb); - check_one_breaker(group, cur_time); - cur = rb_next(cur); - } -} - -const UpstreamAddress *Upstream::get(const ParsedURI& uri) -{ - unsigned int idx; - unsigned int hash_value; - const UpstreamAddress *ua; - - if (is_consistent_)// consistent mode - { - if (consistent_callback_) - hash_value = consistent_callback_(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = __default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - ReadLock lock(rwlock_); - check_all_breaker(); - ua = consistent_hash_select(hash_value); - } - else - { - ReadLock lock(rwlock_); - unsigned int n = (unsigned int)mains_.size(); - - if (n == 0) - return NULL; - else if (n == 1) - idx = 0; - else if (select_callback_) - { - idx = select_callback_(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - if (idx >= n) - idx %= n; - } - else - { - int x = 0; - int s = 0; - int temp_weight = total_weight_; - - if (temp_weight > 0) - x = rand() % temp_weight; - - for (idx = 0; idx < n; idx++) - { - s += mains_[idx]->params.weight; - if (s > x) - break; - } - - if (idx == n) - idx = n - 1; - } - - ua = mains_[idx]; - if (ua->fail_count >= ua->params.max_fails) - { - check_all_breaker(); - ua = __check_get_strong(ua); - - if (!ua && try_another_) - { - if (!select_callback_)// weighted random mode - ua = weighted_random_try_another(); - else// manual mode - { - if (consistent_callback_) - hash_value = consistent_callback_(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = __default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - ua = consistent_hash_select(hash_value); - } - } - } - } - - if (!ua) - ua = default_group_->get_one_backup();//get one backup from group[-1] - - return ua; -} - -int Upstream::set_select_callback(upstream_route_t&& select_callback) -{ - WriteLock lock(rwlock_); - - select_callback_ = std::move(select_callback); - return 0; -} - -int Upstream::set_consistent_mode(upstream_route_t&& consistent_callback) -{ - WriteLock lock(rwlock_); - - is_consistent_ = true; - consistent_callback_ = std::move(consistent_callback); - return 0; -} - -int Upstream::set_attr(bool try_another, upstream_route_t rehash_callback) -{ - WriteLock lock(rwlock_); - - try_another_ = try_another; - consistent_callback_ = std::move(rehash_callback); - return 0; -} - -void Upstream::notify_unavailable(UpstreamAddress *ua) -{ - auto *group = ua->group; - - if (group) - { - std::lock_guard lock(group->mutex); - - if (++ua->fail_count == ua->params.max_fails && ua->group == group && !ua->list.next) - { - ua->broken_timeout = GET_CURRENT_SECOND + MTTR_SECOND; - list_add_tail(&ua->list, &group->breaker_list); - group->nbreak++; - group->upstream->lose_one_server(group, ua); - } - } - else - ++ua->fail_count; -} - -void Upstream::notify_available(UpstreamAddress *ua) -{ - auto *group = ua->group; - - if (group) - { - std::lock_guard lock(group->mutex); - - if (ua->list.next) // in the list - { - if (ua->group == group && ua->fail_count >= ua->params.max_fails) - group->upstream->gain_one_server(group, ua); - - list_del(&ua->list); - ua->list.next = NULL; - } - - ua->fail_count = 0; - } - else - ua->fail_count = 0; -} - -class __UpstreamManager -{ -public: - static __UpstreamManager *get_instance() - { - static __UpstreamManager kInstance; - return &kInstance; - } - - int upstream_create(const std::string& name, - upstream_route_t&& consistent_hash) - { - Upstream *upstream = NULL; - { - WriteLock lock(rwlock_); - - if (upstream_map_.find(name) == upstream_map_.end()) - upstream = &upstream_map_[name]; - } - - if (upstream) - return upstream->set_consistent_mode(std::move(consistent_hash)); - - return -1; - } - - int upstream_create(const std::string& name, bool try_another) - { - Upstream *upstream = NULL; - { - WriteLock lock(rwlock_); - - if (upstream_map_.find(name) == upstream_map_.end()) - upstream = &upstream_map_[name]; - } - - if (upstream) - return upstream->set_attr(try_another, nullptr); - - return -1; - } - - int upstream_create(const std::string& name, - upstream_route_t&& select, - bool try_another, - upstream_route_t&& consistent_hash) - { - Upstream *upstream = NULL; - { - WriteLock lock(rwlock_); - - if (upstream_map_.find(name) == upstream_map_.end()) - upstream = &upstream_map_[name]; - } - - if (upstream) - { - upstream->set_select_callback(std::move(select)); - upstream->set_attr(try_another, std::move(consistent_hash)); - return 0; - } - - return 0; - } - - int upstream_add_server(const std::string& name, - const std::string& address, - const AddressParams *address_params) - { - auto *ua = new UpstreamAddress(address, address_params); - { - WriteLock lock(rwlock_); - - addresses_.push_back(ua); - } - - Upstream *upstream = NULL; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - } - - if (upstream) - return upstream->add(ua); - - return -1; - } - - int upstream_remove_server(const std::string& name, const std::string& address) - { - Upstream *upstream = NULL; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - } - - if (upstream) - return upstream->del(address); - - return -1; - } - - int upstream_replace_server(const std::string& name, - const std::string& address, - const AddressParams *address_params) - { - Upstream *upstream = NULL; - auto *ua = new UpstreamAddress(address, address_params); - WriteLock lock(rwlock_); - - addresses_.push_back(ua); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - - if (upstream) - { - upstream->del(address); - return upstream->add(ua); - } - - return -1; - } - - int upstream_disable_server(const std::string& name, const std::string& address) - { - Upstream *upstream = NULL; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - } - - if (upstream) - { - upstream->disable_server(address); - return 0; - } - - return -1; - } - - int upstream_enable_server(const std::string& name, const std::string& address) - { - Upstream *upstream = NULL; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - } - - if (upstream) - { - upstream->enable_server(address); - return 0; - } - - return -1; - } - - std::vector upstream_main_address_list(const std::string& name) - { - std::vector addr_list; - Upstream *upstream = NULL; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(name); - - if (it != upstream_map_.end()) - upstream = &it->second; - } - - if (upstream) - upstream->get_all_main(addr_list); - - return addr_list; - } - - int upstream_delete(const std::string& name) - { - WriteLock lock(rwlock_); - const auto it = upstream_map_.find(name); - - if (it == upstream_map_.end()) - return 0; - else - upstream_map_.erase(it); - - return 1; - } - - int upstream_choose(ParsedURI& uri, UpstreamManager::UpstreamResult& result) - { - result.cookie = NULL; - result.address_params = NULL; - result.state = UPSTREAM_NOTFOUND; - - if (uri.state != URI_STATE_SUCCESS || !uri.host) - return 0;// UPSTREAM_NOTFOUND - - Upstream *upstream; - { - ReadLock lock(rwlock_); - auto it = upstream_map_.find(uri.host); - - if (it == upstream_map_.end()) - return 0;// UPSTREAM_NOTFOUND - - upstream = &it->second; - } - - const auto *ua = upstream->get(uri); - - if (!ua) - { - result.state = UPSTREAM_ALL_DOWN; - return 0; - } - - char *host = NULL; - char *port = NULL; - - if (!ua->host.empty()) - { - host = strdup(ua->host.c_str()); - if (!host) - return -1; - } - - if (ua->port_value > 0) - { - port = strdup(ua->port.c_str()); - if (!port) - { - free(host); - return -1; - } - - free(uri.port); - uri.port = port; - } - - free(uri.host); - uri.host = host; - - result.state = UPSTREAM_SUCCESS; - result.address_params = &ua->params; - result.cookie = const_cast(ua); - return 0; - } - -private: - __UpstreamManager(): - rwlock_(PTHREAD_RWLOCK_INITIALIZER) - {} - - ~__UpstreamManager() - { - for (auto *ua : addresses_) - delete ua; - } - -private: - pthread_rwlock_t rwlock_; - std::unordered_map upstream_map_; - std::vector addresses_; -}; - int UpstreamManager::upstream_create_consistent_hash(const std::string& name, - upstream_route_t consitent_hash) + upstream_route_t consistent_hash) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_create(name, std::move(consitent_hash)); + UPSConsistentHashPolicy *policy = new UPSConsistentHashPolicy(std::move(consistent_hash)); + pthread_mutex_lock(&this->lock); + this->policies->push_back(policy); + pthread_mutex_unlock(&this->lock); + return WFGlobal::get_name_service()->add_policy(name, policy); } int UpstreamManager::upstream_create_weighted_random(const std::string& name, bool try_another) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_create(name, try_another); + UPSWeightedRandomPolicy *policy = new UPSWeightedRandomPolicy(try_another); + pthread_mutex_lock(&this->lock); + this->policies->push_back(policy); + pthread_mutex_unlock(&this->lock); + return WFGlobal::get_name_service()->add_policy(name, policy); } int UpstreamManager::upstream_create_manual(const std::string& name, @@ -1053,92 +53,93 @@ int UpstreamManager::upstream_create_manual(const std::string& name, bool try_another, upstream_route_t consitent_hash) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_create(name, std::move(select), - try_another, std::move(consitent_hash)); + UPSManualPolicy *policy = new UPSManualPolicy(try_another, + std::move(select), + std::move(consitent_hash)); + pthread_mutex_lock(&this->lock); + this->policies->push_back(policy); + pthread_mutex_unlock(&this->lock); + return WFGlobal::get_name_service()->add_policy(name, policy); } int UpstreamManager::upstream_add_server(const std::string& name, const std::string& address) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_add_server(name, address, &ADDRESS_PARAMS_DEFAULT); + return this->upstream_add_server(name, address, &ADDRESS_PARAMS_DEFAULT); } int UpstreamManager::upstream_add_server(const std::string& name, const std::string& address, const AddressParams *address_params) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_add_server(name, address, address_params); + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_castns->get_policy(name); + if (policy) + { + policy->add_server(address, address_params); + return 0; + } + return -1; } int UpstreamManager::upstream_remove_server(const std::string& name, const std::string& address) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_remove_server(name, address); + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_castns->get_policy(name); + if (policy) + { + policy->remove_server(address); + return 0; + } + return -1; } -int UpstreamManager::upstream_replace_server(const std::string& name, - const std::string& address, - const AddressParams *address_params) +std::vector +UpstreamManager::upstream_main_address_list(const std::string& name) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_replace_server(name, address, address_params); + std::vector address; + //TODO + return address } int UpstreamManager::upstream_disable_server(const std::string& name, const std::string& address) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_disable_server(name, address); + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_castns->get_policy(name); + if (policy) + { + policy->disable_server(address); + return 0; + } + return -1; } int UpstreamManager::upstream_enable_server(const std::string& name, const std::string& address) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_enable_server(name, address); + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_castns->get_policy(name); + if (policy) + { + policy->enable_server(address); + return 0; + } + return -1; } -std::vector UpstreamManager::upstream_main_address_list(const std::string& name) +int UpstreamManager::upstream_replace_server(const std::string& name, + const std::string& address, + const struct AddressParams *address_params) { - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_main_address_list(name); -} - -int UpstreamManager::upstream_delete(const std::string& name) -{ - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_delete(name); -} - -int UpstreamManager::choose(ParsedURI& uri, UpstreamResult& result) -{ - auto *manager = __UpstreamManager::get_instance(); - - return manager->upstream_choose(uri, result); -} - -void UpstreamManager::notify_unavailable(void *cookie) -{ - if (cookie) - Upstream::notify_unavailable((UpstreamAddress *)cookie); -} - -void UpstreamManager::notify_available(void *cookie) -{ - if (cookie) - Upstream::notify_available((UpstreamAddress *)cookie); + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_castns->get_policy(name); + if (policy) + { + policy->replace_server(address, address_params); + return 0; + } + return -1; } diff --git a/src/manager/UpstreamManager.h b/src/manager/UpstreamManager.h index ecc1c81a..63761215 100644 --- a/src/manager/UpstreamManager.h +++ b/src/manager/UpstreamManager.h @@ -23,6 +23,8 @@ #include #include "URIParser.h" #include "EndpointParams.h" +#include "WFGlobal.h" +#include "WFNameService.h" /** * @file UpstreamManager.h @@ -234,6 +236,14 @@ public: const std::string& address, const struct AddressParams *address_params); + UpstreamManager() + { + pthread_mutex_init(&this->lock, NULL); + } +private: + std::vector policies; + pthread_mutex_t lock; +/* public: /// @brief Internal use only class UpstreamResult @@ -267,6 +277,7 @@ public: static void notify_unavailable(void *cookie); /// @brief Internal use only static void notify_available(void *cookie); +*/ }; #endif From 991040b8978c3391dbc8a205abf38275b41fbc3d Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Fri, 22 Jan 2021 14:15:09 +0800 Subject: [PATCH 12/19] update instance for UpstreamManager --- src/manager/CMakeLists.txt | 2 +- src/manager/UPSGroupPolicy.cc | 4 +- src/manager/UPSPolicy.cc | 9 +++ src/manager/UPSPolicy.h | 15 ++-- src/manager/UpstreamManager.cc | 121 +++++++++++++++++++++++++-------- src/manager/UpstreamManager.h | 43 ------------ 6 files changed, 110 insertions(+), 84 deletions(-) diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index c1663af4..c00f7f8c 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -3,7 +3,7 @@ project(manager) set(SRC DNSCache.cc -# UpstreamManager.cc + UpstreamManager.cc RouteManager.cc WFGlobal.cc UPSPolicy.cc diff --git a/src/manager/UPSGroupPolicy.cc b/src/manager/UPSGroupPolicy.cc index 6fa851cc..2dcff4d1 100644 --- a/src/manager/UPSGroupPolicy.cc +++ b/src/manager/UPSGroupPolicy.cc @@ -242,8 +242,8 @@ const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int h for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) { unsigned int dis = std::min - (hash - server->consistent_hash[i], - server->consistent_hash[i] - hash); + (hash - server->consistent_hash[i], + server->consistent_hash[i] - hash); if (dis < min_dis) { diff --git a/src/manager/UPSPolicy.cc b/src/manager/UPSPolicy.cc index 913eb7ee..3352d170 100644 --- a/src/manager/UPSPolicy.cc +++ b/src/manager/UPSPolicy.cc @@ -347,3 +347,12 @@ void UPSPolicy::disable_server(const std::string& address) pthread_rwlock_unlock(&this->rwlock); } +void UPSPolicy::get_main_address(std::vector& addr_list) +{ + pthread_rwlock_rdlock(&this->rwlock); + + for (const EndpointAddress *server : this->servers) + addr_list.push_back(server->address); + + pthread_rwlock_unlock(&this->rwlock); +} diff --git a/src/manager/UPSPolicy.h b/src/manager/UPSPolicy.h index 9da9e543..82f10df2 100644 --- a/src/manager/UPSPolicy.h +++ b/src/manager/UPSPolicy.h @@ -3,6 +3,7 @@ #include "WFDNSResolver.h" #include "WFGlobal.h" #include "WFTaskError.h" +#include "UpstreamManager.h" #include #include @@ -14,17 +15,6 @@ #define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() #define VIRTUAL_GROUP_SIZE 16 -struct AddressParams -{ - struct EndpointParams endpoint_params; - unsigned int dns_ttl_default; - unsigned int dns_ttl_min; - unsigned int max_fails; - unsigned short weight; - int server_type; - int group_id; -}; - class EndpointGroup; class UPSPolicy; class UPSGroupPolicy; @@ -96,6 +86,7 @@ public: virtual void enable_server(const std::string& address); virtual void disable_server(const std::string& address); + virtual void get_main_address(std::vector& addr_list); // virtual void server_list_change(/* std::vector status */) {} public: @@ -157,6 +148,8 @@ class UPSGroupPolicy : public UPSPolicy public: UPSGroupPolicy(); ~UPSGroupPolicy(); + +protected: struct rb_root group_map; EndpointGroup *default_group; diff --git a/src/manager/UpstreamManager.cc b/src/manager/UpstreamManager.cc index 09cbac1e..73f029ce 100644 --- a/src/manager/UpstreamManager.cc +++ b/src/manager/UpstreamManager.cc @@ -17,35 +17,77 @@ */ #include "UpstreamManager.h" -#include "workflow/WFNameService.h" +#include "WFNameService.h" #include "UPSPolicy.h" -// existed: -1; success: 0; -UpstreamManager::~UpstreamManager() +class __UpstreamManager { - for (const NSPolicy *policy : this->policies) - delete policy; - pthread_mutex_destroy(&this->lock); -} +public: + static __UpstreamManager *get_instance() + { + static __UpstreamManager kInstance; + return &kInstance; + } + + void add_policy_name(const std::string& name) + { + pthread_mutex_lock(&this->lock); + this->upstream_names.push_back(name); + pthread_mutex_unlock(&this->lock); + } + +private: + __UpstreamManager() + { + pthread_mutex_init(&this->lock, NULL); + } + + ~__UpstreamManager() + { + WFNSPolicy *policy; + WFNameService *ns = WFGlobal::get_name_service(); + + for (const std::string& name : this->upstream_names) + { + policy = ns->del_policy(name.c_str()); + delete policy; + } + + pthread_mutex_destroy(&this->lock); + } + + pthread_mutex_t lock; + std::vector upstream_names; +}; int UpstreamManager::upstream_create_consistent_hash(const std::string& name, upstream_route_t consistent_hash) { + auto *ns = WFGlobal::get_name_service(); UPSConsistentHashPolicy *policy = new UPSConsistentHashPolicy(std::move(consistent_hash)); - pthread_mutex_lock(&this->lock); - this->policies->push_back(policy); - pthread_mutex_unlock(&this->lock); - return WFGlobal::get_name_service()->add_policy(name, policy); + + if (ns->add_policy(name.c_str(), policy) >= 0) + { + __UpstreamManager::get_instance()->add_policy_name(name); + return 0; + } + + return -1; } int UpstreamManager::upstream_create_weighted_random(const std::string& name, bool try_another) { + auto *ns = WFGlobal::get_name_service(); UPSWeightedRandomPolicy *policy = new UPSWeightedRandomPolicy(try_another); - pthread_mutex_lock(&this->lock); - this->policies->push_back(policy); - pthread_mutex_unlock(&this->lock); - return WFGlobal::get_name_service()->add_policy(name, policy); + + if (ns->add_policy(name.c_str(), policy) >= 0) + { + __UpstreamManager::get_instance()->add_policy_name(name); + return 0; + } + + return -1; } int UpstreamManager::upstream_create_manual(const std::string& name, @@ -53,19 +95,24 @@ int UpstreamManager::upstream_create_manual(const std::string& name, bool try_another, upstream_route_t consitent_hash) { + auto *ns = WFGlobal::get_name_service(); UPSManualPolicy *policy = new UPSManualPolicy(try_another, std::move(select), std::move(consitent_hash)); - pthread_mutex_lock(&this->lock); - this->policies->push_back(policy); - pthread_mutex_unlock(&this->lock); - return WFGlobal::get_name_service()->add_policy(name, policy); + if (ns->add_policy(name.c_str(), policy) >= 0) + { + __UpstreamManager::get_instance()->add_policy_name(name); + return 0; + } + + return -1; } int UpstreamManager::upstream_add_server(const std::string& name, const std::string& address) { - return this->upstream_add_server(name, address, &ADDRESS_PARAMS_DEFAULT); + return UpstreamManager::upstream_add_server(name, address, + &ADDRESS_PARAMS_DEFAULT); } int UpstreamManager::upstream_add_server(const std::string& name, @@ -73,12 +120,15 @@ int UpstreamManager::upstream_add_server(const std::string& name, const AddressParams *address_params) { WFNameService *ns = WFGlobal::get_name_service(); - UPSPolicy *policy = dynamic_castns->get_policy(name); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + if (policy) { policy->add_server(address, address_params); return 0; } + + errno = ENOENT; return -1; } @@ -86,12 +136,15 @@ int UpstreamManager::upstream_remove_server(const std::string& name, const std::string& address) { WFNameService *ns = WFGlobal::get_name_service(); - UPSPolicy *policy = dynamic_castns->get_policy(name); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + if (policy) { policy->remove_server(address); return 0; } + + errno = ENOENT; return -1; } @@ -99,20 +152,28 @@ std::vector UpstreamManager::upstream_main_address_list(const std::string& name) { std::vector address; - //TODO - return address + WFNameService *ns = WFGlobal::get_name_service(); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + + if (policy) + policy->get_main_address(address); + + return address; } int UpstreamManager::upstream_disable_server(const std::string& name, const std::string& address) { WFNameService *ns = WFGlobal::get_name_service(); - UPSPolicy *policy = dynamic_castns->get_policy(name); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + if (policy) { policy->disable_server(address); return 0; } + + errno = ENOENT; return -1; } @@ -120,12 +181,15 @@ int UpstreamManager::upstream_enable_server(const std::string& name, const std::string& address) { WFNameService *ns = WFGlobal::get_name_service(); - UPSPolicy *policy = dynamic_castns->get_policy(name); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + if (policy) { policy->enable_server(address); return 0; } + + errno = ENOENT; return -1; } @@ -134,12 +198,15 @@ int UpstreamManager::upstream_replace_server(const std::string& name, const struct AddressParams *address_params) { WFNameService *ns = WFGlobal::get_name_service(); - UPSPolicy *policy = dynamic_castns->get_policy(name); + UPSPolicy *policy = dynamic_cast(ns->get_policy(name.c_str())); + if (policy) { policy->replace_server(address, address_params); return 0; } + + errno = ENOENT; return -1; } diff --git a/src/manager/UpstreamManager.h b/src/manager/UpstreamManager.h index 63761215..2daa2ef2 100644 --- a/src/manager/UpstreamManager.h +++ b/src/manager/UpstreamManager.h @@ -24,7 +24,6 @@ #include "URIParser.h" #include "EndpointParams.h" #include "WFGlobal.h" -#include "WFNameService.h" /** * @file UpstreamManager.h @@ -236,48 +235,6 @@ public: const std::string& address, const struct AddressParams *address_params); - UpstreamManager() - { - pthread_mutex_init(&this->lock, NULL); - } -private: - std::vector policies; - pthread_mutex_t lock; -/* -public: - /// @brief Internal use only - class UpstreamResult - { -public: - void *cookie; - const struct AddressParams *address_params; -#define UPSTREAM_SUCCESS 0 -#define UPSTREAM_NOTFOUND 1 -#define UPSTREAM_ALL_DOWN 2 - int state; - -public: - UpstreamResult(): - cookie(NULL), - address_params(NULL), - state(UPSTREAM_NOTFOUND) - {} - - void clear() - { - cookie = NULL; - address_params = NULL; - state = UPSTREAM_NOTFOUND; - } - }; - - /// @brief Internal use only - static int choose(ParsedURI& uri, UpstreamResult& result); - /// @brief Internal use only - static void notify_unavailable(void *cookie); - /// @brief Internal use only - static void notify_available(void *cookie); -*/ }; #endif From 6c8e84122a283edfe89f5456d308e28ab1542063 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 24 Jan 2021 04:53:00 +0800 Subject: [PATCH 13/19] move codes --- src/manager/CMakeLists.txt | 2 - src/manager/UPSGroupPolicy.cc | 259 ----------------------- src/manager/UPSPolicy.cc | 366 +++++++++++++++++++++++++++++++++ src/manager/UPSPolicy.h | 1 + src/manager/UPSSelectPolicy.cc | 111 ---------- 5 files changed, 367 insertions(+), 372 deletions(-) delete mode 100644 src/manager/UPSGroupPolicy.cc delete mode 100644 src/manager/UPSSelectPolicy.cc diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index c00f7f8c..dba3b963 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -7,8 +7,6 @@ set(SRC RouteManager.cc WFGlobal.cc UPSPolicy.cc - UPSGroupPolicy.cc - UPSSelectPolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/manager/UPSGroupPolicy.cc b/src/manager/UPSGroupPolicy.cc deleted file mode 100644 index 2dcff4d1..00000000 --- a/src/manager/UPSGroupPolicy.cc +++ /dev/null @@ -1,259 +0,0 @@ -#include "UPSPolicy.h" - -#define VIRTUAL_GROUP_SIZE 16 - -UPSGroupPolicy::UPSGroupPolicy() -{ - this->group_map.rb_node = NULL; - this->default_group = new EndpointGroup(-1, this); - rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); - rb_insert_color(&this->default_group->rb, &this->group_map); -} - -UPSGroupPolicy::~UPSGroupPolicy() -{ - EndpointGroup *group; - - while (this->group_map.rb_node) - { - group = rb_entry(this->group_map.rb_node, EndpointGroup, rb); - rb_erase(this->group_map.rb_node, &this->group_map); - delete group; - } -} - -bool UPSGroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) -{ - pthread_rwlock_rdlock(&this->rwlock); - unsigned int n = (unsigned int)this->servers.size(); - - if (n == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - this->check_breaker(); - if (this->nalives == 0) - { - pthread_rwlock_unlock(&this->rwlock); - return false; - } - - // select_addr == NULL will only happened in consistent_hash - const EndpointAddress *select_addr = this->first_stradegy(uri); - - if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) - { - if (select_addr) - select_addr = select_addr->group->get_one(); - - if (!select_addr && this->try_another) - select_addr = this->another_stradegy(uri); - } - - if (!select_addr) - this->default_group->get_one_backup(); - - pthread_rwlock_unlock(&this->rwlock); - - if (select_addr) - { - *addr = (EndpointAddress *)select_addr; - return true; - } - - return false; -} - -const EndpointAddress *EndpointGroup::get_one() -{ - if (this->nalives == 0) - return NULL; - - const EndpointAddress *addr = NULL; - pthread_mutex_lock(&this->mutex); - - std::random_shuffle(this->mains.begin(), this->mains.end()); - for (size_t i = 0; i < this->mains.size(); i++) - { - if (this->mains[i]->fail_count < this->mains[i]->params.max_fails) - { - addr = this->mains[i]; - break; - } - } - - if (!addr) - { - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (size_t i = 0; i < this->backups.size(); i++) - { - if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) - { - addr = this->backups[i]; - break; - } - } - } - - pthread_mutex_unlock(&this->mutex); - return addr; -} - -const EndpointAddress *EndpointGroup::get_one_backup() -{ - if (this->nalives == 0) - return NULL; - - const EndpointAddress *addr = NULL; - pthread_mutex_lock(&this->mutex); - - std::random_shuffle(this->backups.begin(), this->backups.end()); - for (size_t i = 0; i < this->backups.size(); i++) - { - if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) - { - addr = this->backups[i]; - break; - } - } - - pthread_mutex_unlock(&this->mutex); - return addr; -} - -void UPSGroupPolicy::__add_server(EndpointAddress *addr) -{ - int group_id = addr->params.group_id; - rb_node **p = &this->group_map.rb_node; - rb_node *parent = NULL; - EndpointGroup *group; - - this->addresses.push_back(addr); - this->server_map[addr->address].push_back(addr); - - while (*p) - { - parent = *p; - group = rb_entry(*p, EndpointGroup, rb); - - if (group_id < group->id) - p = &(*p)->rb_left; - else if (group_id > group->id) - p = &(*p)->rb_right; - else - break; - } - - if (*p == NULL) - { - group = new EndpointGroup(group_id, this); - rb_link_node(&group->rb, parent, p); - rb_insert_color(&group->rb, &this->group_map); - } - - pthread_mutex_lock(&group->mutex); - this->recover_one_server(addr); - addr->group = group; - if (addr->params.server_type == 0) - { - group->mains.push_back(addr); - group->weight += addr->params.weight; // TODO - } - else - group->backups.push_back(addr); - pthread_mutex_unlock(&group->mutex); - - return; -} - -int UPSGroupPolicy::__remove_server(const std::string& address) -{ - const auto map_it = this->server_map.find(address); - - if (map_it != this->server_map.cend()) - { - for (EndpointAddress *addr : map_it->second) - { - EndpointGroup *group = addr->group; - std::vector *vec; - - if (addr->params.server_type == 0) - vec = &group->mains; - else - vec = &group->backups; - - //std::lock_guard lock(group->mutex); - pthread_mutex_lock(&group->mutex); - if (addr->fail_count < addr->params.max_fails) - this->fuse_one_server(addr); - - if (addr->params.server_type == 0) - group->weight -= addr->params.weight; // TODO - - for (auto it = vec->begin(); it != vec->end(); ++it) - { - if (*it == addr) - { - vec->erase(it); - break; - } - } - pthread_mutex_unlock(&group->mutex); - } - - this->server_map.erase(map_it); - } - - size_t n = this->servers.size(); - size_t new_n = 0; - - for (size_t i = 0; i < n; i++) - { - if (this->servers[i]->address != address) - { - if (new_n != i) - this->servers[new_n++] = this->servers[i]; - else - new_n++; - } - } - - int ret = 0; - if (new_n < n) - { - this->servers.resize(new_n); - ret = n - new_n; - } - - return ret; -} - -const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash) const -{ - const EndpointAddress *addr = NULL; - unsigned int min_dis = (unsigned int)-1; - - for (const EndpointAddress *server : this->servers) - { - if (this->is_alive_or_group_alive(server)) - { - for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) - { - unsigned int dis = std::min - (hash - server->consistent_hash[i], - server->consistent_hash[i] - hash); - - if (dis < min_dis) - { - min_dis = dis; - addr = server; - } - } - } - } - - return this->check_and_get(addr); -} - diff --git a/src/manager/UPSPolicy.cc b/src/manager/UPSPolicy.cc index 3352d170..2078c5af 100644 --- a/src/manager/UPSPolicy.cc +++ b/src/manager/UPSPolicy.cc @@ -1,5 +1,6 @@ #include "UPSPolicy.h" #include "StringUtil.h" +#include "WFDNSResolver.h" #define DNS_CACHE_LEVEL_1 1 #define DNS_CACHE_LEVEL_2 2 @@ -356,3 +357,368 @@ void UPSPolicy::get_main_address(std::vector& addr_list) pthread_rwlock_unlock(&this->rwlock); } + +UPSGroupPolicy::UPSGroupPolicy() +{ + this->group_map.rb_node = NULL; + this->default_group = new EndpointGroup(-1, this); + rb_link_node(&this->default_group->rb, NULL, &this->group_map.rb_node); + rb_insert_color(&this->default_group->rb, &this->group_map); +} + +UPSGroupPolicy::~UPSGroupPolicy() +{ + EndpointGroup *group; + + while (this->group_map.rb_node) + { + group = rb_entry(this->group_map.rb_node, EndpointGroup, rb); + rb_erase(this->group_map.rb_node, &this->group_map); + delete group; + } +} + +bool UPSGroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) +{ + pthread_rwlock_rdlock(&this->rwlock); + unsigned int n = (unsigned int)this->servers.size(); + + if (n == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + this->check_breaker(); + if (this->nalives == 0) + { + pthread_rwlock_unlock(&this->rwlock); + return false; + } + + // select_addr == NULL will only happened in consistent_hash + const EndpointAddress *select_addr = this->first_stradegy(uri); + + if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) + { + if (select_addr) + select_addr = select_addr->group->get_one(); + + if (!select_addr && this->try_another) + select_addr = this->another_stradegy(uri); + } + + if (!select_addr) + this->default_group->get_one_backup(); + + pthread_rwlock_unlock(&this->rwlock); + + if (select_addr) + { + *addr = (EndpointAddress *)select_addr; + return true; + } + + return false; +} + +const EndpointAddress *EndpointGroup::get_one() +{ + if (this->nalives == 0) + return NULL; + + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); + + std::random_shuffle(this->mains.begin(), this->mains.end()); + for (size_t i = 0; i < this->mains.size(); i++) + { + if (this->mains[i]->fail_count < this->mains[i]->params.max_fails) + { + addr = this->mains[i]; + break; + } + } + + if (!addr) + { + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (size_t i = 0; i < this->backups.size(); i++) + { + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } + } + } + + pthread_mutex_unlock(&this->mutex); + return addr; +} + +const EndpointAddress *EndpointGroup::get_one_backup() +{ + if (this->nalives == 0) + return NULL; + + const EndpointAddress *addr = NULL; + pthread_mutex_lock(&this->mutex); + + std::random_shuffle(this->backups.begin(), this->backups.end()); + for (size_t i = 0; i < this->backups.size(); i++) + { + if (this->backups[i]->fail_count < this->backups[i]->params.max_fails) + { + addr = this->backups[i]; + break; + } + } + + pthread_mutex_unlock(&this->mutex); + return addr; +} + +void UPSGroupPolicy::__add_server(EndpointAddress *addr) +{ + int group_id = addr->params.group_id; + rb_node **p = &this->group_map.rb_node; + rb_node *parent = NULL; + EndpointGroup *group; + + this->addresses.push_back(addr); + this->server_map[addr->address].push_back(addr); + + while (*p) + { + parent = *p; + group = rb_entry(*p, EndpointGroup, rb); + + if (group_id < group->id) + p = &(*p)->rb_left; + else if (group_id > group->id) + p = &(*p)->rb_right; + else + break; + } + + if (*p == NULL) + { + group = new EndpointGroup(group_id, this); + rb_link_node(&group->rb, parent, p); + rb_insert_color(&group->rb, &this->group_map); + } + + pthread_mutex_lock(&group->mutex); + this->recover_one_server(addr); + addr->group = group; + if (addr->params.server_type == 0) + { + group->mains.push_back(addr); + group->weight += addr->params.weight; // TODO + } + else + group->backups.push_back(addr); + pthread_mutex_unlock(&group->mutex); + + return; +} + +int UPSGroupPolicy::__remove_server(const std::string& address) +{ + const auto map_it = this->server_map.find(address); + + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + EndpointGroup *group = addr->group; + std::vector *vec; + + if (addr->params.server_type == 0) + vec = &group->mains; + else + vec = &group->backups; + + //std::lock_guard lock(group->mutex); + pthread_mutex_lock(&group->mutex); + if (addr->fail_count < addr->params.max_fails) + this->fuse_one_server(addr); + + if (addr->params.server_type == 0) + group->weight -= addr->params.weight; // TODO + + for (auto it = vec->begin(); it != vec->end(); ++it) + { + if (*it == addr) + { + vec->erase(it); + break; + } + } + pthread_mutex_unlock(&group->mutex); + } + + this->server_map.erase(map_it); + } + + size_t n = this->servers.size(); + size_t new_n = 0; + + for (size_t i = 0; i < n; i++) + { + if (this->servers[i]->address != address) + { + if (new_n != i) + this->servers[new_n++] = this->servers[i]; + else + new_n++; + } + } + + int ret = 0; + if (new_n < n) + { + this->servers.resize(new_n); + ret = n - new_n; + } + + return ret; +} + +const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash) const +{ + const EndpointAddress *addr = NULL; + unsigned int min_dis = (unsigned int)-1; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) + { + unsigned int dis = std::min + (hash - server->consistent_hash[i], + server->consistent_hash[i] - hash); + + if (dis < min_dis) + { + min_dis = dis; + addr = server; + } + } + } + } + + return this->check_and_get(addr); +} + +const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& uri) +{ + int x = 0; + int s = 0; + int idx, temp_weight; + + temp_weight = this->total_weight; + + if (temp_weight > 0) + x = rand() % temp_weight; + + for (idx = 0; idx < this->servers.size(); idx++) + { + s += this->servers[idx]->params.weight; + if (s > x) + break; + } + if (idx == this->servers.size()) + idx--; + + return this->servers[idx]; +} + +const EndpointAddress *UPSWeightedRandomPolicy::another_stradegy(const ParsedURI& uri) +{ + int temp_weight = this->available_weight; + if (temp_weight == 0) + return NULL; + + const EndpointAddress *addr = NULL; + int x = rand() % temp_weight; + int s = 0; + + for (const EndpointAddress *server : this->servers) + { + if (this->is_alive_or_group_alive(server)) + { + addr = server; + s += server->params.weight; + if (s > x) + break; + } + } + return this->check_and_get(addr); +} + + +void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) +{ + this->nalives++; + if (addr->group->nalives++ == 0 && addr->group->id > 0) + this->available_weight += addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight += addr->params.weight; +} + +void UPSWeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) +{ + this->nalives--; + if (--addr->group->nalives == 0 && addr->group->id > 0) + this->available_weight -= addr->group->weight; + + if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + this->available_weight -= addr->params.weight; +} + +const EndpointAddress *UPSConsistentHashPolicy::first_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->consistent_hash) + hash_value = this->consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = this->default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + +const EndpointAddress *UPSManualPolicy::first_stradegy(const ParsedURI& uri) +{ + int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + + if (idx >= this->servers.size()) + idx %= this->servers.size(); + + return this->servers[idx]; +} + +const EndpointAddress *UPSManualPolicy::another_stradegy(const ParsedURI& uri) +{ + unsigned int hash_value; + + if (this->try_another_select) + hash_value = this->try_another_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + else + hash_value = UPSConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); + return this->consistent_hash_with_group(hash_value); +} + diff --git a/src/manager/UPSPolicy.h b/src/manager/UPSPolicy.h index 82f10df2..71e8cfb5 100644 --- a/src/manager/UPSPolicy.h +++ b/src/manager/UPSPolicy.h @@ -13,6 +13,7 @@ #define MTTR_SECOND 30 #define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() + #define VIRTUAL_GROUP_SIZE 16 class EndpointGroup; diff --git a/src/manager/UPSSelectPolicy.cc b/src/manager/UPSSelectPolicy.cc deleted file mode 100644 index 5d76976d..00000000 --- a/src/manager/UPSSelectPolicy.cc +++ /dev/null @@ -1,111 +0,0 @@ -#include "UPSPolicy.h" - -const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& uri) -{ - int x = 0; - int s = 0; - int idx, temp_weight; - - temp_weight = this->total_weight; - - if (temp_weight > 0) - x = rand() % temp_weight; - - for (idx = 0; idx < this->servers.size(); idx++) - { - s += this->servers[idx]->params.weight; - if (s > x) - break; - } - if (idx == this->servers.size()) - idx--; - - return this->servers[idx]; -} - -const EndpointAddress *UPSWeightedRandomPolicy::another_stradegy(const ParsedURI& uri) -{ - int temp_weight = this->available_weight; - if (temp_weight == 0) - return NULL; - - const EndpointAddress *addr = NULL; - int x = rand() % temp_weight; - int s = 0; - - for (const EndpointAddress *server : this->servers) - { - if (this->is_alive_or_group_alive(server)) - { - addr = server; - s += server->params.weight; - if (s > x) - break; - } - } - return this->check_and_get(addr); -} - - -void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) -{ - this->nalives++; - if (addr->group->nalives++ == 0 && addr->group->id > 0) - this->available_weight += addr->group->weight; - - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO - this->available_weight += addr->params.weight; -} - -void UPSWeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) -{ - this->nalives--; - if (--addr->group->nalives == 0 && addr->group->id > 0) - this->available_weight -= addr->group->weight; - - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO - this->available_weight -= addr->params.weight; -} - -const EndpointAddress *UPSConsistentHashPolicy::first_stradegy(const ParsedURI& uri) -{ - unsigned int hash_value; - - if (this->consistent_hash) - hash_value = this->consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = this->default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - return this->consistent_hash_with_group(hash_value); -} - -const EndpointAddress *UPSManualPolicy::first_stradegy(const ParsedURI& uri) -{ - int idx = this->manual_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - - if (idx >= this->servers.size()) - idx %= this->servers.size(); - - return this->servers[idx]; -} - -const EndpointAddress *UPSManualPolicy::another_stradegy(const ParsedURI& uri) -{ - unsigned int hash_value; - - if (this->try_another_select) - hash_value = this->try_another_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - else - hash_value = UPSConsistentHashPolicy::default_consistent_hash(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); - return this->consistent_hash_with_group(hash_value); -} - From dba49133d88032cbd8ed8f287270034f8b6eb961 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 24 Jan 2021 22:22:04 +0800 Subject: [PATCH 14/19] fix warning --- src/manager/UPSPolicy.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/manager/UPSPolicy.cc b/src/manager/UPSPolicy.cc index 2078c5af..dc408178 100644 --- a/src/manager/UPSPolicy.cc +++ b/src/manager/UPSPolicy.cc @@ -1,3 +1,4 @@ +#include #include "UPSPolicy.h" #include "StringUtil.h" #include "WFDNSResolver.h" @@ -156,7 +157,7 @@ void UPSPolicy::failed(RouteManager::RouteResult *result, void *cookie, EndpointAddress *server = (EndpointAddress *)cookie; pthread_rwlock_rdlock(&this->rwlock); - int fail_count = ++server->fail_count; + size_t fail_count = ++server->fail_count; if (fail_count == server->params.max_fails) this->fuse_server_to_breaker(server); @@ -617,7 +618,8 @@ const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& { int x = 0; int s = 0; - int idx, temp_weight; + size_t idx; + int temp_weight; temp_weight = this->total_weight; @@ -697,9 +699,9 @@ const EndpointAddress *UPSConsistentHashPolicy::first_stradegy(const ParsedURI& const EndpointAddress *UPSManualPolicy::first_stradegy(const ParsedURI& uri) { - int idx = this->manual_select(uri.path ? uri.path : "", - uri.query ? uri.query : "", - uri.fragment ? uri.fragment : ""); + unsigned int idx = this->manual_select(uri.path ? uri.path : "", + uri.query ? uri.query : "", + uri.fragment ? uri.fragment : ""); if (idx >= this->servers.size()) idx %= this->servers.size(); From 5381c1236579459470fb340f53d1fc56fdcf8f8d Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Mon, 25 Jan 2021 16:53:55 +0800 Subject: [PATCH 15/19] add UPSWeightedRandomPolicy __add_server() and __remove_server() --- src/manager/UPSPolicy.cc | 34 +++++++++++++++++++++++++++++----- src/manager/UPSPolicy.h | 5 ++++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/manager/UPSPolicy.cc b/src/manager/UPSPolicy.cc index dc408178..bc8981e4 100644 --- a/src/manager/UPSPolicy.cc +++ b/src/manager/UPSPolicy.cc @@ -511,8 +511,8 @@ void UPSGroupPolicy::__add_server(EndpointAddress *addr) } pthread_mutex_lock(&group->mutex); - this->recover_one_server(addr); addr->group = group; + this->recover_one_server(addr); if (addr->params.server_type == 0) { group->mains.push_back(addr); @@ -614,14 +614,39 @@ const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int h return this->check_and_get(addr); } +void UPSWeightedRandomPolicy::__add_server(EndpointAddress *addr) +{ + UPSGroupPolicy::__add_server(addr); + if (addr->params.server_type == 0) + { + this->total_weight += addr->params.weight; + this->servers.push_back(addr); + } + return; +} + +int UPSWeightedRandomPolicy::__remove_server(const std::string& address) +{ + const auto map_it = this->server_map.find(address); + + if (map_it != this->server_map.cend()) + { + for (EndpointAddress *addr : map_it->second) + { + if (addr->params.server_type == 0) + this->total_weight -= addr->params.weight; + } + } + + return UPSGroupPolicy::__remove_server(address); +} + const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& uri) { int x = 0; int s = 0; size_t idx; - int temp_weight; - - temp_weight = this->total_weight; + int temp_weight = this->total_weight; if (temp_weight > 0) x = rand() % temp_weight; @@ -661,7 +686,6 @@ const EndpointAddress *UPSWeightedRandomPolicy::another_stradegy(const ParsedURI return this->check_and_get(addr); } - void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) { this->nalives++; diff --git a/src/manager/UPSPolicy.h b/src/manager/UPSPolicy.h index 71e8cfb5..58534990 100644 --- a/src/manager/UPSPolicy.h +++ b/src/manager/UPSPolicy.h @@ -168,10 +168,11 @@ private: } // override: select() add_server() remove_server() virtual bool select(const ParsedURI& uri, EndpointAddress **addr); + +protected: virtual void __add_server(EndpointAddress *addr); virtual int __remove_server(const std::string& address); -protected: const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; inline const EndpointAddress *check_and_get(const EndpointAddress *addr) const // check_get_weak @@ -212,6 +213,8 @@ protected: private: virtual void recover_one_server(const EndpointAddress *addr); virtual void fuse_one_server(const EndpointAddress *addr); + virtual void __add_server(EndpointAddress *addr); + virtual int __remove_server(const std::string& address); }; using select_t = std::function; From 0e29ab87f14346c3ce4676299e4d20a067e08610 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 27 Jan 2021 18:43:13 +0800 Subject: [PATCH 16/19] move UpstreamPolicy and fix some implementation --- CMakeLists_Headers.txt | 2 +- src/manager/CMakeLists.txt | 2 +- src/manager/UpstreamManager.cc | 2 +- src/manager/{UPSPolicy.cc => UpstreamPolicy.cc} | 16 ++++++++-------- src/manager/{UPSPolicy.h => UpstreamPolicy.h} | 0 5 files changed, 11 insertions(+), 11 deletions(-) rename src/manager/{UPSPolicy.cc => UpstreamPolicy.cc} (99%) rename src/manager/{UPSPolicy.h => UpstreamPolicy.h} (100%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index 254260fc..f245b52a 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -84,7 +84,7 @@ set(INCLUDE_HEADERS src/factory/WFOperator.h src/factory/WFNameService.h src/factory/WFDNSResolver.h - src/manager/UPSPolicy.h + src/manager/UpstreamPolicy.h ) if(KAFKA STREQUAL "y") diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index dba3b963..43dc7ea1 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -6,7 +6,7 @@ set(SRC UpstreamManager.cc RouteManager.cc WFGlobal.cc - UPSPolicy.cc + UpstreamPolicy.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/manager/UpstreamManager.cc b/src/manager/UpstreamManager.cc index 73f029ce..bad1cf93 100644 --- a/src/manager/UpstreamManager.cc +++ b/src/manager/UpstreamManager.cc @@ -18,7 +18,7 @@ #include "UpstreamManager.h" #include "WFNameService.h" -#include "UPSPolicy.h" +#include "UpstreamPolicy.h" class __UpstreamManager { diff --git a/src/manager/UPSPolicy.cc b/src/manager/UpstreamPolicy.cc similarity index 99% rename from src/manager/UPSPolicy.cc rename to src/manager/UpstreamPolicy.cc index bc8981e4..dc075520 100644 --- a/src/manager/UPSPolicy.cc +++ b/src/manager/UpstreamPolicy.cc @@ -1,5 +1,5 @@ #include -#include "UPSPolicy.h" +#include "UpstreamPolicy.h" #include "StringUtil.h" #include "WFDNSResolver.h" @@ -490,6 +490,9 @@ void UPSGroupPolicy::__add_server(EndpointAddress *addr) this->addresses.push_back(addr); this->server_map[addr->address].push_back(addr); + if (addr->params.server_type == 0) + this->servers.push_back(addr); + while (*p) { parent = *p; @@ -516,7 +519,7 @@ void UPSGroupPolicy::__add_server(EndpointAddress *addr) if (addr->params.server_type == 0) { group->mains.push_back(addr); - group->weight += addr->params.weight; // TODO + group->weight += addr->params.weight; } else group->backups.push_back(addr); @@ -547,7 +550,7 @@ int UPSGroupPolicy::__remove_server(const std::string& address) this->fuse_one_server(addr); if (addr->params.server_type == 0) - group->weight -= addr->params.weight; // TODO + group->weight -= addr->params.weight; for (auto it = vec->begin(); it != vec->end(); ++it) { @@ -618,10 +621,7 @@ void UPSWeightedRandomPolicy::__add_server(EndpointAddress *addr) { UPSGroupPolicy::__add_server(addr); if (addr->params.server_type == 0) - { this->total_weight += addr->params.weight; - this->servers.push_back(addr); - } return; } @@ -692,7 +692,7 @@ void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) if (addr->group->nalives++ == 0 && addr->group->id > 0) this->available_weight += addr->group->weight; - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + if (addr->params.group_id < 0 && addr->params.server_type == 0) this->available_weight += addr->params.weight; } @@ -702,7 +702,7 @@ void UPSWeightedRandomPolicy::fuse_one_server(const EndpointAddress *addr) if (--addr->group->nalives == 0 && addr->group->id > 0) this->available_weight -= addr->group->weight; - if (addr->params.group_id < 0 && addr->params.server_type == 0) // TODO + if (addr->params.group_id < 0 && addr->params.server_type == 0) this->available_weight -= addr->params.weight; } diff --git a/src/manager/UPSPolicy.h b/src/manager/UpstreamPolicy.h similarity index 100% rename from src/manager/UPSPolicy.h rename to src/manager/UpstreamPolicy.h From 4ce5afd7c27dce74ffd989d0419fdafada99b1aa Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Wed, 27 Jan 2021 20:04:13 +0800 Subject: [PATCH 17/19] update add_server_locked() and remove_server_locked() --- CMakeLists_Headers.txt | 2 +- src/manager/CMakeLists.txt | 2 +- src/manager/UpstreamManager.cc | 2 +- ...{UpstreamPolicy.cc => UpstreamPolicies.cc} | 29 ++++++++++--------- .../{UpstreamPolicy.h => UpstreamPolicies.h} | 12 ++++---- 5 files changed, 24 insertions(+), 23 deletions(-) rename src/manager/{UpstreamPolicy.cc => UpstreamPolicies.cc} (95%) rename src/manager/{UpstreamPolicy.h => UpstreamPolicies.h} (94%) diff --git a/CMakeLists_Headers.txt b/CMakeLists_Headers.txt index f245b52a..83dd7457 100644 --- a/CMakeLists_Headers.txt +++ b/CMakeLists_Headers.txt @@ -84,7 +84,7 @@ set(INCLUDE_HEADERS src/factory/WFOperator.h src/factory/WFNameService.h src/factory/WFDNSResolver.h - src/manager/UpstreamPolicy.h + src/manager/UpstreamPolicies.h ) if(KAFKA STREQUAL "y") diff --git a/src/manager/CMakeLists.txt b/src/manager/CMakeLists.txt index 43dc7ea1..2f22b886 100644 --- a/src/manager/CMakeLists.txt +++ b/src/manager/CMakeLists.txt @@ -6,7 +6,7 @@ set(SRC UpstreamManager.cc RouteManager.cc WFGlobal.cc - UpstreamPolicy.cc + UpstreamPolicies.cc ) add_library(${PROJECT_NAME} OBJECT ${SRC}) diff --git a/src/manager/UpstreamManager.cc b/src/manager/UpstreamManager.cc index bad1cf93..e9f55d17 100644 --- a/src/manager/UpstreamManager.cc +++ b/src/manager/UpstreamManager.cc @@ -18,7 +18,7 @@ #include "UpstreamManager.h" #include "WFNameService.h" -#include "UpstreamPolicy.h" +#include "UpstreamPolicies.h" class __UpstreamManager { diff --git a/src/manager/UpstreamPolicy.cc b/src/manager/UpstreamPolicies.cc similarity index 95% rename from src/manager/UpstreamPolicy.cc rename to src/manager/UpstreamPolicies.cc index dc075520..bf4e2461 100644 --- a/src/manager/UpstreamPolicy.cc +++ b/src/manager/UpstreamPolicies.cc @@ -1,6 +1,6 @@ #include -#include "UpstreamPolicy.h" #include "StringUtil.h" +#include "UpstreamPolicies.h" #include "WFDNSResolver.h" #define DNS_CACHE_LEVEL_1 1 @@ -244,7 +244,7 @@ bool UPSPolicy::select(const ParsedURI& uri, EndpointAddress **addr) return false; } -void UPSPolicy::__add_server(EndpointAddress *addr) +void UPSPolicy::add_server_locked(EndpointAddress *addr) { this->addresses.push_back(addr); this->server_map[addr->address].push_back(addr); @@ -252,14 +252,15 @@ void UPSPolicy::__add_server(EndpointAddress *addr) this->recover_one_server(addr); } -int UPSPolicy::__remove_server(const std::string& address) +int UPSPolicy::remove_server_locked(const std::string& address) { const auto map_it = this->server_map.find(address); if (map_it != this->server_map.cend()) { for (EndpointAddress *addr : map_it->second) { - if (addr->fail_count < addr->params.max_fails) // or not: it has already been -- in nalives + // or not: it has already been -- in nalives + if (addr->fail_count < addr->params.max_fails) this->fuse_one_server(addr); } @@ -296,7 +297,7 @@ void UPSPolicy::add_server(const std::string& address, EndpointAddress *addr = new EndpointAddress(address, address_params); pthread_rwlock_wrlock(&this->rwlock); - this->__add_server(addr); + this->add_server_locked(addr); pthread_rwlock_unlock(&this->rwlock); } @@ -304,7 +305,7 @@ int UPSPolicy::remove_server(const std::string& address) { int ret; pthread_rwlock_wrlock(&this->rwlock); - ret = this->__remove_server(address); + ret = this->remove_server_locked(address); pthread_rwlock_unlock(&this->rwlock); return ret; } @@ -316,8 +317,8 @@ int UPSPolicy::replace_server(const std::string& address, EndpointAddress *addr = new EndpointAddress(address, address_params); pthread_rwlock_wrlock(&this->rwlock); - this->__add_server(addr); - ret = this->__remove_server(address); + this->add_server_locked(addr); + ret = this->remove_server_locked(address); pthread_rwlock_unlock(&this->rwlock); return ret; } @@ -480,7 +481,7 @@ const EndpointAddress *EndpointGroup::get_one_backup() return addr; } -void UPSGroupPolicy::__add_server(EndpointAddress *addr) +void UPSGroupPolicy::add_server_locked(EndpointAddress *addr) { int group_id = addr->params.group_id; rb_node **p = &this->group_map.rb_node; @@ -528,7 +529,7 @@ void UPSGroupPolicy::__add_server(EndpointAddress *addr) return; } -int UPSGroupPolicy::__remove_server(const std::string& address) +int UPSGroupPolicy::remove_server_locked(const std::string& address) { const auto map_it = this->server_map.find(address); @@ -617,15 +618,15 @@ const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int h return this->check_and_get(addr); } -void UPSWeightedRandomPolicy::__add_server(EndpointAddress *addr) +void UPSWeightedRandomPolicy::add_server_locked(EndpointAddress *addr) { - UPSGroupPolicy::__add_server(addr); + UPSGroupPolicy::add_server_locked(addr); if (addr->params.server_type == 0) this->total_weight += addr->params.weight; return; } -int UPSWeightedRandomPolicy::__remove_server(const std::string& address) +int UPSWeightedRandomPolicy::remove_server_locked(const std::string& address) { const auto map_it = this->server_map.find(address); @@ -638,7 +639,7 @@ int UPSWeightedRandomPolicy::__remove_server(const std::string& address) } } - return UPSGroupPolicy::__remove_server(address); + return UPSGroupPolicy::remove_server_locked(address); } const EndpointAddress *UPSWeightedRandomPolicy::first_stradegy(const ParsedURI& uri) diff --git a/src/manager/UpstreamPolicy.h b/src/manager/UpstreamPolicies.h similarity index 94% rename from src/manager/UpstreamPolicy.h rename to src/manager/UpstreamPolicies.h index 58534990..f54fd811 100644 --- a/src/manager/UpstreamPolicy.h +++ b/src/manager/UpstreamPolicies.h @@ -121,8 +121,8 @@ private: this->nalives--; } - virtual void __add_server(EndpointAddress *addr); - virtual int __remove_server(const std::string& address); + virtual void add_server_locked(EndpointAddress *addr); + virtual int remove_server_locked(const std::string& address); void recover_server_from_breaker(EndpointAddress *addr); void fuse_server_to_breaker(EndpointAddress *addr); @@ -170,8 +170,8 @@ private: virtual bool select(const ParsedURI& uri, EndpointAddress **addr); protected: - virtual void __add_server(EndpointAddress *addr); - virtual int __remove_server(const std::string& address); + virtual void add_server_locked(EndpointAddress *addr); + virtual int remove_server_locked(const std::string& address); const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; @@ -213,8 +213,8 @@ protected: private: virtual void recover_one_server(const EndpointAddress *addr); virtual void fuse_one_server(const EndpointAddress *addr); - virtual void __add_server(EndpointAddress *addr); - virtual int __remove_server(const std::string& address); + virtual void add_server_locked(EndpointAddress *addr); + virtual int remove_server_locked(const std::string& address); }; using select_t = std::function; From 1879c58da3547ef71694df38424ae9c504ef263e Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Thu, 28 Jan 2021 17:42:44 +0800 Subject: [PATCH 18/19] update codes according to review --- src/manager/UpstreamPolicies.cc | 27 +++++++++--- src/manager/UpstreamPolicies.h | 75 +++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 36 deletions(-) diff --git a/src/manager/UpstreamPolicies.cc b/src/manager/UpstreamPolicies.cc index bf4e2461..e1810eb5 100644 --- a/src/manager/UpstreamPolicies.cc +++ b/src/manager/UpstreamPolicies.cc @@ -1,3 +1,21 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Authors: Wu Jiaxu (wujiaxu@sogou-inc.com) +*/ + #include #include "StringUtil.h" #include "UpstreamPolicies.h" @@ -90,10 +108,10 @@ EndpointAddress::EndpointAddress(const std::string& address, } WFRouterTask *UPSPolicy::create_router_task(const struct WFNSParams *params, - router_callback_t callback) + router_callback_t callback) { - EndpointAddress *addr = NULL; - WFRouterTask *task = NULL; + EndpointAddress *addr; + WFRouterTask *task; if (this->select(params->uri, &addr) && copy_host_port(params->uri, addr)) { @@ -105,12 +123,11 @@ WFRouterTask *UPSPolicy::create_router_task(const struct WFNSParams *params, DNS_CACHE_LEVEL_1; task = this->create(params, dns_cache_level, dns_ttl_default, dns_ttl_min, endpoint_params, std::move(callback)); + task->set_cookie(addr); } else task = new WFSelectorFailTask(std::move(callback)); - task->set_cookie(addr); - return task; } diff --git a/src/manager/UpstreamPolicies.h b/src/manager/UpstreamPolicies.h index f54fd811..9ce5cb95 100644 --- a/src/manager/UpstreamPolicies.h +++ b/src/manager/UpstreamPolicies.h @@ -1,3 +1,24 @@ +/* + Copyright (c) 2021 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Authors: Wu Jiaxu (wujiaxu@sogou-inc.com) +*/ + +#ifndef _UPSTREAM_POLICIES_H_ +#define _UPSTREAM_POLICIES_H_ + #include "EndpointParams.h" #include "WFNameService.h" #include "WFDNSResolver.h" @@ -8,14 +29,11 @@ #include #include -#ifndef _GOVERNANCE_POLICY_H_ -#define _GOVERNANCE_POLICY_H_ - #define MTTR_SECOND 30 -#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() - #define VIRTUAL_GROUP_SIZE 16 +#define GET_CURRENT_SECOND std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() + class EndpointGroup; class UPSPolicy; class UPSGroupPolicy; @@ -51,18 +69,13 @@ public: std::atomic nalives; int weight; - EndpointGroup(int group_id, UPSGroupPolicy *policy) + EndpointGroup(int group_id, UPSGroupPolicy *policy) : + mutex(PTHREAD_MUTEX_INITIALIZER) { this->id = group_id; this->policy = policy; this->nalives = 0; this->weight = 0; - pthread_mutex_init(&this->mutex, NULL); - } - - ~EndpointGroup() - { - pthread_mutex_destroy(&this->mutex); } public: @@ -80,7 +93,6 @@ public: virtual void failed(RouteManager::RouteResult *result, void *cookie, CommTarget *target); - // UPSPolicy *gp = dynamic_cast(policy); gp->add_server(...); void add_server(const std::string& address, const AddressParams *address_params); int remove_server(const std::string& address); int replace_server(const std::string& address, const AddressParams *address_params); @@ -91,19 +103,17 @@ public: // virtual void server_list_change(/* std::vector status */) {} public: - UPSPolicy() + UPSPolicy() : + breaker_lock(PTHREAD_MUTEX_INITIALIZER), + rwlock(PTHREAD_RWLOCK_INITIALIZER) { this->nalives = 0; this->try_another = false; - pthread_rwlock_init(&this->rwlock, NULL); - pthread_mutex_init(&this->breaker_lock, NULL); INIT_LIST_HEAD(&this->breaker_list); } - ~UPSPolicy() + virtual ~UPSPolicy() { - pthread_mutex_destroy(&this->breaker_lock); - pthread_rwlock_destroy(&this->rwlock); for (EndpointAddress *addr : this->addresses) delete addr; } @@ -166,7 +176,7 @@ private: this->nalives--; addr->group->nalives--; } - // override: select() add_server() remove_server() + virtual bool select(const ParsedURI& uri, EndpointAddress **addr); protected: @@ -175,9 +185,11 @@ protected: const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; - inline const EndpointAddress *check_and_get(const EndpointAddress *addr) const // check_get_weak + // check_get_weak + inline const EndpointAddress *check_and_get(const EndpointAddress *addr) const { - if (addr && addr->fail_count >= addr->params.max_fails && addr->params.group_id >= 0) + if (addr && addr->fail_count >= addr->params.max_fails && + addr->params.group_id >= 0) { const auto *ret = addr->group->get_one(); @@ -189,8 +201,10 @@ protected: inline bool is_alive_or_group_alive(const EndpointAddress *addr) const { - return ((addr->params.group_id < 0 && addr->fail_count < addr->params.max_fails) || - (addr->params.group_id >= 0 && addr->group->nalives > 0)); + return ((addr->params.group_id < 0 && + addr->fail_count < addr->params.max_fails) || + (addr->params.group_id >= 0 && + addr->group->nalives > 0)); } }; @@ -217,8 +231,6 @@ private: virtual int remove_server_locked(const std::string& address); }; -using select_t = std::function; - class UPSConsistentHashPolicy : public UPSGroupPolicy { public: @@ -227,7 +239,7 @@ public: this->consistent_hash = this->default_consistent_hash; } - UPSConsistentHashPolicy(select_t consistent_hash) + UPSConsistentHashPolicy(upstream_route_t consistent_hash) { this->consistent_hash = std::move(consistent_hash); } @@ -236,7 +248,7 @@ protected: const EndpointAddress *first_stradegy(const ParsedURI& uri); private: - select_t consistent_hash; + upstream_route_t consistent_hash; public: static unsigned int default_consistent_hash(const char *path, @@ -255,7 +267,8 @@ public: class UPSManualPolicy : public UPSGroupPolicy { public: - UPSManualPolicy(bool try_another, select_t select, select_t try_another_select) + UPSManualPolicy(bool try_another, upstream_route_t select, + upstream_route_t try_another_select) { this->try_another = try_another; this->manual_select = select; @@ -266,8 +279,8 @@ public: const EndpointAddress *another_stradegy(const ParsedURI& uri); private: - select_t manual_select; - select_t try_another_select; + upstream_route_t manual_select; + upstream_route_t try_another_select; }; #endif From 1798a7913f2315697219f58b6cd281311b65cd16 Mon Sep 17 00:00:00 2001 From: holmes1412 Date: Sun, 31 Jan 2021 03:47:03 +0800 Subject: [PATCH 19/19] add upstream_unittest.cc and fix check_get_strong() --- src/manager/UpstreamPolicies.cc | 38 ++++- src/manager/UpstreamPolicies.h | 17 +-- test/CMakeLists.txt | 1 + test/upstream_unittest.cc | 237 ++++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 19 deletions(-) create mode 100644 test/upstream_unittest.cc diff --git a/src/manager/UpstreamPolicies.cc b/src/manager/UpstreamPolicies.cc index e1810eb5..af309b28 100644 --- a/src/manager/UpstreamPolicies.cc +++ b/src/manager/UpstreamPolicies.cc @@ -76,6 +76,8 @@ EndpointAddress::EndpointAddress(const std::string& address, std::vector arr = StringUtil::split(address, ':'); this->params = *address_params; this->address = address; + this->list.next = NULL; + this->fail_count = 0; static std::hash std_hash; for (int i = 0; i < VIRTUAL_GROUP_SIZE; i++) @@ -421,10 +423,13 @@ bool UPSGroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) if (!select_addr || select_addr->fail_count >= select_addr->params.max_fails) { if (select_addr) - select_addr = select_addr->group->get_one(); + select_addr = this->check_and_get(select_addr, true); if (!select_addr && this->try_another) + { select_addr = this->another_stradegy(uri); + select_addr = this->check_and_get(select_addr, false); + } } if (!select_addr) @@ -441,6 +446,30 @@ bool UPSGroupPolicy::select(const ParsedURI& uri, EndpointAddress **addr) return false; } +// flag true : guarantee addr != NULL, and please return an available one +// flag false : means addr maybe useful but want one any way. addr may be NULL +inline const EndpointAddress *UPSGroupPolicy::check_and_get(const EndpointAddress *addr, + bool flag) +{ + if (flag == true) // && addr->fail_count >= addr->params.max_fails + { + if (addr->params.group_id == -1) + return NULL; + + return addr->group->get_one(); + } + + if (addr && addr->fail_count >= addr->params.max_fails && + addr->params.group_id >= 0) + { + const EndpointAddress *tmp = addr->group->get_one(); + if (tmp) + addr = tmp; + } + + return addr; +} + const EndpointAddress *EndpointGroup::get_one() { if (this->nalives == 0) @@ -608,7 +637,7 @@ int UPSGroupPolicy::remove_server_locked(const std::string& address) return ret; } -const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash) const +const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int hash) { const EndpointAddress *addr = NULL; unsigned int min_dis = (unsigned int)-1; @@ -632,7 +661,7 @@ const EndpointAddress *UPSGroupPolicy::consistent_hash_with_group(unsigned int h } } - return this->check_and_get(addr); + return this->check_and_get(addr, false); } void UPSWeightedRandomPolicy::add_server_locked(EndpointAddress *addr) @@ -701,7 +730,8 @@ const EndpointAddress *UPSWeightedRandomPolicy::another_stradegy(const ParsedURI break; } } - return this->check_and_get(addr); + + return this->check_and_get(addr, false); } void UPSWeightedRandomPolicy::recover_one_server(const EndpointAddress *addr) diff --git a/src/manager/UpstreamPolicies.h b/src/manager/UpstreamPolicies.h index 9ce5cb95..c228d2ea 100644 --- a/src/manager/UpstreamPolicies.h +++ b/src/manager/UpstreamPolicies.h @@ -183,21 +183,8 @@ protected: virtual void add_server_locked(EndpointAddress *addr); virtual int remove_server_locked(const std::string& address); - const EndpointAddress *consistent_hash_with_group(unsigned int hash) const; - - // check_get_weak - inline const EndpointAddress *check_and_get(const EndpointAddress *addr) const - { - if (addr && addr->fail_count >= addr->params.max_fails && - addr->params.group_id >= 0) - { - const auto *ret = addr->group->get_one(); - - if (ret) - addr = ret; - } - return addr; - } + const EndpointAddress *consistent_hash_with_group(unsigned int hash); + const EndpointAddress *check_and_get(const EndpointAddress *addr, bool flag); inline bool is_alive_or_group_alive(const EndpointAddress *addr) const { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 17e0a9c8..68c3bd57 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,6 +37,7 @@ set(TEST_LIST facilities_unittest graph_unittest memory_unittest + upstream_unittest ) if (APPLE) diff --git a/test/upstream_unittest.cc b/test/upstream_unittest.cc new file mode 100644 index 00000000..b57fad25 --- /dev/null +++ b/test/upstream_unittest.cc @@ -0,0 +1,237 @@ +/* + Copyright (c) 2020 Sogou, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Author: Li Yingxin (liyingxin@sogou-inc.com) +*/ + +#include +#include "workflow/UpstreamManager.h" +#include "workflow/WFHttpServer.h" +#include "workflow/WFTaskFactory.h" +#include "workflow/WFFacilities.h" + +#define REDIRECT_MAX 3 +#define RETRY_MAX 3 +#define MTTR 30 +#define MAX_FAILS 200 + +static void __http_process1(WFHttpTask *task) +{ + auto *resp = task->get_resp(); + resp->add_header_pair("Content-Type", "text/plain"); + resp->append_output_body_nocopy("server1", 7); +} + +static void __http_process2(WFHttpTask *task) +{ + auto *resp = task->get_resp(); + resp->add_header_pair("Content-Type", "text/plain"); + resp->append_output_body_nocopy("server2", 7); +} + +WFHttpServer http_server1(__http_process1); +WFHttpServer http_server2(__http_process2); + +void register_upstream_hosts() +{ + UpstreamManager::upstream_create_weighted_random("weighted.random", false); + AddressParams address_params = ADDRESS_PARAMS_DEFAULT; + address_params.weight = 1000; + UpstreamManager::upstream_add_server("weighted.random", + "127.0.0.1:8001", + &address_params); + address_params.weight = 1; + UpstreamManager::upstream_add_server("weighted.random", + "127.0.0.1:8002", + &address_params); + + UpstreamManager::upstream_create_consistent_hash( + "hash", + [](const char *path, const char *query, const char *fragment) -> unsigned int { + return 1; + }); + UpstreamManager::upstream_add_server("hash", "127.0.0.1:8001"); + UpstreamManager::upstream_add_server("hash", "127.0.0.1:8002"); + + UpstreamManager::upstream_create_manual( + "manual", + [](const char *path, const char *query, const char *fragment) -> unsigned int { + return 0; + }, + false, nullptr); + UpstreamManager::upstream_add_server("manual", "127.0.0.1:8001"); + UpstreamManager::upstream_add_server("manual", "127.0.0.1:8002"); + + UpstreamManager::upstream_create_weighted_random("try_another", true); + address_params.weight = 1000; + UpstreamManager::upstream_add_server("try_another", + "127.0.0.1:8001", + &address_params); + address_params.weight = 1; + UpstreamManager::upstream_add_server("try_another", + "127.0.0.1:8002", + &address_params); +} + +void basic_callback(WFHttpTask *task, std::string& message) +{ + auto state = task->get_state(); + EXPECT_EQ(state, WFT_STATE_SUCCESS); + if (state == WFT_STATE_SUCCESS && message.compare("")) + { + const void *body; + size_t body_len; + task->get_resp()->get_parsed_body(&body, &body_len); + std::string buffer((char *)body, body_len); + EXPECT_EQ(buffer, message); + } + WFFacilities::WaitGroup *wait_group = (WFFacilities::WaitGroup *)task->user_data; + wait_group->done(); +} + +TEST(upstream_unittest, BasicPolicy) +{ + WFFacilities::WaitGroup wait_group(3); + + register_upstream_hosts(); + + char url[3][30] = {"http://weighted.random", "http://hash", "http://manual"}; + + http_callback_t cb = std::bind(basic_callback, std::placeholders::_1, + std::string("server1")); + for (int i = 0; i < 3; i++) + { + WFHttpTask *task = WFTaskFactory::create_http_task(url[i], + REDIRECT_MAX, RETRY_MAX, cb); + task->user_data = &wait_group; + task->start(); + } + + wait_group.wait(); +} + +TEST(upstream_unittest, EnableAndDisable) +{ + WFFacilities::WaitGroup wait_group(1); + + UpstreamManager::upstream_disable_server("weighted.random", "127.0.0.1:8001"); + + //fprintf(stderr, "disable server and try......................\n"); + std::string url = "http://weighted.random"; + WFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, + [&wait_group, &url](WFHttpTask *task){ + auto state = task->get_state(); + EXPECT_EQ(state, WFT_STATE_TASK_ERROR); + EXPECT_EQ(task->get_error(), WFT_ERR_UPSTREAM_UNAVAILABLE); + UpstreamManager::upstream_enable_server("weighted.random", "127.0.0.1:8001"); + //fprintf(stderr, "ensable server and try......................\n"); + auto *task2 = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, + std::bind(basic_callback, + std::placeholders::_1, + std::string("server1"))); + task2->user_data = &wait_group; + series_of(task)->push_back(task2); + }); + task->user_data = &wait_group; + task->start(); + + wait_group.wait(); +} + +TEST(upstream_unittest, FuseAndRecover) +{ + WFFacilities::WaitGroup wait_group(1); + WFHttpTask *task; + SeriesWork *series; + protocol::HttpRequest *req; + std::string url = "http://weighted.random"; + int batch = MAX_FAILS + 50; + int timeout = (MTTR + 3) * 1000000; + + http_server1.stop(); + fprintf(stderr, "server 1 stopped start %d tasks to fuse it\n", batch); + ParallelWork *pwork = Workflow::create_parallel_work( + [](const ParallelWork *pwork) { + fprintf(stderr, "parallel finished\n"); + }); + + for (int i = 0; i < batch; i++) + { + task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, + nullptr); + req = task->get_req(); + req->add_header_pair("Connection", "keep-alive"); + series = Workflow::create_series_work(task, nullptr); + pwork->add_series(series); + } + + series = Workflow::create_series_work(pwork, nullptr); + + WFTimerTask *timer = WFTaskFactory::create_timer_task(timeout, + [](WFTimerTask *task) { + fprintf(stderr, "timer_finished and start server1\n"); + EXPECT_TRUE(http_server1.start("127.0.0.1", 8001) == 0) + << "http server start failed"; + }); + + series->push_back(timer); + + task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, + std::bind(basic_callback, + std::placeholders::_1, + std::string("server1"))); + task->user_data = &wait_group; + series->push_back(task); + + series->start(); + wait_group.wait(); +} + +TEST(upstream_unittest, TryAnother) +{ + WFFacilities::WaitGroup wait_group(1); + + UpstreamManager::upstream_disable_server("try_another", "127.0.0.1:8001"); + + std::string url = "http://try_another"; + WFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, + std::bind(basic_callback, + std::placeholders::_1, + std::string("server2"))); + + task->user_data = &wait_group; + task->start(); + wait_group.wait(); + UpstreamManager::upstream_enable_server("try_another", "127.0.0.1:8001"); +} + +int main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + + EXPECT_TRUE(http_server1.start("127.0.0.1", 8001) == 0) + << "http server start failed"; + + EXPECT_TRUE(http_server2.start("127.0.0.1", 8002) == 0) + << "http server start failed"; + + EXPECT_EQ(RUN_ALL_TESTS(), 0); + + http_server1.stop(); + http_server2.stop(); + + return 0; +} +