Merge pull request #14 from sogou/master

merge from sogou/workflow
2026-10-22 15:16:20 +08:00 · 2020-09-27 20:15:26 +08:00
parent 71a1df5a84 196b6ecec4
commit 42c400640a
28 changed files with 1332 additions and 232 deletions
@@ -1,11 +1,11 @@
 cmake_minimum_required(VERSION 3.6)

 set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "build type")
-set(CMAKE_INSTALL_PREFIX /opt/sogou CACHE PATH "install prefix")
 set(CMAKE_SKIP_RPATH TRUE)

-project(workflow
-	VERSION 1.12.6
+project(
+	workflow
+	VERSION 0.9.1
 	LANGUAGES C CXX
 )

@@ -26,7 +26,6 @@ set(LIB_DIR ${PROJECT_SOURCE_DIR}/_lib CACHE PATH "workflow lib")

 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LIB_DIR})
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIB_DIR})
-#set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LIB_DIR})

 add_custom_target(
 	LINK_HEADERS ALL
@@ -49,7 +48,6 @@ add_custom_command(
 )

 foreach(header_file ${INCLUDE_HEADERS} ${INCLUDE_KERNEL_HEADERS})
-	#file(COPY ${PROJECT_SOURCE_DIR}/${header_file} DESTINATION ${INC_DIR}/${PROJECT_NAME})
 	string(REPLACE "/" ";" arr ${header_file})
 	list(GET arr -1 file_name)
 	makeLink(${PROJECT_SOURCE_DIR}/${header_file} ${INC_DIR}/${PROJECT_NAME}/${file_name} LINK_HEADERS)
@@ -103,37 +101,6 @@ install(
 	RENAME ${PROJECT_NAME}-config.cmake
 )

-#### PACK
-
-set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Sogou C++ Workflow")
-set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION})
-
-# set(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/pre_script.sh)
-# set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/post_script.sh)
-
-execute_process(
-	COMMAND git log -n1 --format=%ad --date=short
-	RESULT_VARIABLE GIT_LOG_RET
-	OUTPUT_VARIABLE GIT_YMD
-	OUTPUT_STRIP_TRAILING_WHITESPACE
-)
-
-if (NOT GIT_LOG_RET EQUAL 0)
-	set(GIT_YMD 0)
-endif()
-
-string(REPLACE "-" "" RELEASE_LABEL ${GIT_YMD})
-set(CPACK_RPM_PACKAGE_RELEASE ${RELEASE_LABEL})
-
-set(CPACK_RPM_COMPONENT_INSTALL ON)
-set(CPACK_COMPONENTS_ALL devel)
-set(CPACK_RPM_FILE_NAME RPM-DEFAULT)
-set(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
-
-set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /usr/lib64/cmake)
-
-set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "openssl-devel >= 1.0.1")
-
 install(
 	FILES ${INCLUDE_HEADERS} ${INCLUDE_KERNEL_HEADERS}
 	DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}
@@ -145,9 +112,3 @@ install(
 	DESTINATION "${CMAKE_INSTALL_DOCDIR}-${PROJECT_VERSION}"
 	COMPONENT devel
 )
-
-set(CPACK_RPM_SPEC_MORE_DEFINE "%define __spec_install_post /bin/true") # disable strip
-set(CPACK_GENERATOR "RPM")
-
-include(CPack)
-
@@ -1,5 +1,5 @@
 ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
-ALL_TARGETS := all base check install preinstall package rpm clean tutorial
+ALL_TARGETS := all base check install preinstall clean tutorial
 MAKE_FILE := Makefile

 DEFAULT_BUILD_DIR := build
@@ -14,7 +14,9 @@ all: base
 base:
 	mkdir -p $(BUILD_DIR)
 ifeq ($(DEBUG),y)
-	cd $(BUILD_DIR) && $(CMAKE3) -D CMAKE_BUILD_TYPE=Debug $(ROOT_DIR)
+	cd $(BUILD_DIR) && $(CMAKE3) -DCMAKE_BUILD_TYPE=Debug $(ROOT_DIR)
+else ifneq ("${INSTALL_PREFIX}install_prefix", "install_prefix")
+	cd $(BUILD_DIR) && $(CMAKE3) -DCMAKE_INSTALL_PREFIX:STRING=${INSTALL_PREFIX} $(ROOT_DIR)
 else
 	cd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)
 endif
@@ -25,16 +27,11 @@ tutorial: all
 check: all
 	make -C test check

-install preinstall package: base
+install preinstall: base
 	mkdir -p $(BUILD_DIR)
 	cd $(BUILD_DIR) && $(CMAKE3) $(ROOT_DIR)
 	make -C $(BUILD_DIR) -f Makefile $@

-rpm: package
-ifneq ($(BUILD_DIR),.)
-	mv $(BUILD_DIR)/*.rpm ./
-endif
-
 clean:
 ifeq (build, $(wildcard build))
 	-make -C build clean
@@ -44,8 +41,6 @@ endif
 	rm -rf $(DEFAULT_BUILD_DIR)
 	rm -rf _include
 	rm -rf _lib
-	rm -f SRCINFO SRCNUMVER SRCVERSION
-	rm -f ./*.rpm
 	find . -name CMakeCache.txt | xargs rm -f
 	find . -name Makefile       | xargs rm -f
 	find . -name "*.cmake"      | xargs rm -f
@@ -28,7 +28,7 @@ int main()
 ~~~
 * 作为万能异步客户端。目前支持``http``，``redis``，``mysql``和``kafka``协议。
 * 实现自定义协议client/server，构建自己的RPC系统。
-  * 搜狗RPC就是以它为基础，作为独立项目开源。支持``srpc``，``brpc``和``thrift``等协议（[benchmark](https://github.com/holmes1412/sogou-rpc-benchmark)）。
+  * [srpc](https://github.com/sogou/srpc)就是以它为基础，作为独立项目开源。支持``srpc``，``brpc``和``thrift``等协议。
 * 构建异步任务流，支持常用的串并联，也支持更加复杂的DAG结构。
 * 作为并行编程工具使用。除了网络任务，我们也包含计算任务的调度。所有类型的任务都可以放入同一个流中。
 * 在``Linux``系统下作为文件异步IO工具使用，性能超过任何标准调用。磁盘IO也是一种任务。
@@ -1,14 +1,15 @@
 [中文版](README.md)

 ## Sogou C++ Workflow
-[![license MIT](https://img.shields.io/badge/License-Apache-yellow.svg)](https://git.sogou-inc.com/wujiaxu/Filter/blob/master/LICENSE)
-[![C++](https://img.shields.io/badge/language-c++-red.svg)](https://en.cppreference.com/)
-[![platform](https://img.shields.io/badge/platform-linux%20%7C%20macos-lightgrey.svg)](#%E9%A1%B9%E7%9B%AE%E7%9A%84%E4%B8%80%E4%BA%9B%E8%AE%BE%E8%AE%A1%E7%89%B9%E7%82%B9)

-As **Sogou`s C++ server engine**, workflow supports almost all **back-end C++ online services** of Sogou, including all search services, cloud input method，online advertisements, etc., handling more than **10 billion** requests every day. This is an **enterprise-level programming engine** with light and elegantly designed which can satisfy most C++ back-end development requirements.
+[![C++](https://img.shields.io/badge/License-Apache-yellow.svg)](https://git.sogou-inc.com/wujiaxu/Filter/blob/master/LICENSE "license MIT") [![](https://img.shields.io/badge/language-c++-red.svg)](https://en.cppreference.com/ "platform") [![](https://img.shields.io/badge/platform-linux%20%7C%20macos-lightgrey.svg)](#%E9%A1%B9%E7%9B%AE%E7%9A%84%E4%B8%80%E4%BA%9B%E8%AE%BE%E8%AE%A1%E7%89%B9%E7%82%B9)
+
+As **Sogou\`s C++ server engine**, Sogou C++ Workflow supports almost all **back-end C++ online services** of Sogou, including all search services, cloud input method，online advertisements, etc., handling more than **10 billion** requests every day. This is an **enterprise-level programming engine** in light and elegant design which can satisfy most C++ back-end development requirements.

 #### You can use it:
-* To quickly build an **Http server**:
+
+* To quickly build an **HTTP server**:
+
 ~~~cpp
 #include <stdio.h>
 #include "workflow/WFHttpServer.h"
@@ -19,7 +20,7 @@ int main()
        task->get_resp()->append_output_body("<html>Hello World!</html>");
    });

-    if (server.start(8888) == 0) {  // start server on port 8888
+    if (server.start(8888) == 0) { // start server on port 8888
        getchar(); // press "Enter" to end.
        server.stop();
    }
@@ -27,101 +28,107 @@ int main()
    return 0;
 }
 ~~~
-* As a **powerful asynchronous client**. Currently supports ``http``, ``redis``, ``mysql`` and ``kafka`` protocols.
-* To realize **user-defined protocol client/server** and build your own **RPC system**.
-  * Sogou RPC is based on it and open source as an independent project, which supports srpc, brpc and thrift protocol ([benchmark](https://github.com/holmes1412/sogou-rpc-benchmark)).
-* To build **asynchronous task flow**, support common **series** and **parallel** structures, and also support more complex **DAG** structures.
-* As a **parallel programming tool**. In addition to **network tasks**, we also include **the scheduling of computing tasks**. All types of tasks can be put into **the same** task flow.
-* As a **file asynchronous IO tool** under ``Linux`` system, with a high performance exceeding any system call. Disk IO is also a task.
-* To realize any **high-performance** and **high-concurrency** back-end service with a very complex relationship between computing and communication.
+
+* As a **multifunctional asynchronous client**, it currently supports `HTTP`, `Redis`, `MySQL` and `Kafka` protocols.
+* To implement **client/server on user-defined protocol** and build your own **RPC system**.
+  * [srpc](https://github.com/sogou/srpc) is based on it and it is an independent open source project, which supports srpc, brpc and thrift protocols.
+* To build **asynchronous workflow**; support common **series** and **parallel** structures, and also support any **DAG** structures.
+* As a **parallel programming tool**. In addition to **networking tasks**, Sogou C++ Workflow also includes **the scheduling of computing tasks**. All types of tasks can be put into **the same** flow.
+* As a **asynchronous file IO tool** in `Linux` system, with high performance exceeding any system call. Disk file IO is also a task.
+* To realize any **high-performance** and **high-concurrency** back-end service with a very complex relationship between computing and networking.
 * To build a **service mesh** system.
-  * The project has built-in **service governance** and **load balancing** features.
+  * This project has built-in **service governance** and **load balancing** features.

-#### Compile and run environment
+#### Compiling and running environment

-* This project supports ``Linux``, ``macOS``, ``Windows`` and other operating systems.
-  * ``Windows`` version is temporarily released as an independent branch, using ``iocp`` to implement asynchronous networking. All user interfaces are consistent with the ``Linux`` version.
-* Supports all CPU platforms, including 32 or 64-bit ``x86`` processors, big-endian or little-endian ``arm`` processors.
-* Relies on ``OpenSSL``, recommending ``OpenSSL 1.1`` and above.
-* Uses the ``C++11`` standard and therefore, needs to be compiled with a compiler which supports ``C++11``. Does not rely on ``boost`` or ``asio``.
-* No other dependencies. However, it contains the unmodified source code of several compression libraries such as ``lz4``, ``zstd`` and ``snappy`` (required by the ``Kafka`` protocol).
+* This project supports `Linux`, `macOS`, `Windows` and other operating systems.
+  * `Windows` version is currently released as an independent branch, using `iocp` to implement asynchronous networking. All user interfaces are consistent with the `Linux` version.
+* Supports all CPU platforms, including 32 or 64-bit `x86` processors, big-endian or little-endian `arm` processors.
+* Relies on `OpenSSL`; `OpenSSL 1.1` and above is recommended.
+* Uses the `C++11` standard and therefore, it should be compiled with a compiler which supports `C++11`. Does not rely on `boost` or `asio`.
+* No other dependencies. However, if you need `Kafka` protocol, some compression libraries should be installed, including `lz4`, `zstd` and `snappy`.

 # Try it!
-  * Client
-    * [Create your first task：wget](docs/tutorial-01-wget.md)
-    * [Implement redis set and get：redis_cli](docs/tutorial-02-redis_cli.md)
-    * [More features about series：wget_to_redis](docs/tutorial-03-wget_to_redis.md)
-  * Server
-    * [first server：http_echo_server](docs/tutorial-04-http_echo_server.md)
-    * [asynchronous server：http_proxy](docs/tutorial-05-http_proxy.md)
-  * Parallel task and Series　
-    * [A simple parallel wget：parallel_wget](docs/tutorial-06-parallel_wget.md)
-  * Important topics
-    * [About error](docs/about-error.md)
-    * [About timeout](docs/about-timeout.md)
-    * [About global configuration](docs/about-config.md)
-    * [About DNS](docs/about-dns.md)
-    * [About exit](docs/about-exit.md)
-  * Computing tasks
-    * [Using the build-in algorithm factory：sort_task](docs/tutorial-07-sort_task.md)
-    * [User-define computing task：matrix_multiply](docs/tutorial-08-matrix_multiply.md)
-  * File asynchronous IO tasks
-    * [Http server with file IO：http_file_server](docs/tutorial-09-http_file_server.md)
-  * User-defined protocol basic usage
-    * [A simple user-defined portocol client/server](docs/tutorial-10-user_defined_protocol.md)
-  * Timing tasks and counting tasks
-    * [About timer](docs/about-timer.md)
-    * [About counter](docs/about-counter.md)
-  * Service governance
-    * [About service governance](docs/about-service-management.md)
-    * [More documents about upstream](docs/about-upstream.md)
-  * Connection context
-    * [About connection context](docs/about-connection-context.md)
-  * Built-in protocols
-    * [Asynchronous MySQL client：mysql_cli](docs/tutorial-12-mysql_cli.md)
+
+* Client
+  * [Creating your first task：wget](docs/en/tutorial-01-wget.md)
+  * [Implementing Redis set and get：redis\_cli](docs/en/tutorial-02-redis_cli.md)
+  * [More features about series：wget\_to\_redis](docs/en/tutorial-03-wget_to_redis.md)
+* Server
+  * [First server：http\_echo\_server](docs/en/tutorial-04-http_echo_server.md)
+  * [Asynchronous server：http\_proxy](docs/en/tutorial-05-http_proxy.md)
+* Parallel tasks and Series　
+  * [A simple parallel wget：parallel\_wget](docs/en/tutorial-06-parallel_wget.md)
+* Important topics
+  * [About error](docs/en/about-error.md)
+  * [About timeout](docs/en/about-timeout.md)
+  * [About global configuration](docs/en/about-config.md)
+  * [About DNS](docs/en/about-dns.md)
+  * [About exit](docs/en/about-exit.md)
+* Computing tasks
+  * [Using the build-in algorithm factory：sort\_task](docs/en/tutorial-07-sort_task.md)
+  * [User-defined computing task：matrix\_multiply](docs/en/tutorial-08-matrix_multiply.md)
+* Asynchronous File IO tasks
+  * [Http server with file IO：http\_file\_server](docs/en/tutorial-09-http_file_server.md)
+* User-defined protocol
+  * [A simple user-defined portocol: client/server](docs/en/tutorial-10-user_defined_protocol.md)
+* Timing tasks and counting tasks
+  * [About timer](docs/en/about-timer.md)
+  * [About counter](docs/en/about-counter.md)
+* Service governance
+  * [About service governance](docs/en/about-service-management.md)
+  * [More documents about upstream](docs/en/about-upstream.md)
+* Connection context
+  * [About connection context](docs/en/about-connection-context.md)
+* Built-in protocols
+  * [Asynchronous MySQL client：mysql\_cli](docs/en/tutorial-12-mysql_cli.md)

 #### System design features

 We believe that a typical back-end program consists of the following three parts and should be developed completely independently.
+
 * Protocol
-  * In most cases, users use built-in common network protocols, such as http, redis or various rpc.
-  * Users can also easily customize user-defined network protocol,  at the mean time they only need to provide serialization and deserialization functions to define their own client/server.
+  * In most cases, users use built-in common network protocols, such as HTTP, Redis or various rpc.
+  * Users can also easily customize user-defined network protocol. In the customization, they only need to provide serialization and deserialization functions to define their own client/server.
 * Algorithm
-  * In our design, algorithm is a symmetrical concept with protocol.
+  * In our design, the algorithm is a concept symmetrical to the protocol.
    * If protocol call is rpc, then algorithm call is an apc (Async Procedure Call).
  * We have provided some general algorithms, such as sort, merge, psort, reduce, which can be used directly.
-  * Compared with user-defined protocol, user-defined algorithm is much more common. Any complex calculation with clear boundaries should be packaged into an algorithm.
-* Task flow
-  * Task flow is the actual bussiness logic, which is to put the protocols and algorithms into the flow graph for use.
-  * The typical task flow is a closed series-parallel graph. Complex business logic may be a non-closed DAG.
-  * The task flow graph can be constructed directly or dynamically generated based on the results of each step. All tasks are executed asynchronously.
+  * Compared with a user-defined protocol, a user-defined algorithm is much more common. Any complicated computation with clear boundaries should be packaged into an algorithm.
+* Workflow
+  * Workflow is the actual bussiness logic, which is to put the protocols and algorithms into the flow graph for use.
+  * The typical workflow is a closed series-parallel graph. Complex business logic may be a non-closed DAG.
+  * The workflow graph can be constructed directly or dynamically generated based on the results of each step. All tasks are executed asynchronously.

 Basic task, task factory and complex task
-* Our system contains six basic tasks: communication, file IO, CPU, GPU, timer, and counter.
+
+* Our system contains six basic tasks: networking, file IO, CPU, GPU, timer, and counter.
 * All tasks are generated by the task factory and automatically recycled after callback.
-  * Server task is one kind of special communication task, generated by the framework which calls the task factory, and handed over to the user through the process function.
-  * In most cases, the task generated by the user through the task factory is a complex task, which has no necessary to be perceived by the user.
-  * For example, an Http request may include many asynchronous processes (DNS, redirection), but for the user, it is just a communication task.
-  * File sorting seems to be an algorithm, but it actually includes many complex interaction processes between file IO and CPU calculation.
+  * Server task is one kind of special networking task, generated by the framework which calls the task factory, and handed over to the user through the process function.
+* In most cases, the task generated by the user through the task factory is a complex task, which is transparent to the user.
+  * For example, an HTTP request may include many asynchronous processes (DNS, redirection), but for user, it is just a networking task.
+  * File sorting seems to be an algorithm, but it actually includes many complex interaction processes between file IO and CPU computation.
  * If you think of business logic as building circuits with well-designed electronic components, then each electronic component may be a complex circuit.

-Asynchrony and encapsulation based on ``C++11 std::function``
+Asynchrony and encapsulation based on `C++11 std::function`

 * Not based on user mode coroutines. Users need to know that they are writing asynchronous programs.
-* All calls are executed asynchronously, and there are almost no operations to wait for threads.
-  * Although we also provide some convenient semi-synchronous interfaces, they are not core features.
-* Please avoid derivation.Try to encapsulate user behavior with ``std::function`` instead, including:
+* All calls are executed asynchronously, and there are almost no operation that occupys a thread.
+  * Although we also provide some facilities with semi-synchronous interfaces, they are not core features.
+* We try to avoid user's derivations, and encapsulate user behavior with `std::function` instead, including:
  * The callback of any task.
-  * Any server process. This conforms to the ``FaaS`` (Function as a Service) idea.
-  * The realization of an algorithm is simply a ``std::function``. But the algorithm can also be implemented by derivative.
+  * Any server's process. This conforms to the `FaaS` (Function as a Service) idea.
+  * The realization of an algorithm is simply a `std::function`. But the algorithm can also be implemented by derivation.

 Memory reclamation mechanism
-* Every task will be automatically reclaimed after the callback. If a task is created but does not want to run, the user needs to release it through the dismiss method.
-* Any data in the task, such as the response of the network request, will also be recycled with the task. At this time, the user can use ``std::move()`` to move the required data.
+
+* Every task will be automatically reclaimed after the callback. If a task is created but a user does not want to run it, the user needs to release it through the dismiss method.
+* Any data in the task, such as the response of the network request, will also be recycled with the task. At this time, the user can use `std::move()` to move the required data.
 * SeriesWork and ParallelWork are two kinds of framework objects, which are also recycled after their callback.
-* This project doesn’t use ``std::shared_ptr`` to manage memory.
+* This project doesn’t use `std::shared_ptr` to manage memory.

 #### More design documents
+
 To be continued...

 ## Authors
@@ -31,7 +31,9 @@ static constexpr struct WFGlobalSettings GLOBAL_SETTINGS_DEFAULT =
    .compute_threads    =   -1,
 };
 ~~~
+
 其中EndpointParams结构体和默认值在[EndpointParams.h](../src/manager/EndpointParams.h)文件里：
+
 ~~~cpp

 struct EndpointParams
@@ -50,7 +52,9 @@ static constexpr struct EndpointParams ENDPOINT_PARAMS_DEFAULT =
    .ssl_connect_timeout    = 10 * 1000,
 };
 ~~~
+
 举个例子，把默认的连接超时改为5秒，dns默认ttl改为1小时，用于消息反序列化的poller线程增加到10个：
+
 ~~~cpp
 #include "workflow/WFGlobal.h"

@@ -65,11 +69,12 @@ int main()

    ...
 }
-
 ~~~
+
 大多数参数的意义都比较清晰。注意dns ttl相关参数，单位是**秒**。endpoint相关超时参数单位是**毫秒**，并且可以用-1表示无限。  
 dns_threads表示并行访问dns的线程数。目前我们dns是通过系统函数getaddrinfo访问的。如果对dns并发性能有需求，可增加这个值。  
 compute_threads表示用于计算的线程数，默认-1代表与当前节点CPU核数相同。  
+
 与网络性能相关的两个参数为poller_threads和handler_threads：
 * poller线程主要负责epoll（kqueue）和消息反序列化。
 * handler线程是网络任务callback和process所在线程。
@@ -172,11 +172,9 @@ struct AddressParams
 * - If max_fails is set to 1, it means server would out of upstream selection in 30 seconds when failed only once
 */
    unsigned int max_fails;                ///< [1, INT32_MAX] max_fails = 0 means max_fails = 1
-    unsigned short weight;                 ///< [1, 65535] weight = 0 means weight = 1. only for master
-#define SERVER_TYPE_MASTER    0
-#define SERVER_TYPE_SLAVE     1
-    int server_type;                       ///< default is SERVER_TYPE_MASTER
-    int group_id;                          ///< -1 means no group. Slave without group will backup for any master
+    unsigned short weight;                 ///< [1, 65535] weight = 0 means weight = 1. only for main server
+    int server_type;                       ///< 0 for main and 1 for backup
+    int group_id;                          ///< -1 means no group. Backup without group will backup for any main node
 };
 ~~~
 大多数参数的作用一眼了然。其中endpoint_params和dns相关参数，可以覆盖全局的配置。  
@@ -190,7 +188,7 @@ struct AddressParams
 max_fails参数为最大出错次数，如果选取目标连续出错达到max_fails则熔断，如果upstream的try_another属性为false，则任务失败，  
 在任务callback里，get_state()=WFT_STATE_TASK_ERROR，get_error()=WFT_ERR_UPSTREAM_UNAVAILABLE。  
 如果try_another为true，并且所有server都熔断的话，会得到同样错误。熔断时间为30秒。  
-server_type和group_id用于主备功能。所有upstream必需有type为MASTER的server，否则upstream不可用。  
-类型为SLAVE的server，会在同group_id的MASTER熔断情况下被使用。  
+server_type和group_id用于主备功能。所有upstream必需有type为0(主节点)的server，否则upstream不可用。  
+类型为1（备份节点）的server，会在同group_id的主节点熔断情况下被使用。  

 更多upstream功能查询：[about-upstream.md](./about-upstream.md)。
@@ -106,7 +106,7 @@ public:
 ~~~
 其中，set_send_timeout()设置发送完整消息的超时，默认值为-1。  
 set_receive_timeout()只对client任务有效，指接收完整server回复的超时，默认值为-1。  
-  * server任务的receive_timeout在server启动配置里。所有被用户处理的server任务，都已经成功接收了完整请求。
+  * server任务的receive_timeout在server启动配置里。对server任务设置receive_timeout没有意义，因为消息已经接收完成。

 set_keep_alive()接口设置连接保持超时。一般来讲，框架能很好的处理连接保持的问题，用户不需要调用。  
 如果是http协议，client或server想要使用短连接，可通过添加HTTP header来完成，尽量不要用这个接口去修改。  
@@ -139,7 +139,7 @@ public:

 ### 超时功能的实现

-框架内部，需要处理的超时种类比我们在这里展现的还要更多。除了wait_timeout，全都是依赖于Linux的timer_fd，每个epoll线程一个。  
-默认配置下，epoll线程数为2，可以满足大多数应用的需要了。  
+框架内部，需要处理的超时种类比我们在这里展现的还要更多。除了wait_timeout，全都是依赖于Linux的timerfd或kqueue的timer事件。  
+每个poller线程包含一个timerfd，默认配置下，poller线程数为4，可以满足大多数应用的需要了。  
 目前的超时算法利用了链表+红黑树的数据结构，时间复杂度在O(1)和O(logn)之间，其中n为epoll线程的fd数量。  
 超时处理目前看不是瓶颈所在，因为Linux内核epoll相关调用也是O(logn)时间复杂度，我们把超时都做到O(1)也区别不大。
@@ -85,7 +85,7 @@ http_task->start();
 基本原理
 1. 随机选择一个目标
 2. 如果try_another配置为true，那么将在所有存活的目标中随机选择一个
-3. 仅在master中选择，选中目标所在group的主备和无group的备都视为有效的可选对象
+3. 仅在main中选择，选中目标所在group的主备和无group的备都视为有效的可选对象

 ### 例2 在多个目标中按照权重大小随机访问
 配置一个本地反向代理，将本地发出的weighted.random所有请求按照5/20/1的权重分配打到3个目标server上
@@ -107,7 +107,7 @@ http_task->start();
 基本原理
 1. 按照权重分配，随机选择一个目标，权重越大概率越大
 2. 如果try_another配置为true，那么将在所有存活的目标中按照权重分配随机选择一个
-3. 仅在master中选择，选中目标所在group的主备和无group的备都视为有效的可选对象
+3. 仅在main中选择，选中目标所在group的主备和无group的备都视为有效的可选对象

 ### 例3 在多个目标中按照框架默认的一致性哈希访问
 ~~~cpp
@@ -125,11 +125,11 @@ auto *http_task = WFTaskFactory::create_http_task("http://abc.local/service/meth
 http_task->start();
 ~~~
 基本原理
-1. 每1个master视为16个虚拟节点
+1. 每1个main视为16个虚拟节点
 2. 框架会使用std::hash对所有节点的address+虚拟index进行运算，作为一致性哈希的node值
 3. 框架会使用std::hash对path+query+fragment进行运算，作为一致性哈希data值
 4. 每次都选择存活node最近的值作为目标
-5. 对于每一个master、只要有存活group内master/有存活group内slave/有存活no group slave，即视为存活
+5. 对于每一个main、只要有存活group内main/有存活group内backup/有存活no group backup，即视为存活

 ### 例4 自定义一致性哈希函数
 ~~~cpp
@@ -183,8 +183,8 @@ auto *http_task = WFTaskFactory::create_http_task("http://xyz.cdn/sompath?key=so
 http_task->start();
 ~~~
 基本原理
-1. 框架首先依据用户提供的普通选取函数、按照取模，在master列表中确定选取
-2. 对于每一个master、只要有存活group内master/有存活group内slave/有存活no group slave，即视为存活
+1. 框架首先依据用户提供的普通选取函数、按照取模，在main列表中确定选取
+2. 对于每一个main、只要有存活group内main/有存活group内backup/有存活no group backup，即视为存活
 3. 如果选中目标不再存活且try_another设为true，将再使用一致性哈希函数进行二次选取
 4. 如果触发二次选取，一致性哈希将保证一定会选择一个存活目标、除非全部机器都被熔断掉

@@ -195,10 +195,10 @@ UpstreamManager::upstream_create_weighted_random(
    true);//一主一备这项设什么没区别

 AddressParams address_params = ADDRESS_PARAMS_DEFAULT;
-address_params.server_type = SERVER_TYPE_MASTER;
-UpstreamManager::upstream_add_server("simple.name", "master01.test.ted.bj.sogou", &address_params);//主
-address_params.server_type = SERVER_TYPE_SLAVE;
-UpstreamManager::upstream_add_server("simple.name", "slave01.test.ted.gd.sogou", &address_params);//备
+address_params.server_type = 0;
+UpstreamManager::upstream_add_server("simple.name", "main01.test.ted.bj.sogou", &address_params);//主
+address_params.server_type = 1;
+UpstreamManager::upstream_add_server("simple.name", "backup01.test.ted.gd.sogou", &address_params);//备

 auto *http_task = WFTaskFactory::create_http_task("http://simple.name/request", 0, 0, nullptr);
 auto *redis_task = WFTaskFactory::create_redis_task("redis://simple.name/2", 0, nullptr);
@@ -219,33 +219,33 @@ UpstreamManager::upstream_create_consistent_hash(
    nullptr);//nullptr代表使用框架默认的一致性哈希函数

 AddressParams address_params = ADDRESS_PARAMS_DEFAULT;
-address_params.server_type = SERVER_TYPE_MASTER;
+address_params.server_type = 0;
 address_params.group_id = 1001;
-UpstreamManager::upstream_add_server("abc.local", "192.168.2.100:8081", &address_params);//master in group 1001
-address_params.server_type = SERVER_TYPE_SLAVE;
+UpstreamManager::upstream_add_server("abc.local", "192.168.2.100:8081", &address_params);//main in group 1001
+address_params.server_type = 1;
 address_params.group_id = 1001;
-UpstreamManager::upstream_add_server("abc.local", "192.168.2.100:8082", &address_params);//slave for group 1001
-address_params.server_type = SERVER_TYPE_MASTER;
+UpstreamManager::upstream_add_server("abc.local", "192.168.2.100:8082", &address_params);//backup for group 1001
+address_params.server_type = 0;
 address_params.group_id = 1002;
-UpstreamManager::upstream_add_server("abc.local", "master01.test.ted.bj.sogou", &address_params);//master in group 1002
-address_params.server_type = SERVER_TYPE_SLAVE;
+UpstreamManager::upstream_add_server("abc.local", "main01.test.ted.bj.sogou", &address_params);//main in group 1002
+address_params.server_type = 1;
 address_params.group_id = 1002;
-UpstreamManager::upstream_add_server("abc.local", "slave01.test.ted.gd.sogou", &address_params);//slave for group 1002
-address_params.server_type = SERVER_TYPE_SLAVE;
+UpstreamManager::upstream_add_server("abc.local", "backup01.test.ted.gd.sogou", &address_params);//backup for group 1002
+address_params.server_type = 1;
 address_params.group_id = -1;
-UpstreamManager::upstream_add_server("abc.local", "test.sogou.com:8080", &address_params);//slave for no group mean slave for all group and no group
-UpstreamManager::upstream_add_server("abc.local", "abc.sogou.com");//master, no group
+UpstreamManager::upstream_add_server("abc.local", "test.sogou.com:8080", &address_params);//backup for no group mean backup for all group and no group
+UpstreamManager::upstream_add_server("abc.local", "abc.sogou.com");//main, no group

 auto *http_task = WFTaskFactory::create_http_task("http://abc.local/service/method", 0, 0, nullptr);
 http_task->start();
 ~~~
 基本原理
 1. 组号-1代表无组，这种目标不属于任何组
-2. 无组的master之间是平等的，甚至可以视为同一个组。但与有组的master之间是隔离的
-3. 无组的slave可以为全局任何组目标/任何无组目标作为备
+2. 无组的main之间是平等的，甚至可以视为同一个组。但与有组的main之间是隔离的
+3. 无组的backup可以为全局任何组目标/任何无组目标作为备
 4. 组号可以区分哪些主备是在一起工作的
-5. 不同组之间的备是相互隔离的，只为本组的master服务
-6. 添加目标的默认组号-1，type为master
+5. 不同组之间的备是相互隔离的，只为本组的main服务
+6. 添加目标的默认组号-1，type为0，表示主节点。

 # Upstream选择策略

@@ -287,8 +287,6 @@ struct AddressParams
    unsigned int dns_ttl_min;
    unsigned int max_fails;
    unsigned short weight;
-#define SERVER_TYPE_MASTER    0
-#define SERVER_TYPE_SLAVE     1
    int server_type;
    int group_id;
 };
@@ -299,8 +297,8 @@ static constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =
    .dns_ttl_default    =    12 * 3600,
    .dns_ttl_min        =    180,
    .max_fails          =    200,
-    .weight             =    1,    //only for master of UPSTREAM_WEIGHTED_RANDOM
-    .server_type        =    SERVER_TYPE_MASTER,
+    .weight             =    1,    //only for main of UPSTREAM_WEIGHTED_RANDOM
+    .server_type        =    0,
    .group_id           =    -1,
 };
 ~~~
@@ -309,7 +307,7 @@ static constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =
  * dns_ttl_default：dns cache中默认的ttl，单位秒，默认12小时，dns cache是针对当前进程的，即进程退出就会消失，配置也仅对当前进程有效
  * dns_ttl_min：dns最短生效时间，单位秒，默认3分钟，用于在通信失败重试时是否进行重新dns的决策
  * max_fails：触发熔断的【连续】失败次数（注：每次通信成功，计数会清零）
-  * weight：权重，默认1，仅对master有效，用于Upstream随机策略选取，权重越大越容易被选中；其他策略下此参数无意义
+  * weight：权重，默认1，仅对main有效，用于Upstream随机策略选取，权重越大越容易被选中；其他策略下此参数无意义
  * server_type：主备配置，默认主。无论什么时刻，同组的主优先级永远高于其他的备
  * group_id：分组依据，默认-1。-1代表无分组(游离)，游离的备可视为任何主的备，有组的备优先级永远高于游离的备。

@@ -343,8 +341,8 @@ Address在熔断期间，一旦被策略选中，Upstream会根据具体配置
 2. 一致性哈希策略，全部目标都处于熔断期
 3. 手动策略 && try_another==true，全部目标都处于熔断期  
 4. 手动策略 && try_another==false，且同时满足下面三个条件：  
-  1). select函数选中的master处于熔断期，，且游离的备都处于熔断期  
-  2). 这个master是游离的主，或者这个master所在的group其他目标都处于熔断期  
+  1). select函数选中的main处于熔断期，，且游离的备都处于熔断期  
+  2). 这个main是游离的主，或者这个main所在的group其他目标都处于熔断期  
  3). 所有游离的备都处于熔断期  

 # Upstream端口优先级
@@ -0,0 +1,102 @@
+# Creating your first task: wget
+
+# Sample code
+
+[tutorial-01-wget.cc](/tutorial/tutorial-01-wget.cc)
+
+# About wget
+
+wget reads HTTP/HTTPS URLs from stdin, crawls the webpages and then print the content to stdout. It also outputs the HTTP headers of the request and the response to stderr.   
+For convenience, wget exits with Ctrl-C, but it will ensure that all resources are completely released first.
+
+# Creating and starting an HTTP task
+
+~~~cpp
+WFHttpTask *task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX, wget_callback);
+protocol::HttpRequest *req = task->get_req();
+req->add_header_pair("Accept", "*/*");
+req->add_header_pair("User-Agent", "Wget/1.14 (gnu-linux)");
+req->add_header_pair("Connection", "close");
+task->start();
+pause();
+~~~
+
+**WFTaskFactory::create\_http\_task()** generates an HTTP task. In [WFTaskFactory.h](/src/factory/WFTaskFactory.h), the prototype is defined as follows:
+
+~~~cpp
+WFHttpTask *create_http_task(const std::string& url,
+                             int redirect_max, int retry_max,
+                             http_callback_t callback);
+~~~
+
+The first few parameters are self-explanatory. **http\_callback\_t** is the callback of an HTTP task, which is defined below:
+
+~~~cpp
+using http_callback_t = std::function<void (WFHttpTask *)>;
+~~~
+
+To put it simply, it’s the funtion that has **Task** as one parameter and does not return any value. You can pass NULL to this callback, indicating that there is no callback. The callback in all tasks follows the same rule.   
+Please note that all factory functions do not return failure, so even if the URL is illegal, don't worry that the task is a null pointer. All errors are handled in the callback.   
+You can use **task->get\_req()** to get the request of the task. The default method is GET via HTTP/1.1 on long connections. The framework automatically adds request\_uri, Host and other parameters. The framework will add other HTTP header fields automatically according to the actual requirements, including Content-Length or Connection before sending the request. You may also use **add\_header\_pair()** to add your own header. For more interfaces on HTTP messages, please see [HttpMessage.h](/src/protocol/HttpMessage.h).   
+**task->start()** starts the task. It’s non-blocking and will not fail. Then the callback of the task will be called. As it’s an asynchronous task, obviously you cannot use the task pointer after **start()**.   
+To make the example as simple as possible, call **pause()** after **start()** to prevent the program from exiting. You can press Ctrl-C to exit the program.
+
+# Handling crawled HTTP results
+
+This example demonstrates how to handle the results with a general function. Of course, **std::function** supports more features.
+
+~~~cpp
+void wget_callback(WFHttpTask *task)
+{
+    protocol::HttpRequest *req = task->get_req();
+    protocol::HttpResponse *resp = task->get_resp();
+    int state = task->get_state();
+    int error = task->get_error();
+
+    // handle error states
+    ...
+
+    std::string name;
+    std::string value;
+    // print request to stderr
+    fprintf(stderr, "%s %s %s\r\n", req->get_method(), req->get_http_version(), req->get_request_uri());
+    protocol::HttpHeaderCursor req_cursor(req);
+    while (req_cursor.next(name, value))
+        fprintf(stderr, "%s: %s\r\n", name.c_str(), value.c_str());
+    fprintf(stderr, "\r\n");
+    
+    // print response header to stderr
+    ...
+
+    // print response body to stdin
+    void *body;
+    size_t body_len;
+    resp->get_parsed_body(&body, &body_len); // always success.
+    fwrite(body, 1, body_len, stdout);
+    fflush(stdout);
+}
+~~~
+
+In this callback, the task is generated by the factory.   
+You can use **task->get\_state()** and **task->get\_error()** to obtain the running status and the error code of the task respectively. Let's skip the error handling first.  
+Use **task->get\_resp()** to get the response of the task, which is slightly different from the request, as they are both derived from HttpMessage.   
+Then, use the HttpHeaderCursor to scan the headers of the request and the response. [HttpUtil.h](/src/protocol/HttpUtil.h) contains the definition of the Cursor.
+
+~~~cpp
+class HttpHeaderCursor
+{
+public:
+    HttpHeaderCursor(const HttpMessage *message);
+    ...
+    void rewind();
+    ...
+    bool next(std::string& name, std::string& value);
+    bool find(const std::string& name, std::string& value);
+    ...
+};
+~~~
+
+There should be no doubt about the use of this cursor.   
+The next line **resp->get\_parsed\_body()** obtains the HTTP body of the response. This call always returns true when the task is successful, and the body points to the data area.   
+The call gets the raw HTTP body, and does not decode the chunk. If you want to decode the chunk, you can use the HttpChunkCursor in [HttpUtil.h](/src/protocol/HttpUtil.h). 
+In addition, **find()** will change the pointer inside the cursor. If you want to iterate over the header after you use **find()**, please use **rewind()** to return to the cursor header.
@@ -0,0 +1,169 @@
+# First server: http\_echo\_server
+
+# Sample code
+
+[tutorial-04-http\_echo\_server.cc](/tutorial/tutorial-04-http_echo_server.cc)
+
+# About http\_echo\_server
+
+It is an HTTP server that returns an HTML page, which displays the header data in the HTTP request sent by the browser.   
+The log of the program contains the client address and the sequence of the request (the number of requests on the current connection). When 10 requests are completed on the same connection, the server actively closes the connection.   
+The program exits normally after users press Ctrl-C, and all resources are completely reclaimed.
+
+# Creating and starting an HTTP server
+
+In this example, we use the default parameters of an HTTP server. It is very simple to create and start an HTTP server.
+
+~~~cpp
+WFHttpServer server(process);
+port = atoi(argv[1]);
+if (server.start(port) == 0)
+{
+    pause();
+    server.stop();
+}
+...
+~~~
+
+The procedure is too simple to explain. Please note that the start process is non-blocking, so please pause the program. Obviously you can start several server objects and then pause.   
+After a server is started, you can use **stop()** interface to shut down the server at any time. Stopping a server is non-violent and will be done until all the processing requests in the server are completed.   
+Therefore, **stop** is a blocking operation. If non-blocking shutdown is required, please use **shutdown+wait\_finish** interface.   
+There are several overloaded functions with **start()**. [WFServer.h](/src/server/WFServer.h) contains the following interfaces:
+
+~~~cpp
+class WFServerBase
+{
+public:
+    /* To start TCP server. */
+    int start(unsigned short port);
+    int start(int family, unsigned short port);
+    int start(const char *host, unsigned short port);
+    int start(int family, const char *host, unsigned short port);
+    int start(const struct sockaddr *bind_addr, socklen_t addrlen);
+
+    /* To start an SSL server */
+    int start(unsigned short port, const char *cert_file, const char *key_file);
+    int start(int family, unsigned short port,
+              const char *cert_file, const char *key_file);
+    int start(const char *host, unsigned short port,
+              const char *cert_file, const char *key_file);
+    int start(int family, const char *host, unsigned short port,
+              const char *cert_file, const char *key_file);
+    int start(const struct sockaddr *bind_addr, socklen_t addrlen,
+              const char *cert_file, const char *key_file);
+
+    /* For graceful restart. */
+    int serve(int listen_fd);
+    int serve(int listen_fd, const char *cert_file, const char *key_file);
+};
+~~~
+
+There interfaces are easy to understand. When you start an SSL server, the cert\_file and key\_file should be in PEM format.   
+The last two **serve()** interfaces have the parameter **listen\_fd**, which is used for graceful restart or for building a simple non-TCP (such as SCTP) server.   
+Please note that one server object corresponds to one **listen\_fd**. If  the server is running on both IPv4 and IPv6 protocols, you should:
+
+~~~cpp
+{
+    WFHttpServer server_v4(process);
+    WFHttpServer server_v6(process);
+    server_v4.start(AF_INET, port);
+    server_v6.start(AF_INET6, port);
+    ...
+    // now stop...
+    server_v4.shutdown();   /* shutdown() is nonblocking */
+    server_v6.shutdown();
+    server_v4.wait_finish();
+    server_v6.wait_finish();
+}
+~~~
+
+In the above code, the two servers cannot share the connection counter. Therefore, it is recommended to start the IPv6 server only, because the IPv6 server can accept IPv4 connection.
+
+# Business logic of an HTTP echo server
+
+When you build an HTTP server, you pass a process parameter, which is also an **std::function**, as defined below:
+
+~~~cpp
+using http_process_t = std::function<void (WFHttpTask *)>;
+using WFHttpServer = WFServer<protocol::HttpRequest, protocol::HttpResponse>;
+
+template<>
+WFHttpServer::WFServer(http_process_t proc) :
+    WFServerBase(&HTTP_SERVER_PARAMS_DEFAULT),
+    process(std::move(proc))
+{
+}
+~~~
+
+Actually, the type of **http\_proccess\_t** and the type of **http\_callback\_t** are exactly the same. Both are used to handle WFHttpTask.   
+The job of the server is to populate the response based on the request.   
+Similarly, we use an ordinary function to implement the process. The process iterates over the HTTP header of the request line by line and then writes them into an HTML page.
+
+~~~cpp
+void process(WFHttpTask *server_task)
+{
+    protocol::HttpRequest *req = server_task->get_req();
+    protocol::HttpResponse *resp = server_task->get_resp();
+    long seq = server_task->get_task_seq();
+    protocol::HttpHeaderCursor cursor(req);
+    std::string name;
+    std::string value;
+    char buf[8192];
+    int len;
+
+    /* Set response message body. */
+    resp->append_output_body_nocopy("<html>", 6);
+    len = snprintf(buf, 8192, "<p>%s %s %s</p>", req->get_method(),
+                   req->get_request_uri(), req->get_http_version());
+    resp->append_output_body(buf, len);
+
+    while (cursor.next(name, value))
+    {
+        len = snprintf(buf, 8192, "<p>%s: %s</p>", name.c_str(), value.c_str());
+        resp->append_output_body(buf, len);
+    }
+
+    resp->append_output_body_nocopy("</html>", 7);
+
+    /* Set status line if you like. */
+    resp->set_http_version("HTTP/1.1");
+    resp->set_status_code("200");
+    resp->set_reason_phrase("OK");
+
+    resp->add_header_pair("Content-Type", "text/html");
+    resp->add_header_pair("Server", "Sogou WFHttpServer");
+    if (seq == 9) /* no more than 10 requests on the same connection. */
+        resp->add_header_pair("Connection", "close");
+
+    // print log
+    ...
+}
+~~~
+
+You have learned most of the HttpMessage related operations. The only new operation here is **append\_output\_body()**.   
+Obviously, it is not very efficient for the users to generate a complete HTTP body and pass it to the framework. The user only needs to call the **append** interface to append the discrete data to the message block by block.   
+**append\_output\_body()** operation will move the data, and another interface with the suffix **\_nocopy** will directly use the reference to the pointer. Please do not make it point to the local variables when you use it.   
+[HttpMessage.h](../src/protocol/HttpMessage.h) contains the declaration of relevant calls. 
+
+~~~cpp
+class HttpMessage
+{
+public:
+    bool append_output_body(const void *buf, size_t size);
+    bool append_output_body_nocopy(const void *buf, size_t size);
+    ...
+    bool append_output_body(const std::string& buf);
+};
+~~~
+
+Once again, please note that when you use **append\_output\_body\_nocopy()**, the lifecycle of the data referenced by the buf must at least be extended to the callback of the task.   
+Another variable seq in the function is obtained by **server\_task->get\_task\_seq()**, which indicates the number of requests on the current connection, starting from 0.   
+In the program, the connection is forcibly closed after 10 requests are completed, thus:
+
+~~~cpp
+    if (seq == 9) /* no more than 10 requests on the same connection. */
+        resp->add_header_pair("Connection", "close");
+~~~
+
+You can also use **task->set\_keep\_alive()** to close the connection. However, for the connection using HTTP protocol, it is recommended to set the “close” option in HTTP header.   
+In this example, because the response page is very small, we didn't pay attention to the reply status. In the next tutorial **http\_proxy**, you will learn how to get the reply status.
@@ -0,0 +1,204 @@
+# Asynchronous server: http\_proxy
+
+# Sample code
+
+[tutorial-05-http\_proxy.cc](/tutorial/tutorial-05-http_proxy.cc)
+
+# About http\_proxy
+
+It is an HTTP proxy server. You can use it in a browser after proper configuration. It supports all HTTP methods.   
+As HTTPS proxy follows different principles, this example does not support HTTPS proxy. You can only browse HTTP websites.   
+In the implementation, this proxy must crawl the entire HTTP page and then forward it. Therefore, there will be noticeable latency when you upload/download a large file.
+
+# Changing server configuration
+
+In the previous example, we use the default parameters of an HTTP server. In this tutorial, we will made some changes and limit the size of the request so as to prevent malicious attack.
+
+~~~cpp
+int main(int argc, char *argv[])
+{
+    ...
+    struct WFServerParams params = HTTP_SERVER_PARAMS_DEFAULT;
+    params.request_size_limit = 8 * 1024 * 1024;
+
+    WFHttpServer server(&params, process);
+    if (server.start(port) == 0)
+    {
+        pause();
+        server.stop();
+    }
+    else
+    {
+        perror("cannot start server");
+        exit(1);
+    }
+
+    return 0;   
+}
+~~~
+Unlike the previous example, we pass an additional parameter to the server struct. Let’s see the configuration items in the HTTP server.   
+In [WFHttpServer.h](/src/server/WFHttpServer.h), the default parameters for an HTTP server include:
+~~~cpp
+static constexpr struct WFServerParams HTTP_SERVER_PARAMS_DEFAULT =
+{
+    .max_connections        =    2000,
+    .peer_response_timeout  =    10 * 1000,
+    .receive_timeout        =    -1,
+    .keep_alive_timeout     =    60 * 1000,
+    .request_size_limit     =    (size_t)-1,
+    .ssl_accept_timeout     =    10 * 1000,
+};
+~~~
+
+**max\_connections**: the maximum number of connections is 2000. When it is exceeded, the least recently used keep-alive connection will be closed. If there is no keep-alive connection, the server will refuse new connections.  
+**peer\_response\_timeout**: set the maximum duration for reading or sending out a block of data. The default setting is 10 seconds.   
+**receive\_timeout**: set the maximum duration for receiving a complete request; -1 means unlimited time.   
+**keep\_alive\_timeout**: set the maximum duration for maintaining a connection. The default setting is 1 minute.   
+**request\_size\_limit**: set the maximum size of a request packet. The default setting is unlimited packet size.   
+**ssl\_accept\_timeout**: set the maximum duration for an SSL handshake. The default setting is 10 seconds.   
+There is no **send\_timeout** in the parameters. **send\_timeout** sets the timeout for sending a complete response. This parameter should be determined according to the size of the response packet.
+
+# Business logic of a proxy server
+
+Essentially, this proxy server forwards a user's request intactly to the corresponding web server, and then forwards the reply from the web server intactly to the user. 
+In the request sent by a browser to the proxy, the Request URL contains scheme, host and port, which should be removed before forwarding.   
+For example, when the browser visits `http://www.sogou.com/`, the first line of the request sent by the browser to the proxy is:
+ `GET` `http://www.sogou.com/` `HTTP/1.1`  
+which should be rewritten as:  
+`GET` `/` `HTTP/1.1`
+
+~~~cpp
+void process(WFHttpTask *proxy_task)
+{
+    auto *req = proxy_task->get_req();
+    SeriesWork *series = series_of(proxy_task);
+    WFHttpTask *http_task; /* for requesting remote webserver. */
+
+    tutorial_series_context *context = new tutorial_series_context;
+    context->url = req->get_request_uri();
+    context->proxy_task = proxy_task;
+
+    series->set_context(context);
+    series->set_callback([](const SeriesWork *series) {
+        delete (tutorial_series_context *)series->get_context();
+    });
+
+    http_task = WFTaskFactory::create_http_task(req->get_request_uri(), 0, 0,
+                                                http_callback);
+
+    const void *body;
+    size_t len;
+
+    /* Copy user's request to the new task's reuqest using std::move() */
+    req->set_request_uri(http_task->get_req()->get_request_uri());
+    req->get_parsed_body(&body, &len);
+    req->append_output_body_nocopy(body, len);
+    *http_task->get_req() = std::move(*req);
+
+    /* also, limit the remote webserver response size. */
+    http_task->get_resp()->set_size_limit(200 * 1024 * 1024);
+
+    *series << http_task;
+}
+~~~
+
+The above contains the entire content of the process. It first parses the struct of an HTTP request sent by a web server.   
+**req->get\_request\_uri()** is used to get the complete URL of the request sent by a browser. And then build a HTTP task to the server based on this URL.   
+Both the retry times and the redirection times of this HTTP task is 0, because the redirection is handled by the browser and the browser will be resend the request when it meets 302, etc.
+
+~~~cpp
+    req->set_request_uri(http_task->get_req()->get_request_uri());
+    req->get_parsed_body(&body, &len);
+    req->append_output_body_nocopy(body, len);
+    *http_task->get_req() = std::move(*req);
+~~~
+
+In fact, the above four lines generates a HTTP request to the web server. req is the received HTTP request, and it will be moved directly to the new request via **std::move()**.   
+The first line removes the `http://host:port` in the request\_uri and keeps the part after the path.   
+The second line and the third line specify the parsed HTTP body as the HTTP body for output. The reason for this operation is that in the HttpMessage implementation, the http body obtained by parsing and the http body to send out are two fields, so we need to simply set it here, without copying the memory.   
+The fourth line transfers the request content to the request sent to the web server at one time. After the HTTP request is constructed, the request is placed at the end of the current series, and the process function ends.
+
+# Principles behind an asynchronous server
+
+Obviously, the process function is only part of the proxy logic. We also need to handle the HTTP response returned from the web server and generates the response for the browser.   
+In the example of echo server, we populate the response page directly without network communication. However, in the proxy server, we have to wait for the response from the web server.   
+Of course, we can occupy the thread of this process function and wait for the returned result, but this synchronous waiting mode is obviously not desirable.   
+Thus, it is better that we reply to the user's request asynchronously after receiving the results for the request, and no thread is occupied while we are waiting for the result.   
+Therefore,  we set a context for the current series in the head of the process, which contains the proxy\_task itself. In this way, we can populate the results asynchronously.
+
+~~~cpp
+struct tutorial_series_context
+{
+    std::string url;
+    WFHttpTask *proxy_task;
+    bool is_keep_alive;
+};
+
+void process(WFHttpTask *proxy_task)
+{
+    SeriesWork *series = series_of(proxy_task);
+    ...
+    tutorial_series_context *context = new tutorial_series_context;
+    context->url = req->get_request_uri();
+    context->proxy_task = proxy_task;
+
+    series->set_context(context);
+    series->set_callback([](const SeriesWork *series) {
+        delete (tutorial_series_context *)series->get_context();
+    });
+    ...
+}
+~~~
+
+In the previous client example, we said that any running task is in a series, and the server task is no exception.   
+Thus, we can get the current series and set the context. In which the URL is mainly used for the subsequent logs, and the proxy\_task is the main content, which is used for resp later.   
+Next, Let’s see how to handle the responses from the  web server.
+
+~~~cpp
+void http_callback(WFHttpTask *task)
+{
+    int state = task->get_state();
+    auto *resp = task->get_resp();
+    SeriesWork *series = series_of(task);
+    tutorial_series_context *context =
+        (tutorial_series_context *)series->get_context();
+    auto *proxy_resp = context->proxy_task->get_resp();
+
+    ...
+    if (state == WFT_STATE_SUCCESS)
+    {
+        const void *body;
+        size_t len;
+
+        /* set a callback for getting reply status. */
+        context->proxy_task->set_callback(reply_callback);
+
+        /* Copy the remote webserver's response, to proxy response. */
+        if (resp->get_parsed_body(&body, &len))
+            resp->append_output_body_nocopy(body, len);
+        *proxy_resp = std::move(*resp);
+        ...
+    }
+    else
+    {
+        // return a "404 Not found" page
+        ...
+    }
+}
+~~~
+
+Here we focus on the successful cases only. If the proxy gets a complete HTTP page from the web server, no matter what the return code is, it is considered a success. All failure will simply return a 404 page. 
+Because the data returned to the user may be very large, the maximum size is set to 200MB in this example. Therefore, unlike the previous examples, we need to check the success/failure status of the reply.   
+The type of an HTTP server task is identical to the type of an HTTP client task created by ourselves. Both are WFHttpTask. The difference is that a server task is created by the framework, and its callback is initially empty.   
+The callback of a server task is the same as that of a client. Both are called after an HTTP interaction is completed. Therefore, for all server tasks, the callback is called after the reply is completed.   
+The following three lines of code are explained before. They transfer the response packets from the web server to the proxy response packets without copying.   
+After the **http\_callback** function is ended, the reply to the browser is sent out. Everything is done asynchronously.   
+The remaining function **reply\_callback()**  is used just to print some logs here. The proxy task will be automatically deleted after this callback is finished.   
+Finally, the context is destroyed in the callback of the series.
+
+# Timing of a server reply
+
+Please note that the reply message is sent automatically after all other tasks in the series are finished, so there is no **task->reply()** interface.   
+However, there is a **task->noreply()**. If this interface is called for the server task, the connection will be closed directly at the original reply time. But the callback will still be called (its state is NOREPLY).   
+In the callback of a server task, you can also call **series\_of()** to get the series of that server task. Then, you can still add new tasks to this series, although the reply has finished.   
+If you needs to continue to add tasks to the series, please see the instructions in [About exit](/docs/en/about-exit.md), because doing so may cause unfinished tasks after the server is shut down.
@@ -0,0 +1,136 @@
+# A simple parallel wget: parallel\_wget
+
+# Sample code
+
+[tutorial-06-parallel\_wget.cc](/tutorial/tutorial-06-parallel_wget.cc)
+
+# About parallel\_wget
+
+It is our first example on parallel tasks.   
+The program reads multiple HTTP URLs (separated by spaces) from the command line, crawls these URLs in parallel, and prints the crawled results to the standard output according to the input order.
+
+# Creating a parallel task
+
+In the previous example, you have already learned the SeriesWork class.
+
+* SeriesWork consists of a series of tasks that are executed sequentially. The series finishes when all its tasks finish.
+* ParallelWork class, corresponding to the SeriesWork, consists of multiple series that are executed in parallel. The parallel work finishes when all its series finish.
+* ParallelWork is a task.
+
+According to the above definition, you can generate any complex workflow dynamically or statically.   
+The Workflow class has two interfaces for generating parallel tasks:
+
+~~~cpp
+class Workflow
+{
+    ...
+public:
+    static ParallelWork *
+    create_parallel_work(parallel_callback_t callback);
+
+    static ParallelWork *
+    create_parallel_work(SeriesWork *const all_series[], size_t n,
+                         parallel_callback_t callback);
+
+    ...
+};
+~~~
+
+The first interface creates an empty parallel task, and the second interface creates parallel tasks with a series array.   
+Before you start the parallel work, you can use  **add\_series()** interface of the ParallelWork to add series to the parallel tasks generated by either interface.   
+In the sample code, we create an empty parallel task and then add the series one by one.
+
+~~~cpp
+int main(int argc, char *argv[])
+{
+    ParallelWork *pwork = Workflow::create_parallel_work(callback);
+    SeriesWork *series;
+    WFHttpTask *task;
+    HttpRequest *req;
+    tutorial_series_context *ctx;
+    int i;
+
+    for (i = 1; i < argc; i++)
+    {
+        std::string url(argv[i]);
+        ...
+        task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,
+            [](WFHttpTask *task)
+        {
+            // store resp to ctx.
+        });
+
+        req = task->get_req();
+        // add some headers.
+        ...
+
+        ctx = new tutorial_series_context;
+        ctx->url = std::move(url);
+        series = Workflow::create_series_work(task, nullptr);
+        series->set_context(ctx);
+        pwork->add_series(series);
+    }
+    ...
+}
+~~~
+
+You can see that we first create an HTTP task in the code, but the HTTP task cannot be directly added to the parallel task, so we need to use it to create a series first.   
+Each series has its own context, which is used to save the URL and the crawled results. You can learn related methods in our previous examples.
+
+# Saving and using the crawled results
+
+The callback of an HTTP task is a simple lambda function, which saves the crawled result in its own series context, so that it can be retrieved by the parallel task.
+
+~~~cpp
+    task = WFTaskFactory::create_http_task(url, REDIRECT_MAX, RETRY_MAX,
+        [](WFHttpTask *task)
+    {
+        tutorial_series_context *ctx =
+            (tutorial_series_context *)series_of(task)->get_context();
+        ctx->state = task->get_state();
+        ctx->error = task->get_error();
+        ctx->resp = std::move(*task->get_resp());
+    });
+~~~
+
+This is necessary, because HTTP tasks will be recycled after the callback, so we have to use **std::move()** to move the resp.   
+In the callback of parallel tasks, we can easily get the results:
+
+~~~cpp
+void callback(const ParallelWork *pwork)
+{
+    tutorial_series_context *ctx;
+    const void *body;
+    size_t size;
+    size_t i;
+
+    for (i = 0; i < pwork->size(); i++)
+    {
+        ctx = (tutorial_series_context *)pwork->series_at(i)->get_context();
+        printf("%s\n", ctx->url.c_str());
+        if (ctx->state == WFT_STATE_SUCCESS)
+        {
+            ctx->resp.get_parsed_body(&body, &size);
+            printf("%zu%s\n", size, ctx->resp.is_chunked() ? " chunked" : "");
+            fwrite(body, 1, size, stdout);
+            printf("\n");
+        }
+        else
+            printf("ERROR! state = %d, error = %d\n", ctx->state, ctx->error);
+
+        delete ctx;
+    }
+}
+~~~
+
+Here, you can see the two new interfaces of ParallelWork, **size()** and **series\_at(i)**, which are used to obtain the number of the series in parallel and the ith parallel series respectively.   
+You can use **series->get\_context()** to get the context of the series and print out the results.The printing order must be the same as with the order you add the series into the work.   
+In this example, there is no other work after the parallel tasks finish.   
+As we said above, ParallelWork is a kind of tasks, so you can use **series\_of()** to get its series and add a new task.   
+However, if the crawled results are used in the new task, you need to use **std::move()** to move the data to the context of the series of that parallel task.
+
+# Starting a parallel task
+
+As a parallel task is a kind of tasks, so there is nothing special in starting a parallel task. You can call **start()** directly, or you can use it to build or start a series.   
+In this example, we start a series, wake up the main process in the callback of this series, and exit the program normally.   
+We can also wake up the main process in the callback of parallel tasks, and there is little difference in the program behaviors. However, it is more formal to wake up the main process in the callback of the series.
@@ -0,0 +1,152 @@
+# Using the built-in algorithm factory: sort\_task
+
+# Sample code
+
+[tutorial-07-sort\_task.cc](/tutorial/tutorial-07-sort_task.cc)
+
+# About sort\_task
+
+The program reads a number n from the command line,  sorts the random n positive integers in ascending order, and then sorts the results in descending order. You can add the second parameter "p” to the program, and then it can be sorted in parallel. For example:  
+ ./sort\_task 100000000 p   
+The above command will sort 100 million integers in ascending order and then in descending order. The two sortings are done in parallel respectively.
+
+# About computing tasks
+
+Computing tasks (or thread tasks) is a very important function in the framework. When you use the task flow, it is not recommended to directly perform very complicated computation in the callback.   
+All the computations that consume a lot of CPU time can be encapsulated into computing tasks and handed over to the system for scheduling. There is no difference in the usage between computing tasks and networking tasks.   
+The algorithm factory of the system provides some common computing tasks, such as sorting, merging and so on. You can also easily define your own computing tasks.
+
+# Creating sorting tasks in ascending order 
+
+~~~cpp
+int main(int argc, char *argv[])
+{
+    ...
+    WFSortTask<int> *task;
+    if (use_parallel_sort)
+        task = WFAlgoTaskFactory::create_psort_task("sort", array, end, callback);
+    else
+        task = WFAlgoTaskFactory::create_sort_task("sort", array, end, callback);
+    ...
+    task->start();
+    ...
+}
+~~~
+
+Unlike WFHttpTask or WFRedisTask, the sorting task has one more template parameter to represent the type of array data to be sorted.   
+**create\_sort\_task** and **create\_psort\_task** produce a common sorting task and a parallel sorting task respectively.   
+Their ****parameters and return values are the same.****   
+The only thing that needs special explanation is the first parameter "sort", which is the name of the computation queue. It is used to instruct the internal task scheduling. The latter part in this article explains the usage of the queue name.   
+There is no difference in the starting methods and usage between computing tasks and networking tasks.
+
+# Handling results
+
+Like a networking task, the results are handled in the callback. In this example, the ascending sorting is followed by one descending sorting.
+
+~~~cpp
+using namespace algorithm;
+
+void callback(void SortTask<int> *task)
+{
+    SortInput<T> *input = task->get_input();
+    int *first = input->first;
+    int *last = input->last;
+
+    // print result
+    ...
+    
+    if (task->user_data == NULL)
+    {
+        auto cmp = [](int a1, int a2){ return a2 < a1; };
+        WFSortTask<int> *reverse;
+
+        if (use_parallel_sort)
+            reverse = WFAlgoTaskFactory::create_psort_task("sort", first, last, cmp, callback);
+        else
+            reverse = WFAlgoTaskFactory::create_psort_task("sort", first, last, cmp, callback);
+            
+        reverse->user_data = (void *)1; /* as a flag */
+        series_of(task)->push_back(reverse);
+    }
+    else
+    {
+        // all done. Signal main thread to exit.
+        ... 
+    }
+}
+~~~
+
+You can use **get\_input ()** interface of a computing task to get the input data, and use **get\_output ()** to get the output data. For sorting tasks, the input and output are of the same type, and the content are exactly the same.   
+[WFAlgoTaskFactory.h](/src/factory/WFAlgoTaskFactory.h) contains the definitions of the input and output of sorting tasks.
+
+~~~cpp
+namespace algorithm
+{
+
+template <typename T>
+struct SortInput
+{
+    T *first;
+    T *last;
+};
+
+template <typename T>
+using SortOutput = SortInput<T>;
+
+}
+
+template <typename T>
+using WFSortTask = WFThreadTask<algorithm::SortInput<T>,
+                                algorithm::SortOutput<T>>;
+
+template <typename T>
+using sort_callback_t = std::function<void (WFSortTask<T> *)>;
+
+~~~
+
+Obviously, the first and last in the input or output mean the head pointer and the tail pointer of the array to be sorted.   
+Next, we will create a descending sorting task. In this case, we need to pass in a comparison function.
+
+~~~cpp
+        auto cmp = [](int a1, int a2)->bool{ return a2 < a1; };
+        reverse = WFAlgoTaskFactory::create_sort_task("sort", first, last, cmp, callback);
+~~~
+
+Our usage differs slightly from **std::sort()**. Our first and last are pointers, not iterators.   
+Similarly, you can use **create\_psort\_task()** to create a parallel sorting task. And the use of series in the sorting task is no different from that in the networking task.
+
+# About the configuration of the computing threads
+
+If you don't make any configuration, the calculation scheduler will set the number of threads as the number of the CPU cores in the machine. You can change the value with the following method:
+
+~~~cpp
+#include "workflow/WFGlobal.h"
+
+int main()
+{
+    struct WFGlobalSettings settings = GLOBAL_SETTINGS_DEFAULT;
+    settings.compute_threads = 16;
+    WORKFLOW_library_init(&settings);
+    ...
+}
+~~~
+
+With the above configuration, the system will create 16 threads for computations.
+
+# About the parallel sorting algorithm
+
+The built-in parallel sorting algorithm use block+two-way merge. Its space complexity is O(1).   
+The algorithm uses globally configured computing threads for computation, but at most 128 threads can be used. Because no extra space is used, the speedup ratio will be smaller than the number of threads, and the average CPU usage will be smaller.   
+For the detailed implementation, please see [WFAlgoTaskFactory.inl](/src/factory/WFAlgoTaskFactory.inl).
+
+# About the name of a calculation task queue
+
+The computing task does not have priority levels. The only thing that can affect the scheduling order is the queue name of a computing task. In this example, the queue name is a string "sort".   
+To name a queue is very simple. Please note the following items:
+
+* The queue name is a static string, and new queue names cannot be generated infinitely. For example, you cannot generate the queue name according to the request id, because each queue is allocated a small block of resources internally.
+* If the computing threads are not 100% occupied, all tasks are started in real time, and the queue names have no effect.
+* If there are multiple computing steps in a service flow and they are interspersed among multiple network communications, you can simply give each calculation step a name, which is better than using one name as a whole.
+  * If all computing tasks use the same name, the scheduling order of all tasks is consistent with the  order of submission, which will affect the average response time in some scenarios.
+  * If each kind of computing task has an independent name, it means that they are scheduled fairly. And the same kind of tasks are scheduled sequentially, the practical effect is better.
+* In a word, unless the computing load of the machine is already very heavy, you do not need to pay special attention to the queue name and you can just give each kind of task a name.
@@ -0,0 +1,169 @@
+# User-defined computing tasks: matrix\_multiply
+
+# Sample code
+
+[tutorial-08-matrix\_multiply.cc](/tutorial/tutorial-08-matrix_multiply.cc)
+
+# About matrix\_multiply
+
+The program multiplies two matrices and prints the results on the screen.   
+The main purpose of the example is to show how to implement a user-defined CPU computing task.
+
+# About computing tasks
+
+You need to provide three types of basic information when you define a computer task: INPUT, OUTPUT, and routine.   
+INPUT and OUTPUT are two template parameters, which can be of any type. routine means the process from INPUT to OUTPUT, which is defined as follows:
+
+~~~cpp
+template <class INPUT, class OUTPUT>
+class __WFThreadTask
+{
+    ...
+    std::function<void (INPUT *, OUTPUT *)> routine;
+    ...
+};
+~~~
+
+It can be seen that routine is a simple computing process from INPUT to OUTPUT. The INPUT pointer is not necessarily be const, but you can also pass the function of const INPUT \*.   
+For example, to implement an adding task, you can:
+
+~~~cpp
+struct add_input
+{
+    int x;
+    int y;
+};
+
+struct add_ouput
+{
+    int res;
+};
+
+void add_routine(const add_input *input, add_output *output)
+{
+    output->res = input->x + input->y;
+}
+
+typedef WFThreadTask<add_input, add_output> add_task;
+~~~
+
+In the example of matrix multiplication, the input is two matrices and the output is one matrix. They are defined as follows:
+
+~~~cpp
+namespace algorithm
+{
+
+using Matrix = std::vector<std::vector<double>>;
+
+struct MMInput
+{
+    Matrix a;
+    Matrix b;
+};
+
+struct MMOutput
+{
+    int error;
+    size_t m, n, k;
+    Matrix c;
+};
+
+void matrix_multiply(const MMInput *in, MMOutput *out)
+{
+    ...
+}
+
+}
+~~~
+
+As the input matrices may be illegal in matrix multiplication, so there is an error field in the output to indicate errors.
+
+# Generating computing tasks
+
+After you define the types of input and output and the algorithm process, you can use  WFThreadTaskFactory  to generate a computing task.   
+In [WFTaskFactory.h](/src/factory/WFTaskFactory.h), the computing task factory is defined as follows:
+
+~~~cpp
+template <class INPUT, class OUTPUT>
+class WFThreadTaskFactory
+{
+private:
+    using T = WFThreadTask<INPUT, OUTPUT>;
+
+public:
+    static T *create_thread_task(const std::string& queue_name,
+                                 std::function<void (INPUT *, OUTPUT *)> routine,
+                                 std::function<void (T *)> callback);
+    ...
+};
+~~~
+
+Slightly different from the previous network factory class or the algorithm factory class, this factory requires two template parameters: INPUT and OUTPUT.   
+queue\_name is explained in the previous example. routine is the computation process, and callback means the callback.   
+In our example, we see this call:
+
+~~~cpp
+using MMTask = WFThreadTask<algorithm::MMInput,
+                            algorithm::MMOutput>;
+
+using namespace algorithm;
+
+int main()
+{
+    typedef WFThreadTaskFactory<MMInput, MMOutput> MMFactory;
+    MMTask *task = MMFactory::create_thread_task("matrix_multiply_task",
+                                                 matrix_multiply,
+                                                 callback);
+
+    MMInput *input = task->get_input();
+
+    input->a = {{1, 2, 3}, {4, 5, 6}};
+    input->b = {{7, 8}, {9, 10}, {11, 12}};
+    ...
+}
+~~~
+
+After the task is generated, use **get\_input()** interface to get the pointer of the input data. This is similar to the **get\_req()** in a network task.   
+The start and the end of a task is the same as those of a network task. Similarly, the callback is very simple:
+
+~~~cpp
+void callback(MMTask *task)     // MMtask = WFThreadTask<MMInput, MMOutput>
+{
+    MMInput *input = task->get_input();
+    MMOutput *output = task->get_output();
+
+    assert(task->get_state() == WFT_STATE_SUCCESS);
+
+    if (output->error)
+        printf("Error: %d %s\n", output->error, strerror(output->error));
+    else
+    {
+        printf("Matrix A\n");
+        print_matrix(input->a, output->m, output->k);
+        printf("Matrix B\n");
+        print_matrix(input->b, output->k, output->n);
+        printf("Matrix A * Matrix B =>\n");
+        print_matrix(output->c, output->m, output->n);
+    }
+}
+~~~
+
+You can ignore the the possibility of failure in the ordinary computing tasks, and the end state is always SUCCESS.   
+The callback simply prints out the input and the output. If the input data are illegal, the error will be printed out.
+
+# Symmetry of the algorithm and the protocol
+
+In our system, algorithms and protocols are highly symmetrical on a very abstract level.   
+There are thread tasks with user-defined algorithms,  obviously there are network tasks with user-defined protocols.   
+A user-defined algorithm requires the user to provide the algorithm procedure, and a user-defined protocol requires the user to provide the procedure of serialization and deserialization. You can see an introduction in [Simple client/server based on user-defined protocols](/tutorial-10-user_defined_protocol.md)   
+For the user-defined algorithms and the user-defined protocols, both must be very pure .   
+For example, an algorithm is just a conversion procedure from INPUT to OUPUT, and the algorithm does not know the existence of task, series, etc.   
+The implementation of an HTTP protocol only cares about serialization and deserialization, and does not need to care about the task definition. Instead, the HTTP protocol is referred to in an http task.
+
+# Composite features of thread tasks and network tasks
+
+In this example, we use WFThreadTaskFactory to build a thread task. This is the simplest way to get a computing task, and it is sufficient in most cases.   
+Similarly, you can simply define a server and a client with a user-defined protocol.   
+However, in the previous example, we can use the algorithm factory to generate a parallel sorting task, which is obviously not possible with a routine.   
+For a network task, such as a Kafka task, interactions with several machines may be required to get results, but it is completely transparent to users.   
+Therefore, our tasks are composite. If you use our framework skillfully, you can design many composite components.
@@ -0,0 +1,153 @@
+# Http server with file IO: http\_file\_server
+
+# Sample code
+
+[tutorial-09-http\_file\_server.cc](/tutorial/tutorial-09-http_file_server.cc)
+
+# About http\_file\_server
+
+http\_file\_server is a web server. You can start a web server after specifying the startup port and the root path (the default setting is the current path).   
+You can also specify a certificate file and a key file in PEM format to start an HTTPS web server.   
+The program mainly demonstrates how to use disk IO tasks. In the Linux system, we use the aio interface in the kernel of Linux, and the file reading is completely asynchronous.
+
+# Starting a server
+
+For starting a server, the steps are almost the same as those when starting an echo server or an HTTP proxy. There is one more way to start an SSL server here:
+
+~~~cpp
+class WFServerBase
+{
+    ...
+    int start(unsigned short port, const char *cert_file, const char *key_file);
+    ...
+};
+~~~
+
+In other words, you can specify a cert file and a key file in PEM format to start an SSL server.   
+In addition, when you define a server, you can use **std::bind()** to bind a root parameter to the process. The root parameter means the root path of the service.
+
+~~~cpp
+void process(WFHttpTask *server_task, const char *root)
+{
+    ...
+}
+
+int main(int argc, char *argv[])
+{
+    ...
+    const char *root = (argc >= 3 ? argv[2] : ".");
+    auto&& proc = std::bind(process, std::placeholders::_1, root);
+    WFHttpServer server(proc);
+
+    // start server
+    ...
+}
+~~~
+
+# Handling requests
+
+Similar to http\_proxy, no threads are occupied in file reading. Instead, an asynchronous task is generated to read files, and a reply to the request is generated after the reading is completed.   
+Please note again that the complete reply data should be read into the memory before the reply message is sent. Therefore, it is not suitable for transferring very large files.
+
+~~~cpp
+void process(WFHttpTask *server_task, const char *root)
+{
+    // generate abs path.
+    ...
+
+    int fd = open(abs_path.c_str(), O_RDONLY);
+    if (fd >= 0)
+    {
+        size_t size = lseek(fd, 0, SEEK_END);
+        void *buf = malloc(size);        /* As an example, assert(buf != NULL); */
+        WFFileIOTask *pread_task;
+
+        pread_task = WFTaskFactory::create_pread_task(fd, buf, size, 0,
+                                                      pread_callback);
+        /* To implement a more complicated server, please use series' context
+         * instead of tasks' user_data to pass/store internal data. */
+        pread_task->user_data = resp;    /* pass resp pointer to pread task. */
+        server_task->user_data = buf;    /* to free() in callback() */
+        server_task->set_callback([](WFHttpTask *t){ free(t->user_data); });
+        series_of(server_task)->push_back(pread_task);
+    }
+    else
+    {
+        resp->set_status_code("404");
+        resp->append_output_body("<html>404 Not Found.</html>");
+    }
+}
+~~~
+
+Unlike http\_proxy that generates a new HTTP client task, here a pread task is generated by the factory.   
+[WFAlgoTaskFactory.h](/src/factory/WFTaskFactory.h) contains the definitions of relevant interfaces.
+
+~~~cpp
+struct FileIOArgs
+{
+    int fd;
+    void *buf;
+    size_t count;
+    off_t offset;
+};
+
+...
+using WFFileIOTask = WFFileTask<struct FileIOArgs>;
+using fio_callback_t = std::function<void (WFFileIOTask *)>;
+...
+
+class WFTaskFactory
+{
+public:
+    ...
+    static WFFileIOTask *create_pread_task(int fd, void *buf, size_t count, off_t offset,
+                                           fio_callback_t callback);
+
+    static WFFileIOTask *create_pwrite_task(int fd, void *buf, size_t count, off_t offset,
+                                            fio_callback_t callback);
+    ...
+};
+~~~
+
+Both pread and pwrite return WFFileIOTask. We do not distinguish between sort and psort, and we do not distinguish between client and server task. They all follow the same principle.   
+In addition to these two interfaces, preadv and pwritev return WFFileVIOTask; fsync and fdsync return WFFileSyncTask. You can see the details in the header file.   
+Currently, our interface requires users to open and close fd by themselves. We are developing a set of file management interfaces. In the future, users only need to pass file names, which is more friendly to cross-platform applications.   
+The example uses the user\_data field of the task to save the global data of the service. For larger services, we recommend to use series context. You can see the [proxy examples](/tutorial/tutorial-05-http_proxy.cc) for details.
+
+# Handling file reading results
+
+~~~cpp
+using namespace protocol;
+
+void pread_callback(WFFileIOTask *task)
+{
+    FileIOArgs *args = task->get_args();
+    long ret = task->get_retval();
+    HttpResponse *resp = (HttpResponse *)task->user_data;
+
+    close(args->fd);
+    if (ret < 0)
+    {
+        resp->set_status_code("503");
+        resp->append_output_body("<html>503 Internal Server Error.</html>");
+    }
+    else /* Use '_nocopy' carefully. */
+        resp->append_output_body_nocopy(args->buf, ret);
+}
+~~~
+
+Use **get\_args()** of the file task to get the input parameters. Here it is a FileIOArgs struct.   
+Use **get\_retval()** to get the return value of the operation. If ret < 0, the task fails. Otherwise, the ret is the size of the read data.   
+In the file task, ret < 0 and task->get\_state()! = WFT\_STATE\_SUCCESS are completely equivalent.   
+The memory of the buf domain is managed by ourselves. You can use  **append\_output\_body\_nocopy()** to pass that memory to resp.   
+After the reply is completed, we will **free()** this block of memory with this line in the process:   
+server\_task->set\_callback(\[](WFHttpTask \*t){ free(t->user\_data); });
+
+# About the implementation of the file IO
+
+Linux operating system supports a set of asynchronous IO system calls with high efficiency and very little CPU occupation. If you use our framework in a Linux system, this set of interfaces are used by default.   
+We have implemented a set of posix aio interfaces to support other UNIX systems, and used the sigevent notification method of threads, but it is no longer in use because of its low efficiency.   
+Currently, for non-Linux systems, asynchronous IO is always simulated by multi-threading. When an IO task arrives, a thread is created in real time to execute IO tasks, and then a callback is used to return to the handler thread pool.   
+Multi-threaded IO is also the only choice in macOS, because macOS does not have good sigevent support and posix aio will not work in macOS.   
+Multi-threaded IO does not support preadv and pwritev tasks. When these two tasks are created and run, you will get an ENOSYS error in the callback.   
+Some UNIX systems do not support fdatasync. In this case, an fdsync task is equivalent to an fsync task.
@@ -29,8 +29,8 @@ using http_callback_t = std::function<void (WFHttpTask *)>;
 ~~~
 说白了，就是一个参数为Task本身，没有返回值的函数。这个callback可以传NULL，表示无需callback。我们一切任务的callback都是这个风格。  
 需要说明的是，所有工厂函数不会返回失败，所以不用担心task为空指针，哪怕是url不合法。一切错误都在callback再处理。  
-task->get_req()函数得到任务的request，默认是GET方法，HTTP/1.1，长连接。框架会自动加上request_uri，Host等。
-并在发送前根据需要加上Content-Length或Connection这些http header。用户也要以通过add_header_pair()方法添加自己的header。
+task->get_req()函数得到任务的request，默认是GET方法，HTTP/1.1，长连接。框架会自动加上request_uri，Host等。  
+框架会在发送前根据需要自动加上Content-Length或Connection这些http header。用户也可以通过add_header_pair()方法添加自己的header。  
 关于http消息的更多接口，可以在[HttpMessage.h](../src/protocol/HttpMessage.h)中查看。  
 task->start()启动任务，非阻塞，并且不会失败。之后callback必然会在被调用。因为异步的原因，start()以后显然不能再用task指针了。  
 为了让示例尽量简单，start()之后调用pause()防止程序退出，用户需要Ctrl-C结束程序。
@@ -81,18 +81,23 @@ CommMessageOut *ComplexHttpTask::message_out()
 			strncasecmp((const char *)header.value, "identity", 8) == 0);
 	}

-	const char *method = req->get_method();
-	if (!chunked && (strcmp(method, HttpMethodPost) == 0 || strcmp(method, HttpMethodPut) == 0))
+	if (!chunked)
 	{
-		header.name = "Content-Length";
-		header.name_len = 14;
-		req_cursor.rewind();
-		if (!req_cursor.find(&header))
+		size_t body_size = req->get_output_body_size();
+		const char *method = req->get_method();
+
+		if (body_size != 0 || strcmp(method, "POST") == 0 || strcmp(method, "PUT") == 0)
 		{
-			size_t body_size = req->get_output_body_size();
-			char buf[32];
-			snprintf(buf, 32, "%zu", body_size);
-			req->add_header_pair("Content-Length", buf);
+			header.name = "Content-Length";
+			header.name_len = 14;
+			req_cursor.rewind();
+			if (!req_cursor.find(&header))
+			{
+				char buf[32];
+				header.value = buf;
+				header.value_len = sprintf(buf, "%zu", body_size);
+				req->add_header(&header);
+			}
 		}
 	}

@@ -449,7 +454,7 @@ CommMessageOut *WFHttpServerTask::message_out()

 	size_t body_size = resp->get_output_body_size();

-	if (!chunked || body_size == 0)
+	if (!chunked)
 	{
 		header.name = "Content-Length";
 		header.name_len = 14;
@@ -457,8 +462,9 @@ CommMessageOut *WFHttpServerTask::message_out()
 		if (!resp_cursor.find(&header))
 		{
 			char buf[32];
-			snprintf(buf, 32, "%zu", body_size);
-			resp->add_header_pair("Content-Length", buf);
+			header.value = buf;
+			header.value_len = sprintf(buf, "%zu", body_size);
+			resp->add_header(&header);
 		}
 	}

@@ -226,6 +226,7 @@ int ComplexMySQLTask::keep_alive_timeout()
 		{
 			this->resp = std::move(*static_cast<MySQLResponse *>(resp));
 			succ_ = false;
+			return 0;
 		}
 		else
 		{
@@ -1686,6 +1686,11 @@ int Communicator::sleep(SleepSession *session)
 	return -1;
 }

+int Communicator::is_handler_thread()
+{
+	return thrdpool_in_pool(this->thrdpool);
+}
+
 extern "C" void __thrdpool_schedule(const struct thrdpool_task *, void *,
 									thrdpool_t *);

@@ -27,10 +27,7 @@
 #include <pthread.h>
 #include <openssl/ssl.h>
 #include "list.h"
-#include "msgqueue.h"
-#include "thrdpool.h"
 #include "poller.h"
-#include "mpoller.h"

 class CommConnection
 {
@@ -263,13 +260,13 @@ public:
 	void io_unbind(IOService *service);

 public:
+	int is_handler_thread();
 	int increase_handler_thread();
-	int is_handler_thread() { return thrdpool_in_pool(this->thrdpool); }

 private:
-	msgqueue_t *queue;
-	mpoller_t *mpoller;
-	thrdpool_t *thrdpool;
+	struct __mpoller *mpoller;
+	struct __msgqueue *queue;
+	struct __thrdpool *thrdpool;
 	int stop_flag;

 private:
@@ -22,7 +22,6 @@
 #include <stddef.h>
 #include <pthread.h>
 #include "list.h"
-#include "thrdpool.h"

 class ExecQueue
 {
@@ -69,7 +68,7 @@ public:
 	int request(ExecSession *session, ExecQueue *queue);

 private:
-	thrdpool_t *thrdpool;
+	struct __thrdpool *thrdpool;

 private:
 	static void executor_thread_routine(void *context);
@@ -20,6 +20,7 @@
 #define _SLEEPREQUEST_H_

 #include "SubTask.h"
+#include "Communicator.h"
 #include "CommScheduler.h"

 class SleepRequest : public SubTask, public SleepSession
@@ -16,6 +16,16 @@
  Author: Xie Han (xiehan@sogou-inc.com)
 */

+/*
+ * This message queue originates from the project of Sogou C++ Workflow:
+ * https://github.com/sogou/workflow
+ *
+ * The idea of this implementation is quite simple and abvious. When the
+ * get_list is not empty, consumer takes a message. Otherwise the consumer
+ * waits till put_list is not empty, and swap two lists. This method performs
+ * well when the queue is very busy, and the number of consumers is big.
+ */
+
 #include <errno.h>
 #include <stdlib.h>
 #include <pthread.h>
@@ -48,6 +48,8 @@ static void *__thrdpool_routine(void *arg)
 	thrdpool_t *pool = (thrdpool_t *)arg;
 	struct list_head **pos = &pool->task_queue.next;
 	struct __thrdpool_task_entry *entry;
+	void (*task_routine)(void *);
+	void *task_context;
 	pthread_t tid;

 	pthread_setspecific(pool->key, pool);
@@ -64,10 +66,20 @@ static void *__thrdpool_routine(void *arg)
 		list_del(*pos);
 		pthread_mutex_unlock(&pool->mutex);

-		entry->task.routine(entry->task.context);
+		task_routine = entry->task.routine;
+		task_context = entry->task.context;
 		free(entry);
+		task_routine(task_context);
+
+		if (pool->nthreads == 0)
+		{
+			/* Thread pool was destroyed by the task. */
+			free(pool);
+			return NULL;
+		}
 	}

+	/* One thread joins another. Don't need to keep all thread IDs. */
 	tid = pool->tid;
 	pool->tid = pthread_self();
 	if (--pool->nthreads == 0)
@@ -104,13 +116,21 @@ static void __thrdpool_destroy_locks(thrdpool_t *pool)
 	pthread_cond_destroy(&pool->cond);
 }

-static void __thrdpool_terminate(thrdpool_t *pool)
+static void __thrdpool_terminate(int in_pool, thrdpool_t *pool)
 {
 	pthread_cond_t term = PTHREAD_COND_INITIALIZER;

 	pthread_mutex_lock(&pool->mutex);
 	pool->terminate = &term;
 	pthread_cond_broadcast(&pool->cond);
+
+	if (in_pool)
+	{
+		/* Thread pool destroyed in a pool thread is legal. */
+		pthread_detach(pthread_self());
+		pool->nthreads--;
+	}
+
 	while (pool->nthreads > 0)
 		pthread_cond_wait(&term, &pool->mutex);

@@ -144,7 +164,7 @@ static int __thrdpool_create_threads(size_t nthreads, thrdpool_t *pool)
 		if (pool->nthreads == nthreads)
 			return 0;

-		__thrdpool_terminate(pool);
+		__thrdpool_terminate(0, pool);
 	}

 	errno = ret;
@@ -156,7 +176,7 @@ thrdpool_t *thrdpool_create(size_t nthreads, size_t stacksize)
 	thrdpool_t *pool;
 	int ret;

-	pool = (struct __thrdpool *)malloc(sizeof (struct __thrdpool));
+	pool = (thrdpool_t *)malloc(sizeof (thrdpool_t));
 	if (pool)
 	{
 		if (__thrdpool_init_locks(pool) >= 0)
@@ -194,8 +214,8 @@ inline void __thrdpool_schedule(const struct thrdpool_task *task, void *buf,
 	entry->task = *task;
 	pthread_mutex_lock(&pool->mutex);
 	list_add_tail(&entry->list, &pool->task_queue);
-	pthread_mutex_unlock(&pool->mutex);
 	pthread_cond_signal(&pool->cond);
+	pthread_mutex_unlock(&pool->mutex);
 }

 int thrdpool_schedule(const struct thrdpool_task *task, thrdpool_t *pool)
@@ -238,18 +258,19 @@ int thrdpool_increase(thrdpool_t *pool)
 	return -1;
 }

-int thrdpool_in_pool(thrdpool_t *pool)
+inline int thrdpool_in_pool(thrdpool_t *pool)
 {
-	return pthread_getspecific(pool->key) == (void *)pool;
+	return pthread_getspecific(pool->key) == pool;
 }

 void thrdpool_destroy(void (*pending)(const struct thrdpool_task *),
 					  thrdpool_t *pool)
 {
+	int in_pool = thrdpool_in_pool(pool);
 	struct __thrdpool_task_entry *entry;
 	struct list_head *pos, *tmp;

-	__thrdpool_terminate(pool);
+	__thrdpool_terminate(in_pool, pool);
 	list_for_each_safe(pos, tmp, &pool->task_queue)
 	{
 		entry = list_entry(pos, struct __thrdpool_task_entry, list);
@@ -262,6 +283,7 @@ void thrdpool_destroy(void (*pending)(const struct thrdpool_task *),

 	pthread_key_delete(pool->key);
 	__thrdpool_destroy_locks(pool);
-	free(pool);
+	if (!in_pool)
+		free(pool);
 }

@@ -34,6 +34,18 @@ extern "C"
 {
 #endif

+/*
+ * Thread pool originates from project Sogou C++ Workflow
+ * https://github.com/sogou/workflow
+ *
+ * A thread task can be scheduled by another task, which is very important,
+ * even if the pool is being destroyed. Because thread task is hard to know
+ * what's happening to the pool.
+ * The thread pool can also be destroyed by a thread task. This may sound
+ * strange, but it's very logical. Destroying thread pool in thread task
+ * does not end the task thread. It'll run till the end of task.
+ */
+
 thrdpool_t *thrdpool_create(size_t nthreads, size_t stacksize);
 int thrdpool_schedule(const struct thrdpool_task *task, thrdpool_t *pool);
 int thrdpool_increase(thrdpool_t *pool);
@@ -16,8 +16,8 @@
  Authors: Wu Jiaxu (wujiaxu@sogou-inc.com)
 */

-#ifndef _CONNECTIONPARAMS_H_
-#define _CONNECTIONPARAMS_H_
+#ifndef _ENDPOINTPARAMS_H_
+#define _ENDPOINTPARAMS_H_

 #include <stddef.h>

@@ -96,7 +96,7 @@ public:
 	struct rb_node rb;
 	std::mutex mutex;
 	std::vector<UpstreamAddress *> masters;
-	std::vector<UpstreamAddress *> slaves;
+	std::vector<UpstreamAddress *> backups;
 	struct list_head breaker_list;
 	std::atomic<int> nbreak;
 	std::atomic<int> nalive;
@@ -115,7 +115,7 @@ public:
 	}

 	const UpstreamAddress *get_one();
-	const UpstreamAddress *get_one_slave();
+	const UpstreamAddress *get_one_backup();
 };

 class Upstream
@@ -173,28 +173,28 @@ const UpstreamAddress *UpstreamGroup::get_one()
 			return master;
 	}

-	std::random_shuffle(this->slaves.begin(), this->slaves.end());
-	for (const auto *slave : this->slaves)
+	std::random_shuffle(this->backups.begin(), this->backups.end());
+	for (const auto *backup : this->backups)
 	{
-		if (slave->fail_count < slave->params.max_fails)
-			return slave;
+		if (backup->fail_count < backup->params.max_fails)
+			return backup;
 	}

 	return NULL;
 }

-const UpstreamAddress *UpstreamGroup::get_one_slave()
+const UpstreamAddress *UpstreamGroup::get_one_backup()
 {
 	if (this->nalive == 0)
 		return NULL;

 	std::lock_guard<std::mutex> lock(this->mutex);

-	std::random_shuffle(this->slaves.begin(), this->slaves.end());
-	for (const auto *slave : this->slaves)
+	std::random_shuffle(this->backups.begin(), this->backups.end());
+	for (const auto *backup : this->backups)
 	{
-		if (slave->fail_count < slave->params.max_fails)
-			return slave;
+		if (backup->fail_count < backup->params.max_fails)
+			return backup;
 	}

 	return NULL;
@@ -283,7 +283,7 @@ void Upstream::lose_one_server(UpstreamGroup *group, const UpstreamAddress *ua)
 	if (--group->nalive == 0 && ua->params.group_id >= 0)
 		available_weight_ -= group->weight;

-	if (ua->params.group_id < 0 && ua->params.server_type == SERVER_TYPE_MASTER)
+	if (ua->params.group_id < 0 && ua->params.server_type == 0)
 		available_weight_ -= ua->params.weight;
 }

@@ -292,7 +292,7 @@ void Upstream::gain_one_server(UpstreamGroup *group, const UpstreamAddress *ua)
 	if (group->nalive++ == 0 && ua->params.group_id >= 0)
 		available_weight_ += group->weight;

-	if (ua->params.group_id < 0 && ua->params.server_type == SERVER_TYPE_MASTER)
+	if (ua->params.group_id < 0 && ua->params.server_type == 0)
 		available_weight_ += ua->params.weight;
 }

@@ -325,7 +325,7 @@ int Upstream::add(UpstreamAddress *ua)
 		rb_insert_color(&group->rb, &group_map_);
 	}

-	if (ua->params.server_type == SERVER_TYPE_MASTER)
+	if (ua->params.server_type == 0)
 	{
 		total_weight_ += ua->params.weight;
 		masters_.push_back(ua);
@@ -334,13 +334,13 @@ int Upstream::add(UpstreamAddress *ua)
 	group->mutex.lock();
 	gain_one_server(group, ua);
 	ua->group = group;
-	if (ua->params.server_type == SERVER_TYPE_MASTER)
+	if (ua->params.server_type == 0)
 	{
 		group->weight += ua->params.weight;
 		group->masters.push_back(ua);
 	}
 	else
-		group->slaves.push_back(ua);
+		group->backups.push_back(ua);

 	group->mutex.unlock();

@@ -359,13 +359,13 @@ int Upstream::del(const std::string& address)
 			auto *group = ua->group;
 			std::vector<UpstreamAddress *> *vec;

-			if (ua->params.server_type == SERVER_TYPE_MASTER)
+			if (ua->params.server_type == 0)
 			{
 				total_weight_ -= ua->params.weight;
 				vec = &group->masters;
 			}
 			else
-				vec = &group->slaves;
+				vec = &group->backups;

 			std::lock_guard<std::mutex> lock(group->mutex);

@@ -373,7 +373,7 @@ int Upstream::del(const std::string& address)
 			if (ua->fail_count < ua->params.max_fails)
 				lose_one_server(group, ua);

-			if (ua->params.server_type == SERVER_TYPE_MASTER)
+			if (ua->params.server_type == 0)
 				group->weight -= ua->params.weight;

 			for (auto it = vec->begin(); it != vec->end(); ++it)
@@ -679,7 +679,7 @@ const UpstreamAddress *Upstream::get(const ParsedURI& uri)
 	}

 	if (!ua)
-		ua = default_group_->get_one_slave();//get one slave from group[-1]
+		ua = default_group_->get_one_backup();//get one backup from group[-1]

 	return ua;
 }
@@ -58,11 +58,9 @@ struct AddressParams
 * - If max_fails is set to 1, it means server would out of upstream selection in 30 seconds when failed only once
 */
 	unsigned int max_fails;                ///< [1, INT32_MAX] max_fails = 0 means max_fails = 1
-	unsigned short weight;                 ///< [1, 65535] weight = 0 means weight = 1. only for master
-#define SERVER_TYPE_MASTER	0
-#define SERVER_TYPE_SLAVE	1
-	int server_type;                       ///< default is SERVER_TYPE_MASTER
-	int group_id;                          ///< -1 means no group. Slave without group will backup for any master
+	unsigned short weight;                 ///< [1, 65535] weight = 0 means weight = 1. only for main server
+	int server_type;                       ///< 0 for main and 1 for backup
+	int group_id;                          ///< -1 means no group. Backup without group will be backup for any main
 };

 /**
@@ -75,7 +73,7 @@ static constexpr struct AddressParams ADDRESS_PARAMS_DEFAULT =
 	.dns_ttl_min		=	180,
 	.max_fails			=	200,
 	.weight				=	1,
-	.server_type		=	SERVER_TYPE_MASTER,
+	.server_type		=	0,	/* 0 for main and 1 for backup. */
 	.group_id			=	-1,
 };