diff --git a/docs/cn/mysql_client.md b/docs/cn/mysql_client.md new file mode 100644 index 0000000000..a7efe88a26 --- /dev/null +++ b/docs/cn/mysql_client.md @@ -0,0 +1,562 @@ +[MySQL](https://www.mysql.com/)是著名的开源的关系型数据库，为了使用户更快捷地访问mysql并充分利用bthread的并发能力，brpc直接支持mysql协议。示例程序：[example/mysql_c++](https://github.com/brpc/brpc/tree/master/example/mysql_c++/) + +**注意**：只支持MySQL 4.1 及之后的版本的文本协议，支持事务，不支持Prepared statement。目前支持的鉴权方式为mysql_native_password，使用事务的时候不支持single模式。 + +相比使用[libmysqlclient](https://dev.mysql.com/downloads/connector/c/)(官方client)的优势有： + +- 线程安全。用户不需要为每个线程建立独立的client。 +- 支持同步、异步、半同步等访问方式，能使用[ParallelChannel等](combo_channel.md)组合访问方式。 +- 支持多种[连接方式](client.md#连接方式)。支持超时、backup request、取消、tracing、内置服务等一系列brpc提供的福利。 +- 明确的返回类型校验，如果使用了不正确的变量接受mysql的数据类型，将抛出异常。 +- 调用mysql标准库会阻塞框架的并发能力，使用本实现将能充分利用brpc框架的并发能力。 +- 使用brpc实现的mysql不会造成pthread的阻塞，使用libmysqlclient会阻塞pthread [线程相关](bthread.md)，使用mysql的异步api会使编程变得很复杂。 +# 访问mysql + +创建一个访问mysql的Channel： + +```c++ +# include +# include +# include + +brpc::ChannelOptions options; +options.protocol = brpc::PROTOCOL_MYSQL; +options.connection_type = FLAGS_connection_type; +options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; +options.max_retry = FLAGS_max_retry; +options.auth = new brpc::policy::MysqlAuthenticator("yangliming01", "123456", "test", + "charset=utf8&collation_connection=utf8_unicode_ci"); +if (channel.Init("127.0.0.1", 3306, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; +} +``` + +向mysql发起命令。 + +```c++ +// 执行各种mysql命令，可以批量执行命令如："select * from tab1;select * from tab2" +std::string command = "show databases"; // select,delete,update,insert,create,drop ... +brpc::MysqlRequest request; +if (!request.Query(command)) { + LOG(ERROR) << "Fail to add command"; + return false; +} +brpc::MysqlResponse response; +brpc::Controller cntl; +channel.CallMethod(NULL, &cntl, &request, &response, NULL); +if (!cntl.Failed()) { + std::cout << response << std::endl; +} else { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return false; +} +return true; +``` + +上述代码的说明： + +- 请求类型必须为MysqlRequest，回复类型必须为MysqlResponse，否则CallMethod会失败。不需要stub，直接调用channel.CallMethod，method填NULL。 +- 调用request.Query()传入要执行的命令，可以批量执行命令，多个命令用分号隔开。 +- 依次调用response.reply(X)弹出操作结果，根据返回类型的不同，选择不同的类型接收，如：MysqlReply::Ok，MysqlReply::Error，const MysqlReply::Columnconst MysqlReply::Row等。 +- 如果只有一条命令则reply为1个，如果为批量操作返回的reply为多个。 + +目前支持的请求操作有： + +```c++ +bool Query(const butil::StringPiece& command); +``` + +对应的回复操作： + +```c++ +// 返回不同类型的结果 +const MysqlReply::Auth& auth() const; +const MysqlReply::Ok& ok() const; +const MysqlReply::Error& error() const; +const MysqlReply::Eof& eof() const; +// 对result set结果集的操作 +// get column number +uint64_t MysqlReply::column_number() const; +// get one column +const MysqlReply::Column& MysqlReply::column(const uint64_t index) const; +// get row number +uint64_t MysqlReply::row_number() const; +// get one row +const MysqlReply::Row& MysqlReply::next() const; +// 结果集中每个字段的操作 +const MysqlReply::Field& MysqlReply::Row::field(const uint64_t index) const; +``` + +# 事务操作 + +事务可以保证在一个事务中的多个RPC请求最终要么都成功，要么都失败。 + +```c++ +rpc::Channel channel; +// Initialize the channel, NULL means using default options. +brpc::ChannelOptions options; +options.protocol = brpc::PROTOCOL_MYSQL; +options.connection_type = FLAGS_connection_type; +options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; +options.connect_timeout_ms = FLAGS_connect_timeout_ms; +options.max_retry = FLAGS_max_retry; +options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params); +if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; +} + +// create transaction +brpc::MysqlTransactionOptions options; +options.readonly = FLAGS_readonly; +options.isolation_level = brpc::MysqlIsolationLevel(FLAGS_isolation_level); +auto tx(brpc::NewMysqlTransaction(channel, options)); +if (tx == NULL) { + LOG(ERROR) << "Fail to create transaction"; + return false; +} + +brpc::MysqlRequest request(tx.get()); +if (!request.Query(*it)) { + LOG(ERROR) << "Fail to add command"; + tx->rollback(); + return false; +} +brpc::MysqlResponse response; +brpc::Controller cntl; +channel.CallMethod(NULL, &cntl, &request, &response, NULL); +if (cntl.Failed()) { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + tx->rollback(); + return false; +} +// handle response +std::cout << response << std::endl; +bool rc = tx->commit(); +``` + +# Prepared Statement + +Prepared statement对于一个需要执行很多次的SQL语句，它把这个SQL语句注册到mysql-server，避免了每次请求在mysql-server端都去解析这个SQL语句，能得到性能上的提升。 + +```c++ +rpc::Channel channel; +// Initialize the channel, NULL means using default options. +brpc::ChannelOptions options; +options.protocol = brpc::PROTOCOL_MYSQL; +options.connection_type = FLAGS_connection_type; +options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; +options.connect_timeout_ms = FLAGS_connect_timeout_ms; +options.max_retry = FLAGS_max_retry; +options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params); +if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; +} + +auto stmt(brpc::NewMysqlStatement(channel, "select * from tb where name=?")); +if (stmt == NULL) { + LOG(ERROR) << "Fail to create mysql statement"; + return -1; +} + +brpc::MysqlRequest request(stmt.get()); +if (!request.AddParam("lilei")) { + LOG(ERROR) << "Fail to add name param"; + return NULL; +} + +brpc::MysqlResponse response; +brpc::Controller cntl; +channel->CallMethod(NULL, &cntl, &request, &response, NULL); +if (cntl.Failed()) { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return NULL; +} + +std::cout << response << std::endl; +``` + + + +# 性能测试 + +我在example/mysql_c++目录下面写了两个测试程序，mysql_press.cpp mysqlclient_press.cpp，mysql_go_press.go 一个是使用了brpc框架，一个是使用了的libmysqlclient访问mysql，一个是使用[go-sql-driver](https://github.com/go-sql-driver)/**go-mysql**访问mysql + +启动单线程测试 + +##### brpc框架访问mysql（单线程） + +./mysql_press -thread_num=1 -op_type=0 // insert + +``` +qps=3071 latency=320 +qps=3156 latency=311 +qps=3166 latency=310 +qps=3151 latency=312 +qps=3093 latency=317 +qps=3146 latency=312 +qps=3139 latency=313 +qps=3114 latency=315 +qps=3055 latency=321 +qps=3135 latency=313 +qps=2611 latency=376 +qps=3072 latency=320 +qps=3026 latency=324 +qps=2792 latency=352 +qps=3181 latency=309 +qps=3181 latency=309 +qps=3197 latency=307 +qps=3024 latency=325 +``` + +./mysql_press -thread_num=1 -op_type=1 + +``` +qps=6414 latency=151 +qps=5292 latency=182 +qps=6700 latency=144 +qps=6858 latency=141 +qps=6915 latency=140 +qps=6822 latency=142 +qps=6722 latency=144 +qps=6852 latency=141 +qps=6713 latency=144 +qps=6741 latency=144 +qps=6734 latency=144 +qps=6611 latency=146 +qps=6554 latency=148 +qps=6810 latency=142 +qps=6787 latency=143 +qps=6737 latency=144 +qps=6579 latency=147 +qps=6634 latency=146 +qps=6716 latency=144 +qps=6711 latency=144 +``` + +./mysql_press -thread_num=1 -op_type=2 // update + +``` +qps=3090 latency=318 +qps=3452 latency=284 +qps=3239 latency=303 +qps=3328 latency=295 +qps=3218 latency=305 +qps=3251 latency=302 +qps=2516 latency=391 +qps=2874 latency=342 +qps=3366 latency=292 +qps=3249 latency=302 +qps=3346 latency=294 +qps=3486 latency=282 +qps=3457 latency=284 +qps=3439 latency=286 +qps=3386 latency=290 +qps=3352 latency=293 +qps=3253 latency=302 +qps=3341 latency=294 +``` + +##### libmysqlclient实现（单线程） + +./mysqlclient_press -thread_num=1 -op_type=0 // insert + +``` +qps=3166 latency=313 +qps=3157 latency=314 +qps=2941 latency=337 +qps=3270 latency=303 +qps=3305 latency=300 +qps=3445 latency=287 +qps=3455 latency=287 +qps=3449 latency=287 +qps=3486 latency=284 +qps=3551 latency=279 +qps=3517 latency=281 +qps=3283 latency=302 +qps=3353 latency=295 +qps=2564 latency=386 +qps=3243 latency=305 +qps=3333 latency=297 +qps=3598 latency=275 +qps=3714 latency=267 +``` + +./mysqlclient_press -thread_num=1 -op_type=1 + +``` +qps=8209 latency=120 +qps=8022 latency=123 +qps=7879 latency=125 +qps=8083 latency=122 +qps=8504 latency=116 +qps=8112 latency=121 +qps=8278 latency=119 +qps=8698 latency=113 +qps=8817 latency=112 +qps=8755 latency=112 +qps=8734 latency=113 +qps=8390 latency=117 +qps=8230 latency=120 +qps=8486 latency=116 +qps=8038 latency=122 +qps=8640 latency=114 +``` + +./mysqlclient_press -thread_num=1 -op_type=2 // update + +``` +qps=3583 latency=276 +qps=3530 latency=280 +qps=3610 latency=274 +qps=3492 latency=283 +qps=3508 latency=282 +qps=3465 latency=286 +qps=3543 latency=279 +qps=3610 latency=274 +qps=3567 latency=278 +qps=3381 latency=293 +qps=3514 latency=282 +qps=3461 latency=286 +qps=3456 latency=286 +qps=3517 latency=281 +qps=3492 latency=284 +``` + +##### golang访问mysql（单线程） + +go run test.go -thread_num=1 + +``` +qps = 6905 latency = 144 +qps = 6922 latency = 143 +qps = 6931 latency = 143 +qps = 6998 latency = 142 +qps = 6780 latency = 146 +qps = 6980 latency = 142 +qps = 6901 latency = 144 +qps = 6887 latency = 144 +qps = 6943 latency = 143 +qps = 6880 latency = 144 +qps = 6815 latency = 146 +qps = 6089 latency = 163 +qps = 6626 latency = 150 +qps = 6361 latency = 156 +qps = 6783 latency = 146 +qps = 6789 latency = 146 +qps = 6883 latency = 144 +qps = 6795 latency = 146 +qps = 6724 latency = 148 +qps = 6861 latency = 145 +qps = 6878 latency = 144 +qps = 6842 latency = 146 +``` + +从以上测试结果看来，使用brpc实现的mysql协议和使用libmysqlclient在插入、修改、删除操作上性能是类似的，但是在查询操作看会逊色于libmysqlclient，查询的性能和golang实现的mysql类似。 + +##### brpc框架访问mysql（50线程） + +./mysql_press -thread_num=50 -op_type=1 -use_bthread=true + +``` +qps=18843 latency=2656 +qps=22426 latency=2226 +qps=22536 latency=2203 +qps=22560 latency=2193 +qps=22270 latency=2226 +qps=22302 latency=2247 +qps=22147 latency=2225 +qps=22517 latency=2228 +qps=22762 latency=2176 +qps=23061 latency=2162 +qps=23819 latency=2070 +qps=23852 latency=2077 +qps=22682 latency=2214 +qps=22381 latency=2213 +qps=24041 latency=2069 +qps=24562 latency=2022 +qps=24874 latency=2004 +qps=24821 latency=1988 +qps=24209 latency=2073 +qps=21706 latency=2281 +``` + +##### libmysqlclient实现（50线程） + +./mysql_press -thread_num=50 -op_type=1 -use_bthread=true + +``` +qps=23656 latency=378 +qps=16190 latency=555 +qps=20136 latency=445 +qps=22238 latency=401 +qps=22229 latency=403 +qps=19109 latency=470 +qps=22569 latency=394 +qps=26250 latency=343 +qps=28208 latency=318 +qps=29649 latency=301 +qps=29874 latency=301 +qps=30033 latency=301 +qps=25911 latency=345 +qps=28048 latency=317 +qps=27398 latency=329 +``` + +##### golang访问mysql（50协程） + +go run ../mysql_go_press.go -thread_num=50 + +``` +qps = 23660 latency = 2049 +qps = 23198 latency = 2160 +qps = 23765 latency = 2181 +qps = 23323 latency = 2149 +qps = 14833 latency = 2136 +qps = 23822 latency = 2853 +qps = 20389 latency = 2474 +qps = 23290 latency = 2151 +qps = 23526 latency = 2153 +qps = 21426 latency = 2613 +qps = 23339 latency = 2155 +qps = 25623 latency = 2084 +qps = 23048 latency = 2210 +qps = 20694 latency = 2423 +qps = 23705 latency = 2122 +qps = 23445 latency = 2125 +qps = 24368 latency = 2054 +qps = 23027 latency = 2175 +qps = 24307 latency = 2063 +qps = 23227 latency = 2096 +qps = 23646 latency = 2173 +``` + +以上是启动50并发的查询请求，看上去qps都比较相似，但是libmysqlclient延时明显低。 + +##### brpc框架访问mysql（100线程） + +./mysql_press -thread_num=100 -op_type=1 -use_bthread=true + +``` +qps=26428 latency=3764 +qps=26305 latency=3780 +qps=26390 latency=3779 +qps=26278 latency=3787 +qps=26326 latency=3787 +qps=26266 latency=3792 +qps=26394 latency=3773 +qps=26263 latency=3797 +qps=26250 latency=3783 +qps=26362 latency=3782 +qps=26212 latency=3796 +qps=26260 latency=3800 +qps=24666 latency=4035 +qps=25569 latency=3896 +qps=26223 latency=3794 +qps=25538 latency=3890 +qps=20065 latency=4958 +qps=23023 latency=4331 +qps=25808 latency=3875 +``` + +##### libmysqlclient实现（100线程） + +./mysql_press -thread_num=50 -op_type=1 -use_bthread=true + +``` +qps=29467 latency=304 +qps=29413 latency=305 +qps=29459 latency=304 +qps=29562 latency=302 +qps=30657 latency=291 +qps=30445 latency=295 +qps=30179 latency=298 +qps=30072 latency=297 +qps=29802 latency=299 +qps=29752 latency=301 +qps=29701 latency=304 +qps=29731 latency=301 +qps=29622 latency=299 +qps=29440 latency=304 +qps=29495 latency=306 +qps=29297 latency=303 +qps=29626 latency=306 +qps=29482 latency=300 +qps=28649 latency=313 +qps=29537 latency=305 +qps=29634 latency=299 +``` + +##### golang访问mysql（100协程） + +go run ../mysql_go_press.go -thread_num=100 + +``` +qps = 22108 latency = 4553 +qps = 21930 latency = 4536 +qps = 20653 latency = 4906 +qps = 22100 latency = 4443 +qps = 21091 latency = 4850 +qps = 21718 latency = 4600 +qps = 21444 latency = 4488 +qps = 17832 latency = 5859 +qps = 18296 latency = 5378 +qps = 20463 latency = 4963 +qps = 21611 latency = 4880 +qps = 18441 latency = 5424 +qps = 20731 latency = 4834 +qps = 20611 latency = 4837 +qps = 20188 latency = 4979 +qps = 15450 latency = 5723 +qps = 20927 latency = 5328 +qps = 19893 latency = 5027 +qps = 21080 latency = 4782 +qps = 20192 latency = 4970 +``` + +以上是启动100并发的查询请求，看上去qps都比较相似，但是libmysqlclient延时明显低。 + +并发调整到150的时候，mysql-server已经报错"Too many connections"。 + +为什么并发数50或者100的时候libmysqlclient的延时会那么低呢？因为libmysqlclient使用的IO模式为阻塞模式，我们运行的mysql_press和mysqlclient_press都是使用的bthread模式（-use_bthread=true），底层默认都是9个pthread，使用阻塞模式的libmysqlclient和mysql交互的相当于并发度是9个线程，mysql会启动9个线程，使用非阻塞模式的rpc访问mysql并发度相当于100个，mysql会启动100个线程，所以会造成mysql的频繁上线文切换。 + +如果将libmysqlclient的执行方式改为不使用bthread，那么100个线程的执行效果为如下： + +``` +qps=26919 latency=1927 +qps=27155 latency=2037 +qps=28054 latency=1784 +qps=26738 latency=1856 +qps=27807 latency=1781 +qps=26734 latency=1730 +qps=26562 latency=1939 +qps=27473 latency=1845 +qps=26677 latency=1806 +qps=27369 latency=1948 +qps=27955 latency=1618 +qps=26574 latency=2151 +qps=27343 latency=1777 +qps=26705 latency=1822 +qps=26668 latency=1807 +qps=25347 latency=2104 +qps=26651 latency=1560 +qps=27815 latency=1979 +qps=27221 latency=1762 +qps=26516 latency=2017 +``` + +这个结果就和brpc框架启动100个bthread访问mysql的效果类似了。 + +##### 内存使用 + +在内存占用上，mysql_press和mysqlclient_press都运行了一个晚上，两个程序的内存占用 + +![libmysqlclient](../images/mysql_memory.png) + + + +以上为我的一些简单测试，以及一些简单的分析，在低并发的情况下同步IO的效率高于异步IO，可以阅读[IO相关的内容](io.md)有更多解释，后续还将继续分析性能问题，优化协议，给出更多测试。 \ No newline at end of file diff --git a/docs/images/mysql_memory.png b/docs/images/mysql_memory.png new file mode 100644 index 0000000000..bf7fe1caa9 Binary files /dev/null and b/docs/images/mysql_memory.png differ diff --git a/docs/images/mysql_select.png b/docs/images/mysql_select.png new file mode 100644 index 0000000000..ace48aba57 Binary files /dev/null and b/docs/images/mysql_select.png differ diff --git a/docs/images/mysqlclient_select.png b/docs/images/mysqlclient_select.png new file mode 100644 index 0000000000..508567bbe5 Binary files /dev/null and b/docs/images/mysqlclient_select.png differ diff --git a/example/mysql_c++/CMakeLists.txt b/example/mysql_c++/CMakeLists.txt new file mode 100644 index 0000000000..1e0b953180 --- /dev/null +++ b/example/mysql_c++/CMakeLists.txt @@ -0,0 +1,148 @@ +cmake_minimum_required(VERSION 2.8.10) +project(mysql_c++ C CXX) + +# Install dependencies: +# With apt: +# sudo apt-get install libreadline-dev +# sudo apt-get install ncurses-dev +# With yum: +# sudo yum install readline-devel +# sudo yum install ncurses-devel + +option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" OFF) + +execute_process( + COMMAND bash -c "find ${PROJECT_SOURCE_DIR}/../.. -type d -regex \".*output/include$\" | head -n1 | xargs dirname | tr -d '\n'" + OUTPUT_VARIABLE OUTPUT_PATH +) + +set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) + +include(FindThreads) +include(FindProtobuf) + +# Search for libthrift* by best effort. If it is not found and brpc is +# compiled with thrift protocol enabled, a link error would be reported. +find_library(THRIFT_LIB NAMES thrift) +if (NOT THRIFT_LIB) + set(THRIFT_LIB "") +endif() +find_library(THRIFTNB_LIB NAMES thriftnb) +if (NOT THRIFTNB_LIB) + set(THRIFTNB_LIB "") +endif() + +find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) +if(EXAMPLE_LINK_SO) + find_library(BRPC_LIB NAMES brpc) +else() + find_library(BRPC_LIB NAMES libbrpc.a brpc) +endif() +if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) + message(FATAL_ERROR "Fail to find brpc") +endif() +include_directories(${BRPC_INCLUDE_PATH}) + +find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) +find_library(GFLAGS_LIBRARY NAMES gflags libgflags) +if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) + message(FATAL_ERROR "Fail to find gflags") +endif() +include_directories(${GFLAGS_INCLUDE_PATH}) + +execute_process( + COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" + OUTPUT_VARIABLE GFLAGS_NS +) +if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") + execute_process( + COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" + OUTPUT_VARIABLE GFLAGS_NS + ) +endif() +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + include(CheckFunctionExists) + CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) + if(NOT HAVE_CLOCK_GETTIME) + set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") + endif() +endif() + +set(CMAKE_CPP_FLAGS "${DEFINE_CLOCK_GETTIME} -DGFLAGS_NS=${GFLAGS_NS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") + +if(CMAKE_VERSION VERSION_LESS "3.1.3") + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() +else() + set(CMAKE_CXX_STANDARD 11) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +endif() + +find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) +find_library(LEVELDB_LIB NAMES leveldb) +if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) + message(FATAL_ERROR "Fail to find leveldb") +endif() +include_directories(${LEVELDB_INCLUDE_PATH}) + +find_library(SSL_LIB NAMES ssl) +if (NOT SSL_LIB) + message(FATAL_ERROR "Fail to find ssl") +endif() + +find_library(CRYPTO_LIB NAMES crypto) +if (NOT CRYPTO_LIB) + message(FATAL_ERROR "Fail to find crypto") +endif() + +# find_path(MYSQL_INCLUDE_PATH NAMES mysql/mysql.h) +# find_library(MYSQL_LIB NAMES mysqlclient) +# if (NOT MYSQL_LIB) +# message(FATAL_ERROR "Fail to find mysqlclient") +# endif() +# include_directories(${MYSQL_INCLUDE_PATH}) + +set(DYNAMIC_LIB + ${CMAKE_THREAD_LIBS_INIT} + ${GFLAGS_LIBRARY} + ${PROTOBUF_LIBRARIES} + ${LEVELDB_LIB} + ${SSL_LIB} + ${CRYPTO_LIB} + ${THRIFT_LIB} + ${THRIFTNB_LIB} +# ${MYSQL_LIB} + dl + ) + +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(DYNAMIC_LIB ${DYNAMIC_LIB} + pthread + "-framework CoreFoundation" + "-framework CoreGraphics" + "-framework CoreData" + "-framework CoreText" + "-framework Security" + "-framework Foundation" + "-Wl,-U,_MallocExtension_ReleaseFreeMemory" + "-Wl,-U,_ProfilerStart" + "-Wl,-U,_ProfilerStop") +endif() + +add_executable(mysql_cli mysql_cli.cpp) +add_executable(mysql_tx mysql_tx.cpp) +add_executable(mysql_stmt mysql_stmt.cpp) +add_executable(mysql_press mysql_press.cpp) +# add_executable(mysqlclient_press mysqlclient_press.cpp) + +set(AUX_LIB readline ncurses) +target_link_libraries(mysql_cli ${BRPC_LIB} ${DYNAMIC_LIB} ${AUX_LIB}) +target_link_libraries(mysql_tx ${BRPC_LIB} ${DYNAMIC_LIB}) +target_link_libraries(mysql_stmt ${BRPC_LIB} ${DYNAMIC_LIB}) +target_link_libraries(mysql_press ${BRPC_LIB} ${DYNAMIC_LIB}) +# target_link_libraries(mysqlclient_press ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/mysql_c++/mysql_cli.cpp b/example/mysql_c++/mysql_cli.cpp new file mode 100644 index 0000000000..776f847a9c --- /dev/null +++ b/example/mysql_c++/mysql_cli.cpp @@ -0,0 +1,168 @@ +// Copyright (c) 2014 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A brpc based command-line interface to talk with mysql-server + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short"); +DEFINE_string(server, "127.0.0.1", "IP Address of server"); +DEFINE_int32(port, 3306, "Port of server"); +DEFINE_string(user, "brpcuser", "user name"); +DEFINE_string(password, "12345678", "password"); +DEFINE_string(schema, "brpc_test", "schema"); +DEFINE_string(params, "", "params"); +DEFINE_string(collation, "utf8mb4_general_ci", "collation"); +DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)"); + +namespace brpc { +const char* logo(); +} + +// Send `command' to mysql-server via `channel' +static bool access_mysql(brpc::Channel& channel, const char* command) { + brpc::MysqlRequest request; + if (!request.Query(command)) { + LOG(ERROR) << "Fail to add command"; + return false; + } + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (!cntl.Failed()) { + std::cout << response << std::endl; + } else { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return false; + } + return true; +} + +// For freeing the memory returned by readline(). +struct Freer { + void operator()(char* mem) { + free(mem); + } +}; + +static void dummy_handler(int) {} + +// The getc for readline. The default getc retries reading when meeting +// EINTR, which is not what we want. +static bool g_canceled = false; +static int cli_getc(FILE* stream) { + int c = getc(stream); + if (c == EOF && errno == EINTR) { + g_canceled = true; + return '\n'; + } + return c; +} + +int main(int argc, char* argv[]) { + // Parse gflags. We recommend you to use gflags as well. + GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); + + // A Channel represents a communication line to a Server. Notice that + // Channel is thread-safe and can be shared by all threads in your program. + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; + options.connect_timeout_ms = FLAGS_connect_timeout_ms; + options.max_retry = FLAGS_max_retry; + options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation); + if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + + if (argc <= 1) { // interactive mode + // We need this dummy signal hander to interrupt getc (and returning + // EINTR), SIG_IGN did not work. + signal(SIGINT, dummy_handler); + + // Hook getc of readline. + rl_getc_function = cli_getc; + + // Print welcome information. + printf("%s\n", brpc::logo()); + printf( + "This command-line tool mimics the look-n-feel of official " + "mysql-cli, as a demostration of brpc's capability of" + " talking to mysql-server. The output and behavior is " + "not exactly same with the official one.\n\n"); + + for (;;) { + char prompt[128]; + snprintf(prompt, sizeof(prompt), "mysql %s> ", FLAGS_server.c_str()); + std::unique_ptr command(readline(prompt)); + if (command == NULL || *command == '\0') { + if (g_canceled) { + // No input after the prompt and user pressed Ctrl-C, + // quit the CLI. + return 0; + } + // User entered an empty command by just pressing Enter. + continue; + } + if (g_canceled) { + // User entered sth. and pressed Ctrl-C, start a new prompt. + g_canceled = false; + continue; + } + // Add user's command to history so that it's browse-able by + // UP-key and search-able by Ctrl-R. + add_history(command.get()); + + if (!strcmp(command.get(), "help")) { + printf("This is a mysql CLI written in brpc.\n"); + continue; + } + if (!strcmp(command.get(), "quit")) { + // Although quit is a valid mysql command, it does not make + // too much sense to run it in this CLI, just quit. + return 0; + } + access_mysql(channel, command.get()); + } + } else { + std::string command; + command.reserve(argc * 16); + for (int i = 1; i < argc; ++i) { + if (i != 1) { + command.push_back(';'); + } + command.append(argv[i]); + } + if (!access_mysql(channel, command.c_str())) { + return -1; + } + } + return 0; +} diff --git a/example/mysql_c++/mysql_go_press.go b/example/mysql_c++/mysql_go_press.go new file mode 100644 index 0000000000..b68f9d78b6 --- /dev/null +++ b/example/mysql_c++/mysql_go_press.go @@ -0,0 +1,63 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + _ "github.com/go-sql-driver/mysql" + "log" + "sync/atomic" + "time" +) + +var thread_num int + +func init() { + flag.IntVar(&thread_num, "thread_num", 1, "thread number") +} + +var cost int64 +var qps int64 = 1 + +func main() { + flag.Parse() + + db, err := sql.Open("mysql", "brpcuser:12345678@tcp(127.0.0.1:3306)/brpc_test?charset=utf8") + if err != nil { + log.Fatal(err) + } + + for i := 0; i < thread_num; i++ { + go func() { + for { + var ( + id int + col1 string + col2 string + col3 string + col4 string + ) + start := time.Now() + rows, err := db.Query("select * from brpc_press where id = 1") + if err != nil { + log.Fatal(err) + } + for rows.Next() { + if err := rows.Scan(&id, &col1, &col2, &col3, &col4); err != nil { + log.Fatal(err) + } + } + atomic.AddInt64(&cost, time.Since(start).Nanoseconds()) + atomic.AddInt64(&qps, 1) + } + }() + } + + var q int64 = 0 + for { + fmt.Println("qps =", qps-q, "latency =", cost/(qps-q)/1000) + q = atomic.LoadInt64(&qps) + atomic.StoreInt64(&cost, 0) + time.Sleep(1 * time.Second) + } +} diff --git a/example/mysql_c++/mysql_press.cpp b/example/mysql_c++/mysql_press.cpp new file mode 100644 index 0000000000..d58500ff75 --- /dev/null +++ b/example/mysql_c++/mysql_press.cpp @@ -0,0 +1,237 @@ +// Copyright (c) 2014 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A brpc based command-line interface to talk with mysql-server + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short"); +DEFINE_string(server, "127.0.0.1", "IP Address of server"); +DEFINE_int32(port, 3306, "Port of server"); +DEFINE_string(user, "brpcuser", "user name"); +DEFINE_string(password, "12345678", "password"); +DEFINE_string(schema, "brpc_test", "schema"); +DEFINE_string(params, "", "params"); +DEFINE_string(collation, "utf8mb4_general_ci", "collation"); +DEFINE_string(data, "ABCDEF", "data"); +DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); +DEFINE_int32(thread_num, 50, "Number of threads to send requests"); +DEFINE_bool(use_bthread, false, "Use bthread to send requests"); +DEFINE_int32(dummy_port, -1, "port of dummy server(for monitoring)"); +DEFINE_int32(op_type, 0, "CRUD operation, 0:INSERT, 1:SELECT, 2:UPDATE"); +DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); + +bvar::LatencyRecorder g_latency_recorder("client"); +bvar::Adder g_error_count("client_error_count"); + +struct SenderArgs { + int base_index; + brpc::Channel* mysql_channel; +}; + +const std::string insert = + "insert into brpc_press(col1,col2,col3,col4) values " + "('" + "ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCA" + "BCABCABCABCABCABCABCA', '" + + FLAGS_data + + "' ,1.5, " + "now())"; +// Send `command' to mysql-server via `channel' +static void* sender(void* void_args) { + SenderArgs* args = (SenderArgs*)void_args; + std::stringstream command; + if (FLAGS_op_type == 0) { + command << insert; + } else if (FLAGS_op_type == 1) { + command << "select * from brpc_press where id = " << args->base_index + 1; + } else if (FLAGS_op_type == 2) { + command << "update brpc_press set col2 = '" + FLAGS_data + "' where id = " + << args->base_index + 1; + } else { + LOG(ERROR) << "wrong op type " << FLAGS_op_type; + } + + brpc::MysqlRequest request; + if (!request.Query(command.str())) { + LOG(ERROR) << "Fail to execute command"; + return NULL; + } + + while (!brpc::IsAskedToQuit()) { + brpc::MysqlResponse response; + brpc::Controller cntl; + args->mysql_channel->CallMethod(NULL, &cntl, &request, &response, NULL); + const int64_t elp = cntl.latency_us(); + if (!cntl.Failed()) { + g_latency_recorder << elp; + if (FLAGS_op_type == 0) { + CHECK_EQ(response.reply(0).is_ok(), true); + } else if (FLAGS_op_type == 1) { + CHECK_EQ(response.reply(0).row_count(), 1); + } else if (FLAGS_op_type == 2) { + CHECK_EQ(response.reply(0).is_ok(), true); + } + } else { + g_error_count << 1; + CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail) + << "error=" << cntl.ErrorText() << " latency=" << elp; + // We can't connect to the server, sleep a while. Notice that this + // is a specific sleeping to prevent this thread from spinning too + // fast. You should continue the business logic in a production + // server rather than sleeping. + bthread_usleep(50000); + } + } + return NULL; +} + +int main(int argc, char* argv[]) { + // Parse gflags. We recommend you to use gflags as well. + GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); + + // A Channel represents a communication line to a Server. Notice that + // Channel is thread-safe and can be shared by all threads in your program. + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; + options.connect_timeout_ms = FLAGS_connect_timeout_ms; + options.max_retry = FLAGS_max_retry; + options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation); + if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + + // create table brpc_press + { + brpc::MysqlRequest request; + if (!request.Query( + "CREATE TABLE IF NOT EXISTS `brpc_press`(`id` INT UNSIGNED AUTO_INCREMENT, `col1` " + "VARCHAR(100) NOT NULL, `col2` VARCHAR(1024) NOT NULL, `col3` decimal(10,0) NOT " + "NULL, `col4` DATE, PRIMARY KEY ( `id` )) ENGINE=InnoDB DEFAULT CHARSET=utf8;")) { + LOG(ERROR) << "Fail to create table"; + return -1; + } + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (!cntl.Failed()) { + std::cout << response << std::endl; + } else { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return -1; + } + } + + // truncate table + { + brpc::MysqlRequest request; + if (!request.Query("truncate table brpc_press")) { + LOG(ERROR) << "Fail to truncate table"; + return -1; + } + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (!cntl.Failed()) { + std::cout << response << std::endl; + } else { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return -1; + } + } + + // prepare data for select, update + if (FLAGS_op_type != 0) { + for (int i = 0; i < FLAGS_thread_num; ++i) { + brpc::MysqlRequest request; + if (!request.Query(insert)) { + LOG(ERROR) << "Fail to execute command"; + return -1; + } + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << cntl.ErrorText(); + return -1; + } + if (!response.reply(0).is_ok()) { + LOG(ERROR) << "prepare data failed"; + return -1; + } + } + } + + if (FLAGS_dummy_port >= 0) { + brpc::StartDummyServerAt(FLAGS_dummy_port); + } + + // test CRUD operations + std::vector bids; + std::vector pids; + bids.resize(FLAGS_thread_num); + pids.resize(FLAGS_thread_num); + std::vector args; + args.resize(FLAGS_thread_num); + for (int i = 0; i < FLAGS_thread_num; ++i) { + args[i].base_index = i; + args[i].mysql_channel = &channel; + if (!FLAGS_use_bthread) { + if (pthread_create(&pids[i], NULL, sender, &args[i]) != 0) { + LOG(ERROR) << "Fail to create pthread"; + return -1; + } + } else { + if (bthread_start_background(&bids[i], NULL, sender, &args[i]) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + } + } + + while (!brpc::IsAskedToQuit()) { + sleep(1); + + LOG(INFO) << "Accessing mysql-server at qps=" << g_latency_recorder.qps(1) + << " latency=" << g_latency_recorder.latency(1); + } + + LOG(INFO) << "mysql_client is going to quit"; + for (int i = 0; i < FLAGS_thread_num; ++i) { + if (!FLAGS_use_bthread) { + pthread_join(pids[i], NULL); + } else { + bthread_join(bids[i], NULL); + } + } + + return 0; +} diff --git a/example/mysql_c++/mysql_stmt.cpp b/example/mysql_c++/mysql_stmt.cpp new file mode 100644 index 0000000000..8e15f2ba8e --- /dev/null +++ b/example/mysql_c++/mysql_stmt.cpp @@ -0,0 +1,204 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A brpc based mysql transaction example +#include +#include +#include +#include +#include +#include + +DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short"); +DEFINE_string(server, "127.0.0.1", "IP Address of server"); +DEFINE_int32(port, 3306, "Port of server"); +DEFINE_string(user, "brpcuser", "user name"); +DEFINE_string(password, "12345678", "password"); +DEFINE_string(schema, "brpc_test", "schema"); +DEFINE_string(params, "", "params"); +DEFINE_string(collation, "utf8mb4_general_ci", "collation"); +DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)"); +DEFINE_int32(thread_num, 1, "Number of threads to send requests"); +DEFINE_int32(count, 1, "Number of request to send pre thread"); + +namespace brpc { +const char* logo(); +} + +struct SenderArgs { + brpc::Channel* mysql_channel; + brpc::MysqlStatement* mysql_stmt; + std::vector commands; +}; + +// Send `command' to mysql-server via `channel' +static void* access_mysql(void* void_args) { + SenderArgs* args = (SenderArgs*)void_args; + brpc::Channel* channel = args->mysql_channel; + brpc::MysqlStatement* stmt = args->mysql_stmt; + const std::vector& commands = args->commands; + + for (int i = 0; i < FLAGS_count; ++i) { + // for (;;) { + brpc::MysqlRequest request(stmt); + for (size_t i = 1; i < commands.size(); i += 2) { + if (commands[i] == "int8") { + int8_t val = strtol(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add int8 param"; + return NULL; + } + } else if (commands[i] == "uint8") { + uint8_t val = strtoul(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add uint8 param"; + return NULL; + } + } else if (commands[i] == "int16") { + int16_t val = strtol(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add uint16 param"; + return NULL; + } + } else if (commands[i] == "uint16") { + uint16_t val = strtoul(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add uint16 param"; + return NULL; + } + } else if (commands[i] == "int32") { + int32_t val = strtol(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add int32 param"; + return NULL; + } + } else if (commands[i] == "uint32") { + uint32_t val = strtoul(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add uint32 param"; + return NULL; + } + } else if (commands[i] == "int64") { + int64_t val = strtol(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add int64 param"; + return NULL; + } + } else if (commands[i] == "uint64") { + uint64_t val = strtoul(commands[i + 1].c_str(), NULL, 10); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add uint64 param"; + return NULL; + } + } else if (commands[i] == "float") { + float val = strtof(commands[i + 1].c_str(), NULL); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add float param"; + return NULL; + } + } else if (commands[i] == "double") { + double val = strtod(commands[i + 1].c_str(), NULL); + if (!request.AddParam(val)) { + LOG(ERROR) << "Fail to add double param"; + return NULL; + } + } else if (commands[i] == "string") { + if (!request.AddParam(commands[i + 1])) { + LOG(ERROR) << "Fail to add string param"; + return NULL; + } + } else { + LOG(ERROR) << "Wrong param type " << commands[i]; + } + } + + brpc::MysqlResponse response; + brpc::Controller cntl; + channel->CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + return NULL; + } + + // if (response.reply(0).is_error()) { + // check response + std::cout << response << std::endl; + // } + } + + return NULL; +} + +int main(int argc, char* argv[]) { + // Parse gflags. We recommend you to use gflags as well. + GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); + + // A Channel represents a communication line to a Server. Notice that + // Channel is thread-safe and can be shared by all threads in your program. + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; + options.connect_timeout_ms = FLAGS_connect_timeout_ms; + options.max_retry = FLAGS_max_retry; + options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation); + if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + + if (argc <= 1) { + LOG(ERROR) << "No sql statement args"; + } else { + std::vector commands; + commands.reserve(argc * 16); + for (int i = 1; i < argc; ++i) { + commands.push_back(argv[i]); + } + auto stmt(brpc::NewMysqlStatement(channel, commands[0])); + if (stmt == NULL) { + LOG(ERROR) << "Fail to create mysql statement"; + return -1; + } + + std::vector args; + std::vector bids; + args.resize(FLAGS_thread_num); + bids.resize(FLAGS_thread_num); + + for (int i = 0; i < FLAGS_thread_num; ++i) { + args[i].mysql_channel = &channel; + args[i].mysql_stmt = stmt.get(); + args[i].commands = commands; + if (bthread_start_background(&bids[i], NULL, access_mysql, &args[i]) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + } + + for (int i = 0; i < FLAGS_thread_num; ++i) { + bthread_join(bids[i], NULL); + } + } + + return 0; +} + +/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/example/mysql_c++/mysql_tx.cpp b/example/mysql_c++/mysql_tx.cpp new file mode 100644 index 0000000000..af9077c2f5 --- /dev/null +++ b/example/mysql_c++/mysql_tx.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A brpc based mysql transaction example +#include +#include +#include +#include +#include + +DEFINE_string(connection_type, "pooled", "Connection type. Available values: pooled, short"); +DEFINE_string(server, "127.0.0.1", "IP Address of server"); +DEFINE_int32(port, 3306, "Port of server"); +DEFINE_string(user, "brpcuser", "user name"); +DEFINE_string(password, "12345678", "password"); +DEFINE_string(schema, "brpc_test", "schema"); +DEFINE_string(params, "", "params"); +DEFINE_string(collation, "utf8mb4_general_ci", "collation"); +DEFINE_int32(timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(connect_timeout_ms, 5000, "RPC timeout in milliseconds"); +DEFINE_int32(max_retry, 0, "Max retries(not including the first RPC)"); +DEFINE_bool(readonly, false, "readonly transaction"); +DEFINE_int32(isolation_level, 0, "transaction isolation level"); + +namespace brpc { +const char* logo(); +} + +// Send `command' to mysql-server via `channel' +static bool access_mysql(brpc::Channel& channel, const std::vector& commands) { + brpc::MysqlTransactionOptions options; + options.readonly = FLAGS_readonly; + options.isolation_level = brpc::MysqlIsolationLevel(FLAGS_isolation_level); + auto tx(brpc::NewMysqlTransaction(channel, options)); + if (tx == NULL) { + LOG(ERROR) << "Fail to create transaction"; + return false; + } + + for (auto it = commands.begin(); it != commands.end(); ++it) { + brpc::MysqlRequest request(tx.get()); + if (!request.Query(*it)) { + LOG(ERROR) << "Fail to add command"; + tx->rollback(); + return false; + } + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (cntl.Failed()) { + LOG(ERROR) << "Fail to access mysql, " << cntl.ErrorText(); + tx->rollback(); + return false; + } + // check response + std::cout << response << std::endl; + for (size_t i = 0; i < response.reply_size(); ++i) { + if (response.reply(i).is_error()) { + tx->rollback(); + return false; + } + } + } + tx->commit(); + return true; +} + +int main(int argc, char* argv[]) { + // Parse gflags. We recommend you to use gflags as well. + GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); + + // A Channel represents a communication line to a Server. Notice that + // Channel is thread-safe and can be shared by all threads in your program. + brpc::Channel channel; + + // Initialize the channel, NULL means using default options. + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms /*milliseconds*/; + options.connect_timeout_ms = FLAGS_connect_timeout_ms; + options.max_retry = FLAGS_max_retry; + options.auth = new brpc::policy::MysqlAuthenticator( + FLAGS_user, FLAGS_password, FLAGS_schema, FLAGS_params, FLAGS_collation); + if (channel.Init(FLAGS_server.c_str(), FLAGS_port, &options) != 0) { + LOG(ERROR) << "Fail to initialize channel"; + return -1; + } + + if (argc <= 1) { + LOG(ERROR) << "No sql statement args"; + } else { + std::vector commands; + commands.reserve(argc * 16); + for (int i = 1; i < argc; ++i) { + commands.push_back(argv[i]); + } + if (!access_mysql(channel, commands)) { + return -1; + } + } + return 0; +} + +/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/example/mysql_c++/mysqlclient_press.cpp b/example/mysql_c++/mysqlclient_press.cpp new file mode 100644 index 0000000000..b1f27a8c9c --- /dev/null +++ b/example/mysql_c++/mysqlclient_press.cpp @@ -0,0 +1,239 @@ +// Copyright (c) 2014 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A brpc based command-line interface to talk with mysql-server + +#include +#include +#include +extern "C" { +#include +} +#include +#include +#include +#include + +DEFINE_string(server, "127.0.0.1", "IP Address of server"); +DEFINE_int32(port, 3306, "Port of server"); +DEFINE_string(user, "brpcuser", "user name"); +DEFINE_string(password, "12345678", "password"); +DEFINE_string(schema, "brpc_test", "schema"); +DEFINE_string(params, "", "params"); +DEFINE_string(data, "ABCDEF", "data"); +DEFINE_int32(thread_num, 50, "Number of threads to send requests"); +DEFINE_bool(use_bthread, false, "Use bthread to send requests"); +DEFINE_int32(dummy_port, -1, "port of dummy server(for monitoring)"); +DEFINE_int32(op_type, 0, "CRUD operation, 0:INSERT, 1:SELECT, 3:UPDATE"); +DEFINE_bool(dont_fail, false, "Print fatal when some call failed"); + +bvar::LatencyRecorder g_latency_recorder("client"); +bvar::Adder g_error_count("client_error_count"); + +struct SenderArgs { + int base_index; + MYSQL* mysql_conn; +}; + +const std::string insert = + "insert into mysqlclient_press(col1,col2,col3,col4) values " + "('" + "ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCA" + "BCABCABCABCABCABCABCA', '" + + FLAGS_data + + "' ,1.5, " + "now())"; +// Send `command' to mysql-server via `channel' +static void* sender(void* void_args) { + SenderArgs* args = (SenderArgs*)void_args; + std::stringstream command; + if (FLAGS_op_type == 0) { + command << insert; + } else if (FLAGS_op_type == 1) { + command << "select * from mysqlclient_press where id = " << args->base_index + 1; + } else if (FLAGS_op_type == 2) { + command << "update brpc_press set col2 = '" + FLAGS_data + "' where id = " + << args->base_index + 1; + } else { + LOG(ERROR) << "wrong op type " << FLAGS_op_type; + } + + std::string command_str = command.str(); + + while (!brpc::IsAskedToQuit()) { + const int64_t begin_time_us = butil::cpuwide_time_us(); + const int rc = mysql_real_query(args->mysql_conn, command_str.c_str(), command_str.size()); + if (rc != 0) { + goto ERROR; + } + + if (mysql_errno(args->mysql_conn) == 0) { + if (FLAGS_op_type == 0) { + CHECK_EQ(mysql_affected_rows(args->mysql_conn), 1); + } else if (FLAGS_op_type == 1) { + MYSQL_RES* res = mysql_store_result(args->mysql_conn); + if (res == NULL) { + LOG(INFO) << "not found"; + } else { + CHECK_EQ(mysql_num_rows(res), 1); + mysql_free_result(res); + } + } else if (FLAGS_op_type == 2) { + } + const int64_t elp = butil::cpuwide_time_us() - begin_time_us; + g_latency_recorder << elp; + } else { + goto ERROR; + } + + if (false) { + ERROR: + const int64_t elp = butil::cpuwide_time_us() - begin_time_us; + g_error_count << 1; + CHECK(brpc::IsAskedToQuit() || !FLAGS_dont_fail) + << "error=" << mysql_error(args->mysql_conn) << " latency=" << elp; + // We can't connect to the server, sleep a while. Notice that this + // is a specific sleeping to prevent this thread from spinning too + // fast. You should continue the business logic in a production + // server rather than sleeping. + bthread_usleep(50000); + } + } + return NULL; +} + +int main(int argc, char* argv[]) { + // Parse gflags. We recommend you to use gflags as well. + GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); + + if (FLAGS_dummy_port >= 0) { + brpc::StartDummyServerAt(FLAGS_dummy_port); + } + + MYSQL* conn = mysql_init(NULL); + if (!mysql_real_connect(conn, + FLAGS_server.c_str(), + FLAGS_user.c_str(), + FLAGS_password.c_str(), + FLAGS_schema.c_str(), + FLAGS_port, + NULL, + 0)) { + LOG(ERROR) << mysql_error(conn); + return -1; + } + + // create table mysqlclient_press + { + const char* sql = + "CREATE TABLE IF NOT EXISTS `mysqlclient_press`(`id` INT UNSIGNED AUTO_INCREMENT, " + "`col1` " + "VARCHAR(100) NOT NULL, `col2` VARCHAR(1024) NOT NULL, `col3` decimal(10,0) NOT " + "NULL, `col4` DATE, PRIMARY KEY ( `id` )) ENGINE=InnoDB DEFAULT CHARSET=utf8;"; + const int rc = mysql_real_query(conn, sql, strlen(sql)); + if (rc != 0) { + LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn); + return -1; + } + + if (mysql_errno(conn) != 0) { + LOG(ERROR) << "Fail to store result, " << mysql_error(conn); + return -1; + } + } + + // truncate table + { + const char* sql = "truncate table mysqlclient_press"; + const int rc = mysql_real_query(conn, sql, strlen(sql)); + if (rc != 0) { + LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn); + return -1; + } + + if (mysql_errno(conn) != 0) { + LOG(ERROR) << "Fail to store result, " << mysql_error(conn); + return -1; + } + } + + // prepare data for select, update + if (FLAGS_op_type != 0) { + for (int i = 0; i < FLAGS_thread_num; ++i) { + const int rc = mysql_real_query(conn, insert.c_str(), insert.size()); + if (rc != 0) { + LOG(ERROR) << "Fail to execute sql, " << mysql_error(conn); + return -1; + } + + if (mysql_errno(conn) != 0) { + LOG(ERROR) << "Fail to store result, " << mysql_error(conn); + return -1; + } + } + } + + // test CRUD operations + std::vector bids; + std::vector pids; + bids.resize(FLAGS_thread_num); + pids.resize(FLAGS_thread_num); + std::vector args; + args.resize(FLAGS_thread_num); + for (int i = 0; i < FLAGS_thread_num; ++i) { + MYSQL* conn = mysql_init(NULL); + if (!mysql_real_connect(conn, + FLAGS_server.c_str(), + FLAGS_user.c_str(), + FLAGS_password.c_str(), + FLAGS_schema.c_str(), + FLAGS_port, + NULL, + 0)) { + LOG(ERROR) << mysql_error(conn); + return -1; + } + args[i].base_index = i; + args[i].mysql_conn = conn; + if (!FLAGS_use_bthread) { + if (pthread_create(&pids[i], NULL, sender, &args[i]) != 0) { + LOG(ERROR) << "Fail to create pthread"; + return -1; + } + } else { + if (bthread_start_background(&bids[i], NULL, sender, &args[i]) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + } + } + + while (!brpc::IsAskedToQuit()) { + sleep(1); + + LOG(INFO) << "Accessing mysql-server at qps=" << g_latency_recorder.qps(1) + << " latency=" << g_latency_recorder.latency(1); + } + + LOG(INFO) << "mysql_client is going to quit"; + for (int i = 0; i < FLAGS_thread_num; ++i) { + if (!FLAGS_use_bthread) { + pthread_join(pids[i], NULL); + } else { + bthread_join(bids[i], NULL); + } + } + + return 0; +} diff --git a/src/brpc/controller.cpp b/src/brpc/controller.cpp index b6c8e750fe..ccec56e67f 100644 --- a/src/brpc/controller.cpp +++ b/src/brpc/controller.cpp @@ -278,6 +278,8 @@ void Controller::ResetPods() { _request_stream = INVALID_STREAM_ID; _response_stream = INVALID_STREAM_ID; _remote_stream_settings = NULL; + _bind_sock_action = BIND_SOCK_NONE; + _stmt = NULL; _auth_flags = 0; } @@ -308,6 +310,7 @@ void Controller::Call::Reset() { peer_id = INVALID_SOCKET_ID; begin_time_us = 0; sending_sock.reset(NULL); + bind_sock_action = BIND_SOCK_NONE; stream_user_data = NULL; } @@ -760,8 +763,14 @@ void Controller::Call::OnComplete( // Otherwise in-flight responses may come back in future and break the // assumption that one pooled connection cannot have more than one // message at the same time. + // If bind sock action is active take the ownship of sending sock + // If bind sock is not INVALID, RPC complete, reset the sock to INVALID if (sending_sock != NULL && (error_code == 0 || responded)) { - if (!sending_sock->is_read_progressive()) { + if (bind_sock_action == BIND_SOCK_ACTIVE) { + c->_bind_sock.reset(sending_sock.release()); + } else if (bind_sock_action == BIND_SOCK_USE) { + // do nothing + } else if (!sending_sock->is_read_progressive()) { // Normally-read socket which will not be used after RPC ends, // safe to return. Notice that Socket::is_read_progressive may // differ from Controller::is_response_read_progressively() @@ -778,7 +787,11 @@ void Controller::Call::OnComplete( case CONNECTION_TYPE_SHORT: if (sending_sock != NULL) { // Check the comment in CONNECTION_TYPE_POOLED branch. - if (!sending_sock->is_read_progressive()) { + if (bind_sock_action == BIND_SOCK_ACTIVE) { + c->_bind_sock.reset(sending_sock.release()); + } else if (bind_sock_action == BIND_SOCK_USE) { + // do nothing + } else if (!sending_sock->is_read_progressive()) { if (c->_stream_creator == NULL) { sending_sock->SetFailed(); } @@ -845,6 +858,8 @@ void Controller::EndRPC(const CompletionInfo& info) { } // TODO: Replace this with stream_creator. HandleStreamConnection(_current_call.sending_sock.get()); + // Only bind the success sock of the RPC + _current_call.bind_sock_action = _bind_sock_action; _current_call.OnComplete(this, _error_code, info.responded, true); } else { // Even if _unfinished_call succeeded, we don't use EBACKUPREQUEST @@ -1017,7 +1032,17 @@ void Controller::IssueRPC(int64_t start_realtime_us) { _current_call.need_feedback = false; _current_call.enable_circuit_breaker = has_enabled_circuit_breaker(); SocketUniquePtr tmp_sock; - if (SingleServer()) { + if ((_connection_type & CONNECTION_TYPE_POOLED_AND_SHORT) && + _bind_sock_action == BIND_SOCK_USE) { + tmp_sock.reset(_bind_sock.release()); + if (!tmp_sock || (!is_health_check_call() && !tmp_sock->IsAvailable())) { + SetFailed(EHOSTDOWN, "Not connected to bind socket yet, server_id=%" PRIu64, + tmp_sock->id()); + tmp_sock.reset(); // Release ref ASAP + return HandleSendFailed(); + } + _current_call.peer_id = tmp_sock->id(); + } else if (SingleServer()) { // Don't use _current_call.peer_id which is set to -1 after construction // of the backup call. const int rc = Socket::Address(_single_server_id, &tmp_sock); @@ -1083,7 +1108,10 @@ void Controller::IssueRPC(int64_t start_realtime_us) { _current_call.sending_sock->set_preferred_index(_preferred_index); } else { int rc = 0; - if (_connection_type == CONNECTION_TYPE_POOLED) { + // if _bind_sock is not NULL, use it + if (_bind_sock_action == BIND_SOCK_USE) { + _current_call.sending_sock.reset(tmp_sock.release()); + } else if (_connection_type == CONNECTION_TYPE_POOLED) { rc = tmp_sock->GetPooledSocket(&_current_call.sending_sock); } else if (_connection_type == CONNECTION_TYPE_SHORT) { rc = tmp_sock->GetShortSocket(&_current_call.sending_sock); @@ -1105,7 +1133,7 @@ void Controller::IssueRPC(int64_t start_realtime_us) { _current_call.sending_sock->set_preferred_index(_preferred_index); // Set preferred_index of main_socket as well to make it easier to // debug and observe from /connections. - if (tmp_sock->preferred_index() < 0) { + if (tmp_sock && tmp_sock->preferred_index() < 0) { tmp_sock->set_preferred_index(_preferred_index); } tmp_sock.reset(); diff --git a/src/brpc/controller.h b/src/brpc/controller.h index 658cc6957c..92fe7c76a2 100644 --- a/src/brpc/controller.h +++ b/src/brpc/controller.h @@ -104,6 +104,14 @@ enum StopStyle { const int32_t UNSET_MAGIC_NUM = -123456789; +// if controller want to reserve a sock after RPC, set BIND_SOCK_ACTIVE +enum BindSockAction { + BIND_SOCK_ACTIVE, + BIND_SOCK_USE, + BIND_SOCK_NONE, +}; +// mysql prepared statement declare +class MysqlStatementStub; // A Controller mediates a single method call. The primary purpose of // the controller is to provide a way to manipulate settings per RPC-call // and to find out about RPC-level errors. @@ -674,6 +682,7 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // CONNECTION_TYPE_SINGLE. Otherwise, it may be a temporary // socket fetched from socket pool SocketUniquePtr sending_sock; + BindSockAction bind_sock_action; StreamUserData* stream_user_data; }; @@ -815,6 +824,13 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Defined at both sides StreamSettings *_remote_stream_settings; + // controller bind socket action + BindSockAction _bind_sock_action; + // controller bind sock + SocketUniquePtr _bind_sock; + // sql prepare statement + MysqlStatementStub *_stmt; + // Thrift method name, only used when thrift protocol enabled std::string _thrift_method_name; diff --git a/src/brpc/details/controller_private_accessor.h b/src/brpc/details/controller_private_accessor.h index 1be7df8bb8..5c130054d5 100644 --- a/src/brpc/details/controller_private_accessor.h +++ b/src/brpc/details/controller_private_accessor.h @@ -152,6 +152,22 @@ class ControllerPrivateAccessor { return *this; } + // Set bind socket action + void set_bind_sock_action(BindSockAction action) { _cntl->_bind_sock_action = action; } + // Transfer ownership to other + void get_bind_sock(SocketUniquePtr* ptr) { + _cntl->_bind_sock->ReAddress(ptr); + } + // Use a external socket + void use_bind_sock(SocketId sock_id) { + _cntl->_bind_sock_action = BIND_SOCK_USE; + Socket::Address(sock_id, &_cntl->_bind_sock); + } + // set prepare statement + void set_stmt(MysqlStatementStub *stmt) { _cntl->_stmt = stmt; } + // get prepare statement + MysqlStatementStub* get_stmt() { return _cntl->_stmt; } + private: Controller* _cntl; }; diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 30c2f1a3b9..aedeea6c8c 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -72,6 +72,7 @@ #include "brpc/policy/nshead_mcpack_protocol.h" #include "brpc/policy/rtmp_protocol.h" #include "brpc/policy/esp_protocol.h" +#include "brpc/policy/mysql_protocol.h" #ifdef ENABLE_THRIFT_FRAMED_PROTOCOL # include "brpc/policy/thrift_protocol.h" #endif @@ -582,6 +583,20 @@ static void GlobalInitializeOrDieImpl() { exit(1); } + Protocol mysql_protocol = {ParseMysqlMessage, + SerializeMysqlRequest, + PackMysqlRequest, + NULL, + ProcessMysqlResponse, + NULL, + NULL, + GetMysqlMethodName, + CONNECTION_TYPE_POOLED_AND_SHORT, + "mysql"}; + if (RegisterProtocol(PROTOCOL_MYSQL, mysql_protocol) != 0) { + exit(1); + } + std::vector protocols; ListProtocols(&protocols); for (size_t i = 0; i < protocols.size(); ++i) { diff --git a/src/brpc/mysql.cpp b/src/brpc/mysql.cpp new file mode 100644 index 0000000000..5f3501a2e4 --- /dev/null +++ b/src/brpc/mysql.cpp @@ -0,0 +1,695 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#define INTERNAL_SUPPRESS_PROTOBUF_FIELD_DEPRECATION +#include +#include +#include +#include +#include +#include +#include +#include +#include "butil/string_printf.h" +#include "butil/macros.h" +#include "brpc/controller.h" +#include "brpc/mysql.h" +#include "brpc/mysql_common.h" + +namespace brpc { + +DEFINE_int32(mysql_multi_replies_size, 10, "multi replies size in one MysqlResponse"); + +// Internal implementation detail -- do not call these. +void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_impl(); +void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); +void protobuf_AssignDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); +void protobuf_ShutdownFile_baidu_2frpc_2fmysql_5fbase_2eproto(); + +namespace { + +const ::google::protobuf::Descriptor* MysqlRequest_descriptor_ = NULL; +const ::google::protobuf::Descriptor* MysqlResponse_descriptor_ = NULL; + +} // namespace + +void protobuf_AssignDesc_baidu_2frpc_2fmysql_5fbase_2eproto() { + protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + const ::google::protobuf::FileDescriptor* file = + ::google::protobuf::DescriptorPool::generated_pool()->FindFileByName( + "baidu/rpc/mysql_base.proto"); + GOOGLE_CHECK(file != NULL); + MysqlRequest_descriptor_ = file->message_type(0); + MysqlResponse_descriptor_ = file->message_type(1); +} + +namespace { + +GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AssignDescriptors_once_); +inline void protobuf_AssignDescriptorsOnce() { + ::google::protobuf::GoogleOnceInit(&protobuf_AssignDescriptors_once_, + &protobuf_AssignDesc_baidu_2frpc_2fmysql_5fbase_2eproto); +} + +void protobuf_RegisterTypes(const ::std::string&) { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + MysqlRequest_descriptor_, &MysqlRequest::default_instance()); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedMessage( + MysqlResponse_descriptor_, &MysqlResponse::default_instance()); +} + +} // namespace + +void protobuf_ShutdownFile_baidu_2frpc_2fmysql_5fbase_2eproto() { + delete MysqlRequest::default_instance_; + delete MysqlResponse::default_instance_; +} + +void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_impl() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + +#if GOOGLE_PROTOBUF_VERSION >= 3002000 + ::google::protobuf::internal::InitProtobufDefaults(); +#else + ::google::protobuf::protobuf_AddDesc_google_2fprotobuf_2fdescriptor_2eproto(); +#endif + ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( + "\n\032baidu/rpc/mysql_base.proto\022\tbaidu.rpc\032" + " google/protobuf/descriptor.proto\"\016\n\014Mys" + "qlRequest\"\017\n\rMysqlResponseB\003\200\001\001", + 111); + ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile("baidu/rpc/mysql_base.proto", + &protobuf_RegisterTypes); + MysqlRequest::default_instance_ = new MysqlRequest(); + MysqlResponse::default_instance_ = new MysqlResponse(); + MysqlRequest::default_instance_->InitAsDefaultInstance(); + MysqlResponse::default_instance_->InitAsDefaultInstance(); + ::google::protobuf::internal::OnShutdown( + &protobuf_ShutdownFile_baidu_2frpc_2fmysql_5fbase_2eproto); +} + +GOOGLE_PROTOBUF_DECLARE_ONCE(protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_once); +void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto() { + ::google::protobuf::GoogleOnceInit(&protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_once, + &protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_impl); +} + +// Force AddDescriptors() to be called at static initialization time. +struct StaticDescriptorInitializer_baidu_2frpc_2fmysql_5fbase_2eproto { + StaticDescriptorInitializer_baidu_2frpc_2fmysql_5fbase_2eproto() { + protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + } +} static_descriptor_initializer_baidu_2frpc_2fmysql_5fbase_2eproto_; + + +// =================================================================== + +#ifndef _MSC_VER +#endif // !_MSC_VER + +butil::Status MysqlStatementStub::PackExecuteCommand(butil::IOBuf* outbuf, uint32_t stmt_id) { + butil::Status st; + // long data + for (const auto& i : _long_data) { + st = MysqlMakeLongDataPacket(outbuf, stmt_id, i.param_id, i.long_data); + if (!st.ok()) { + LOG(ERROR) << "make long data header error " << st; + return st; + } + } + _long_data.clear(); + // execute data + st = MysqlMakeExecutePacket(outbuf, stmt_id, _execute_data); + if (!st.ok()) { + LOG(ERROR) << "make execute header error " << st; + return st; + } + _execute_data.clear(); + _null_mask.mask.clear(); + _null_mask.area = butil::IOBuf::INVALID_AREA; + _param_types.types.clear(); + _param_types.area = butil::IOBuf::INVALID_AREA; + + return st; +} + +MysqlRequest::MysqlRequest() : ::google::protobuf::Message() { + SharedCtor(); +} + +MysqlRequest::MysqlRequest(const MysqlTransaction* tx) : ::google::protobuf::Message() { + SharedCtor(); + _tx = tx; +} + +MysqlRequest::MysqlRequest(MysqlStatement* stmt) : ::google::protobuf::Message() { + SharedCtor(); + _stmt = new MysqlStatementStub(stmt); +} + +MysqlRequest::MysqlRequest(const MysqlTransaction* tx, MysqlStatement* stmt) + : ::google::protobuf::Message() { + SharedCtor(); + _tx = tx; + _stmt = new MysqlStatementStub(stmt); +} + +void MysqlRequest::InitAsDefaultInstance() {} + +MysqlRequest::MysqlRequest(const MysqlRequest& from) : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); +} + +void MysqlRequest::SharedCtor() { + _has_error = false; + _cached_size_ = 0; + _has_command = false; + _tx = NULL; + _stmt = NULL; + _param_index = 0; +} + +MysqlRequest::~MysqlRequest() { + SharedDtor(); + if (_stmt != NULL) { + delete _stmt; + } + _stmt = NULL; +} + +void MysqlRequest::SharedDtor() { + if (this != default_instance_) { + } +} + +void MysqlRequest::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const ::google::protobuf::Descriptor* MysqlRequest::descriptor() { + protobuf_AssignDescriptorsOnce(); + return MysqlRequest_descriptor_; +} + +const MysqlRequest& MysqlRequest::default_instance() { + if (default_instance_ == NULL) { + protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + } + return *default_instance_; +} + +MysqlRequest* MysqlRequest::default_instance_ = NULL; + +MysqlRequest* MysqlRequest::New() const { + return new MysqlRequest; +} + +void MysqlRequest::Clear() { + _has_error = false; + _buf.clear(); + _has_command = false; + _tx = NULL; + _stmt = NULL; +} + +bool MysqlRequest::MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream*) { + LOG(WARNING) << "You're not supposed to parse a MysqlRequest"; + return true; +} + +void MysqlRequest::SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream*) const { + LOG(WARNING) << "You're not supposed to serialize a MysqlRequest"; +} + +::google::protobuf::uint8* MysqlRequest::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + return target; +} + +int MysqlRequest::ByteSize() const { + int total_size = _buf.size(); + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = total_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void MysqlRequest::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const MysqlRequest* source = + ::google::protobuf::internal::dynamic_cast_if_available(&from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void MysqlRequest::MergeFrom(const MysqlRequest& from) { + // TODO: maybe need to optimize + // GOOGLE_CHECK_NE(&from, this); + // const int header_size = 4; + // const uint32_t size_l = from._buf.size() - header_size - 1; // payload - type + // const uint32_t size_r = _buf.size() - header_size + 1; // payload + seqno + // const uint32_t payload_size = butil::ByteSwapToLE32(size_l + size_r); + // if (payload_size > mysql_max_package_size) { + // CHECK(false) + // << "[MysqlRequest::MergeFrom] statement size is too big, merge from do nothing"; + // return; + // } + // butil::IOBuf buf; + // butil::IOBuf result; + // _has_error = _has_error || from._has_error; + // buf.append(from._buf); + // buf.pop_front(header_size + 1); + // _buf.pop_front(header_size - 1); + // result.append(&payload_size, 3); + // result.append(_buf); + // result.append(buf); + // _buf = result; + // _has_command = _has_command || from._has_command; +} + +void MysqlRequest::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) + return; + Clear(); + MergeFrom(from); +} + +void MysqlRequest::CopyFrom(const MysqlRequest& from) { + if (&from == this) + return; + Clear(); + MergeFrom(from); +} + +void MysqlRequest::Swap(MysqlRequest* other) { + if (other != this) { + _buf.swap(other->_buf); + std::swap(_has_error, other->_has_error); + std::swap(_cached_size_, other->_cached_size_); + std::swap(_has_command, other->_has_command); + } +} + +bool MysqlRequest::SerializeTo(butil::IOBuf* buf) const { + if (_has_error) { + LOG(ERROR) << "Reject serialization due to error in CommandXXX[V]"; + return false; + } + *buf = _buf; + return true; +} + +::google::protobuf::Metadata MysqlRequest::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = MysqlRequest_descriptor_; + metadata.reflection = NULL; + return metadata; +} + +bool MysqlRequest::Query(const butil::StringPiece& command) { + if (_has_error) { + return false; + } + + if (_has_command) { + return false; + } + + const butil::Status st = MysqlMakeCommand(&_buf, MYSQL_COM_QUERY, command); + if (st.ok()) { + _has_command = true; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} + +bool MysqlRequest::AddParam(int8_t p) { + if (_has_error) { + return false; + } + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_TINY); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(uint8_t p) { + const butil::Status st = + MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_TINY, true); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(int16_t p) { + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_SHORT); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(uint16_t p) { + const butil::Status st = + MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_SHORT, true); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(int32_t p) { + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_LONG); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(uint32_t p) { + const butil::Status st = + MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_LONG, true); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(int64_t p) { + const butil::Status st = + MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_LONGLONG); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(uint64_t p) { + const butil::Status st = + MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_LONGLONG, true); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(float p) { + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_FLOAT); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(double p) { + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_DOUBLE); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} +bool MysqlRequest::AddParam(const butil::StringPiece& p) { + const butil::Status st = MysqlMakeExecuteData(_stmt, _param_index, &p, MYSQL_FIELD_TYPE_STRING); + if (st.ok()) { + ++_param_index; + return true; + } else { + CHECK(st.ok()) << st; + _has_error = true; + return false; + } +} + +void MysqlRequest::Print(std::ostream& os) const { + butil::IOBuf cp = _buf; + { + uint8_t buf[3]; + cp.cutn(buf, 3); + os << "size:" << mysql_uint3korr(buf) << ","; + } + { + uint8_t buf; + cp.cut1((char*)&buf); + os << "sequence:" << (unsigned)buf << ","; + } + os << "payload(hex):"; + while (!cp.empty()) { + uint8_t buf; + cp.cut1((char*)&buf); + os << std::hex << (unsigned)buf; + } +} + +std::ostream& operator<<(std::ostream& os, const MysqlRequest& r) { + r.Print(os); + return os; +} + +// =================================================================== + +#ifndef _MSC_VER +#endif // !_MSC_VER + +MysqlResponse::MysqlResponse() : ::google::protobuf::Message() { + SharedCtor(); +} + +void MysqlResponse::InitAsDefaultInstance() {} + +MysqlResponse::MysqlResponse(const MysqlResponse& from) : ::google::protobuf::Message() { + SharedCtor(); + MergeFrom(from); +} + +void MysqlResponse::SharedCtor() { + _nreply = 0; + _cached_size_ = 0; +} + +MysqlResponse::~MysqlResponse() { + SharedDtor(); +} + +void MysqlResponse::SharedDtor() { + if (this != default_instance_) { + } +} + +void MysqlResponse::SetCachedSize(int size) const { + _cached_size_ = size; +} +const ::google::protobuf::Descriptor* MysqlResponse::descriptor() { + protobuf_AssignDescriptorsOnce(); + return MysqlResponse_descriptor_; +} + +const MysqlResponse& MysqlResponse::default_instance() { + if (default_instance_ == NULL) { + protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + } + return *default_instance_; +} + +MysqlResponse* MysqlResponse::default_instance_ = NULL; + +MysqlResponse* MysqlResponse::New() const { + return new MysqlResponse; +} + +void MysqlResponse::Clear() {} + +bool MysqlResponse::MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream*) { + LOG(WARNING) << "You're not supposed to parse a MysqlResponse"; + return true; +} + +void MysqlResponse::SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream*) const { + LOG(WARNING) << "You're not supposed to serialize a MysqlResponse"; +} + +::google::protobuf::uint8* MysqlResponse::SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* target) const { + return target; +} + +int MysqlResponse::ByteSize() const { + return _cached_size_; +} + +void MysqlResponse::MergeFrom(const ::google::protobuf::Message& from) { + GOOGLE_CHECK_NE(&from, this); + const MysqlResponse* source = + ::google::protobuf::internal::dynamic_cast_if_available(&from); + if (source == NULL) { + ::google::protobuf::internal::ReflectionOps::Merge(from, this); + } else { + MergeFrom(*source); + } +} + +void MysqlResponse::MergeFrom(const MysqlResponse& from) { + GOOGLE_CHECK_NE(&from, this); +} + +void MysqlResponse::CopyFrom(const ::google::protobuf::Message& from) { + if (&from == this) + return; + Clear(); + MergeFrom(from); +} + +void MysqlResponse::CopyFrom(const MysqlResponse& from) { + if (&from == this) + return; + Clear(); + MergeFrom(from); +} + +bool MysqlResponse::IsInitialized() const { + return true; +} + +void MysqlResponse::Swap(MysqlResponse* other) { + if (other != this) { + _first_reply.Swap(other->_first_reply); + std::swap(_other_replies, other->_other_replies); + _arena.swap(other->_arena); + std::swap(_nreply, other->_nreply); + std::swap(_cached_size_, other->_cached_size_); + } +} + +::google::protobuf::Metadata MysqlResponse::GetMetadata() const { + protobuf_AssignDescriptorsOnce(); + ::google::protobuf::Metadata metadata; + metadata.descriptor = MysqlResponse_descriptor_; + metadata.reflection = NULL; + return metadata; +} + +// =================================================================== + +ParseError MysqlResponse::ConsumePartialIOBuf(butil::IOBuf& buf, + bool is_auth, + MysqlStmtType stmt_type) { + bool more_results = true; + size_t oldsize = 0; + while (more_results) { + oldsize = buf.size(); + if (reply_size() == 0) { + ParseError err = + _first_reply.ConsumePartialIOBuf(buf, &_arena, is_auth, stmt_type, &more_results); + if (err != PARSE_OK) { + return err; + } + } else { + const int32_t replies_size = + FLAGS_mysql_multi_replies_size > 1 ? FLAGS_mysql_multi_replies_size : 10; + if (_other_replies.size() < reply_size()) { + MysqlReply* replies = + (MysqlReply*)_arena.allocate(sizeof(MysqlReply) * (replies_size - 1)); + if (replies == NULL) { + LOG(ERROR) << "Fail to allocate MysqlReply[" << replies_size - 1 << "]"; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + _other_replies.reserve(replies_size - 1); + for (int i = 0; i < replies_size - 1; ++i) { + new (&replies[i]) MysqlReply; + _other_replies.push_back(&replies[i]); + } + } + ParseError err = _other_replies[_nreply - 1]->ConsumePartialIOBuf( + buf, &_arena, is_auth, stmt_type, &more_results); + if (err != PARSE_OK) { + return err; + } + } + + const size_t newsize = buf.size(); + _cached_size_ += oldsize - newsize; + oldsize = newsize; + ++_nreply; + } + + if (oldsize == 0) { + return PARSE_OK; + } else { + LOG(ERROR) << "Parse protocol finished, but IOBuf has more data"; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } +} + +std::ostream& operator<<(std::ostream& os, const MysqlResponse& response) { + os << "\n-----MYSQL REPLY BEGIN-----\n"; + if (response.reply_size() == 0) { + os << ""; + } else if (response.reply_size() == 1) { + os << response.reply(0); + } else { + for (size_t i = 0; i < response.reply_size(); ++i) { + os << "\nreply(" << i << ")----------"; + os << response.reply(i); + } + } + os << "\n-----MYSQL REPLY END-----\n"; + + return os; +} + +} // namespace brpc diff --git a/src/brpc/mysql.h b/src/brpc/mysql.h new file mode 100644 index 0000000000..17fd5e6785 --- /dev/null +++ b/src/brpc/mysql.h @@ -0,0 +1,286 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_H +#define BRPC_MYSQL_H + +#include +#include +#include + +#include +#include +#include +#include +#include "google/protobuf/descriptor.pb.h" + +#include "butil/iobuf.h" +#include "butil/strings/string_piece.h" +#include "butil/arena.h" +#include "parse_result.h" +#include "mysql_command.h" +#include "mysql_reply.h" +#include "mysql_transaction.h" +#include "mysql_statement.h" + +namespace brpc { +// Request to mysql. +// Notice that you can pipeline multiple commands in one request and sent +// them to ONE mysql-server together. +// Example: +// MysqlRequest request; +// request.Query("select * from table"); +// MysqlResponse response; +// channel.CallMethod(NULL, &controller, &request, &response, NULL/*done*/); +// if (!cntl.Failed()) { +// LOG(INFO) << response.reply(0); +// } + +class MysqlStatementStub { +public: + MysqlStatementStub(MysqlStatement* stmt); + MysqlStatement* stmt(); + butil::IOBuf& execute_data(); + butil::Status PackExecuteCommand(butil::IOBuf* outbuf, uint32_t stmt_id); + // prepare statement null mask + struct NullMask { + NullMask() : area(butil::IOBuf::INVALID_AREA) {} + std::vector mask; + butil::IOBuf::Area area; + }; + // prepare statement param types + struct ParamTypes { + ParamTypes() : area(butil::IOBuf::INVALID_AREA) {} + std::vector types; + butil::IOBuf::Area area; + }; + // null mask and param types + NullMask& null_mask(); + ParamTypes& param_types(); + // save long data + void save_long_data(uint16_t param_id, const butil::StringPiece& value); + +private: + MysqlStatement* _stmt; + butil::IOBuf _execute_data; + NullMask _null_mask; + ParamTypes _param_types; + // long data + struct LongData { + uint16_t param_id; + butil::IOBuf long_data; + }; + std::vector _long_data; +}; + +inline MysqlStatementStub::MysqlStatementStub(MysqlStatement* stmt) : _stmt(stmt) {} + +inline MysqlStatement* MysqlStatementStub::stmt() { + return _stmt; +} + +inline butil::IOBuf& MysqlStatementStub::execute_data() { + return _execute_data; +} + +inline MysqlStatementStub::NullMask& MysqlStatementStub::null_mask() { + return _null_mask; +} + +inline MysqlStatementStub::ParamTypes& MysqlStatementStub::param_types() { + return _param_types; +} + +inline void MysqlStatementStub::save_long_data(uint16_t param_id, const butil::StringPiece& value) { + LongData d; + d.param_id = param_id; + d.long_data.append(value.data(), value.size()); + _long_data.push_back(d); +} + +class MysqlRequest : public ::google::protobuf::Message { +public: + MysqlRequest(); + MysqlRequest(const MysqlTransaction* tx); + MysqlRequest(MysqlStatement* stmt); + MysqlRequest(const MysqlTransaction* tx, MysqlStatement* stmt); + virtual ~MysqlRequest(); + MysqlRequest(const MysqlRequest& from); + inline MysqlRequest& operator=(const MysqlRequest& from) { + CopyFrom(from); + return *this; + } + void Swap(MysqlRequest* other); + + // Serialize the request into `buf'. Return true on success. + bool SerializeTo(butil::IOBuf* buf) const; + + // Protobuf methods. + MysqlRequest* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const MysqlRequest& from); + void MergeFrom(const MysqlRequest& from); + void Clear(); + + int ByteSize() const; + bool MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* output) const; + int GetCachedSize() const { + return _cached_size_; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const MysqlRequest& default_instance(); + ::google::protobuf::Metadata GetMetadata() const; + + // call query command + bool Query(const butil::StringPiece& command); + // add statement params + bool AddParam(int8_t p); + bool AddParam(uint8_t p); + bool AddParam(int16_t p); + bool AddParam(uint16_t p); + bool AddParam(int32_t p); + bool AddParam(uint32_t p); + bool AddParam(int64_t p); + bool AddParam(uint64_t p); + bool AddParam(float p); + bool AddParam(double p); + bool AddParam(const butil::StringPiece& p); + + // True if previous command failed. + bool has_error() const { + return _has_error; + } + + const MysqlTransaction* get_tx() const { + return _tx; + } + + MysqlStatementStub* get_stmt() const { + return _stmt; + } + + void Print(std::ostream&) const; + +private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + + bool _has_command; // request has command + bool _has_error; // previous AddCommand had error + butil::IOBuf _buf; // the serialized request. + mutable int _cached_size_; // ByteSize + const MysqlTransaction* _tx; // transaction + MysqlStatementStub* _stmt; // statement + uint16_t _param_index; // statement param index + + friend void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_impl(); + friend void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + friend void protobuf_AssignDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + friend void protobuf_ShutdownFile_baidu_2frpc_2fmysql_5fbase_2eproto(); + + void InitAsDefaultInstance(); + static MysqlRequest* default_instance_; +}; + +// Response from Mysql. +// Notice that a MysqlResponse instance may contain multiple replies +// due to pipelining. +class MysqlResponse : public ::google::protobuf::Message { +public: + MysqlResponse(); + virtual ~MysqlResponse(); + MysqlResponse(const MysqlResponse& from); + inline MysqlResponse& operator=(const MysqlResponse& from) { + CopyFrom(from); + return *this; + } + void Swap(MysqlResponse* other); + // Parse and consume intact replies from the buf, actual reply size may less then max_count, if + // some command execute failed + // Returns PARSE_OK on success. + // Returns PARSE_ERROR_NOT_ENOUGH_DATA if data in `buf' is not enough to parse. + // Returns PARSE_ERROR_ABSOLUTELY_WRONG if the parsing + // failed. + ParseError ConsumePartialIOBuf(butil::IOBuf& buf, bool is_auth, MysqlStmtType stmt_type); + + // Number of replies in this response. + // (May have more than one reply due to pipeline) + size_t reply_size() const { + return _nreply; + } + + const MysqlReply& reply(size_t index) const { + if (index < reply_size()) { + return (index == 0 ? _first_reply : *_other_replies[index - 1]); + } + static MysqlReply mysql_nil; + return mysql_nil; + } + // implements Message ---------------------------------------------- + + MysqlResponse* New() const; + void CopyFrom(const ::google::protobuf::Message& from); + void MergeFrom(const ::google::protobuf::Message& from); + void CopyFrom(const MysqlResponse& from); + void MergeFrom(const MysqlResponse& from); + void Clear(); + bool IsInitialized() const; + + int ByteSize() const; + bool MergePartialFromCodedStream(::google::protobuf::io::CodedInputStream* input); + void SerializeWithCachedSizes(::google::protobuf::io::CodedOutputStream* output) const; + ::google::protobuf::uint8* SerializeWithCachedSizesToArray( + ::google::protobuf::uint8* output) const; + int GetCachedSize() const { + return 0; + } + + static const ::google::protobuf::Descriptor* descriptor(); + static const MysqlResponse& default_instance(); + ::google::protobuf::Metadata GetMetadata() const; + +private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + + MysqlReply _first_reply; + std::vector _other_replies; + butil::Arena _arena; + size_t _nreply; + mutable int _cached_size_; + + friend void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto_impl(); + friend void protobuf_AddDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + friend void protobuf_AssignDesc_baidu_2frpc_2fmysql_5fbase_2eproto(); + friend void protobuf_ShutdownFile_baidu_2frpc_2fmysql_5fbase_2eproto(); + + void InitAsDefaultInstance(); + static MysqlResponse* default_instance_; +}; + +std::ostream& operator<<(std::ostream& os, const MysqlRequest&); +std::ostream& operator<<(std::ostream& os, const MysqlResponse&); + +} // namespace brpc + +#endif // BRPC_MYSQL_H diff --git a/src/brpc/mysql_command.cpp b/src/brpc/mysql_command.cpp new file mode 100644 index 0000000000..c4f4debf6a --- /dev/null +++ b/src/brpc/mysql_command.cpp @@ -0,0 +1,260 @@ +// Copyright (c) 2015 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include "butil/sys_byteorder.h" +#include "butil/logging.h" // LOG() +#include "brpc/mysql_command.h" +#include "brpc/mysql_common.h" +#include "brpc/mysql.h" + +namespace brpc { + +namespace { +const uint32_t max_allowed_packet = 67108864; +const uint32_t max_packet_size = 16777215; + +template +butil::Status MakePacket(butil::IOBuf* outbuf, const H& head, const F& func, const D& data) { + long pkg_len = head.size() + data.size(); + if (pkg_len > max_allowed_packet) { + return butil::Status( + EINVAL, + "[MakePacket] statement size is too big, maxAllowedPacket = %d, pkg_len = %ld", + max_allowed_packet, + pkg_len); + } + uint32_t size, header; + uint8_t seq = 0; + size_t offset = 0; + for (; pkg_len > 0; pkg_len -= max_packet_size, ++seq) { + if (pkg_len > max_packet_size) { + size = max_packet_size; + } else { + size = pkg_len; + } + header = butil::ByteSwapToLE32(size); + ((uint8_t*)&header)[3] = seq; + outbuf->append(&header, 4); + if (seq == 0) { + const uint32_t old_size = outbuf->size(); + outbuf->append(head); + size -= outbuf->size() - old_size; + } + func(outbuf, data, size, offset); + offset += size; + } + + return butil::Status::OK(); +} + +} // namespace + +butil::Status MysqlMakeCommand(butil::IOBuf* outbuf, + const MysqlCommandType type, + const butil::StringPiece& command) { + if (outbuf == NULL || command.size() == 0) { + return butil::Status(EINVAL, "[MysqlMakeCommand] Param[outbuf] or [stmt] is NULL"); + } + auto func = + [](butil::IOBuf* outbuf, const butil::StringPiece& command, size_t size, size_t offset) { + outbuf->append(command.data() + offset, size); + }; + butil::IOBuf head; + head.push_back(type); + return MakePacket(outbuf, head, func, command); +} + +butil::Status MysqlMakeExecutePacket(butil::IOBuf* outbuf, + uint32_t stmt_id, + const butil::IOBuf& edata) { + butil::IOBuf head; // cmd_type + stmt_id + flag + reserved + body_size + head.push_back(MYSQL_COM_STMT_EXECUTE); + const uint32_t si = butil::ByteSwapToLE32(stmt_id); + head.append(&si, 4); + head.push_back('\0'); + head.push_back((char)0x01); + head.push_back('\0'); + head.push_back('\0'); + head.push_back('\0'); + auto func = [](butil::IOBuf* outbuf, const butil::IOBuf& data, size_t size, size_t offset) { + data.append_to(outbuf, size, offset); + }; + return MakePacket(outbuf, head, func, edata); +} + +butil::Status MysqlMakeExecuteData(MysqlStatementStub* stmt, + uint16_t index, + const void* value, + MysqlFieldType type, + bool is_unsigned) { + const uint16_t n = stmt->stmt()->param_count(); + uint32_t long_data_size = max_allowed_packet / (n + 1); + if (long_data_size < 64) { + long_data_size = 64; + } + // if param count is zero finished. + if (n == 0) { + return butil::Status::OK(); + } + butil::IOBuf& buf = stmt->execute_data(); + MysqlStatementStub::NullMask& null_mask = stmt->null_mask(); + MysqlStatementStub::ParamTypes& param_types = stmt->param_types(); + // else param number larger than zero. + if (index >= n) { + LOG(ERROR) << "too many params"; + return butil::Status(EINVAL, "[MysqlMakeExecuteData] too many params"); + } + // reserve null mask and param types packing at first param + if (index == 0) { + const size_t mask_len = (n + 7) / 8; + const size_t types_len = 2 * n; + null_mask.mask.resize(mask_len, 0); + null_mask.area = buf.reserve(mask_len); + buf.push_back((char)0x01); + param_types.types.reserve(types_len); + param_types.area = buf.reserve(types_len); + } + // pack param value + switch (type) { + case MYSQL_FIELD_TYPE_TINY: + if (is_unsigned) { + param_types.types[index + index] = MYSQL_FIELD_TYPE_TINY; + param_types.types[index + index + 1] = 0x80; + } else { + param_types.types[index + index] = MYSQL_FIELD_TYPE_TINY; + param_types.types[index + index + 1] = 0x00; + } + buf.append(value, 1); + break; + case MYSQL_FIELD_TYPE_SHORT: + if (is_unsigned) { + param_types.types[index + index] = MYSQL_FIELD_TYPE_SHORT; + param_types.types[index + index + 1] = 0x80; + } else { + param_types.types[index + index] = MYSQL_FIELD_TYPE_SHORT; + param_types.types[index + index + 1] = 0x00; + } + { + uint16_t v = butil::ByteSwapToLE16(*(uint16_t*)value); + buf.append(&v, 2); + } + break; + case MYSQL_FIELD_TYPE_LONG: + if (is_unsigned) { + param_types.types[index + index] = MYSQL_FIELD_TYPE_LONG; + param_types.types[index + index + 1] = 0x80; + + } else { + param_types.types[index + index] = MYSQL_FIELD_TYPE_LONG; + param_types.types[index + index + 1] = 0x00; + } + { + uint32_t v = butil::ByteSwapToLE32(*(uint32_t*)value); + buf.append(&v, 4); + } + break; + case MYSQL_FIELD_TYPE_LONGLONG: + if (is_unsigned) { + param_types.types[index + index] = MYSQL_FIELD_TYPE_LONGLONG; + param_types.types[index + index + 1] = 0x80; + } else { + param_types.types[index + index] = MYSQL_FIELD_TYPE_LONGLONG; + param_types.types[index + index + 1] = 0x00; + } + { + uint64_t v = butil::ByteSwapToLE64(*(uint64_t*)value); + buf.append(&v, 8); + } + break; + case MYSQL_FIELD_TYPE_FLOAT: + param_types.types[index + index] = MYSQL_FIELD_TYPE_FLOAT; + param_types.types[index + index + 1] = 0x00; + buf.append(value, 4); + break; + case MYSQL_FIELD_TYPE_DOUBLE: + param_types.types[index + index] = MYSQL_FIELD_TYPE_DOUBLE; + param_types.types[index + index + 1] = 0x00; + buf.append(value, 8); + break; + case MYSQL_FIELD_TYPE_STRING: { + const butil::StringPiece* p = (butil::StringPiece*)value; + if (p == NULL || p->data() == NULL) { + param_types.types[index + index] = MYSQL_FIELD_TYPE_NULL; + param_types.types[index + index + 1] = 0x00; + null_mask.mask[index / 8] |= 1 << (index & 7); + } else { + param_types.types[index + index] = MYSQL_FIELD_TYPE_STRING; + param_types.types[index + index + 1] = 0x00; + if (p->size() < long_data_size) { + std::string len = pack_encode_length(p->size()); + buf.append(len); + buf.append(p->data(), p->size()); + } else { + stmt->save_long_data(index, *p); + } + } + } break; + case MYSQL_FIELD_TYPE_NULL: { + param_types.types[index + index] = MYSQL_FIELD_TYPE_NULL; + param_types.types[index + index + 1] = 0x00; + null_mask.mask[index / 8] |= 1 << (index & 7); + } break; + default: + LOG(ERROR) << "wrong param type"; + return butil::Status(EINVAL, "[MysqlMakeExecuteData] wrong param type"); + } + + // all args have been building + if (index + 1 == n) { + buf.unsafe_assign(null_mask.area, null_mask.mask.data()); + buf.unsafe_assign(param_types.area, param_types.types.data()); + } + + return butil::Status::OK(); +} + +butil::Status MysqlMakeLongDataPacket(butil::IOBuf* outbuf, + uint32_t stmt_id, + uint16_t param_id, + const butil::IOBuf& ldata) { + butil::IOBuf head; + head.push_back(MYSQL_COM_STMT_SEND_LONG_DATA); + const uint32_t si = butil::ByteSwapToLE32(stmt_id); + outbuf->append(&si, 4); + const uint16_t pi = butil::ByteSwapToLE16(param_id); + outbuf->append(&pi, 2); + size_t len, pos = 0; + for (size_t pkg_len = ldata.size(); pkg_len > 0; pkg_len -= max_allowed_packet) { + if (pkg_len < max_allowed_packet) { + len = pkg_len; + } else { + len = max_allowed_packet; + } + butil::IOBuf data; + ldata.append_to(&data, len, pos); + pos += pkg_len; + auto func = [](butil::IOBuf* outbuf, const butil::IOBuf& data, size_t size, size_t offset) { + data.append_to(outbuf, size, offset); + }; + auto rc = MakePacket(outbuf, head, func, data); + if (!rc.ok()) { + return rc; + } + } + return butil::Status::OK(); +} + +} // namespace brpc diff --git a/src/brpc/mysql_command.h b/src/brpc/mysql_command.h new file mode 100644 index 0000000000..9d6c65604b --- /dev/null +++ b/src/brpc/mysql_command.h @@ -0,0 +1,92 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_COMMAND_H +#define BRPC_MYSQL_COMMAND_H + +#include +#include "butil/iobuf.h" +#include "butil/status.h" +#include "brpc/mysql_common.h" + +namespace brpc { +// mysql command types +enum MysqlCommandType : unsigned char { + MYSQL_COM_SLEEP, + MYSQL_COM_QUIT, + MYSQL_COM_INIT_DB, + MYSQL_COM_QUERY, + MYSQL_COM_FIELD_LIST, + MYSQL_COM_CREATE_DB, + MYSQL_COM_DROP_DB, + MYSQL_COM_REFRESH, + MYSQL_COM_SHUTDOWN, + MYSQL_COM_STATISTICS, + MYSQL_COM_PROCESS_INFO, + MYSQL_COM_CONNECT, + MYSQL_COM_PROCESS_KILL, + MYSQL_COM_DEBUG, + MYSQL_COM_PING, + MYSQL_COM_TIME, + MYSQL_COM_DELAYED_INSERT, + MYSQL_COM_CHANGE_USER, + MYSQL_COM_BINLOG_DUMP, + MYSQL_COM_TABLE_DUMP, + MYSQL_COM_CONNECT_OUT, + MYSQL_COM_REGISTER_SLAVE, + MYSQL_COM_STMT_PREPARE, + MYSQL_COM_STMT_EXECUTE, + MYSQL_COM_STMT_SEND_LONG_DATA, + MYSQL_COM_STMT_CLOSE, + MYSQL_COM_STMT_RESET, + MYSQL_COM_SET_OPTION, + MYSQL_COM_STMT_FETCH, + MYSQL_COM_DAEMON, + MYSQL_COM_BINLOG_DUMP_GTID, + MYSQL_COM_RESET_CONNECTION, +}; + +butil::Status MysqlMakeCommand(butil::IOBuf* outbuf, + const MysqlCommandType type, + const butil::StringPiece& stmt); + +// Prepared Statement Protocol +// an prepared statement has a unique statement id in one connection (in brpc SocketId), an prepared +// statement can be executed in many connections, so ever connection has a different statement id. +// In bprc, we can only get a connection in the stage of PackXXXRequest which is behind our building +// mysql protocol stage, but building prepared statement need the statement id of a connection, so +// we will need to building this fragment at PackXXXRequest stage. + +// maybe we can Add a wrapper function, call CallMethod many times use bind_sock +class MysqlStatementStub; +// prepared statement execute command header, will be called at PackXXXRequest stage. +butil::Status MysqlMakeExecutePacket(butil::IOBuf* outbuf, + uint32_t stmt_id, + const butil::IOBuf& body); +// prepared statement execute command body, will be called at building mysql protocol stage. +butil::Status MysqlMakeExecuteData(MysqlStatementStub* stmt, + uint16_t index, + const void* value, + MysqlFieldType type, + bool is_unsigned = false); +// prepared statement long data header +butil::Status MysqlMakeLongDataPacket(butil::IOBuf* outbuf, + uint32_t stmt_id, + uint16_t param_id, + const butil::IOBuf& body); + +} // namespace brpc +#endif diff --git a/src/brpc/mysql_common.cpp b/src/brpc/mysql_common.cpp new file mode 100644 index 0000000000..368c0e245f --- /dev/null +++ b/src/brpc/mysql_common.cpp @@ -0,0 +1,86 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include "mysql_common.h" + +namespace brpc { + +const char* MysqlDefaultCollation = "utf8mb4_general_ci"; +const char* MysqlBinaryCollation = "binary"; + +const char* MysqlFieldTypeToString(MysqlFieldType type) { + switch (type) { + case MYSQL_FIELD_TYPE_DECIMAL: + case MYSQL_FIELD_TYPE_TINY: + return "tiny"; + case MYSQL_FIELD_TYPE_SHORT: + return "short"; + case MYSQL_FIELD_TYPE_LONG: + return "long"; + case MYSQL_FIELD_TYPE_FLOAT: + return "float"; + case MYSQL_FIELD_TYPE_DOUBLE: + return "double"; + case MYSQL_FIELD_TYPE_NULL: + return "null"; + case MYSQL_FIELD_TYPE_TIMESTAMP: + return "timestamp"; + case MYSQL_FIELD_TYPE_LONGLONG: + return "longlong"; + case MYSQL_FIELD_TYPE_INT24: + return "int24"; + case MYSQL_FIELD_TYPE_DATE: + return "date"; + case MYSQL_FIELD_TYPE_TIME: + return "time"; + case MYSQL_FIELD_TYPE_DATETIME: + return "datetime"; + case MYSQL_FIELD_TYPE_YEAR: + return "year"; + case MYSQL_FIELD_TYPE_NEWDATE: + return "new date"; + case MYSQL_FIELD_TYPE_VARCHAR: + return "varchar"; + case MYSQL_FIELD_TYPE_BIT: + return "bit"; + case MYSQL_FIELD_TYPE_JSON: + return "json"; + case MYSQL_FIELD_TYPE_NEWDECIMAL: + return "new decimal"; + case MYSQL_FIELD_TYPE_ENUM: + return "enum"; + case MYSQL_FIELD_TYPE_SET: + return "set"; + case MYSQL_FIELD_TYPE_TINY_BLOB: + return "tiny blob"; + case MYSQL_FIELD_TYPE_MEDIUM_BLOB: + return "blob"; + case MYSQL_FIELD_TYPE_LONG_BLOB: + return "long blob"; + case MYSQL_FIELD_TYPE_BLOB: + return "blob"; + case MYSQL_FIELD_TYPE_VAR_STRING: + return "var string"; + case MYSQL_FIELD_TYPE_STRING: + return "string"; + case MYSQL_FIELD_TYPE_GEOMETRY: + return "geometry"; + default: + return "Unknown Field Type"; + } +} + +} // namespace brpc diff --git a/src/brpc/mysql_common.h b/src/brpc/mysql_common.h new file mode 100644 index 0000000000..621388137b --- /dev/null +++ b/src/brpc/mysql_common.h @@ -0,0 +1,419 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_COMMON_H +#define BRPC_MYSQL_COMMON_H + +#include +#include +#include "butil/logging.h" // LOG() + +namespace brpc { +// Msql Collation +extern const char* MysqlDefaultCollation; +extern const char* MysqlBinaryCollation; +const std::map MysqlCollations = { + {"big5_chinese_ci", 1}, + {"latin2_czech_cs", 2}, + {"dec8_swedish_ci", 3}, + {"cp850_general_ci", 4}, + {"latin1_german1_ci", 5}, + {"hp8_english_ci", 6}, + {"koi8r_general_ci", 7}, + {"latin1_swedish_ci", 8}, + {"latin2_general_ci", 9}, + {"swe7_swedish_ci", 10}, + {"ascii_general_ci", 11}, + {"ujis_japanese_ci", 12}, + {"sjis_japanese_ci", 13}, + {"cp1251_bulgarian_ci", 14}, + {"latin1_danish_ci", 15}, + {"hebrew_general_ci", 16}, + {"tis620_thai_ci", 18}, + {"euckr_korean_ci", 19}, + {"latin7_estonian_cs", 20}, + {"latin2_hungarian_ci", 21}, + {"koi8u_general_ci", 22}, + {"cp1251_ukrainian_ci", 23}, + {"gb2312_chinese_ci", 24}, + {"greek_general_ci", 25}, + {"cp1250_general_ci", 26}, + {"latin2_croatian_ci", 27}, + {"gbk_chinese_ci", 28}, + {"cp1257_lithuanian_ci", 29}, + {"latin5_turkish_ci", 30}, + {"latin1_german2_ci", 31}, + {"armscii8_general_ci", 32}, + {"utf8_general_ci", 33}, + {"cp1250_czech_cs", 34}, + //{"ucs2_general_ci", 35}, + {"cp866_general_ci", 36}, + {"keybcs2_general_ci", 37}, + {"macce_general_ci", 38}, + {"macroman_general_ci", 39}, + {"cp852_general_ci", 40}, + {"latin7_general_ci", 41}, + {"latin7_general_cs", 42}, + {"macce_bin", 43}, + {"cp1250_croatian_ci", 44}, + {"utf8mb4_general_ci", 45}, + {"utf8mb4_bin", 46}, + {"latin1_bin", 47}, + {"latin1_general_ci", 48}, + {"latin1_general_cs", 49}, + {"cp1251_bin", 50}, + {"cp1251_general_ci", 51}, + {"cp1251_general_cs", 52}, + {"macroman_bin", 53}, + //{"utf16_general_ci", 54}, + //{"utf16_bin", 55}, + //{"utf16le_general_ci", 56}, + {"cp1256_general_ci", 57}, + {"cp1257_bin", 58}, + {"cp1257_general_ci", 59}, + //{"utf32_general_ci", 60}, + //{"utf32_bin", 61}, + //{"utf16le_bin", 62}, + {"binary", 63}, + {"armscii8_bin", 64}, + {"ascii_bin", 65}, + {"cp1250_bin", 66}, + {"cp1256_bin", 67}, + {"cp866_bin", 68}, + {"dec8_bin", 69}, + {"greek_bin", 70}, + {"hebrew_bin", 71}, + {"hp8_bin", 72}, + {"keybcs2_bin", 73}, + {"koi8r_bin", 74}, + {"koi8u_bin", 75}, + {"utf8_tolower_ci", 76}, + {"latin2_bin", 77}, + {"latin5_bin", 78}, + {"latin7_bin", 79}, + {"cp850_bin", 80}, + {"cp852_bin", 81}, + {"swe7_bin", 82}, + {"utf8_bin", 83}, + {"big5_bin", 84}, + {"euckr_bin", 85}, + {"gb2312_bin", 86}, + {"gbk_bin", 87}, + {"sjis_bin", 88}, + {"tis620_bin", 89}, + //"{ucs2_bin", 90}, + {"ujis_bin", 91}, + {"geostd8_general_ci", 92}, + {"geostd8_bin", 93}, + {"latin1_spanish_ci", 94}, + {"cp932_japanese_ci", 95}, + {"cp932_bin", 96}, + {"eucjpms_japanese_ci", 97}, + {"eucjpms_bin", 98}, + {"cp1250_polish_ci", 99}, + // {"utf16_unicode_ci", 101}, + // {"utf16_icelandic_ci", 102}, + // {"utf16_latvian_ci", 103}, + // {"utf16_romanian_ci", 104}, + // {"utf16_slovenian_ci", 105}, + // {"utf16_polish_ci", 106}, + // {"utf16_estonian_ci", 107}, + // {"utf16_spanish_ci", 108}, + // {"utf16_swedish_ci", 109}, + // {"utf16_turkish_ci", 110}, + // {"utf16_czech_ci", 111}, + // {"utf16_danish_ci", 112}, + // {"utf16_lithuanian_ci", 113}, + // {"utf16_slovak_ci", 114}, + // {"utf16_spanish2_ci", 115}, + // {"utf16_roman_ci", 116}, + // {"utf16_persian_ci", 117}, + // {"utf16_esperanto_ci", 118}, + // {"utf16_hungarian_ci", 119}, + // {"utf16_sinhala_ci", 120}, + // {"utf16_german2_ci", 121}, + // {"utf16_croatian_ci", 122}, + // {"utf16_unicode_520_ci", 123}, + // {"utf16_vietnamese_ci", 124}, + // {"ucs2_unicode_ci", 128}, + // {"ucs2_icelandic_ci", 129}, + // {"ucs2_latvian_ci", 130}, + // {"ucs2_romanian_ci", 131}, + // {"ucs2_slovenian_ci", 132}, + // {"ucs2_polish_ci", 133}, + // {"ucs2_estonian_ci", 134}, + // {"ucs2_spanish_ci", 135}, + // {"ucs2_swedish_ci", 136}, + // {"ucs2_turkish_ci", 137}, + // {"ucs2_czech_ci", 138}, + // {"ucs2_danish_ci", 139}, + // {"ucs2_lithuanian_ci", 140}, + // {"ucs2_slovak_ci", 141}, + // {"ucs2_spanish2_ci", 142}, + // {"ucs2_roman_ci", 143}, + // {"ucs2_persian_ci", 144}, + // {"ucs2_esperanto_ci", 145}, + // {"ucs2_hungarian_ci", 146}, + // {"ucs2_sinhala_ci", 147}, + // {"ucs2_german2_ci", 148}, + // {"ucs2_croatian_ci", 149}, + // {"ucs2_unicode_520_ci", 150}, + // {"ucs2_vietnamese_ci", 151}, + // {"ucs2_general_mysql500_ci", 159}, + // {"utf32_unicode_ci", 160}, + // {"utf32_icelandic_ci", 161}, + // {"utf32_latvian_ci", 162}, + // {"utf32_romanian_ci", 163}, + // {"utf32_slovenian_ci", 164}, + // {"utf32_polish_ci", 165}, + // {"utf32_estonian_ci", 166}, + // {"utf32_spanish_ci", 167}, + // {"utf32_swedish_ci", 168}, + // {"utf32_turkish_ci", 169}, + // {"utf32_czech_ci", 170}, + // {"utf32_danish_ci", 171}, + // {"utf32_lithuanian_ci", 172}, + // {"utf32_slovak_ci", 173}, + // {"utf32_spanish2_ci", 174}, + // {"utf32_roman_ci", 175}, + // {"utf32_persian_ci", 176}, + // {"utf32_esperanto_ci", 177}, + // {"utf32_hungarian_ci", 178}, + // {"utf32_sinhala_ci", 179}, + // {"utf32_german2_ci", 180}, + // {"utf32_croatian_ci", 181}, + // {"utf32_unicode_520_ci", 182}, + // {"utf32_vietnamese_ci", 183}, + {"utf8_unicode_ci", 192}, + {"utf8_icelandic_ci", 193}, + {"utf8_latvian_ci", 194}, + {"utf8_romanian_ci", 195}, + {"utf8_slovenian_ci", 196}, + {"utf8_polish_ci", 197}, + {"utf8_estonian_ci", 198}, + {"utf8_spanish_ci", 199}, + {"utf8_swedish_ci", 200}, + {"utf8_turkish_ci", 201}, + {"utf8_czech_ci", 202}, + {"utf8_danish_ci", 203}, + {"utf8_lithuanian_ci", 204}, + {"utf8_slovak_ci", 205}, + {"utf8_spanish2_ci", 206}, + {"utf8_roman_ci", 207}, + {"utf8_persian_ci", 208}, + {"utf8_esperanto_ci", 209}, + {"utf8_hungarian_ci", 210}, + {"utf8_sinhala_ci", 211}, + {"utf8_german2_ci", 212}, + {"utf8_croatian_ci", 213}, + {"utf8_unicode_520_ci", 214}, + {"utf8_vietnamese_ci", 215}, + {"utf8_general_mysql500_ci", 223}, + {"utf8mb4_unicode_ci", 224}, + {"utf8mb4_icelandic_ci", 225}, + {"utf8mb4_latvian_ci", 226}, + {"utf8mb4_romanian_ci", 227}, + {"utf8mb4_slovenian_ci", 228}, + {"utf8mb4_polish_ci", 229}, + {"utf8mb4_estonian_ci", 230}, + {"utf8mb4_spanish_ci", 231}, + {"utf8mb4_swedish_ci", 232}, + {"utf8mb4_turkish_ci", 233}, + {"utf8mb4_czech_ci", 234}, + {"utf8mb4_danish_ci", 235}, + {"utf8mb4_lithuanian_ci", 236}, + {"utf8mb4_slovak_ci", 237}, + {"utf8mb4_spanish2_ci", 238}, + {"utf8mb4_roman_ci", 239}, + {"utf8mb4_persian_ci", 240}, + {"utf8mb4_esperanto_ci", 241}, + {"utf8mb4_hungarian_ci", 242}, + {"utf8mb4_sinhala_ci", 243}, + {"utf8mb4_german2_ci", 244}, + {"utf8mb4_croatian_ci", 245}, + {"utf8mb4_unicode_520_ci", 246}, + {"utf8mb4_vietnamese_ci", 247}, + {"gb18030_chinese_ci", 248}, + {"gb18030_bin", 249}, + {"gb18030_unicode_520_ci", 250}, + {"utf8mb4_0900_ai_ci", 255}, +}; + +enum MysqlFieldType : uint8_t { + MYSQL_FIELD_TYPE_DECIMAL = 0x00, + MYSQL_FIELD_TYPE_TINY = 0x01, + MYSQL_FIELD_TYPE_SHORT = 0x02, + MYSQL_FIELD_TYPE_LONG = 0x03, + MYSQL_FIELD_TYPE_FLOAT = 0x04, + MYSQL_FIELD_TYPE_DOUBLE = 0x05, + MYSQL_FIELD_TYPE_NULL = 0x06, + MYSQL_FIELD_TYPE_TIMESTAMP = 0x07, + MYSQL_FIELD_TYPE_LONGLONG = 0x08, + MYSQL_FIELD_TYPE_INT24 = 0x09, + MYSQL_FIELD_TYPE_DATE = 0x0A, + MYSQL_FIELD_TYPE_TIME = 0x0B, + MYSQL_FIELD_TYPE_DATETIME = 0x0C, + MYSQL_FIELD_TYPE_YEAR = 0x0D, + MYSQL_FIELD_TYPE_NEWDATE = 0x0E, + MYSQL_FIELD_TYPE_VARCHAR = 0x0F, + MYSQL_FIELD_TYPE_BIT = 0x10, + MYSQL_FIELD_TYPE_JSON = 0xF5, + MYSQL_FIELD_TYPE_NEWDECIMAL = 0xF6, + MYSQL_FIELD_TYPE_ENUM = 0xF7, + MYSQL_FIELD_TYPE_SET = 0xF8, + MYSQL_FIELD_TYPE_TINY_BLOB = 0xF9, + MYSQL_FIELD_TYPE_MEDIUM_BLOB = 0xFA, + MYSQL_FIELD_TYPE_LONG_BLOB = 0xFB, + MYSQL_FIELD_TYPE_BLOB = 0xFC, + MYSQL_FIELD_TYPE_VAR_STRING = 0xFD, + MYSQL_FIELD_TYPE_STRING = 0xFE, + MYSQL_FIELD_TYPE_GEOMETRY = 0xFF, +}; + +enum MysqlFieldFlag : uint16_t { + MYSQL_NOT_NULL_FLAG = 0x0001, + MYSQL_PRI_KEY_FLAG = 0x0002, + MYSQL_UNIQUE_KEY_FLAG = 0x0004, + MYSQL_MULTIPLE_KEY_FLAG = 0x0008, + MYSQL_BLOB_FLAG = 0x0010, + MYSQL_UNSIGNED_FLAG = 0x0020, + MYSQL_ZEROFILL_FLAG = 0x0040, + MYSQL_BINARY_FLAG = 0x0080, + MYSQL_ENUM_FLAG = 0x0100, + MYSQL_AUTO_INCREMENT_FLAG = 0x0200, + MYSQL_TIMESTAMP_FLAG = 0x0400, + MYSQL_SET_FLAG = 0x0800, +}; + +enum MysqlServerStatus : uint16_t { + MYSQL_SERVER_STATUS_IN_TRANS = 1, + MYSQL_SERVER_STATUS_AUTOCOMMIT = 2, /* Server in auto_commit mode */ + MYSQL_SERVER_MORE_RESULTS_EXISTS = 8, /* Multi query - next query exists */ + MYSQL_SERVER_QUERY_NO_GOOD_INDEX_USED = 16, + MYSQL_SERVER_QUERY_NO_INDEX_USED = 32, + /** + The server was able to fulfill the clients request and opened a + read-only non-scrollable cursor for a query. This flag comes + in reply to COM_STMT_EXECUTE and COM_STMT_FETCH commands. + */ + MYSQL_SERVER_STATUS_CURSOR_EXISTS = 64, + /** + This flag is sent when a read-only cursor is exhausted, in reply to + COM_STMT_FETCH command. + */ + MYSQL_SERVER_STATUS_LAST_ROW_SENT = 128, + MYSQL_SERVER_STATUS_DB_DROPPED = 256, /* A database was dropped */ + MYSQL_SERVER_STATUS_NO_BACKSLASH_ESCAPES = 512, + /** + Sent to the client if after a prepared statement reprepare + we discovered that the new statement returns a different + number of result set columns. + */ + MYSQL_SERVER_STATUS_METADATA_CHANGED = 1024, + MYSQL_SERVER_QUERY_WAS_SLOW = 2048, + + /** + To mark ResultSet containing output parameter values. + */ + MYSQL_SERVER_PS_OUT_PARAMS = 4096, + + /** + Set at the same time as MYSQL_SERVER_STATUS_IN_TRANS if the started + multi-statement transaction is a read-only transaction. Cleared + when the transaction commits or aborts. Since this flag is sent + to clients in OK and EOF packets, the flag indicates the + transaction status at the end of command execution. + */ + MYSQL_SERVER_STATUS_IN_TRANS_READONLY = 8192, + MYSQL_SERVER_SESSION_STATE_CHANGED = 1UL << 14, +}; + +// 1. normal statement 2. prepared statement 3. need prepare statement +enum MysqlStmtType : uint32_t { + MYSQL_NORMAL_STATEMENT = 1, + MYSQL_PREPARED_STATEMENT = 2, + MYSQL_NEED_PREPARE = 3, +}; + +const char* MysqlFieldTypeToString(MysqlFieldType); + +inline std::string pack_encode_length(const uint64_t value) { + std::stringstream ss; + if (value <= 250) { + ss.put((char)value); + } else if (value <= 0xffff) { + ss.put((char)0xfc).put((char)value).put((char)(value >> 8)); + } else if (value <= 0xffffff) { + ss.put((char)0xfd).put((char)value).put((char)(value >> 8)).put((char)(value >> 16)); + } else { + ss.put((char)0xfd) + .put((char)value) + .put((char)(value >> 8)) + .put((char)(value >> 16)) + .put((char)(value >> 24)) + .put((char)(value >> 32)) + .put((char)(value >> 40)) + .put((char)(value >> 48)) + .put((char)(value >> 56)); + } + return ss.str(); +} + +// little endian order to host order +#if !defined(ARCH_CPU_LITTLE_ENDIAN) + +inline uint16_t mysql_uint2korr(const uint8_t* A) { + return (uint16_t)(((uint16_t)(A[0])) + ((uint16_t)(A[1]) << 8)); +} + +inline uint32_t mysql_uint3korr(const uint8_t* A) { + return (uint32_t)(((uint32_t)(A[0])) + (((uint32_t)(A[1])) << 8) + (((uint32_t)(A[2])) << 16)); +} + +inline uint32_t mysql_uint4korr(const uint8_t* A) { + return (uint32_t)(((uint32_t)(A[0])) + (((uint32_t)(A[1])) << 8) + (((uint32_t)(A[2])) << 16) + + (((uint32_t)(A[3])) << 24)); +} + +inline uint64_t mysql_uint8korr(const uint8_t* A) { + return (uint64_t)(((uint64_t)(A[0])) + (((uint64_t)(A[1])) << 8) + (((uint64_t)(A[2])) << 16) + + (((uint64_t)(A[3])) << 24) + (((uint64_t)(A[4])) << 32) + + (((uint64_t)(A[5])) << 40) + (((uint64_t)(A[6])) << 48) + + (((uint64_t)(A[7])) << 56)); +} + +#else + +inline uint16_t mysql_uint2korr(const uint8_t* A) { + return *(uint16_t*)A; +} + +inline uint32_t mysql_uint3korr(const uint8_t* A) { + return (uint32_t)(((uint32_t)(A[0])) + (((uint32_t)(A[1])) << 8) + (((uint32_t)(A[2])) << 16)); +} + +inline uint32_t mysql_uint4korr(const uint8_t* A) { + return *(uint32_t*)A; +} + +inline uint64_t mysql_uint8korr(const uint8_t* A) { + return *(uint64_t*)A; +} + +#endif + +} // namespace brpc +#endif diff --git a/src/brpc/mysql_reply.cpp b/src/brpc/mysql_reply.cpp new file mode 100644 index 0000000000..78b96d0c22 --- /dev/null +++ b/src/brpc/mysql_reply.cpp @@ -0,0 +1,1189 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include "brpc/mysql_common.h" +#include "brpc/mysql_reply.h" + +namespace brpc { + +#define MY_ALLOC_CHECK(expr) \ + do { \ + if ((expr) == false) { \ + return PARSE_ERROR_ABSOLUTELY_WRONG; \ + } \ + } while (0) + +#define MY_PARSE_CHECK(expr) \ + do { \ + ParseError rc = (expr); \ + if (rc != PARSE_OK) { \ + return rc; \ + } \ + } while (0) + +template +inline bool my_alloc_check(butil::Arena* arena, const size_t n, Type*& pointer) { + if (pointer == NULL) { + pointer = (Type*)arena->allocate(sizeof(Type) * n); + if (pointer == NULL) { + return false; + } + for (size_t i = 0; i < n; ++i) { + new (pointer + i) Type; + } + } + return true; +} + +template <> +inline bool my_alloc_check(butil::Arena* arena, const size_t n, char*& pointer) { + if (pointer == NULL) { + pointer = (char*)arena->allocate(sizeof(char) * n); + if (pointer == NULL) { + return false; + } + } + return true; +} + +namespace { +struct MysqlHeader { + uint32_t payload_size; + uint32_t seq; +}; +const char* digits01 = + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123" + "456789"; +const char* digits10 = + "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999" + "999999"; +} // namespace + +const char* MysqlRspTypeToString(MysqlRspType type) { + switch (type) { + case MYSQL_RSP_OK: + return "ok"; + case MYSQL_RSP_ERROR: + return "error"; + case MYSQL_RSP_RESULTSET: + return "resultset"; + case MYSQL_RSP_EOF: + return "eof"; + case MYSQL_RSP_AUTH: + return "auth"; + case MYSQL_RSP_PREPARE_OK: + return "prepare_ok"; + default: + return "Unknown Response Type"; + } +} + +// check if the buf is contain a full package +inline bool is_full_package(const butil::IOBuf& buf) { + uint8_t header[4]; + const uint8_t* p = (const uint8_t*)buf.fetch(header, sizeof(header)); + if (p == NULL) { + return false; + } + uint32_t payload_size = mysql_uint3korr(p); + if (buf.size() < payload_size + 4) { + return false; + } + return true; +} +// if is eof package +inline bool is_an_eof(const butil::IOBuf& buf) { + uint8_t tmp[5]; + const uint8_t* p = (const uint8_t*)buf.fetch(tmp, sizeof(tmp)); + if (p == NULL) { + return false; + } + uint8_t type = p[4]; + if (type == MYSQL_RSP_EOF) { + return true; + } else { + return false; + } +} +// parse header +inline bool parse_header(butil::IOBuf& buf, MysqlHeader* value) { + if (!is_full_package(buf)) { + return false; + } + { + uint8_t tmp[3]; + buf.cutn(tmp, sizeof(tmp)); + value->payload_size = mysql_uint3korr(tmp); + } + { + uint8_t tmp; + buf.cut1((char*)&tmp); + value->seq = tmp; + } + return true; +} +// use this carefully, we depending on parse_header for checking IOBuf contain full package +inline uint64_t parse_encode_length(butil::IOBuf& buf) { + if (buf.size() == 0) { + return 0; + } + + uint64_t value = 0; + uint8_t f = 0; + buf.cut1((char*)&f); + if (f <= 250) { + value = f; + } else if (f == 251) { + value = 0; + } else if (f == 252) { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + value = mysql_uint2korr(tmp); + } else if (f == 253) { + uint8_t tmp[3]; + buf.cutn(tmp, sizeof(tmp)); + value = mysql_uint3korr(tmp); + } else if (f == 254) { + uint8_t tmp[8]; + buf.cutn(tmp, sizeof(tmp)); + value = mysql_uint8korr(tmp); + } + return value; +} + +ParseError MysqlReply::ConsumePartialIOBuf(butil::IOBuf& buf, + butil::Arena* arena, + bool is_auth, + MysqlStmtType stmt_type, + bool* more_results) { + *more_results = false; + if (!is_full_package(buf)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + uint8_t header[4 + 1]; // use the extra byte to judge message type + const uint8_t* p = (const uint8_t*)buf.fetch(header, sizeof(header)); + uint8_t type = (_type == MYSQL_RSP_UNKNOWN) ? p[4] : (uint8_t)_type; + if (is_auth && type != 0x00 && type != 0xFF) { + _type = MYSQL_RSP_AUTH; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.auth)); + MY_PARSE_CHECK(_data.auth->Parse(buf, arena)); + return PARSE_OK; + } + if (type == 0x00 && (is_auth || stmt_type != MYSQL_NEED_PREPARE)) { + _type = MYSQL_RSP_OK; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.ok)); + MY_PARSE_CHECK(_data.ok->Parse(buf, arena)); + *more_results = _data.ok->status() & MYSQL_SERVER_MORE_RESULTS_EXISTS; + } else if ((type == 0x00 && stmt_type == MYSQL_NEED_PREPARE) || type == MYSQL_RSP_PREPARE_OK) { + _type = MYSQL_RSP_PREPARE_OK; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.prepare_ok)); + MY_PARSE_CHECK(_data.prepare_ok->Parse(buf, arena)); + } else if (type == 0xFF) { + _type = MYSQL_RSP_ERROR; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.error)); + MY_PARSE_CHECK(_data.error->Parse(buf, arena)); + } else if (type == 0xFE) { + _type = MYSQL_RSP_EOF; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.eof)); + MY_PARSE_CHECK(_data.eof->Parse(buf)); + *more_results = _data.eof->status() & MYSQL_SERVER_MORE_RESULTS_EXISTS; + } else if (type >= 0x01 && type <= 0xFA) { + _type = MYSQL_RSP_RESULTSET; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, _data.result_set)); + MY_PARSE_CHECK(_data.result_set->Parse(buf, arena, !(stmt_type == MYSQL_NORMAL_STATEMENT))); + *more_results = _data.result_set->_eof2.status() & MYSQL_SERVER_MORE_RESULTS_EXISTS; + } else { + LOG(ERROR) << "Unknown Response Type " + << "type=" << unsigned(type) << " buf_size=" << buf.size(); + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + return PARSE_OK; +} + +void MysqlReply::Print(std::ostream& os) const { + if (_type == MYSQL_RSP_AUTH) { + const Auth& auth = *_data.auth; + os << "\nprotocol:" << (unsigned)auth._protocol << "\nversion:" << auth._version + << "\nthread_id:" << auth._thread_id << "\nsalt:" << auth._salt + << "\ncapacity:" << auth._capability << "\nlanguage:" << (unsigned)auth._collation + << "\nstatus:" << auth._status << "\nextended_capacity:" << auth._extended_capability + << "\nauth_plugin_length:" << auth._auth_plugin_length << "\nsalt2:" << auth._salt2 + << "\nauth_plugin:" << auth._auth_plugin; + } else if (_type == MYSQL_RSP_OK) { + const Ok& ok = *_data.ok; + os << "\naffect_row:" << ok._affect_row << "\nindex:" << ok._index + << "\nstatus:" << ok._status << "\nwarning:" << ok._warning << "\nmessage:" << ok._msg; + } else if (_type == MYSQL_RSP_ERROR) { + const Error& err = *_data.error; + os << "\nerrcode:" << err._errcode << "\nstatus:" << err._status + << "\nmessage:" << err._msg; + } else if (_type == MYSQL_RSP_RESULTSET) { + const ResultSet& r = *_data.result_set; + os << "\nheader.column_count:" << r._header._column_count; + for (uint64_t i = 0; i < r._header._column_count; ++i) { + os << "\ncolumn[" << i << "].catalog:" << r._columns[i]._catalog << "\ncolumn[" << i + << "].database:" << r._columns[i]._database << "\ncolumn[" << i + << "].table:" << r._columns[i]._table << "\ncolumn[" << i + << "].origin_table:" << r._columns[i]._origin_table << "\ncolumn[" << i + << "].name:" << r._columns[i]._name << "\ncolumn[" << i + << "].origin_name:" << r._columns[i]._origin_name << "\ncolumn[" << i + << "].charset:" << (uint16_t)r._columns[i]._charset << "\ncolumn[" << i + << "].length:" << r._columns[i]._length << "\ncolumn[" << i + << "].type:" << (unsigned)r._columns[i]._type << "\ncolumn[" << i + << "].flag:" << (unsigned)r._columns[i]._flag << "\ncolumn[" << i + << "].decimal:" << (unsigned)r._columns[i]._decimal; + } + os << "\neof1.warning:" << r._eof1._warning; + os << "\neof1.status:" << r._eof1._status; + int n = 0; + for (const Row* row = r._first->_next; row != r._last->_next; row = row->_next) { + os << "\nrow(" << n++ << "):"; + for (uint64_t j = 0; j < r._header._column_count; ++j) { + if (row->field(j).is_nil()) { + os << "NULL\t"; + continue; + } + switch (row->field(j)._type) { + case MYSQL_FIELD_TYPE_NULL: + os << "NULL"; + break; + case MYSQL_FIELD_TYPE_TINY: + if (r._columns[j]._flag & MYSQL_UNSIGNED_FLAG) { + os << unsigned(row->field(j).tiny()); + } else { + os << signed(row->field(j).stiny()); + } + break; + case MYSQL_FIELD_TYPE_SHORT: + case MYSQL_FIELD_TYPE_YEAR: + if (r._columns[j]._flag & MYSQL_UNSIGNED_FLAG) { + os << unsigned(row->field(j).small()); + } else { + os << signed(row->field(j).ssmall()); + } + break; + case MYSQL_FIELD_TYPE_INT24: + case MYSQL_FIELD_TYPE_LONG: + if (r._columns[j]._flag & MYSQL_UNSIGNED_FLAG) { + os << row->field(j).integer(); + } else { + os << row->field(j).sinteger(); + } + break; + case MYSQL_FIELD_TYPE_LONGLONG: + if (r._columns[j]._flag & MYSQL_UNSIGNED_FLAG) { + os << row->field(j).bigint(); + } else { + os << row->field(j).sbigint(); + } + break; + case MYSQL_FIELD_TYPE_FLOAT: + os << row->field(j).float32(); + break; + case MYSQL_FIELD_TYPE_DOUBLE: + os << row->field(j).float64(); + break; + case MYSQL_FIELD_TYPE_DECIMAL: + case MYSQL_FIELD_TYPE_NEWDECIMAL: + case MYSQL_FIELD_TYPE_VARCHAR: + case MYSQL_FIELD_TYPE_BIT: + case MYSQL_FIELD_TYPE_ENUM: + case MYSQL_FIELD_TYPE_SET: + case MYSQL_FIELD_TYPE_TINY_BLOB: + case MYSQL_FIELD_TYPE_MEDIUM_BLOB: + case MYSQL_FIELD_TYPE_LONG_BLOB: + case MYSQL_FIELD_TYPE_BLOB: + case MYSQL_FIELD_TYPE_VAR_STRING: + case MYSQL_FIELD_TYPE_STRING: + case MYSQL_FIELD_TYPE_GEOMETRY: + case MYSQL_FIELD_TYPE_JSON: + case MYSQL_FIELD_TYPE_TIME: + case MYSQL_FIELD_TYPE_DATE: + case MYSQL_FIELD_TYPE_NEWDATE: + case MYSQL_FIELD_TYPE_TIMESTAMP: + case MYSQL_FIELD_TYPE_DATETIME: + os << row->field(j).string(); + break; + default: + os << "Unknown field type"; + } + os << "\t"; + } + } + os << "\neof2.warning:" << r._eof2._warning; + os << "\neof2.status:" << r._eof2._status; + } else if (_type == MYSQL_RSP_EOF) { + const Eof& e = *_data.eof; + os << "\nwarning:" << e._warning << "\nstatus:" << e._status; + } else if (_type == MYSQL_RSP_PREPARE_OK) { + const PrepareOk& prep = *_data.prepare_ok; + os << "\nstmt_id:" << prep._header._stmt_id + << "\ncolumn_count:" << prep._header._column_count + << "\nparam_count:" << prep._header._param_count; + for (uint16_t i = 0; i < prep._header._param_count; ++i) { + os << "\nparam[" << i << "].catalog:" << prep._params[i]._catalog << "\nparam[" << i + << "].database:" << prep._params[i]._database << "\nparam[" << i + << "].table:" << prep._params[i]._table << "\nparam[" << i + << "].origin_table:" << prep._params[i]._origin_table << "\nparam[" << i + << "].name:" << prep._params[i]._name << "\nparam[" << i + << "].origin_name:" << prep._params[i]._origin_name << "\nparam[" << i + << "].charset:" << (uint16_t)prep._params[i]._charset << "\nparam[" << i + << "].length:" << prep._params[i]._length << "\nparam[" << i + << "].type:" << (unsigned)prep._params[i]._type << "\nparam[" << i + << "].flag:" << (unsigned)prep._params[i]._flag << "\nparam[" << i + << "].decimal:" << (unsigned)prep._params[i]._decimal; + } + for (uint16_t i = 0; i < prep._header._column_count; ++i) { + os << "\ncolumn[" << i << "].catalog:" << prep._columns[i]._catalog << "\ncolumn[" << i + << "].database:" << prep._columns[i]._database << "\ncolumn[" << i + << "].table:" << prep._columns[i]._table << "\ncolumn[" << i + << "].origin_table:" << prep._columns[i]._origin_table << "\ncolumn[" << i + << "].name:" << prep._columns[i]._name << "\ncolumn[" << i + << "].origin_name:" << prep._columns[i]._origin_name << "\ncolumn[" << i + << "].charset:" << (uint16_t)prep._columns[i]._charset << "\ncolumn[" << i + << "].length:" << prep._columns[i]._length << "\ncolumn[" << i + << "].type:" << (unsigned)prep._columns[i]._type << "\ncolumn[" << i + << "].flag:" << (unsigned)prep._columns[i]._flag << "\ncolumn[" << i + << "].decimal:" << (unsigned)prep._columns[i]._decimal; + } + } else { + os << "Unknown response type"; + } +} + +ParseError MysqlReply::Auth::Parse(butil::IOBuf& buf, butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + const std::string delim(1, 0x00); + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + buf.cut1((char*)&_protocol); + { + butil::IOBuf version; + buf.cut_until(&version, delim); + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, version.size(), d)); + version.copy_to(d); + _version.set(d, version.size()); + } + { + uint8_t tmp[4]; + buf.cutn(tmp, sizeof(tmp)); + _thread_id = mysql_uint4korr(tmp); + } + { + butil::IOBuf salt; + buf.cut_until(&salt, delim); + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, salt.size(), d)); + salt.copy_to(d); + _salt.set(d, salt.size()); + } + { + uint8_t tmp[2]; + buf.cutn(&tmp, sizeof(tmp)); + _capability = mysql_uint2korr(tmp); + } + buf.cut1((char*)&_collation); + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _status = mysql_uint2korr(tmp); + } + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _extended_capability = mysql_uint2korr(tmp); + } + buf.cut1((char*)&_auth_plugin_length); + buf.pop_front(10); + { + butil::IOBuf salt2; + buf.cut_until(&salt2, delim); + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, salt2.size(), d)); + salt2.copy_to(d); + _salt2.set(d, salt2.size()); + } + { + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, _auth_plugin_length, d)); + buf.cutn(d, _auth_plugin_length); + _auth_plugin.set(d, _auth_plugin_length); + } + buf.clear(); // consume all buf + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::ResultSetHeader::Parse(butil::IOBuf& buf) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + uint64_t old_size, new_size; + old_size = buf.size(); + _column_count = parse_encode_length(buf); + new_size = buf.size(); + if (old_size - new_size < header.payload_size) { + _extra_msg = parse_encode_length(buf); + } else { + _extra_msg = 0; + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Column::Parse(butil::IOBuf& buf, butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + + uint64_t len = parse_encode_length(buf); + char* catalog = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, catalog)); + buf.cutn(catalog, len); + _catalog.set(catalog, len); + + len = parse_encode_length(buf); + char* database = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, database)); + buf.cutn(database, len); + _database.set(database, len); + + len = parse_encode_length(buf); + char* table = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, table)); + buf.cutn(table, len); + _table.set(table, len); + + len = parse_encode_length(buf); + char* origin_table = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, origin_table)); + buf.cutn(origin_table, len); + _origin_table.set(origin_table, len); + + len = parse_encode_length(buf); + char* name = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, name)); + buf.cutn(name, len); + _name.set(name, len); + + len = parse_encode_length(buf); + char* origin_name = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, origin_name)); + buf.cutn(origin_name, len); + _origin_name.set(origin_name, len); + buf.pop_front(1); + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _charset = mysql_uint2korr(tmp); + } + { + uint8_t tmp[4]; + buf.cutn(tmp, sizeof(tmp)); + _length = mysql_uint4korr(tmp); + } + buf.cut1((char*)&_type); + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _flag = (MysqlFieldFlag)mysql_uint2korr(tmp); + } + buf.cut1((char*)&_decimal); + buf.pop_front(2); + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Ok::Parse(butil::IOBuf& buf, butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + + uint64_t old_size, new_size; + old_size = buf.size(); + buf.pop_front(1); + + _affect_row = parse_encode_length(buf); + _index = parse_encode_length(buf); + buf.cutn(&_status, 2); + buf.cutn(&_warning, 2); + + new_size = buf.size(); + if (old_size - new_size < header.payload_size) { + const int64_t len = header.payload_size - (old_size - new_size); + char* msg = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, msg)); + buf.cutn(msg, len); + _msg.set(msg, len); + // buf.pop_front(1); // Null + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Eof::Parse(butil::IOBuf& buf) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + buf.pop_front(1); + buf.cutn(&_warning, 2); + buf.cutn(&_status, 2); + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Error::Parse(butil::IOBuf& buf, butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + buf.pop_front(1); // 0xFF + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _errcode = mysql_uint2korr(tmp); + } + buf.pop_front(1); // '#' + // 5 byte server status + char* status = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, 5, status)); + buf.cutn(status, 5); + _status.set(status, 5); + // error message, Null-Terminated string + uint64_t len = header.payload_size - 9; + char* msg = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, msg)); + buf.cutn(msg, len); + _msg.set(msg, len); + // buf.pop_front(1); // Null + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Row::Parse(butil::IOBuf& buf, + const MysqlReply::Column* columns, + uint64_t column_count, + MysqlReply::Field* fields, + bool binary, + butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + if (!binary) { // mysql text protocol + for (uint64_t i = 0; i < column_count; ++i) { + MY_PARSE_CHECK(fields[i].Parse(buf, columns + i, arena)); + } + } else { // mysql binary protocol + uint8_t hdr = 0; + buf.cut1((char*)&hdr); + if (hdr != 0x00) { + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + // NULL-bitmap, [(column-count + 7 + 2) / 8 bytes] + const uint64_t size = ((column_count + 7 + 2) >> 3); + uint8_t null_mask[size]; + for (size_t i = 0; i < sizeof(null_mask); ++i) { + null_mask[i] = 0; + } + buf.cutn(null_mask, size); + for (uint64_t i = 0; i < column_count; ++i) { + MY_PARSE_CHECK(fields[i].Parse(buf, columns + i, i, column_count, null_mask, arena)); + } + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Field::Parse(butil::IOBuf& buf, + const MysqlReply::Column* column, + butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + // field type + _type = column->_type; + // is unsigned flag set + _unsigned = column->_flag & MYSQL_UNSIGNED_FLAG; + // parse encode length + const uint64_t len = parse_encode_length(buf); + // is it null? + if (len == 0 && !(column->_flag & MYSQL_NOT_NULL_FLAG)) { + _is_nil = true; + set_parsed(); + return PARSE_OK; + } + // field is not null + butil::IOBuf str; + buf.cutn(&str, len); + switch (_type) { + case MYSQL_FIELD_TYPE_NULL: + _is_nil = true; + break; + case MYSQL_FIELD_TYPE_TINY: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + _data.tiny = strtoul(str.to_string().c_str(), NULL, 10); + } else { + _data.stiny = strtol(str.to_string().c_str(), NULL, 10); + } + break; + case MYSQL_FIELD_TYPE_SHORT: + case MYSQL_FIELD_TYPE_YEAR: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + _data.small = strtoul(str.to_string().c_str(), NULL, 10); + } else { + _data.ssmall = strtol(str.to_string().c_str(), NULL, 10); + } + break; + case MYSQL_FIELD_TYPE_INT24: + case MYSQL_FIELD_TYPE_LONG: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + _data.integer = strtoul(str.to_string().c_str(), NULL, 10); + } else { + _data.sinteger = strtol(str.to_string().c_str(), NULL, 10); + } + break; + case MYSQL_FIELD_TYPE_LONGLONG: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + _data.bigint = strtoul(str.to_string().c_str(), NULL, 10); + } else { + _data.sbigint = strtol(str.to_string().c_str(), NULL, 10); + } + break; + case MYSQL_FIELD_TYPE_FLOAT: + _data.float32 = strtof(str.to_string().c_str(), NULL); + break; + case MYSQL_FIELD_TYPE_DOUBLE: + _data.float64 = strtod(str.to_string().c_str(), NULL); + break; + case MYSQL_FIELD_TYPE_DECIMAL: + case MYSQL_FIELD_TYPE_NEWDECIMAL: + case MYSQL_FIELD_TYPE_VARCHAR: + case MYSQL_FIELD_TYPE_BIT: + case MYSQL_FIELD_TYPE_ENUM: + case MYSQL_FIELD_TYPE_SET: + case MYSQL_FIELD_TYPE_TINY_BLOB: + case MYSQL_FIELD_TYPE_MEDIUM_BLOB: + case MYSQL_FIELD_TYPE_LONG_BLOB: + case MYSQL_FIELD_TYPE_BLOB: + case MYSQL_FIELD_TYPE_VAR_STRING: + case MYSQL_FIELD_TYPE_STRING: + case MYSQL_FIELD_TYPE_GEOMETRY: + case MYSQL_FIELD_TYPE_JSON: + case MYSQL_FIELD_TYPE_TIME: + case MYSQL_FIELD_TYPE_DATE: + case MYSQL_FIELD_TYPE_NEWDATE: + case MYSQL_FIELD_TYPE_TIMESTAMP: + case MYSQL_FIELD_TYPE_DATETIME: { + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, d)); + str.copy_to(d); + _data.str.set(d, len); + } break; + default: + LOG(ERROR) << "Unknown field type"; + set_parsed(); + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Field::Parse(butil::IOBuf& buf, + const MysqlReply::Column* column, + uint64_t column_index, + uint64_t column_count, + const uint8_t* null_mask, + butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + // field type + _type = column->_type; + // is unsigned flag set + _unsigned = column->_flag & MYSQL_UNSIGNED_FLAG; + // (byte >> bit-pos) % 2 == 1 + if (((null_mask[(column_index + 2) >> 3] >> ((column_index + 2) & 7)) & 1) == 1) { + _is_nil = true; + set_parsed(); + return PARSE_OK; + } + + switch (_type) { + case MYSQL_FIELD_TYPE_NULL: + _is_nil = true; + break; + case MYSQL_FIELD_TYPE_TINY: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + buf.cut1((char*)&_data.tiny); + } else { + buf.cut1((char*)&_data.stiny); + } + break; + case MYSQL_FIELD_TYPE_SHORT: + case MYSQL_FIELD_TYPE_YEAR: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + uint8_t* p = (uint8_t*)&_data.small; + buf.cutn(p, 2); + _data.small = mysql_uint2korr(p); + } else { + uint8_t* p = (uint8_t*)&_data.ssmall; + buf.cutn(p, 2); + _data.ssmall = (int16_t)mysql_uint2korr(p); + } + break; + case MYSQL_FIELD_TYPE_INT24: + case MYSQL_FIELD_TYPE_LONG: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + uint8_t* p = (uint8_t*)&_data.integer; + buf.cutn(p, 4); + _data.integer = mysql_uint4korr(p); + } else { + uint8_t* p = (uint8_t*)&_data.sinteger; + buf.cutn(p, 4); + _data.sinteger = (int32_t)mysql_uint4korr(p); + } + break; + case MYSQL_FIELD_TYPE_LONGLONG: + if (column->_flag & MYSQL_UNSIGNED_FLAG) { + uint8_t* p = (uint8_t*)&_data.bigint; + buf.cutn(p, 8); + _data.bigint = mysql_uint8korr(p); + } else { + uint8_t* p = (uint8_t*)&_data.sbigint; + buf.cutn(p, 8); + _data.sbigint = (int64_t)mysql_uint8korr(p); + } + break; + case MYSQL_FIELD_TYPE_FLOAT: { + uint8_t* p = (uint8_t*)&_data.float32; + buf.cutn(p, 4); + } break; + case MYSQL_FIELD_TYPE_DOUBLE: { + uint8_t* p = (uint8_t*)&_data.float64; + buf.cutn(p, 8); + } break; + case MYSQL_FIELD_TYPE_DECIMAL: + case MYSQL_FIELD_TYPE_NEWDECIMAL: + case MYSQL_FIELD_TYPE_VARCHAR: + case MYSQL_FIELD_TYPE_BIT: + case MYSQL_FIELD_TYPE_ENUM: + case MYSQL_FIELD_TYPE_SET: + case MYSQL_FIELD_TYPE_TINY_BLOB: + case MYSQL_FIELD_TYPE_MEDIUM_BLOB: + case MYSQL_FIELD_TYPE_LONG_BLOB: + case MYSQL_FIELD_TYPE_BLOB: + case MYSQL_FIELD_TYPE_VAR_STRING: + case MYSQL_FIELD_TYPE_STRING: + case MYSQL_FIELD_TYPE_GEOMETRY: + case MYSQL_FIELD_TYPE_JSON: { + const uint64_t len = parse_encode_length(buf); + // is it null? + if (len == 0 && !(column->_flag & MYSQL_NOT_NULL_FLAG)) { + _is_nil = true; + set_parsed(); + return PARSE_OK; + } + // field is not null + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, len, d)); + buf.cutn(d, len); + _data.str.set(d, len); + } break; + case MYSQL_FIELD_TYPE_NEWDATE: // Date YYYY-MM-DD + case MYSQL_FIELD_TYPE_DATE: // Date YYYY-MM-DD + case MYSQL_FIELD_TYPE_DATETIME: // Timestamp YYYY-MM-DD HH:MM:SS[.fractal] + case MYSQL_FIELD_TYPE_TIMESTAMP: { // Timestamp YYYY-MM-DD HH:MM:SS[.fractal] + ParseError rc = ParseBinaryDataTime(buf, column, _data.str, arena); + if (rc != PARSE_OK) { + return rc; + } + } break; + case MYSQL_FIELD_TYPE_TIME: { // Time [-][H]HH:MM:SS[.fractal] + ParseError rc = ParseBinaryTime(buf, column, _data.str, arena); + if (rc != PARSE_OK) { + return rc; + } + } break; + default: + LOG(ERROR) << "Unknown field type"; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::Field::ParseBinaryTime(butil::IOBuf& buf, + const MysqlReply::Column* column, + butil::StringPiece& str, + butil::Arena* arena) { + + const uint64_t len = parse_encode_length(buf); + if (len == 0) { + _is_nil = true; + return PARSE_OK; + } + + if (len != 8 && len != 12) { + LOG(ERROR) << "invalid TIME packet length " << len; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + + uint8_t dstlen; + switch (column->_decimal) { + case 0x00: + case 0x1f: + dstlen = 8; + break; + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + dstlen = 8 + 1 + column->_decimal; + break; + default: + LOG(ERROR) << "protocol error, illegal decimals value " << column->_decimal; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + + size_t i = 0; + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, dstlen + 2, d)); + d[dstlen] = '\0'; + d[dstlen + 1] = '\0'; + uint32_t day; + uint8_t neg, hour, min, sec; + + buf.cut1((char*)&neg); + if (neg == 1) { + d[i++] = '-'; + } + + buf.cutn(&day, 4); + day = mysql_uint4korr((uint8_t*)&day); + buf.cut1((char*)&hour); + hour += day * 24; + if (hour >= 100) { + std::ostringstream os; + os << hour; + std::string s = os.str(); + for (const auto& v : s) { + d[i++] = v; + } + } else { + d[i++] = digits10[hour]; + d[i++] = digits01[hour]; + } + + buf.cut1((char*)&min); + buf.cut1((char*)&sec); + + d[i++] = ':'; + d[i++] = digits10[min]; + d[i++] = digits01[min]; + d[i++] = ':'; + d[i++] = digits10[sec]; + d[i++] = digits01[sec]; + + ParseError rc = ParseMicrosecs(buf, column->_decimal, d + i); + if (rc == PARSE_OK) { + str.set(d, dstlen + 2); + } + return rc; +} + +ParseError MysqlReply::Field::ParseBinaryDataTime(butil::IOBuf& buf, + const MysqlReply::Column* column, + butil::StringPiece& str, + butil::Arena* arena) { + const uint64_t len = parse_encode_length(buf); + if (len == 0) { + _is_nil = true; + return PARSE_OK; + } + + if (len != 4 && len != 7 && len != 11) { + LOG(ERROR) << "illegal date time length " << len; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + + uint8_t dstlen; + if (column->_type == MYSQL_FIELD_TYPE_DATE) { + dstlen = 10; + } else { + switch (column->_decimal) { + case 0x00: + case 0x1f: + dstlen = 19; + break; + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + dstlen = 19 + 1 + column->_decimal; + break; + default: + LOG(ERROR) << "protocol error, illegal decimal value " << column->_decimal; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + } + + size_t i = 0; + char* d = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, dstlen, d)); + uint16_t year; + uint8_t pt, p1, p2, p3; + buf.cutn(&year, 2); // year + year = mysql_uint2korr((uint8_t*)&year); + pt = year / 100; + p1 = year - (100 * pt); + buf.cut1((char*)&p2); + buf.cut1((char*)&p3); + + d[i++] = digits10[pt]; + d[i++] = digits01[pt]; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = '-'; + d[i++] = digits10[p2]; + d[i++] = digits01[p2]; + d[i++] = '-'; + d[i++] = digits10[p3]; + d[i++] = digits01[p3]; + + if (len == 4) { + str.set(d, dstlen); + return PARSE_OK; + } + + d[i++] = ' '; + buf.cut1((char*)&p1); // hour + buf.cut1((char*)&p2); // min + buf.cut1((char*)&p3); // sec + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = ':'; + d[i++] = digits10[p2]; + d[i++] = digits01[p2]; + d[i++] = ':'; + d[i++] = digits10[p3]; + d[i++] = digits01[p3]; + + ParseError rc = ParseMicrosecs(buf, column->_decimal, d + i); + if (rc == PARSE_OK) { + str.set(d, dstlen); + } + return rc; +} + +ParseError MysqlReply::Field::ParseMicrosecs(butil::IOBuf& buf, uint8_t decimal, char* d) { + if (decimal <= 0) { + return PARSE_OK; + } + + size_t i = 0; + uint32_t microsecs; + uint8_t p1, p2, p3; + buf.cutn((char*)µsecs, 4); + microsecs = mysql_uint4korr((uint8_t*)µsecs); + p1 = microsecs / 10000; + microsecs -= 10000 * p1; + p2 = microsecs / 100; + microsecs -= 100 * p2; + p3 = microsecs; + + switch (decimal) { + case 1: + d[i++] = '.'; + d[i++] = digits10[p1]; + break; + case 2: + d[i++] = '.'; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + break; + case 3: + d[i++] = '.'; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = digits10[p2]; + break; + case 4: + d[i++] = '.'; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = digits10[p2]; + d[i++] = digits01[p2]; + break; + case 5: + d[i++] = '.'; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = digits10[p2]; + d[i++] = digits01[p2]; + d[i++] = digits10[p3]; + break; + default: + d[i++] = '.'; + d[i++] = digits10[p1]; + d[i++] = digits01[p1]; + d[i++] = digits10[p2]; + d[i++] = digits01[p2]; + d[i++] = digits10[p3]; + d[i++] = digits01[p3]; + } + return PARSE_OK; +} + +ParseError MysqlReply::ResultSet::Parse(butil::IOBuf& buf, butil::Arena* arena, bool binary) { + if (is_parsed()) { + return PARSE_OK; + } + // parse header + MY_PARSE_CHECK(_header.Parse(buf)); + // parse colunms + MY_ALLOC_CHECK(my_alloc_check(arena, _header._column_count, _columns)); + for (uint64_t i = 0; i < _header._column_count; ++i) { + MY_PARSE_CHECK(_columns[i].Parse(buf, arena)); + } + // parse eof1 + MY_PARSE_CHECK(_eof1.Parse(buf)); + // parse row + std::vector rows; + for (;;) { + // if not full package reread + if (!is_full_package(buf)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + // if eof break loops for row + if (is_an_eof(buf)) { + break; + } + // allocate memory for row and fields + Row* row = NULL; + Field* fields = NULL; + MY_ALLOC_CHECK(my_alloc_check(arena, 1, row)); + MY_ALLOC_CHECK(my_alloc_check(arena, _header._column_count, fields)); + row->_fields = fields; + row->_field_count = _header._column_count; + _last->_next = row; + _last = row; + // parse row and fields + MY_PARSE_CHECK(row->Parse(buf, _columns, _header._column_count, fields, binary, arena)); + // add row count + ++_row_count; + } + // parse eof2 + MY_PARSE_CHECK(_eof2.Parse(buf)); + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::PrepareOk::Parse(butil::IOBuf& buf, butil::Arena* arena) { + if (is_parsed()) { + return PARSE_OK; + } + + MY_PARSE_CHECK(_header.Parse(buf)); + + if (_header._param_count > 0) { + MY_ALLOC_CHECK(my_alloc_check(arena, _header._param_count, _params)); + for (uint16_t i = 0; i < _header._param_count; ++i) { + MY_PARSE_CHECK(_params[i].Parse(buf, arena)); + } + MY_PARSE_CHECK(_eof1.Parse(buf)); + } + + if (_header._column_count > 0) { + MY_ALLOC_CHECK(my_alloc_check(arena, _header._column_count, _columns)); + for (uint16_t i = 0; i < _header._column_count; ++i) { + MY_PARSE_CHECK(_columns[i].Parse(buf, arena)); + } + MY_PARSE_CHECK(_eof2.Parse(buf)); + } + set_parsed(); + return PARSE_OK; +} + +ParseError MysqlReply::PrepareOk::Header::Parse(butil::IOBuf& buf) { + if (is_parsed()) { + return PARSE_OK; + } + + MysqlHeader header; + if (!parse_header(buf, &header)) { + return PARSE_ERROR_NOT_ENOUGH_DATA; + } + + buf.pop_front(1); + { + uint8_t tmp[4]; + buf.cutn(tmp, sizeof(tmp)); + _stmt_id = mysql_uint4korr(tmp); + } + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _column_count = mysql_uint2korr(tmp); + } + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _param_count = mysql_uint2korr(tmp); + } + buf.pop_front(1); + { + uint8_t tmp[2]; + buf.cutn(tmp, sizeof(tmp)); + _warning = mysql_uint2korr(tmp); + } + + set_parsed(); + return PARSE_OK; +} + +} // namespace brpc diff --git a/src/brpc/mysql_reply.h b/src/brpc/mysql_reply.h new file mode 100644 index 0000000000..4a52537743 --- /dev/null +++ b/src/brpc/mysql_reply.h @@ -0,0 +1,801 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_REPLY_H +#define BRPC_MYSQL_REPLY_H + +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/arena.h" +#include "butil/sys_byteorder.h" +#include "butil/logging.h" // LOG() +#include "brpc/parse_result.h" +#include "brpc/mysql_common.h" + +namespace brpc { + +class CheckParsed { +public: + CheckParsed() : _is_parsed(false) {} + bool is_parsed() const { + return _is_parsed; + } + void set_parsed() { + _is_parsed = true; + } + +private: + bool _is_parsed; +}; + +enum MysqlRspType : uint8_t { + MYSQL_RSP_OK = 0x00, + MYSQL_RSP_ERROR = 0xFF, + MYSQL_RSP_RESULTSET = 0x01, + MYSQL_RSP_EOF = 0xFE, + MYSQL_RSP_AUTH = 0xFB, // add for mysql auth + MYSQL_RSP_PREPARE_OK = 0xFC, // add for prepared statement + MYSQL_RSP_UNKNOWN = 0xFD, // add for other case +}; + +const char* MysqlRspTypeToString(MysqlRspType); + +class MysqlReply { +public: + // Mysql Auth package + class Auth : private CheckParsed { + public: + Auth(); + uint8_t protocol() const; + butil::StringPiece version() const; + uint32_t thread_id() const; + butil::StringPiece salt() const; + uint16_t capability() const; + uint8_t collation() const; + uint16_t status() const; + uint16_t extended_capability() const; + uint8_t auth_plugin_length() const; + butil::StringPiece salt2() const; + butil::StringPiece auth_plugin() const; + + private: + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(Auth); + friend class MysqlReply; + + uint8_t _protocol; + butil::StringPiece _version; + uint32_t _thread_id; + butil::StringPiece _salt; + uint16_t _capability; + uint8_t _collation; + uint16_t _status; + uint16_t _extended_capability; + uint8_t _auth_plugin_length; + butil::StringPiece _salt2; + butil::StringPiece _auth_plugin; + }; + // Mysql Prepared Statement Ok + class Column; + // Mysql Eof package + class Eof : private CheckParsed { + public: + Eof(); + uint16_t warning() const; + uint16_t status() const; + + private: + ParseError Parse(butil::IOBuf& buf); + + DISALLOW_COPY_AND_ASSIGN(Eof); + friend class MysqlReply; + + uint16_t _warning; + uint16_t _status; + }; + // Mysql PrepareOk package + class PrepareOk : private CheckParsed { + public: + PrepareOk(); + uint32_t stmt_id() const; + uint16_t column_count() const; + uint16_t param_count() const; + uint16_t warning() const; + const Column& param(uint16_t index) const; + const Column& column(uint16_t index) const; + + private: + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(PrepareOk); + friend class MysqlReply; + + class Header : private CheckParsed { + public: + Header() : _stmt_id(0), _column_count(0), _param_count(0), _warning(0) {} + uint32_t _stmt_id; + uint16_t _column_count; + uint16_t _param_count; + uint16_t _warning; + ParseError Parse(butil::IOBuf& buf); + }; + Header _header; + Column* _params; + Eof _eof1; + Column* _columns; + Eof _eof2; + }; + // Mysql Ok package + class Ok : private CheckParsed { + public: + Ok(); + uint64_t affect_row() const; + uint64_t index() const; + uint16_t status() const; + uint16_t warning() const; + butil::StringPiece msg() const; + + private: + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(Ok); + friend class MysqlReply; + + uint64_t _affect_row; + uint64_t _index; + uint16_t _status; + uint16_t _warning; + butil::StringPiece _msg; + }; + // Mysql Error package + class Error : private CheckParsed { + public: + Error(); + uint16_t errcode() const; + butil::StringPiece status() const; + butil::StringPiece msg() const; + + private: + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(Error); + friend class MysqlReply; + + uint16_t _errcode; + butil::StringPiece _status; + butil::StringPiece _msg; + }; + // Mysql Column + class Column : private CheckParsed { + public: + Column(); + butil::StringPiece catalog() const; + butil::StringPiece database() const; + butil::StringPiece table() const; + butil::StringPiece origin_table() const; + butil::StringPiece name() const; + butil::StringPiece origin_name() const; + uint16_t charset() const; + uint32_t length() const; + MysqlFieldType type() const; + MysqlFieldFlag flag() const; + uint8_t decimal() const; + + private: + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(Column); + friend class MysqlReply; + + butil::StringPiece _catalog; + butil::StringPiece _database; + butil::StringPiece _table; + butil::StringPiece _origin_table; + butil::StringPiece _name; + butil::StringPiece _origin_name; + uint16_t _charset; + uint32_t _length; + MysqlFieldType _type; + MysqlFieldFlag _flag; + uint8_t _decimal; + }; + // Mysql Field + class Field : private CheckParsed { + public: + Field(); + int8_t stiny() const; + uint8_t tiny() const; + int16_t ssmall() const; + uint16_t small() const; + int32_t sinteger() const; + uint32_t integer() const; + int64_t sbigint() const; + uint64_t bigint() const; + float float32() const; + double float64() const; + butil::StringPiece string() const; + bool is_stiny() const; + bool is_tiny() const; + bool is_ssmall() const; + bool is_small() const; + bool is_sinteger() const; + bool is_integer() const; + bool is_sbigint() const; + bool is_bigint() const; + bool is_float32() const; + bool is_float64() const; + bool is_string() const; + bool is_nil() const; + + private: + ParseError Parse(butil::IOBuf& buf, const MysqlReply::Column* column, butil::Arena* arena); + ParseError Parse(butil::IOBuf& buf, + const MysqlReply::Column* column, + uint64_t column_index, + uint64_t column_number, + const uint8_t* null_mask, + butil::Arena* arena); + ParseError ParseBinaryTime(butil::IOBuf& buf, + const MysqlReply::Column* column, + butil::StringPiece& str, + butil::Arena* arena); + ParseError ParseBinaryDataTime(butil::IOBuf& buf, + const MysqlReply::Column* column, + butil::StringPiece& str, + butil::Arena* arena); + ParseError ParseMicrosecs(butil::IOBuf& buf, uint8_t decimal, char* d); + DISALLOW_COPY_AND_ASSIGN(Field); + friend class MysqlReply; + + union { + int8_t stiny; + uint8_t tiny; + int16_t ssmall; + uint16_t small; + int32_t sinteger; + uint32_t integer; + int64_t sbigint; + uint64_t bigint; + float float32; + double float64; + butil::StringPiece str; + } _data = {.str = NULL}; + MysqlFieldType _type; + bool _unsigned; + bool _is_nil; + }; + // Mysql Row + class Row : private CheckParsed { + public: + Row(); + uint64_t field_count() const; + const Field& field(const uint64_t index) const; + + private: + ParseError Parse(butil::IOBuf& buf, + const Column* columns, + uint64_t column_number, + Field* fields, + bool binary, + butil::Arena* arena); + + DISALLOW_COPY_AND_ASSIGN(Row); + friend class MysqlReply; + + Field* _fields; + uint64_t _field_count; + Row* _next; + }; + +public: + MysqlReply(); + ParseError ConsumePartialIOBuf(butil::IOBuf& buf, + butil::Arena* arena, + bool is_auth, + MysqlStmtType stmt_type, + bool* more_results); + void Swap(MysqlReply& other); + void Print(std::ostream& os) const; + // response type + MysqlRspType type() const; + // get auth + const Auth& auth() const; + const Ok& ok() const; + const PrepareOk& prepare_ok() const; + const Error& error() const; + const Eof& eof() const; + // get column number + uint64_t column_count() const; + // get one column + const Column& column(const uint64_t index) const; + // get row number + uint64_t row_count() const; + // get one row + const Row& next() const; + bool is_auth() const; + bool is_ok() const; + bool is_prepare_ok() const; + bool is_error() const; + bool is_eof() const; + bool is_resultset() const; + +private: + // Mysql result set header + struct ResultSetHeader : private CheckParsed { + ResultSetHeader() : _column_count(0), _extra_msg(0) {} + ParseError Parse(butil::IOBuf& buf); + uint64_t _column_count; + uint64_t _extra_msg; + + private: + DISALLOW_COPY_AND_ASSIGN(ResultSetHeader); + }; + // Mysql result set + struct ResultSet : private CheckParsed { + ResultSet() : _columns(NULL), _row_count(0) { + _cur = _first = _last = &_dummy; + } + ParseError Parse(butil::IOBuf& buf, butil::Arena* arena, bool binary); + ResultSetHeader _header; + Column* _columns; + Eof _eof1; + // row list begin + Row* _first; + Row* _last; + Row* _cur; + uint64_t _row_count; + // row list end + Eof _eof2; + + private: + DISALLOW_COPY_AND_ASSIGN(ResultSet); + Row _dummy; + }; + // member values + MysqlRspType _type; + union { + Auth* auth; + ResultSet* result_set; + Ok* ok; + PrepareOk* prepare_ok; + Error* error; + Eof* eof; + uint64_t padding; // For swapping, must cover all bytes. + } _data; + + DISALLOW_COPY_AND_ASSIGN(MysqlReply); +}; + +// mysql reply +inline MysqlReply::MysqlReply() { + _type = MYSQL_RSP_UNKNOWN; + _data.padding = 0; +} +inline void MysqlReply::Swap(MysqlReply& other) { + std::swap(_type, other._type); + std::swap(_data.padding, other._data.padding); +} +inline std::ostream& operator<<(std::ostream& os, const MysqlReply& r) { + r.Print(os); + return os; +} +inline MysqlRspType MysqlReply::type() const { + return _type; +} +inline const MysqlReply::Auth& MysqlReply::auth() const { + if (is_auth()) { + return *_data.auth; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an auth"; + static Auth auth_nil; + return auth_nil; +} +inline const MysqlReply::PrepareOk& MysqlReply::prepare_ok() const { + if (is_prepare_ok()) { + return *_data.prepare_ok; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an ok"; + static PrepareOk prepare_ok_nil; + return prepare_ok_nil; +} +inline const MysqlReply::Ok& MysqlReply::ok() const { + if (is_ok()) { + return *_data.ok; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an ok"; + static Ok ok_nil; + return ok_nil; +} +inline const MysqlReply::Error& MysqlReply::error() const { + if (is_error()) { + return *_data.error; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an error"; + static Error error_nil; + return error_nil; +} +inline const MysqlReply::Eof& MysqlReply::eof() const { + if (is_eof()) { + return *_data.eof; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an eof"; + static Eof eof_nil; + return eof_nil; +} +inline uint64_t MysqlReply::column_count() const { + if (is_resultset()) { + return _data.result_set->_header._column_count; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an resultset"; + return 0; +} +inline const MysqlReply::Column& MysqlReply::column(const uint64_t index) const { + static Column column_nil; + if (is_resultset()) { + if (index < _data.result_set->_header._column_count) { + return _data.result_set->_columns[index]; + } + CHECK(false) << "index " << index << " out of bound [0," + << _data.result_set->_header._column_count << ")"; + return column_nil; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an resultset"; + return column_nil; +} +inline uint64_t MysqlReply::row_count() const { + if (is_resultset()) { + return _data.result_set->_row_count; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an resultset"; + return 0; +} +inline const MysqlReply::Row& MysqlReply::next() const { + static Row row_nil; + if (is_resultset()) { + if (_data.result_set->_row_count == 0) { + CHECK(false) << "there are 0 rows returned"; + return row_nil; + } + if (_data.result_set->_cur == _data.result_set->_last->_next) { + _data.result_set->_cur = _data.result_set->_first->_next; + } else { + _data.result_set->_cur = _data.result_set->_cur->_next; + } + return *_data.result_set->_cur; + } + CHECK(false) << "The reply is " << MysqlRspTypeToString(_type) << ", not an resultset"; + return row_nil; +} +inline bool MysqlReply::is_auth() const { + return _type == MYSQL_RSP_AUTH; +} +inline bool MysqlReply::is_prepare_ok() const { + return _type == MYSQL_RSP_PREPARE_OK; +} +inline bool MysqlReply::is_ok() const { + return _type == MYSQL_RSP_OK; +} +inline bool MysqlReply::is_error() const { + return _type == MYSQL_RSP_ERROR; +} +inline bool MysqlReply::is_eof() const { + return _type == MYSQL_RSP_EOF; +} +inline bool MysqlReply::is_resultset() const { + return _type == MYSQL_RSP_RESULTSET; +} +// mysql auth +inline MysqlReply::Auth::Auth() + : _protocol(0), + _thread_id(0), + _capability(0), + _collation(0), + _status(0), + _extended_capability(0), + _auth_plugin_length(0) {} +inline uint8_t MysqlReply::Auth::protocol() const { + return _protocol; +} +inline butil::StringPiece MysqlReply::Auth::version() const { + return _version; +} +inline uint32_t MysqlReply::Auth::thread_id() const { + return _thread_id; +} +inline butil::StringPiece MysqlReply::Auth::salt() const { + return _salt; +} +inline uint16_t MysqlReply::Auth::capability() const { + return _capability; +} +inline uint8_t MysqlReply::Auth::collation() const { + return _collation; +} +inline uint16_t MysqlReply::Auth::status() const { + return _status; +} +inline uint16_t MysqlReply::Auth::extended_capability() const { + return _extended_capability; +} +inline uint8_t MysqlReply::Auth::auth_plugin_length() const { + return _auth_plugin_length; +} +inline butil::StringPiece MysqlReply::Auth::salt2() const { + return _salt2; +} +inline butil::StringPiece MysqlReply::Auth::auth_plugin() const { + return _auth_plugin; +} +// mysql prepared statement ok +inline MysqlReply::PrepareOk::PrepareOk() : _params(NULL), _columns(NULL) {} +inline uint32_t MysqlReply::PrepareOk::stmt_id() const { + CHECK(_header._stmt_id > 0) << "stmt id is wrong"; + return _header._stmt_id; +} +inline uint16_t MysqlReply::PrepareOk::column_count() const { + return _header._column_count; +} +inline uint16_t MysqlReply::PrepareOk::param_count() const { + return _header._param_count; +} +inline uint16_t MysqlReply::PrepareOk::warning() const { + return _header._warning; +} +inline const MysqlReply::Column& MysqlReply::PrepareOk::param(uint16_t index) const { + if (index < _header._param_count) { + return _params[index]; + } + static Column column_nil; + CHECK(false) << "index " << index << " out of bound [0," << _header._param_count << ")"; + return column_nil; +} +inline const MysqlReply::Column& MysqlReply::PrepareOk::column(uint16_t index) const { + if (index < _header._column_count) { + return _columns[index]; + } + CHECK(false) << "index " << index << " out of bound [0," << _header._column_count << ")"; + static Column column_nil; + return column_nil; +} +// mysql reply ok +inline MysqlReply::Ok::Ok() : _affect_row(0), _index(0), _status(0), _warning(0) {} +inline uint64_t MysqlReply::Ok::affect_row() const { + return _affect_row; +} +inline uint64_t MysqlReply::Ok::index() const { + return _index; +} +inline uint16_t MysqlReply::Ok::status() const { + return _status; +} +inline uint16_t MysqlReply::Ok::warning() const { + return _warning; +} +inline butil::StringPiece MysqlReply::Ok::msg() const { + return _msg; +} +// mysql reply error +inline MysqlReply::Error::Error() : _errcode(0) {} +inline uint16_t MysqlReply::Error::errcode() const { + return _errcode; +} +inline butil::StringPiece MysqlReply::Error::status() const { + return _status; +} +inline butil::StringPiece MysqlReply::Error::msg() const { + return _msg; +} +// mysql reply eof +inline MysqlReply::Eof::Eof() : _warning(0), _status(0) {} +inline uint16_t MysqlReply::Eof::warning() const { + return _warning; +} +inline uint16_t MysqlReply::Eof::status() const { + return _status; +} +// mysql reply column +inline MysqlReply::Column::Column() : _length(0), _type(MYSQL_FIELD_TYPE_NULL), _decimal(0) {} +inline butil::StringPiece MysqlReply::Column::catalog() const { + return _catalog; +} +inline butil::StringPiece MysqlReply::Column::database() const { + return _database; +} +inline butil::StringPiece MysqlReply::Column::table() const { + return _table; +} +inline butil::StringPiece MysqlReply::Column::origin_table() const { + return _origin_table; +} +inline butil::StringPiece MysqlReply::Column::name() const { + return _name; +} +inline butil::StringPiece MysqlReply::Column::origin_name() const { + return _origin_name; +} +inline uint16_t MysqlReply::Column::charset() const { + return _charset; +} +inline uint32_t MysqlReply::Column::length() const { + return _length; +} +inline MysqlFieldType MysqlReply::Column::type() const { + return _type; +} +inline MysqlFieldFlag MysqlReply::Column::flag() const { + return _flag; +} +inline uint8_t MysqlReply::Column::decimal() const { + return _decimal; +} +// mysql reply row +inline MysqlReply::Row::Row() : _fields(NULL), _field_count(0), _next(NULL) {} +inline uint64_t MysqlReply::Row::field_count() const { + return _field_count; +} +inline const MysqlReply::Field& MysqlReply::Row::field(const uint64_t index) const { + if (index < _field_count) { + return _fields[index]; + } + CHECK(false) << "index " << index << " out of bound [0," << _field_count << ")"; + static Field field_nil; + return field_nil; +} +// mysql reply field +inline MysqlReply::Field::Field() + : _type(MYSQL_FIELD_TYPE_NULL), _unsigned(false), _is_nil(false) {} +inline int8_t MysqlReply::Field::stiny() const { + if (is_stiny()) { + return _data.stiny; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an stiny"; + return 0; +} +inline uint8_t MysqlReply::Field::tiny() const { + if (is_tiny()) { + return _data.tiny; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an tiny"; + return 0; +} +inline int16_t MysqlReply::Field::ssmall() const { + if (is_ssmall()) { + return _data.ssmall; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an ssmall"; + return 0; +} +inline uint16_t MysqlReply::Field::small() const { + if (is_small()) { + return _data.small; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an small"; + return 0; +} +inline int32_t MysqlReply::Field::sinteger() const { + if (is_sinteger()) { + return _data.sinteger; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an sinteger"; + return 0; +} +inline uint32_t MysqlReply::Field::integer() const { + if (is_integer()) { + return _data.integer; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an integer"; + return 0; +} +inline int64_t MysqlReply::Field::sbigint() const { + if (is_sbigint()) { + return _data.sbigint; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an sbigint"; + return 0; +} +inline uint64_t MysqlReply::Field::bigint() const { + if (is_bigint()) { + return _data.bigint; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an bigint"; + return 0; +} +inline float MysqlReply::Field::float32() const { + if (is_float32()) { + return _data.float32; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an float32"; + return 0; +} +inline double MysqlReply::Field::float64() const { + if (is_float64()) { + return _data.float64; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an float64"; + return 0; +} +inline butil::StringPiece MysqlReply::Field::string() const { + if (is_string()) { + return _data.str; + } + CHECK(false) << "The reply is " << MysqlFieldTypeToString(_type) << " and " + << (_is_nil ? "NULL" : "NOT NULL") << ", not an string"; + return butil::StringPiece(); +} +inline bool MysqlReply::Field::is_stiny() const { + return _type == MYSQL_FIELD_TYPE_TINY && !_unsigned && !_is_nil; +} +inline bool MysqlReply::Field::is_tiny() const { + return _type == MYSQL_FIELD_TYPE_TINY && _unsigned && !_is_nil; +} +inline bool MysqlReply::Field::is_ssmall() const { + return (_type == MYSQL_FIELD_TYPE_SHORT || _type == MYSQL_FIELD_TYPE_YEAR) && !_unsigned && + !_is_nil; +} +inline bool MysqlReply::Field::is_small() const { + return (_type == MYSQL_FIELD_TYPE_SHORT || _type == MYSQL_FIELD_TYPE_YEAR) && _unsigned && + !_is_nil; +} +inline bool MysqlReply::Field::is_sinteger() const { + return (_type == MYSQL_FIELD_TYPE_INT24 || _type == MYSQL_FIELD_TYPE_LONG) && !_unsigned && + !_is_nil; +} +inline bool MysqlReply::Field::is_integer() const { + return (_type == MYSQL_FIELD_TYPE_INT24 || _type == MYSQL_FIELD_TYPE_LONG) && _unsigned && + !_is_nil; +} +inline bool MysqlReply::Field::is_sbigint() const { + return _type == MYSQL_FIELD_TYPE_LONGLONG && !_unsigned && !_is_nil; +} +inline bool MysqlReply::Field::is_bigint() const { + return _type == MYSQL_FIELD_TYPE_LONGLONG && _unsigned && !_is_nil; +} +inline bool MysqlReply::Field::is_float32() const { + return _type == MYSQL_FIELD_TYPE_FLOAT && !_is_nil; +} +inline bool MysqlReply::Field::is_float64() const { + return _type == MYSQL_FIELD_TYPE_DOUBLE && !_is_nil; +} +inline bool MysqlReply::Field::is_string() const { + return (_type == MYSQL_FIELD_TYPE_DECIMAL || _type == MYSQL_FIELD_TYPE_NEWDECIMAL || + _type == MYSQL_FIELD_TYPE_VARCHAR || _type == MYSQL_FIELD_TYPE_BIT || + _type == MYSQL_FIELD_TYPE_ENUM || _type == MYSQL_FIELD_TYPE_SET || + _type == MYSQL_FIELD_TYPE_TINY_BLOB || _type == MYSQL_FIELD_TYPE_MEDIUM_BLOB || + _type == MYSQL_FIELD_TYPE_LONG_BLOB || _type == MYSQL_FIELD_TYPE_BLOB || + _type == MYSQL_FIELD_TYPE_VAR_STRING || _type == MYSQL_FIELD_TYPE_STRING || + _type == MYSQL_FIELD_TYPE_GEOMETRY || _type == MYSQL_FIELD_TYPE_JSON || + _type == MYSQL_FIELD_TYPE_TIME || _type == MYSQL_FIELD_TYPE_DATE || + _type == MYSQL_FIELD_TYPE_NEWDATE || _type == MYSQL_FIELD_TYPE_TIMESTAMP || + _type == MYSQL_FIELD_TYPE_DATETIME) && + !_is_nil; +} +inline bool MysqlReply::Field::is_nil() const { + return _is_nil; +} + +} // namespace brpc + +#endif // BRPC_MYSQL_REPLY_H diff --git a/src/brpc/mysql_statement.cpp b/src/brpc/mysql_statement.cpp new file mode 100644 index 0000000000..449792e753 --- /dev/null +++ b/src/brpc/mysql_statement.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include +#include +#include "brpc/socket.h" +#include "brpc/mysql_statement.h" + +namespace brpc { +DEFINE_int32(mysql_statment_map_size, + 100, + "Mysql statement map size, usually equal to max bthread number"); + +MysqlStatementUniquePtr NewMysqlStatement(const Channel& channel, const butil::StringPiece& str) { + MysqlStatementUniquePtr ptr(new MysqlStatement(channel, str)); + return ptr; +} + +uint32_t MysqlStatement::StatementId(SocketId socket_id) const { + if (_connection_type == CONNECTION_TYPE_SHORT) { + return 0; + } + MysqlStatementDBD::ScopedPtr ptr; + if (_id_map.Read(&ptr) != 0) { + return 0; + } + const MysqlStatementId* p = ptr->seek(socket_id); + if (p == NULL) { + return 0; + } + SocketUniquePtr socket; + if (Socket::Address(socket_id, &socket) == 0) { + uint64_t fd_version = socket->fd_version(); + if (fd_version == p->version) { + return p->stmt_id; + } + } + return 0; +} + +void MysqlStatement::SetStatementId(SocketId socket_id, uint32_t stmt_id) { + if (_connection_type == CONNECTION_TYPE_SHORT) { + return; + } + SocketUniquePtr socket; + if (Socket::Address(socket_id, &socket) == 0) { + uint64_t fd_version = socket->fd_version(); + MysqlStatementId value{stmt_id, fd_version}; + _id_map.Modify(my_update_kv, socket_id, value); + } +} + +void MysqlStatement::Init(const Channel& channel) { + _param_count = std::count(_str.begin(), _str.end(), '?'); + ChannelOptions opts = channel.options(); + _connection_type = ConnectionType(opts.connection_type); + if (_connection_type != CONNECTION_TYPE_SHORT) { + _id_map.Modify(my_init_kv); + } +} + +} // namespace brpc diff --git a/src/brpc/mysql_statement.h b/src/brpc/mysql_statement.h new file mode 100644 index 0000000000..49dc7ca318 --- /dev/null +++ b/src/brpc/mysql_statement.h @@ -0,0 +1,66 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_STATEMENT_H +#define BRPC_MYSQL_STATEMENT_H +#include +#include +#include "brpc/channel.h" +#include "brpc/mysql_statement_inl.h" + +namespace brpc { +// mysql prepared statement Unique Ptr +class MysqlStatement; +typedef std::unique_ptr MysqlStatementUniquePtr; +// mysql prepared statement +class MysqlStatement { +public: + const butil::StringPiece str() const; + uint16_t param_count() const; + uint32_t StatementId(SocketId sock_id) const; + void SetStatementId(SocketId sock_id, uint32_t stmt_id); + +private: + MysqlStatement(const Channel& channel, const butil::StringPiece& str); + void Init(const Channel& channel); + DISALLOW_COPY_AND_ASSIGN(MysqlStatement); + + friend MysqlStatementUniquePtr NewMysqlStatement(const Channel& channel, + const butil::StringPiece& str); + + const std::string _str; // prepare statement string + uint16_t _param_count; + mutable MysqlStatementDBD _id_map; // SocketId and statement id + ConnectionType _connection_type; +}; + +inline MysqlStatement::MysqlStatement(const Channel& channel, const butil::StringPiece& str) + : _str(str.data(), str.size()), _param_count(0) { + Init(channel); +} + +inline const butil::StringPiece MysqlStatement::str() const { + return butil::StringPiece(_str); +} + +inline uint16_t MysqlStatement::param_count() const { + return _param_count; +} + +MysqlStatementUniquePtr NewMysqlStatement(const Channel& channel, const butil::StringPiece& str); + +} // namespace brpc +#endif diff --git a/src/brpc/mysql_statement_inl.h b/src/brpc/mysql_statement_inl.h new file mode 100644 index 0000000000..97a814c1ee --- /dev/null +++ b/src/brpc/mysql_statement_inl.h @@ -0,0 +1,58 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_STATEMENT_INL_H +#define BRPC_MYSQL_STATEMENT_INL_H +#include +#include "butil/containers/flat_map.h" // FlatMap +#include "butil/containers/doubly_buffered_data.h" +#include "brpc/socket_id.h" + +namespace brpc { +DECLARE_int32(mysql_statment_map_size); + +struct MysqlStatementId { + uint32_t stmt_id; // statement id + uint64_t version; // socket's fd version +}; + +typedef butil::FlatMap MysqlStatementKVMap; +typedef butil::DoublyBufferedData MysqlStatementDBD; + +inline size_t my_init_kv(MysqlStatementKVMap& m) { + if (FLAGS_mysql_statment_map_size < 100) { + FLAGS_mysql_statment_map_size = 100; + } + m.init(FLAGS_mysql_statment_map_size); + return 1; +} + +inline size_t my_update_kv(MysqlStatementKVMap& m, SocketId key, MysqlStatementId value) { + MysqlStatementId* p = m.seek(key); + if (p == NULL) { + m.insert(key, value); + } else { + *p = value; + } + return 1; +} + +inline size_t my_delete_k(MysqlStatementKVMap& m, SocketId key) { + return m.erase(key); +} + +} // namespace brpc +#endif diff --git a/src/brpc/mysql_transaction.cpp b/src/brpc/mysql_transaction.cpp new file mode 100644 index 0000000000..3704f0ad5d --- /dev/null +++ b/src/brpc/mysql_transaction.cpp @@ -0,0 +1,113 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include +#include "butil/logging.h" // LOG() +#include "brpc/mysql_transaction.h" +#include "brpc/mysql.h" +#include "brpc/socket.h" +#include "brpc/details/controller_private_accessor.h" + +namespace brpc { +// mysql transaction isolation level string +const char* mysql_isolation_level[] = { + "REPEATABLE READ", "READ COMMITTED", "READ UNCOMMITTED", "SERIALIZABLE"}; + +SocketId MysqlTransaction::GetSocketId() const { + return _socket->id(); +} + +bool MysqlTransaction::DoneTransaction(const char* command) { + bool rc = false; + MysqlRequest request(this); + if (_socket == NULL) { // must already commit or rollback, return true. + return true; + } else if (!request.Query(command)) { + LOG(ERROR) << "Fail to query command" << command; + } else { + MysqlResponse response; + Controller cntl; + _channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.reply(0).is_ok()) { + rc = true; + } else { + LOG(ERROR) << "Fail " << command << " transaction, " << response; + } + } else { + LOG(ERROR) << "Fail " << command << " transaction, " << cntl.ErrorText(); + } + } + if (rc && _connection_type == CONNECTION_TYPE_POOLED) { + _socket->ReturnToPool(); + } + _socket.reset(); + return rc; +} + +MysqlTransactionUniquePtr NewMysqlTransaction(Channel& channel, + const MysqlTransactionOptions& opts) { + const char* command[2] = {"START TRANSACTION READ ONLY", "START TRANSACTION"}; + + if (channel.options().connection_type == CONNECTION_TYPE_SINGLE) { + LOG(ERROR) << "mysql transaction can't use connection type 'single'"; + return NULL; + } + std::stringstream ss; + // repeatable read is mysql default isolation level, so ignore it. + if (opts.isolation_level != MysqlIsoRepeatableRead) { + ss << "SET TRANSACTION ISOLATION LEVEL " << mysql_isolation_level[opts.isolation_level] + << ";"; + } + + if (opts.readonly) { + ss << command[0]; + } else { + ss << command[1]; + } + + MysqlRequest request; + if (!request.Query(ss.str())) { + LOG(ERROR) << "Fail to query command" << ss.str(); + return NULL; + } + + MysqlTransactionUniquePtr tx; + MysqlResponse response; + Controller cntl; + ControllerPrivateAccessor(&cntl).set_bind_sock_action(BIND_SOCK_ACTIVE); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + if (!cntl.Failed()) { + // repeatable read isolation send one reply, other isolation has two reply + if ((opts.isolation_level == MysqlIsoRepeatableRead && response.reply(0).is_ok()) || + (response.reply(0).is_ok() && response.reply(1).is_ok())) { + SocketUniquePtr socket; + ControllerPrivateAccessor(&cntl).get_bind_sock(&socket); + if (socket == NULL) { + LOG(ERROR) << "Fail create mysql transaction, get bind socket failed"; + } else { + tx.reset(new MysqlTransaction(channel, socket, cntl.connection_type())); + } + } else { + LOG(ERROR) << "Fail create mysql transaction, " << response; + } + } else { + LOG(ERROR) << "Fail create mysql transaction, " << cntl.ErrorText(); + } + return tx; +} + +} // namespace brpc diff --git a/src/brpc/mysql_transaction.h b/src/brpc/mysql_transaction.h new file mode 100644 index 0000000000..8bfae54909 --- /dev/null +++ b/src/brpc/mysql_transaction.h @@ -0,0 +1,89 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_MYSQL_TRANSACTION_H +#define BRPC_MYSQL_TRANSACTION_H + +#include "brpc/socket_id.h" +#include "brpc/channel.h" + +namespace brpc { +// mysql isolation level enum +enum MysqlIsolationLevel { + MysqlIsoRepeatableRead = 0, + MysqlIsoReadCommitted = 1, + MysqlIsoReadUnCommitted = 2, + MysqlIsoSerializable = 3, +}; +// mysql transaction options +struct MysqlTransactionOptions { + // if is readonly transaction + MysqlTransactionOptions() : readonly(false), isolation_level(MysqlIsoRepeatableRead) {} + bool readonly; + MysqlIsolationLevel isolation_level; +}; +// MysqlTransaction Unique Ptr +class MysqlTransaction; +typedef std::unique_ptr MysqlTransactionUniquePtr; +// mysql transaction type +class MysqlTransaction { +public: + ~MysqlTransaction(); + SocketId GetSocketId() const; + // commit transaction + bool commit(); + // rollback transaction + bool rollback(); + +private: + MysqlTransaction(Channel& channel, SocketUniquePtr& socket, ConnectionType connection_type); + bool DoneTransaction(const char* command); + DISALLOW_COPY_AND_ASSIGN(MysqlTransaction); + + friend MysqlTransactionUniquePtr NewMysqlTransaction(Channel& channel, + const MysqlTransactionOptions& opts); + +private: + Channel& _channel; + SocketUniquePtr _socket; + ConnectionType _connection_type; +}; + +inline MysqlTransaction::MysqlTransaction(Channel& channel, + SocketUniquePtr& socket, + ConnectionType connection_type) + : _channel(channel), _connection_type(connection_type) { + _socket.reset(socket.release()); +} + +inline MysqlTransaction::~MysqlTransaction() { + CHECK(rollback()) << "rollback failed"; +} + +inline bool MysqlTransaction::commit() { + return DoneTransaction("COMMIT"); +} + +inline bool MysqlTransaction::rollback() { + return DoneTransaction("ROLLBACK"); +} + +MysqlTransactionUniquePtr NewMysqlTransaction( + Channel& channel, const MysqlTransactionOptions& opts = MysqlTransactionOptions()); + +} // namespace brpc + +#endif diff --git a/src/brpc/options.proto b/src/brpc/options.proto index 3e34b5f6f6..4df7cff13d 100644 --- a/src/brpc/options.proto +++ b/src/brpc/options.proto @@ -64,6 +64,7 @@ enum ProtocolType { PROTOCOL_CDS_AGENT = 24; // Client side only PROTOCOL_ESP = 25; // Client side only PROTOCOL_H2 = 26; + PROTOCOL_MYSQL = 27; // Client side only } enum CompressType { diff --git a/src/brpc/policy/mysql_auth_hash.cpp b/src/brpc/policy/mysql_auth_hash.cpp new file mode 100644 index 0000000000..52e7acd96b --- /dev/null +++ b/src/brpc/policy/mysql_auth_hash.cpp @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2.0, as + * published by the Free Software Foundation. + * + * This program is also distributed with certain software (including + * but not limited to OpenSSL) that is licensed under separate terms, + * as designated in a particular file or component or in included license + * documentation. The authors of MySQL hereby grant you an + * additional permission to link the program and your derivative works + * with the separately licensed software that they have included with + * MySQL. + * + * Without limiting anything contained in the foregoing, this file, + * which is part of MySQL Connector/C++, is also subject to the + * Universal FOSS Exception, version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License, version 2.0, for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "mysql_auth_hash.h" +#include +#include +#include // memset +#include "butil/logging.h" // LOG() + +// Avoid warnings from protobuf +#if defined __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-conversion" +#elif defined _MSC_VER +#pragma warning(push) +#endif + +#include + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined _MSC_VER +#pragma warning(pop) +#endif + + +#define PVERSION41_CHAR '*' +#define SCRAMBLE_LENGTH 20 +#define SHA1_HASH_SIZE 20 +#define SHA256_HASH_SIZE 32 + +typedef unsigned char byte; +typedef size_t length_t; + +class SHA { + SHA_CTX m_sha; + + void init() { + SHA1_Init(&m_sha); + } + +public: + enum { DIGEST_SIZE = SHA1_HASH_SIZE }; // in Bytes + + SHA() { + init(); + } + + void Update(byte* data, length_t length) { + SHA1_Update(&m_sha, + data, +#ifdef WITH_SSL_WOLFSSL + (unsigned long) +#endif + length); + } + + void Final(byte* hash) { + SHA1_Final(hash, &m_sha); + init(); + } + + size_t getDigestSize() const { + return SHA1_HASH_SIZE; + } +}; + +class SHA256 { + SHA256_CTX m_sha; + + void init() { + SHA256_Init(&m_sha); + } + +public: + enum { DIGEST_SIZE = SHA256_HASH_SIZE }; // in Bytes + + SHA256() { + init(); + } + + void Update(byte* data, length_t length) { + SHA256_Update(&m_sha, data, length); + } + + void Final(byte* hash) { + SHA256_Final(hash, &m_sha); + init(); + } + + size_t getDigestSize() const { + return SHA256_HASH_SIZE; + } +}; + + +static void my_crypt(uint8_t* to, const uint8_t* s1, const uint8_t* s2, size_t len) { + const uint8_t* s1_end = s1 + len; + + while (s1 < s1_end) + *to++ = *s1++ ^ *s2++; +} + + +template +static std::string scramble(const std::string& scramble_data, const std::string& password) { + SHA_Crypt sha; + + if (scramble_data.length() != SCRAMBLE_LENGTH) + throw std::invalid_argument("Password scramble data is invalid"); + + byte hash_stage1[SHA_Crypt::DIGEST_SIZE]; + byte hash_stage2[SHA_Crypt::DIGEST_SIZE]; + byte result_buf[SHA_Crypt::DIGEST_SIZE + 1]; + + memset(result_buf, 0, sizeof(result_buf)); + + /* Two stage SHA1 hash of the pwd */ + /* Stage 1: hash pwd */ + sha.Update((byte*)password.data(), (length_t)password.length()); + sha.Final(hash_stage1); + + /* Stage 2 : hash first stage's output. */ + sha.Update(hash_stage1, sha.getDigestSize()); + sha.Final(hash_stage2); + + /* create crypt string as sha1(message, hash_stage2) */; + /* MYSQL41 and SHA256_PASSWORD have different behaviors here! Bug? */ + if (sha.getDigestSize() == SHA1_HASH_SIZE) { + sha.Update((byte*)scramble_data.data(), (length_t)scramble_data.length()); + sha.Update(hash_stage2, sha.getDigestSize()); + } else { + sha.Update(hash_stage2, sha.getDigestSize()); + sha.Update((byte*)scramble_data.data(), (length_t)scramble_data.length()); + } + sha.Final(result_buf); + result_buf[sha.getDigestSize()] = '\0'; + + my_crypt(result_buf, result_buf, hash_stage1, sha.getDigestSize()); + + return std::string((char*)result_buf, sha.getDigestSize()); +} + +static char* octet2hex(char* to, const char* str, size_t len) { + const char* _dig_vec_upper = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + const char* str_end = str + len; + for (; str != str_end; ++str) { + *to++ = _dig_vec_upper[((uint8_t)*str) >> 4]; + *to++ = _dig_vec_upper[((uint8_t)*str) & 0x0F]; + } + *to = '\0'; + return to; +} + +// MYSQL41 specific + +std::string brpc::policy::mysql_build_mysql41_authentication_response(const std::string& salt_data, + const std::string& password) { + std::string data; + std::string password_hash; + if (password.length()) { + password_hash = scramble(salt_data, password); + } + data.append(password_hash); // pass + + return data; +} + +// SHA256_MEMORY specific + +static std::string get_password_from_salt_sha256(const std::string& hash_stage2) { + std::string result(2 * SHA256_HASH_SIZE + 1, '\0'); + + if (hash_stage2.length() != SHA256_HASH_SIZE) + throw std::invalid_argument("Wrong size of binary hash password"); + + octet2hex(&result[0], &hash_stage2[0], SHA256_HASH_SIZE); + + return result; +} + +std::string brpc::policy::mysql_build_sha256_authentication_response(const std::string& salt_data, + const std::string& user, + const std::string& password, + const std::string& schema) { + std::string password_hash; + std::string data; + + password_hash = scramble(salt_data, password); + password_hash = get_password_from_salt_sha256(password_hash); + // REMOVE ending \0 + password_hash.pop_back(); + + data.append(schema).push_back('\0'); // authz + data.append(user).push_back('\0'); // authc + data.append(password_hash); // pass + + return data; +} diff --git a/src/brpc/policy/mysql_auth_hash.h b/src/brpc/policy/mysql_auth_hash.h new file mode 100644 index 0000000000..b037474637 --- /dev/null +++ b/src/brpc/policy/mysql_auth_hash.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2.0, as + * published by the Free Software Foundation. + * + * This program is also distributed with certain software (including + * but not limited to OpenSSL) that is licensed under separate terms, + * as designated in a particular file or component or in included license + * documentation. The authors of MySQL hereby grant you an + * additional permission to link the program and your derivative works + * with the separately licensed software that they have included with + * MySQL. + * + * Without limiting anything contained in the foregoing, this file, + * which is part of MySQL Connector/C++, is also subject to the + * Universal FOSS Exception, version 1.0, a copy of which can be found at + * http://oss.oracle.com/licenses/universal-foss-exception. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License, version 2.0, for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef BRPC_POLICY_MYSQL_AUTH_HASH_H +#define BRPC_POLICY_MYSQL_AUTH_HASH_H + +// #include +#include + +namespace brpc { +namespace policy { + +std::string mysql_build_mysql41_authentication_response(const std::string& salt_data, + const std::string& password); + +std::string mysql_build_sha256_authentication_response(const std::string& salt_data, + const std::string& user, + const std::string& password, + const std::string& schema); +} // namespace policy +} // namespace brpc + +#endif diff --git a/src/brpc/policy/mysql_authenticator.cpp b/src/brpc/policy/mysql_authenticator.cpp new file mode 100644 index 0000000000..1e1fd5fa1f --- /dev/null +++ b/src/brpc/policy/mysql_authenticator.cpp @@ -0,0 +1,176 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author(s): Yang,Liming + +#include +#include "brpc/policy/mysql_authenticator.h" +#include "brpc/policy/mysql_auth_hash.h" +#include "brpc/mysql_command.h" +#include "brpc/mysql_reply.h" +#include "brpc/mysql_common.h" +#include "butil/base64.h" +#include "butil/iobuf.h" +#include "butil/logging.h" // LOG() +#include "butil/sys_byteorder.h" + +namespace brpc { +namespace policy { + +namespace { +const butil::StringPiece mysql_native_password("mysql_native_password"); +const char* auth_param_delim = "\t"; +bool MysqlHandleParams(const butil::StringPiece& params, std::string* param_cmd) { + if (params.empty()) { + return true; + } + const char* delim1 = "&"; + std::vector idx; + for (size_t p = params.find(delim1); p != butil::StringPiece::npos; + p = params.find(delim1, p + 1)) { + idx.push_back(p); + } + + const char* delim2 = "="; + std::stringstream ss; + for (size_t i = 0; i < idx.size() + 1; ++i) { + size_t pos = (i > 0) ? idx[i - 1] + 1 : 0; + size_t len = (i < idx.size()) ? idx[i] - pos : params.size() - pos; + butil::StringPiece raw(params.data() + pos, len); + const size_t p = raw.find(delim2); + if (p != butil::StringPiece::npos) { + butil::StringPiece k(raw.data(), p); + butil::StringPiece v(raw.data() + p + 1, raw.size() - p - 1); + if (k == "charset") { + ss << "SET NAMES " << v << ";"; + } else { + ss << "SET " << k << "=" << v << ";"; + } + } + } + *param_cmd = ss.str(); + return true; +} +}; // namespace + +// user + "\t" + password + "\t" + schema + "\t" + collation + "\t" + param +bool MysqlAuthenticator::SerializeToString(std::string* str) const { + std::stringstream ss; + ss << _user << auth_param_delim; + ss << _passwd << auth_param_delim; + ss << _schema << auth_param_delim; + ss << _collation << auth_param_delim; + std::string param_cmd; + if (MysqlHandleParams(_params, ¶m_cmd)) { + ss << param_cmd; + } else { + LOG(ERROR) << "handle mysql authentication params failed, ignore it"; + return false; + } + *str = ss.str(); + return true; +} + +void MysqlParseAuthenticator(const butil::StringPiece& raw, + std::string* user, + std::string* password, + std::string* schema, + std::string* collation) { + std::vector idx; + idx.reserve(4); + for (size_t p = raw.find(auth_param_delim); p != butil::StringPiece::npos; + p = raw.find(auth_param_delim, p + 1)) { + idx.push_back(p); + } + user->assign(raw.data(), 0, idx[0]); + password->assign(raw.data(), idx[0] + 1, idx[1] - idx[0] - 1); + schema->assign(raw.data(), idx[1] + 1, idx[2] - idx[1] - 1); + collation->assign(raw.data(), idx[2] + 1, idx[3] - idx[2] - 1); +} + +void MysqlParseParams(const butil::StringPiece& raw, std::string* params) { + size_t idx = raw.rfind(auth_param_delim); + params->assign(raw.data(), idx + 1, raw.size() - idx - 1); +} + +int MysqlPackAuthenticator(const MysqlReply::Auth& auth, + const butil::StringPiece& user, + const butil::StringPiece& password, + const butil::StringPiece& schema, + const butil::StringPiece& collation, + std::string* auth_cmd) { + const uint16_t capability = + butil::ByteSwapToLE16((schema == "" ? 0x8285 : 0x828d) & auth.capability()); + const uint16_t extended_capability = butil::ByteSwapToLE16(0x000b & auth.extended_capability()); + butil::IOBuf salt; + salt.append(auth.salt().data(), auth.salt().size()); + salt.append(auth.salt2().data(), auth.salt2().size()); + if (auth.auth_plugin() == mysql_native_password) { + salt = mysql_build_mysql41_authentication_response(salt.to_string(), password.data()); + } else { + LOG(ERROR) << "no support auth plugin [" << auth.auth_plugin() << "]"; + return 1; + } + + butil::IOBuf payload; + payload.append(&capability, 2); + payload.append(&extended_capability, 2); + payload.push_back(0x00); + payload.push_back(0x00); + payload.push_back(0x00); + payload.push_back(0x00); + auto iter = MysqlCollations.find(collation.data()); + if (iter == MysqlCollations.end()) { + LOG(ERROR) << "wrong collation [" << collation << "]"; + return 1; + } + payload.append(&iter->second, 1); + const std::string stuff(23, '\0'); + payload.append(stuff); + payload.append(user.data()); + payload.push_back('\0'); + payload.append(pack_encode_length(salt.size())); + payload.append(salt); + if (schema != "") { + payload.append(schema.data()); + payload.push_back('\0'); + } + if (auth.auth_plugin() == mysql_native_password) { + payload.append(mysql_native_password.data(), mysql_native_password.size()); + payload.push_back('\0'); + } + butil::IOBuf message; + const uint32_t payload_size = butil::ByteSwapToLE32(payload.size()); + // header + message.append(&payload_size, 3); + message.push_back(0x01); + // payload + message.append(payload); + *auth_cmd = message.to_string(); + return 0; +} + +int MysqlPackParams(const butil::StringPiece& params, std::string* param_cmd) { + if (!params.empty()) { + butil::IOBuf buf; + MysqlMakeCommand(&buf, MYSQL_COM_QUERY, params); + buf.copy_to(param_cmd); + return 0; + } + LOG(ERROR) << "empty connection params"; + return 1; +} + +} // namespace policy +} // namespace brpc diff --git a/src/brpc/policy/mysql_authenticator.h b/src/brpc/policy/mysql_authenticator.h new file mode 100644 index 0000000000..e3494b61ac --- /dev/null +++ b/src/brpc/policy/mysql_authenticator.h @@ -0,0 +1,88 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author(s): Yang,Liming + +#ifndef BRPC_POLICY_MYSQL_AUTHENTICATOR_H +#define BRPC_POLICY_MYSQL_AUTHENTICATOR_H + +#include "butil/iobuf.h" +#include "brpc/authenticator.h" +#include "brpc/mysql_reply.h" + +namespace brpc { +namespace policy { +// Request to mysql for authentication. +class MysqlAuthenticator : public Authenticator { +public: + MysqlAuthenticator(const butil::StringPiece& user, + const butil::StringPiece& passwd, + const butil::StringPiece& schema, + const butil::StringPiece& params = "", + const butil::StringPiece& collation = MysqlDefaultCollation) + : _user(user.data(), user.size()), + _passwd(passwd.data(), passwd.size()), + _schema(schema.data(), schema.size()), + _params(params.data(), params.size()), + _collation(collation.data(), collation.size()) {} + + int GenerateCredential(std::string* auth_str) const { + return 0; + } + + int VerifyCredential(const std::string&, const butil::EndPoint&, brpc::AuthContext*) const { + return 0; + } + + const butil::StringPiece user() const; + const butil::StringPiece passwd() const; + const butil::StringPiece schema() const; + const butil::StringPiece params() const; + const butil::StringPiece collation() const; + bool SerializeToString(std::string* str) const; + +private: + DISALLOW_COPY_AND_ASSIGN(MysqlAuthenticator); + + const std::string _user; + const std::string _passwd; + const std::string _schema; + const std::string _params; + const std::string _collation; +}; + +inline const butil::StringPiece MysqlAuthenticator::user() const { + return _user; +} + +inline const butil::StringPiece MysqlAuthenticator::passwd() const { + return _passwd; +} + +inline const butil::StringPiece MysqlAuthenticator::schema() const { + return _schema; +} + +inline const butil::StringPiece MysqlAuthenticator::params() const { + return _params; +} + +inline const butil::StringPiece MysqlAuthenticator::collation() const { + return _collation; +} + +} // namespace policy +} // namespace brpc + +#endif // BRPC_POLICY_COUCHBASE_AUTHENTICATOR_H diff --git a/src/brpc/policy/mysql_protocol.cpp b/src/brpc/policy/mysql_protocol.cpp new file mode 100644 index 0000000000..dd1cf8a1b6 --- /dev/null +++ b/src/brpc/policy/mysql_protocol.cpp @@ -0,0 +1,416 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#include // MethodDescriptor +#include // Message +#include +#include +#include "butil/logging.h" // LOG() +#include "butil/time.h" +#include "butil/iobuf.h" // butil::IOBuf +#include "butil/sys_byteorder.h" +#include "brpc/controller.h" // Controller +#include "brpc/details/controller_private_accessor.h" +#include "brpc/socket.h" // Socket +#include "brpc/server.h" // Server +#include "brpc/details/server_private_accessor.h" +#include "brpc/span.h" +#include "brpc/mysql.h" +#include "brpc/policy/mysql_authenticator.h" +#include "brpc/policy/mysql_protocol.h" + +namespace brpc { + +DECLARE_bool(enable_rpcz); + +namespace policy { + +DEFINE_bool(mysql_verbose, false, "[DEBUG] Print EVERY mysql request/response"); + +void MysqlParseAuthenticator(const butil::StringPiece& raw, + std::string* user, + std::string* password, + std::string* schema, + std::string* collation); +void MysqlParseParams(const butil::StringPiece& raw, std::string* params); +// pack mysql authentication_data +int MysqlPackAuthenticator(const MysqlReply::Auth& auth, + const butil::StringPiece& user, + const butil::StringPiece& password, + const butil::StringPiece& schema, + const butil::StringPiece& collation, + std::string* auth_cmd); +int MysqlPackParams(const butil::StringPiece& params, std::string* param_cmd); + +namespace { +// I really don't want to add a variable in controller, so I use AuthContext group to mark auth +// step. +const char* auth_step[] = {"AUTH_OK", "PARAMS_OK"}; + +struct InputResponse : public InputMessageBase { + bthread_id_t id_wait; + MysqlResponse response; + + // @InputMessageBase + void DestroyImpl() { + delete this; + } +}; + +bool PackRequest(butil::IOBuf* buf, + ControllerPrivateAccessor& accessor, + const butil::IOBuf& request) { + if (accessor.pipelined_count() == MYSQL_PREPARED_STATEMENT) { + Socket* sock = accessor.get_sending_socket(); + if (sock == NULL) { + LOG(ERROR) << "[MYSQL PACK] get sending socket with NULL"; + return false; + } + auto stub = accessor.get_stmt(); + if (stub == NULL) { + LOG(ERROR) << "[MYSQL PACK] get prepare statement with NULL"; + return false; + } + uint32_t stmt_id; + // if can't found stmt_id in this socket, create prepared statement on it, store user + // request. + if ((stmt_id = stub->stmt()->StatementId(sock->id())) == 0) { + butil::IOBuf b; + butil::Status st = MysqlMakeCommand(&b, MYSQL_COM_STMT_PREPARE, stub->stmt()->str()); + if (!st.ok()) { + LOG(ERROR) << "[MYSQL PACK] make prepare statement error " << st; + return false; + } + accessor.set_pipelined_count(MYSQL_NEED_PREPARE); + buf->append(b); + return true; + } + // else pack execute header with stmt_id + butil::Status st = stub->PackExecuteCommand(buf, stmt_id); + if (!st.ok()) { + LOG(ERROR) << "write execute data error " << st; + return false; + } + return true; + } + buf->append(request); + return true; +} + +ParseError HandleAuthentication(const InputResponse* msg, const Socket* socket, PipelinedInfo* pi) { + const bthread_id_t cid = pi->id_wait; + Controller* cntl = NULL; + if (bthread_id_lock(cid, (void**)&cntl) != 0) { + LOG(ERROR) << "[MYSQL PARSE] fail to lock controller"; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + + ParseError parseCode = PARSE_OK; + const AuthContext* ctx = socket->auth_context(); + if (ctx == NULL) { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] auth context is null"; + goto END_OF_AUTH; + } + if (msg->response.reply(0).is_auth()) { + std::string user, password, schema, collation, auth_cmd; + const MysqlReply& reply = msg->response.reply(0); + MysqlParseAuthenticator(ctx->user(), &user, &password, &schema, &collation); + if (MysqlPackAuthenticator(reply.auth(), user, password, schema, collation, &auth_cmd) == + 0) { + butil::IOBuf buf; + buf.append(auth_cmd); + buf.cut_into_file_descriptor(socket->fd()); + const_cast(ctx)->set_group(auth_step[0]); + } else { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] wrong pack authentication data"; + } + } else if (msg->response.reply_size() > 0) { + for (size_t i = 0; i < msg->response.reply_size(); ++i) { + if (!msg->response.reply(i).is_ok()) { + LOG(ERROR) << "[MYSQL PARSE] auth failed " << msg->response; + parseCode = PARSE_ERROR_NO_RESOURCE; + goto END_OF_AUTH; + } + } + std::string params, params_cmd; + MysqlParseParams(ctx->user(), ¶ms); + if (ctx->group() == auth_step[0] && !params.empty()) { + if (MysqlPackParams(params, ¶ms_cmd) == 0) { + butil::IOBuf buf; + buf.append(params_cmd); + buf.cut_into_file_descriptor(socket->fd()); + const_cast(ctx)->set_group(auth_step[1]); + } else { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] wrong pack params data"; + } + } else { + butil::IOBuf raw_req; + raw_req.append(ctx->starter()); + raw_req.cut_into_file_descriptor(socket->fd()); + pi->with_auth = false; + } + } else { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] wrong authentication step"; + } + +END_OF_AUTH: + if (bthread_id_unlock(cid) != 0) { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] fail to unlock controller"; + } + return parseCode; +} + +ParseError HandlePrepareStatement(const InputResponse* msg, + const Socket* socket, + PipelinedInfo* pi) { + if (!msg->response.reply(0).is_prepare_ok()) { + LOG(ERROR) << "[MYSQL PARSE] response is not prepare ok, " << msg->response; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + const MysqlReply::PrepareOk& ok = msg->response.reply(0).prepare_ok(); + const bthread_id_t cid = pi->id_wait; + Controller* cntl = NULL; + if (bthread_id_lock(cid, (void**)&cntl) != 0) { + LOG(ERROR) << "[MYSQL PARSE] fail to lock controller"; + return PARSE_ERROR_ABSOLUTELY_WRONG; + } + ParseError parseCode = PARSE_OK; + butil::IOBuf buf; + butil::Status st; + auto stub = ControllerPrivateAccessor(cntl).get_stmt(); + auto stmt = stub->stmt(); + if (stmt == NULL || stmt->param_count() != ok.param_count()) { + LOG(ERROR) << "[MYSQL PACK] stmt can't be NULL"; + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + goto END_OF_PREPARE; + } + if (stmt->param_count() != ok.param_count()) { + LOG(ERROR) << "[MYSQL PACK] stmt param number " << stmt->param_count() + << " not equal to prepareOk.param_number " << ok.param_count(); + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + goto END_OF_PREPARE; + } + stmt->SetStatementId(socket->id(), ok.stmt_id()); + st = stub->PackExecuteCommand(&buf, ok.stmt_id()); + if (!st.ok()) { + LOG(ERROR) << "[MYSQL PACK] make execute header error " << st; + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + goto END_OF_PREPARE; + } + buf.cut_into_file_descriptor(socket->fd()); + pi->count = MYSQL_PREPARED_STATEMENT; +END_OF_PREPARE: + if (bthread_id_unlock(cid) != 0) { + parseCode = PARSE_ERROR_ABSOLUTELY_WRONG; + LOG(ERROR) << "[MYSQL PARSE] fail to unlock controller"; + } + return parseCode; +} + +} // namespace + +// "Message" = "Response" as we only implement the client for mysql. +ParseResult ParseMysqlMessage(butil::IOBuf* source, + Socket* socket, + bool /*read_eof*/, + const void* /*arg*/) { + if (source->empty()) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + + PipelinedInfo pi; + if (!socket->PopPipelinedInfo(&pi)) { + LOG(WARNING) << "No corresponding PipelinedInfo in socket"; + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + + InputResponse* msg = static_cast(socket->parsing_context()); + if (msg == NULL) { + msg = new InputResponse; + socket->reset_parsing_context(msg); + } + + MysqlStmtType stmt_type = static_cast(pi.count); + ParseError err = msg->response.ConsumePartialIOBuf(*source, pi.with_auth, stmt_type); + if (FLAGS_mysql_verbose) { + LOG(INFO) << "[MYSQL PARSE] " << msg->response; + } + if (err != PARSE_OK) { + if (err == PARSE_ERROR_NOT_ENOUGH_DATA) { + socket->GivebackPipelinedInfo(pi); + } + return MakeParseError(err); + } + if (pi.with_auth) { + ParseError err = HandleAuthentication(msg, socket, &pi); + if (err != PARSE_OK) { + return MakeParseError(err, "Fail to authenticate with Mysql"); + } + DestroyingPtr auth_msg = + static_cast(socket->release_parsing_context()); + socket->GivebackPipelinedInfo(pi); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + if (stmt_type == MYSQL_NEED_PREPARE) { + // store stmt_id, make execute header. + ParseError err = HandlePrepareStatement(msg, socket, &pi); + if (err != PARSE_OK) { + return MakeParseError(err, "Fail to make parepared statement with Mysql"); + } + DestroyingPtr prepare_msg = + static_cast(socket->release_parsing_context()); + socket->GivebackPipelinedInfo(pi); + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + msg->id_wait = pi.id_wait; + socket->release_parsing_context(); + return MakeMessage(msg); +} + +void ProcessMysqlResponse(InputMessageBase* msg_base) { + const int64_t start_parse_us = butil::cpuwide_time_us(); + DestroyingPtr msg(static_cast(msg_base)); + + const bthread_id_t cid = msg->id_wait; + Controller* cntl = NULL; + const int rc = bthread_id_lock(cid, (void**)&cntl); + if (rc != 0) { + LOG_IF(ERROR, rc != EINVAL && rc != EPERM) + << "Fail to lock correlation_id=" << cid << ": " << berror(rc); + return; + } + + ControllerPrivateAccessor accessor(cntl); + Span* span = accessor.span(); + if (span) { + span->set_base_real_us(msg->base_real_us()); + span->set_received_us(msg->received_us()); + span->set_response_size(msg->response.ByteSize()); + span->set_start_parse_us(start_parse_us); + } + const int saved_error = cntl->ErrorCode(); + if (cntl->response() != NULL) { + if (cntl->response()->GetDescriptor() != MysqlResponse::descriptor()) { + cntl->SetFailed(ERESPONSE, "Must be MysqlResponse"); + } else { + // We work around ParseFrom of pb which is just a placeholder. + ((MysqlResponse*)cntl->response())->Swap(&msg->response); + } + } // silently ignore the response. + + // Unlocks correlation_id inside. Revert controller's + // error code if it version check of `cid' fails + msg.reset(); // optional, just release resourse ASAP + accessor.OnResponse(cid, saved_error); +} + +void SerializeMysqlRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request) { + if (request == NULL) { + return cntl->SetFailed(EREQUEST, "request is NULL"); + } + if (request->GetDescriptor() != MysqlRequest::descriptor()) { + return cntl->SetFailed(EREQUEST, "The request is not a MysqlRequest"); + } + const MysqlRequest* rr = (const MysqlRequest*)request; + // We work around SerializeTo of pb which is just a placeholder. + if (!rr->SerializeTo(buf)) { + return cntl->SetFailed(EREQUEST, "Fail to serialize MysqlRequest"); + } + // mysql protocol don't use pipelined count to verify the end of a response, so pipelined count + // is meanless, but we can use it help us to distinguish mysql reply type. In mysql protocol, we + // can't distinguish OK and PreparedOk, so we set pipelined count to 2 to let parse function to + // parse PreparedOk reply + ControllerPrivateAccessor accessor(cntl); + accessor.set_pipelined_count(MYSQL_NORMAL_STATEMENT); + + auto tx = rr->get_tx(); + if (tx != NULL) { + accessor.use_bind_sock(tx->GetSocketId()); + } + auto st = rr->get_stmt(); + if (st != NULL) { + accessor.set_stmt(st); + accessor.set_pipelined_count(MYSQL_PREPARED_STATEMENT); + } + if (FLAGS_mysql_verbose) { + LOG(INFO) << "\n[MYSQL REQUEST] " << *rr; + } +} + +void PackMysqlRequest(butil::IOBuf* buf, + SocketMessage**, + uint64_t /*correlation_id*/, + const google::protobuf::MethodDescriptor*, + Controller* cntl, + const butil::IOBuf& request, + const Authenticator* auth) { + ControllerPrivateAccessor accessor(cntl); + if (auth) { + const MysqlAuthenticator* my_auth(dynamic_cast(auth)); + if (my_auth == NULL) { + LOG(ERROR) << "[MYSQL PACK] there is not MysqlAuthenticator"; + return; + } + Socket* sock = accessor.get_sending_socket(); + if (sock == NULL) { + LOG(ERROR) << "[MYSQL PACK] get sending socket with NULL"; + return; + } + AuthContext* ctx = sock->mutable_auth_context(); + // std::string params; + // if (!MysqlHandleParams(my_auth->params(), ¶ms)) { + // LOG(ERROR) << "[MYSQL PACK] handle params error"; + // return; + // } + // std::stringstream ss; + // ss << my_auth->user() << "\t" << my_auth->passwd() << "\t" << my_auth->schema() << "\t" + // << my_auth->collation() << "\t" << params; + std::string str; + if (!my_auth->SerializeToString(&str)) { + LOG(ERROR) << "[MYSQL PACK] auth param serialize to string failed"; + return; + } + ctx->set_user(str); + butil::IOBuf b; + if (!PackRequest(&b, accessor, request)) { + LOG(ERROR) << "[MYSQL PACK] pack request error"; + return; + } + ctx->set_starter(b.to_string()); + accessor.add_with_auth(); + } else { + if (!PackRequest(buf, accessor, request)) { + LOG(ERROR) << "[MYSQL PACK] pack request error"; + return; + } + } +} + +const std::string& GetMysqlMethodName(const google::protobuf::MethodDescriptor*, + const Controller*) { + const static std::string MYSQL_SERVER_STR = "mysql-server"; + return MYSQL_SERVER_STR; +} + +} // namespace policy +} // namespace brpc diff --git a/src/brpc/policy/mysql_protocol.h b/src/brpc/policy/mysql_protocol.h new file mode 100644 index 0000000000..163629c6cf --- /dev/null +++ b/src/brpc/policy/mysql_protocol.h @@ -0,0 +1,52 @@ +// Copyright (c) 2019 Baidu, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Authors: Yang,Liming (yangliming01@baidu.com) + +#ifndef BRPC_POLICY_MYSQL_PROTOCOL_H +#define BRPC_POLICY_MYSQL_PROTOCOL_H + +#include "brpc/protocol.h" + + +namespace brpc { +namespace policy { + +// Parse mysql response. +ParseResult ParseMysqlMessage(butil::IOBuf* source, Socket* socket, bool read_eof, const void* arg); + +// Actions to a mysql response. +void ProcessMysqlResponse(InputMessageBase* msg); + +// Serialize a mysql request. +void SerializeMysqlRequest(butil::IOBuf* buf, + Controller* cntl, + const google::protobuf::Message* request); + +// Pack `request' to `method' into `buf'. +void PackMysqlRequest(butil::IOBuf* buf, + SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* controller, + const butil::IOBuf& request, + const Authenticator* auth); + +const std::string& GetMysqlMethodName(const google::protobuf::MethodDescriptor*, const Controller*); + +} // namespace policy +} // namespace brpc + + +#endif // BRPC_POLICY_MYSQL_PROTOCOL_H diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp index e3878c198e..f4c6b1da50 100644 --- a/src/brpc/socket.cpp +++ b/src/brpc/socket.cpp @@ -428,6 +428,7 @@ Socket::Socket(Forbidden) , _fd(-1) , _tos(0) , _reset_fd_real_us(-1) + , _fd_version(0) , _on_edge_triggered_events(NULL) , _user(NULL) , _conn(NULL) @@ -533,6 +534,8 @@ int Socket::ResetFileDescriptor(int fd) { _avg_msg_size = 0; // MUST store `_fd' before adding itself into epoll device to avoid // race conditions with the callback function inside epoll + static butil::atomic BAIDU_CACHELINE_ALIGNMENT fd_version(0); + _fd_version = fd_version.fetch_add(1, butil::memory_order_relaxed); _fd.store(fd, butil::memory_order_release); _reset_fd_real_us = butil::gettimeofday_us(); if (!ValidFileDescriptor(fd)) { @@ -1512,7 +1515,7 @@ int Socket::Write(butil::IOBuf* data, const WriteOptions* options_in) { if (options_in) { opt = *options_in; } - if (data->empty()) { + if (data->empty() && !opt.with_auth) { return SetError(opt.id_wait, EINVAL); } if (opt.pipelined_count > MAX_PIPELINED_COUNT) { diff --git a/src/brpc/socket.h b/src/brpc/socket.h index 6f710ee2a9..e839efbe22 100644 --- a/src/brpc/socket.h +++ b/src/brpc/socket.h @@ -283,6 +283,9 @@ friend class policy::H2GlobalStreamCreator; // The file descriptor int fd() const { return _fd.load(butil::memory_order_relaxed); } + // The file descriptor version, used to avoid ABA problem. + uint64_t fd_version() const { return _fd_version; } + // ip/port of the local end of the connection butil::EndPoint local_side() const { return _local_side; } @@ -720,6 +723,7 @@ friend void DereferenceSocket(Socket*); butil::atomic _fd; // -1 when not connected. int _tos; // Type of service which is actually only 8bits. int64_t _reset_fd_real_us; // When _fd was reset, in microseconds. + uint64_t _fd_version; // _fd_version, use only for mysql now. // Address of peer. Initialized by SocketOptions.remote_side. butil::EndPoint _remote_side; diff --git a/test/brpc_mysql_unittest.cpp b/test/brpc_mysql_unittest.cpp new file mode 100644 index 0000000000..7668749a8d --- /dev/null +++ b/test/brpc_mysql_unittest.cpp @@ -0,0 +1,852 @@ +// Copyright (c) 2019 Baidu, Inc. +// Date: Thu Jun 11 14:30:07 CST 2019 + +#include +#include +#include +#include "butil/time.h" +#include +#include +#include "butil/logging.h" // LOG() +#include "butil/strings/string_piece.h" +#include +#include + +namespace brpc { +const std::string MYSQL_connection_type = "pooled"; +const int MYSQL_timeout_ms = 80000; +const int MYSQL_connect_timeout_ms = 80000; + +// const std::string MYSQL_host = "127.0.0.1"; +const std::string MYSQL_host = "db4free.net"; +const std::string MYSQL_port = "3306"; +const std::string MYSQL_user = "brpcuser"; +const std::string MYSQL_password = "12345678"; +const std::string MYSQL_schema = "brpc_test"; +int64_t MYSQL_table_suffix; +} // namespace brpc + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +namespace { +static pthread_once_t check_mysql_server_once = PTHREAD_ONCE_INIT; + +static void CheckMysqlServer() { + brpc::MYSQL_table_suffix = butil::gettimeofday_us(); + puts("Checking mysql-server..."); + std::stringstream ss; + ss << "mysql" + << " -h" << brpc::MYSQL_host << " -P" << brpc::MYSQL_port << " -u" << brpc::MYSQL_user + << " -p" << brpc::MYSQL_password << " -D" << brpc::MYSQL_schema << " -e 'show databases'"; + puts(ss.str().c_str()); + if (system(ss.str().c_str()) != 0) { + std::stringstream ss; + ss << "please startup your mysql-server, then create \nschema:" << brpc::MYSQL_schema + << "\nuser:" << brpc::MYSQL_user << "\npassword:" << brpc::MYSQL_password; + puts(ss.str().c_str()); + return; + } +} + +class MysqlTest : public testing::Test { +protected: + MysqlTest() {} + void SetUp() { + pthread_once(&check_mysql_server_once, CheckMysqlServer); + } + void TearDown() {} +}; + +TEST_F(MysqlTest, auth) { + // config auth + { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + request.Query("show databases"); + + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_RESULTSET, response.reply(0).type()); + } + + // Auth failed + { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = + new brpc::policy::MysqlAuthenticator(brpc::MYSQL_user, "123456789", brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + request.Query("show databases"); + + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_TRUE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(brpc::MYSQL_RSP_UNKNOWN, response.reply(0).type()); + } + + // check noauth. + { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + request.Query("show databases"); + + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_TRUE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(brpc::MYSQL_RSP_UNKNOWN, response.reply(0).type()); + } +} + +TEST_F(MysqlTest, ok) { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "drop table brpc_table_" << brpc::MYSQL_table_suffix; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + } + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "CREATE TABLE IF NOT EXISTS `brpc_table_" << brpc::MYSQL_table_suffix + << "` (`col1` int(11) NOT NULL AUTO_INCREMENT, " + "`col2` varchar(45) DEFAULT NULL, " + "`col3` decimal(6,3) DEFAULT NULL, `col4` datetime DEFAULT NULL, `col5` blob, `col6` " + "binary(6) DEFAULT NULL, `col7` tinyblob, `col8` longblob, `col9` mediumblob, " + "`col10` " + "tinyblob, `col11` varbinary(10) DEFAULT NULL, `col12` date DEFAULT NULL, `col13` " + "datetime(6) DEFAULT NULL, `col14` time DEFAULT NULL, `col15` timestamp(4) NULL " + "DEFAULT NULL, `col16` year(4) DEFAULT NULL, `col17` geometry DEFAULT NULL, `col18` " + "geometrycollection DEFAULT NULL, `col19` linestring DEFAULT NULL, `col20` point " + "DEFAULT NULL, `col21` polygon DEFAULT NULL, `col22` bigint(64) DEFAULT NULL, " + "`col23` " + "decimal(10,0) DEFAULT NULL, `col24` double DEFAULT NULL, `col25` float DEFAULT " + "NULL, " + "`col26` int(7) DEFAULT NULL, `col27` mediumint(18) DEFAULT NULL, `col28` double " + "DEFAULT NULL, `col29` smallint(2) DEFAULT NULL, `col30` tinyint(1) DEFAULT NULL, " + "`col31` char(6) DEFAULT NULL, `col32` varchar(6) DEFAULT NULL, `col33` longtext, " + "`col34` mediumtext, `col35` tinytext, `col36` tinytext, `col37` bit(7) DEFAULT " + "NULL, " + "`col38` tinyint(4) DEFAULT NULL, `col39` varchar(45) DEFAULT NULL, `col40` " + "varchar(45) CHARACTER SET utf8 DEFAULT NULL, `col41` char(4) CHARACTER SET utf8 " + "DEFAULT NULL, `col42` varchar(6) CHARACTER SET utf8 DEFAULT NULL, PRIMARY KEY " + "(`col1`)) ENGINE=InnoDB AUTO_INCREMENT=1157 DEFAULT CHARSET=utf8"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + std::stringstream ss1; + ss1 << "INSERT INTO `brpc_table_" << brpc::MYSQL_table_suffix + << "` " + "(`col2`,`col3`,`col4`,`col5`,`col6`,`col7`,`col8`,`col9`,`col10`,`col11`,`" + "col12`,`col13`,`col14`,`col15`,`col16`,`col17`,`col18`,`col19`,`col20`,`col21`, " + "`col22` " + ",`col23`,`col24`,`col25`,`col26`,`col27`,`col28`,`col29`,`col30`,`col31`,`col32`,`" + "col33`,`col34`,`col35`,`col36`,`col37`,`col38`,`col39`,`col40`,`col41`,`col42`) " + "VALUES ('col2',0.015,'2018-12-01 " + "12:13:14','aaa','bbb','ccc','ddd','eee','fff','ggg','2014-09-18', '2010-12-10 " + "14:12:09.019473' ,'01:06:09','1970-12-08 00:00:00.0001' " + ",2014,NULL,NULL,NULL,NULL,NULL,69,'12.5',16.9,6.7,24,37,69.56,234,6, '" + "col31','col32','col33','col34','col35','col36',NULL,9,'col39','col40','col4' ,'" + "col42')"; + request.Query(ss1.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } +} + +TEST_F(MysqlTest, error) { + { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + request.Query("select nocol from notable"); + + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_ERROR, response.reply(0).type()); + } +} + +TEST_F(MysqlTest, resultset) { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema, "charset=utf8"); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + { + for (int i = 0; i < 50; ++i) { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + + std::stringstream ss1; + ss1 << "INSERT INTO `brpc_table_" << brpc::MYSQL_table_suffix + << "` " + "(`col2`,`col3`,`col4`,`col5`,`col6`,`col7`,`col8`,`col9`,`col10`,`col11`" + ",`" + "col12`,`col13`,`col14`,`col15`,`col16`,`col17`,`col18`,`col19`,`col20`,`col21`," + " " + "`col22` " + ",`col23`,`col24`,`col25`,`col26`,`col27`,`col28`,`col29`,`col30`,`col31`,`" + "col32`,`" + "col33`,`col34`,`col35`,`col36`,`col37`,`col38`,`col39`,`col40`,`col41`,`col42`)" + " VALUES ('col2',0.015,'2018-12-01 " + "12:13:14','aaa','bbb','ccc','ddd','eee','fff','ggg','2014-09-18', '2010-12-10 " + "14:12:09.019473' ,'01:06:09','1970-12-08 00:00:00.0001' " + ",2014,NULL,NULL,NULL,NULL,NULL,69,'12.5',16.9,6.7,24,37,69.56,234,6, '" + "col31','col32','col33','col34','col35','col36',NULL,9,'col39','col40','col4' " + ",'" + "col42')"; + request.Query(ss1.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } + } + + { + std::stringstream ss1; + for (int i = 0; i < 30; ++i) { + ss1 << "INSERT INTO `brpc_table_" << brpc::MYSQL_table_suffix + << "` " + "(`col2`,`col3`,`col4`,`col5`,`col6`,`col7`,`col8`,`col9`,`col10`,`col11`" + ",`" + "col12`,`col13`,`col14`,`col15`,`col16`,`col17`,`col18`,`col19`,`col20`,`col21`," + " " + "`col22` " + ",`col23`,`col24`,`col25`,`col26`,`col27`,`col28`,`col29`,`col30`,`col31`,`" + "col32`,`" + "col33`,`col34`,`col35`,`col36`,`col37`,`col38`,`col39`,`col40`,`col41`,`col42`)" + "VALUES ('col2',0.015,'2018-12-01 " + "12:13:14','aaa','bbb','ccc','ddd','eee','fff','ggg','2014-09-18', '2010-12-10 " + "14:12:09.019473' ,'01:06:09','1970-12-08 00:00:00.0001' " + ",2014,NULL,NULL,NULL,NULL,NULL,69,'12.5',16.9,6.7,24,37,69.56,234,6, '" + "col31','col32','col33','col34','col35','col36',NULL,9,'col39','col40','col4' " + ",'" + "col42');"; + } + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + request.Query(ss1.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(30ul, response.reply_size()); + for (int i = 0; i < 30; ++i) { + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(i).type()); + } + } + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "select count(0) from brpc_table_" << brpc::MYSQL_table_suffix; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + // ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_RESULTSET, response.reply(0).type()); + } + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "select * from brpc_table_" << brpc::MYSQL_table_suffix << " where 1 = 2"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_RESULTSET, response.reply(0).type()); + } + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "select * from brpc_table_" << brpc::MYSQL_table_suffix << " limit 10"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_RESULTSET, response.reply(0).type()); + ASSERT_EQ(42ull, response.reply(0).column_count()); + const brpc::MysqlReply& reply = response.reply(0); + ASSERT_EQ(reply.column(0).name(), "col1"); + ASSERT_EQ(reply.column(0).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(0).type(), brpc::MYSQL_FIELD_TYPE_LONG); + + ASSERT_EQ(reply.column(1).name(), "col2"); + ASSERT_EQ(reply.column(1).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(1).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + ASSERT_EQ(reply.column(2).name(), "col3"); + ASSERT_EQ(reply.column(2).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(2).type(), brpc::MYSQL_FIELD_TYPE_NEWDECIMAL); + + ASSERT_EQ(reply.column(3).name(), "col4"); + ASSERT_EQ(reply.column(3).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(3).type(), brpc::MYSQL_FIELD_TYPE_DATETIME); + + ASSERT_EQ(reply.column(4).name(), "col5"); + ASSERT_EQ(reply.column(4).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(4).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(5).name(), "col6"); + ASSERT_EQ(reply.column(5).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(5).type(), brpc::MYSQL_FIELD_TYPE_STRING); + + ASSERT_EQ(reply.column(6).name(), "col7"); + ASSERT_EQ(reply.column(6).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(6).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(7).name(), "col8"); + ASSERT_EQ(reply.column(7).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(7).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(8).name(), "col9"); + ASSERT_EQ(reply.column(8).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(8).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(9).name(), "col10"); + ASSERT_EQ(reply.column(9).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(9).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(10).name(), "col11"); + ASSERT_EQ(reply.column(10).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(10).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + ASSERT_EQ(reply.column(11).name(), "col12"); + ASSERT_EQ(reply.column(11).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(11).type(), brpc::MYSQL_FIELD_TYPE_DATE); + + ASSERT_EQ(reply.column(12).name(), "col13"); + ASSERT_EQ(reply.column(12).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(12).type(), brpc::MYSQL_FIELD_TYPE_DATETIME); + + ASSERT_EQ(reply.column(13).name(), "col14"); + ASSERT_EQ(reply.column(13).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(13).type(), brpc::MYSQL_FIELD_TYPE_TIME); + + ASSERT_EQ(reply.column(14).name(), "col15"); + ASSERT_EQ(reply.column(14).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(14).type(), brpc::MYSQL_FIELD_TYPE_TIMESTAMP); + + ASSERT_EQ(reply.column(15).name(), "col16"); + ASSERT_EQ(reply.column(15).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(15).type(), brpc::MYSQL_FIELD_TYPE_YEAR); + + ASSERT_EQ(reply.column(16).name(), "col17"); + ASSERT_EQ(reply.column(16).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(16).type(), brpc::MYSQL_FIELD_TYPE_GEOMETRY); + + ASSERT_EQ(reply.column(17).name(), "col18"); + ASSERT_EQ(reply.column(17).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(17).type(), brpc::MYSQL_FIELD_TYPE_GEOMETRY); + + ASSERT_EQ(reply.column(18).name(), "col19"); + ASSERT_EQ(reply.column(18).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(18).type(), brpc::MYSQL_FIELD_TYPE_GEOMETRY); + + ASSERT_EQ(reply.column(19).name(), "col20"); + ASSERT_EQ(reply.column(19).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(19).type(), brpc::MYSQL_FIELD_TYPE_GEOMETRY); + + ASSERT_EQ(reply.column(20).name(), "col21"); + ASSERT_EQ(reply.column(20).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(20).type(), brpc::MYSQL_FIELD_TYPE_GEOMETRY); + + ASSERT_EQ(reply.column(21).name(), "col22"); + ASSERT_EQ(reply.column(21).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(21).type(), brpc::MYSQL_FIELD_TYPE_LONGLONG); + + ASSERT_EQ(reply.column(22).name(), "col23"); + ASSERT_EQ(reply.column(22).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(22).type(), brpc::MYSQL_FIELD_TYPE_NEWDECIMAL); + + ASSERT_EQ(reply.column(23).name(), "col24"); + ASSERT_EQ(reply.column(23).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(23).type(), brpc::MYSQL_FIELD_TYPE_DOUBLE); + + ASSERT_EQ(reply.column(24).name(), "col25"); + ASSERT_EQ(reply.column(24).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(24).type(), brpc::MYSQL_FIELD_TYPE_FLOAT); + + ASSERT_EQ(reply.column(25).name(), "col26"); + ASSERT_EQ(reply.column(25).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(25).type(), brpc::MYSQL_FIELD_TYPE_LONG); + + ASSERT_EQ(reply.column(26).name(), "col27"); + ASSERT_EQ(reply.column(26).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(26).type(), brpc::MYSQL_FIELD_TYPE_INT24); + + ASSERT_EQ(reply.column(27).name(), "col28"); + ASSERT_EQ(reply.column(27).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(27).type(), brpc::MYSQL_FIELD_TYPE_DOUBLE); + + ASSERT_EQ(reply.column(28).name(), "col29"); + ASSERT_EQ(reply.column(28).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(28).type(), brpc::MYSQL_FIELD_TYPE_SHORT); + + ASSERT_EQ(reply.column(29).name(), "col30"); + ASSERT_EQ(reply.column(29).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(29).type(), brpc::MYSQL_FIELD_TYPE_TINY); + + ASSERT_EQ(reply.column(30).name(), "col31"); + ASSERT_EQ(reply.column(30).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(30).type(), brpc::MYSQL_FIELD_TYPE_STRING); + + ASSERT_EQ(reply.column(31).name(), "col32"); + ASSERT_EQ(reply.column(31).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(31).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + ASSERT_EQ(reply.column(32).name(), "col33"); + ASSERT_EQ(reply.column(32).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(32).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(33).name(), "col34"); + ASSERT_EQ(reply.column(33).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(33).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(34).name(), "col35"); + ASSERT_EQ(reply.column(34).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(34).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(35).name(), "col36"); + ASSERT_EQ(reply.column(35).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(35).type(), brpc::MYSQL_FIELD_TYPE_BLOB); + + ASSERT_EQ(reply.column(36).name(), "col37"); + ASSERT_EQ(reply.column(36).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(36).type(), brpc::MYSQL_FIELD_TYPE_BIT); + + ASSERT_EQ(reply.column(37).name(), "col38"); + ASSERT_EQ(reply.column(37).charset(), brpc::MysqlCollations.at("binary")); + ASSERT_EQ(reply.column(37).type(), brpc::MYSQL_FIELD_TYPE_TINY); + + ASSERT_EQ(reply.column(38).name(), "col39"); + ASSERT_EQ(reply.column(38).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(38).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + ASSERT_EQ(reply.column(39).name(), "col40"); + ASSERT_EQ(reply.column(39).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(39).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + ASSERT_EQ(reply.column(40).name(), "col41"); + ASSERT_EQ(reply.column(40).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(40).type(), brpc::MYSQL_FIELD_TYPE_STRING); + + ASSERT_EQ(reply.column(41).name(), "col42"); + ASSERT_EQ(reply.column(41).charset(), brpc::MysqlCollations.at("utf8_general_ci")); + ASSERT_EQ(reply.column(41).type(), brpc::MYSQL_FIELD_TYPE_VAR_STRING); + + for (uint64_t idx = 0; idx < reply.row_count(); ++idx) { + const brpc::MysqlReply::Row& row = reply.next(); + ASSERT_EQ(row.field(1).string(), "col2"); + ASSERT_EQ(row.field(2).string(), "0.015"); + ASSERT_EQ(row.field(3).string(), "2018-12-01 12:13:14"); + ASSERT_EQ(row.field(4).string(), "aaa"); + butil::StringPiece field5 = row.field(5).string(); + ASSERT_EQ(field5.size(), size_t(6)); + ASSERT_EQ(field5[0], 'b'); + ASSERT_EQ(field5[1], 'b'); + ASSERT_EQ(field5[2], 'b'); + ASSERT_EQ(field5[3], '\0'); + ASSERT_EQ(field5[4], '\0'); + ASSERT_EQ(field5[5], '\0'); + ASSERT_EQ(row.field(6).string(), "ccc"); + ASSERT_EQ(row.field(7).string(), "ddd"); + ASSERT_EQ(row.field(8).string(), "eee"); + ASSERT_EQ(row.field(9).string(), "fff"); + ASSERT_EQ(row.field(10).string(), "ggg"); + ASSERT_EQ(row.field(11).string(), "2014-09-18"); + ASSERT_EQ(row.field(12).string(), "2010-12-10 14:12:09.019473"); + ASSERT_EQ(row.field(13).string(), "01:06:09"); + ASSERT_EQ(row.field(14).string(), "1970-12-08 00:00:00.0001"); + ASSERT_EQ(row.field(15).small(), uint16_t(2014)); + ASSERT_EQ(row.field(16).is_nil(), true); + ASSERT_EQ(row.field(17).is_nil(), true); + ASSERT_EQ(row.field(18).is_nil(), true); + ASSERT_EQ(row.field(19).is_nil(), true); + ASSERT_EQ(row.field(20).is_nil(), true); + ASSERT_EQ(row.field(21).sbigint(), int64_t(69)); + ASSERT_EQ(row.field(22).string(), "13"); + ASSERT_EQ(row.field(23).float64(), double(16.9)); + ASSERT_EQ(row.field(24).float32(), float(6.7)); + ASSERT_EQ(row.field(25).sinteger(), int32_t(24)); + ASSERT_EQ(row.field(26).sinteger(), int32_t(37)); + ASSERT_EQ(row.field(27).float64(), double(69.56)); + ASSERT_EQ(row.field(28).ssmall(), int16_t(234)); + ASSERT_EQ(row.field(29).stiny(), 6); + ASSERT_EQ(row.field(30).string(), "col31"); + ASSERT_EQ(row.field(31).string(), "col32"); + ASSERT_EQ(row.field(32).string(), "col33"); + ASSERT_EQ(row.field(33).string(), "col34"); + ASSERT_EQ(row.field(34).string(), "col35"); + ASSERT_EQ(row.field(35).string(), "col36"); + ASSERT_EQ(row.field(36).is_nil(), true); + ASSERT_EQ(row.field(37).stiny(), 9); + ASSERT_EQ(row.field(38).string(), "col39"); + ASSERT_EQ(row.field(39).string(), "col40"); + ASSERT_EQ(row.field(40).string(), "col4"); // size is 4 + ASSERT_EQ(row.field(41).string(), "col42"); + } + } +} + +TEST_F(MysqlTest, transaction) { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "drop table brpc_tx_" << brpc::MYSQL_table_suffix; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + } + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "CREATE TABLE IF NOT EXISTS `brpc_tx_" << brpc::MYSQL_table_suffix + << "` (`Id` int(11) NOT NULL AUTO_INCREMENT,`LastName` " + "varchar(255) DEFAULT " + "NULL,`FirstName` decimal(10,0) DEFAULT NULL,`Address` varchar(255) DEFAULT " + "NULL,`City` varchar(255) DEFAULT NULL, PRIMARY KEY (`Id`)) ENGINE=InnoDB " + "AUTO_INCREMENT=1157 DEFAULT CHARSET=utf8"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } + { + brpc::MysqlTransactionOptions tx_options; + tx_options.readonly = false; + tx_options.isolation_level = brpc::MysqlIsoRepeatableRead; + brpc::MysqlTransactionUniquePtr tx(brpc::NewMysqlTransaction(channel, tx_options)); + ASSERT_FALSE(tx == NULL) << "Fail to create transaction"; + uint64_t idx1, idx2; + { + brpc::MysqlRequest request(tx.get()); + std::stringstream ss; + ss << "insert into brpc_tx_" << brpc::MYSQL_table_suffix + << "(LastName,FirstName, Address) values " + "('lucy',12.5,'beijing')"; + ASSERT_EQ(request.Query(ss.str()), true); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + idx1 = response.reply(0).ok().index(); + } + { + brpc::MysqlRequest request(tx.get()); + std::stringstream ss; + ss << "insert into brpc_tx_" << brpc::MYSQL_table_suffix + << "(LastName,FirstName, Address) values " + "('lilei',12.6,'shanghai')"; + ASSERT_EQ(request.Query(ss.str()), true); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + idx2 = response.reply(0).ok().index(); + } + + LOG(INFO) << "idx1=" << idx1 << " idx2=" << idx2; + // not commit, so return 0 rows + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "select * from brpc_tx_" << brpc::MYSQL_table_suffix << " where id in (" << idx1 + << "," << idx2 << ")"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).row_count(), 0ul); + } + + { ASSERT_EQ(tx->commit(), true); } + // after commit, so return 2 rows + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "select * from brpc_tx_" << brpc::MYSQL_table_suffix << " where id in (" << idx1 + << "," << idx2 << ")"; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).row_count(), 2ul); + } + } + + { + brpc::MysqlTransactionOptions tx_options; + tx_options.readonly = true; + tx_options.isolation_level = brpc::MysqlIsoReadCommitted; + + brpc::MysqlTransactionUniquePtr tx(brpc::NewMysqlTransaction(channel, tx_options)); + ASSERT_FALSE(tx == NULL) << "Fail to create transaction"; + + { + brpc::MysqlRequest request(tx.get()); + std::stringstream ss; + ss << "update brpc_tx_" << brpc::MYSQL_table_suffix + << " set Address = 'hangzhou' where Id=1"; + ASSERT_EQ(request.Query(ss.str()), true); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_ERROR, response.reply(0).type()); + } + } +} + +// mysql prepared statement +TEST_F(MysqlTest, statement) { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + // zero parameter + { + std::stringstream ss; + ss << "select * from brpc_table_" << brpc::MYSQL_table_suffix << " limit 1"; + auto stmt(brpc::NewMysqlStatement(channel, ss.str())); + ASSERT_FALSE(stmt == NULL) << "Fail to create statement"; + { + brpc::MysqlRequest request(stmt.get()); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_resultset(), true); + } + { + brpc::MysqlRequest request(stmt.get()); + ASSERT_EQ(request.AddParam(1157), true); + + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_resultset(), true); + } + } + // one parameter + { + std::stringstream ss; + ss << "select * from brpc_table_" << brpc::MYSQL_table_suffix << " where col1 = ?"; + auto stmt(brpc::NewMysqlStatement(channel, ss.str())); + ASSERT_FALSE(stmt == NULL) << "Fail to create statement"; + { + brpc::MysqlRequest request(stmt.get()); + ASSERT_EQ(request.AddParam(1157), true); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_resultset(), true); + } + { + brpc::MysqlRequest request(stmt.get()); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_error(), true); + } + } + // two parameter + { + std::stringstream ss; + ss << "select * from brpc_table_" << brpc::MYSQL_table_suffix + << " where col1 = ? and col2 = ?"; + auto stmt(brpc::NewMysqlStatement(channel, ss.str())); + ASSERT_FALSE(stmt == NULL) << "Fail to create statement"; + { + brpc::MysqlRequest request(stmt.get()); + ASSERT_EQ(request.AddParam(1157), true); + ASSERT_EQ(request.AddParam("col2"), true); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_resultset(), true); + } + { + brpc::MysqlRequest request(stmt.get()); + brpc::MysqlResponse response; + brpc::Controller cntl; + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(response.reply(0).is_error(), true); + } + } +} + +TEST_F(MysqlTest, drop_table) { + brpc::ChannelOptions options; + options.protocol = brpc::PROTOCOL_MYSQL; + options.connection_type = brpc::MYSQL_connection_type; + options.connect_timeout_ms = brpc::MYSQL_connect_timeout_ms; + options.timeout_ms = brpc::MYSQL_timeout_ms /*milliseconds*/; + options.auth = new brpc::policy::MysqlAuthenticator( + brpc::MYSQL_user, brpc::MYSQL_password, brpc::MYSQL_schema, "charset=utf8"); + std::stringstream ss; + ss << brpc::MYSQL_host + ":" + brpc::MYSQL_port; + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(ss.str().c_str(), &options)); + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "delete from brpc_table_" << brpc::MYSQL_table_suffix; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } + + { + brpc::MysqlRequest request; + brpc::MysqlResponse response; + brpc::Controller cntl; + std::stringstream ss; + ss << "drop table brpc_table_" << brpc::MYSQL_table_suffix; + request.Query(ss.str()); + channel.CallMethod(NULL, &cntl, &request, &response, NULL); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(1ul, response.reply_size()); + ASSERT_EQ(brpc::MYSQL_RSP_OK, response.reply(0).type()); + } +} + +} // namespace