forked from apache/incubator-pegasus
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcommand_helper.h
More file actions
2306 lines (2099 loc) · 103 KB
/
command_helper.h
File metadata and controls
2306 lines (2099 loc) · 103 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
#pragma once
#include <boost/algorithm/string.hpp>
#include <fmt/color.h>
#include <fmt/ostream.h>
#include <geo/lib/geo_client.h>
#include <getopt.h>
#include <pegasus/error.h>
#include <pegasus/git_commit.h>
#include <pegasus/version.h>
#include <rocksdb/db.h>
#include <rocksdb/env.h>
#include <rocksdb/sst_dump_tool.h>
#include <rocksdb/statistics.h>
#include <rrdb/rrdb.code.definition.h>
#include <rrdb/rrdb_types.h>
#include <fstream>
#include <functional>
#include <iomanip>
#include <memory>
#include <queue>
#include <string_view>
#include <thread>
#include <utility>
#include "base/pegasus_key_schema.h"
#include "base/pegasus_utils.h"
#include "base/pegasus_value_schema.h"
#include "client/replication_ddl_client.h"
#include "command_executor.h"
#include "command_utils.h"
#include "common/json_helper.h"
#include "http/http_client.h"
#include "perf_counter/perf_counter_utils.h"
#include "remote_cmd/remote_command.h"
#include "task/async_calls.h"
#include "tools/mutation_log_tool.h"
#include "utils/fmt_utils.h"
#include "utils/errors.h"
#include "utils/metrics.h"
#include "utils/ports.h"
#include "utils/string_conv.h"
#include "utils/strings.h"
#include "utils/synchronize.h"
#include "utils/time_utils.h"
#define SHELL_PRINTLN_ERROR(msg, ...) \
fmt::print(stderr, \
fmt::emphasis::bold | fmt::fg(fmt::color::red), \
"ERROR: {}\n", \
fmt::format(msg, ##__VA_ARGS__))
#define SHELL_PRINT_WARNING_BASE(msg, ...) \
fmt::print(stdout, \
fmt::emphasis::bold | fmt::fg(fmt::color::yellow), \
"WARNING: {}", \
fmt::format(msg, ##__VA_ARGS__))
#define SHELL_PRINT_WARNING(msg, ...) SHELL_PRINT_WARNING_BASE(msg, ##__VA_ARGS__)
#define SHELL_PRINTLN_WARNING(msg, ...) \
SHELL_PRINT_WARNING_BASE("{}\n", fmt::format(msg, ##__VA_ARGS__))
#define SHELL_PRINT_OK_BASE(msg, ...) \
fmt::print(stdout, fmt::emphasis::bold | fmt::fg(fmt::color::green), msg, ##__VA_ARGS__)
#define SHELL_PRINT_OK(msg, ...) SHELL_PRINT_OK_BASE(msg, ##__VA_ARGS__)
#define SHELL_PRINTLN_OK(msg, ...) SHELL_PRINT_OK_BASE("{}\n", fmt::format(msg, ##__VA_ARGS__))
// Print messages to stderr and return false if `exp` is evaluated to false.
#define SHELL_PRINT_AND_RETURN_FALSE_IF_NOT(exp, ...) \
do { \
if (dsn_unlikely(!(exp))) { \
SHELL_PRINTLN_ERROR(__VA_ARGS__); \
return false; \
} \
} while (0)
#define RETURN_FALSE_IF_SAMPLE_INTERVAL_MS_INVALID() \
SHELL_PRINT_AND_RETURN_FALSE_IF_NOT(dsn::buf2uint32(optarg, sample_interval_ms), \
"parse sample_interval_ms({}) failed", \
optarg); \
SHELL_PRINT_AND_RETURN_FALSE_IF_NOT(sample_interval_ms > 0, "sample_interval_ms should be > 0")
DEFINE_TASK_CODE(LPC_SCAN_DATA, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT)
DEFINE_TASK_CODE(LPC_GET_METRICS, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT)
enum scan_data_operator
{
SCAN_COPY,
SCAN_CLEAR,
SCAN_COUNT,
SCAN_GEN_GEO,
SCAN_AND_MULTI_SET
};
USER_DEFINED_ENUM_FORMATTER(scan_data_operator)
class top_container
{
public:
struct top_heap_item
{
std::string hash_key;
std::string sort_key;
long row_size;
top_heap_item(std::string &&hash_key_, std::string &&sort_key_, long row_size_)
: hash_key(std::move(hash_key_)), sort_key(std::move(sort_key_)), row_size(row_size_)
{
}
};
struct top_heap_compare
{
bool operator()(top_heap_item i1, top_heap_item i2) { return i1.row_size < i2.row_size; }
};
typedef std::priority_queue<top_heap_item, std::vector<top_heap_item>, top_heap_compare>
top_heap;
top_container(int count) : _count(count) {}
void push(std::string &&hash_key, std::string &&sort_key, long row_size)
{
dsn::utils::auto_lock<dsn::utils::ex_lock_nr> l(_lock);
if (_heap.size() < _count) {
_heap.emplace(std::move(hash_key), std::move(sort_key), row_size);
} else {
const top_heap_item &top = _heap.top();
if (top.row_size < row_size) {
_heap.pop();
_heap.emplace(std::move(hash_key), std::move(sort_key), row_size);
}
}
}
top_heap &all() { return _heap; }
private:
int _count;
top_heap _heap;
dsn::utils::ex_lock_nr _lock;
};
enum class histogram_type
{
HASH_KEY_SIZE,
SORT_KEY_SIZE,
VALUE_SIZE,
ROW_SIZE
};
struct scan_data_context
{
scan_data_operator op;
int split_id;
int max_batch_count;
int timeout_ms;
bool no_overwrite; // if set true, then use check_and_set() instead of set()
// when inserting data to destination table for copy_data,
// to not overwrite old data if it aleady exist.
pegasus::pegasus_client::filter_type sort_key_filter_type;
std::string sort_key_filter_pattern;
pegasus::pegasus_client::filter_type value_filter_type;
std::string value_filter_pattern;
pegasus::pegasus_client::pegasus_scanner_wrapper scanner;
pegasus::pegasus_client *client;
pegasus::geo::geo_client *geoclient;
std::atomic_bool *error_occurred;
std::atomic_long split_rows;
std::atomic_long split_request_count;
std::atomic_bool split_completed;
bool stat_size;
std::shared_ptr<rocksdb::Statistics> statistics;
int top_count;
top_container top_rows;
bool count_hash_key;
std::string last_hash_key;
std::atomic_long split_hash_key_count;
long data_count;
uint32_t multi_ttl_seconds;
std::unordered_map<std::string, std::map<std::string, std::string>> multi_kvs;
dsn::utils::semaphore sema;
scan_data_context(scan_data_operator op_,
int split_id_,
int max_batch_count_,
int timeout_ms_,
pegasus::pegasus_client::pegasus_scanner_wrapper scanner_,
pegasus::pegasus_client *client_,
pegasus::geo::geo_client *geoclient_,
std::atomic_bool *error_occurred_,
int max_multi_set_concurrency = 20,
bool stat_size_ = false,
std::shared_ptr<rocksdb::Statistics> statistics_ = nullptr,
int top_count_ = 0,
bool count_hash_key_ = false)
: op(op_),
split_id(split_id_),
max_batch_count(max_batch_count_),
timeout_ms(timeout_ms_),
no_overwrite(false),
sort_key_filter_type(pegasus::pegasus_client::FT_NO_FILTER),
value_filter_type(pegasus::pegasus_client::FT_NO_FILTER),
scanner(scanner_),
client(client_),
geoclient(geoclient_),
error_occurred(error_occurred_),
split_rows(0),
split_request_count(0),
split_completed(false),
stat_size(stat_size_),
statistics(statistics_),
top_count(top_count_),
top_rows(top_count_),
count_hash_key(count_hash_key_),
split_hash_key_count(0),
data_count(0),
multi_ttl_seconds(0),
sema(max_multi_set_concurrency)
{
// max_batch_count should > 1 because scan may be terminated
// when split_request_count = 1
CHECK_GT(max_batch_count, 1);
}
void set_sort_key_filter(pegasus::pegasus_client::filter_type type, const std::string &pattern)
{
sort_key_filter_type = type;
sort_key_filter_pattern = pattern;
}
void set_value_filter(pegasus::pegasus_client::filter_type type, const std::string &pattern)
{
value_filter_type = type;
value_filter_pattern = pattern;
}
void set_no_overwrite() { no_overwrite = true; }
};
inline void update_atomic_max(std::atomic_long &max, long value)
{
while (true) {
long old = max.load();
if (value <= old || max.compare_exchange_weak(old, value)) {
break;
}
}
}
inline pegasus::pegasus_client::filter_type parse_filter_type(const std::string &name,
bool include_exact)
{
if (include_exact && name == "exact")
return pegasus::pegasus_client::FT_MATCH_EXACT;
else
return (pegasus::pegasus_client::filter_type)type_from_string(
dsn::apps::_filter_type_VALUES_TO_NAMES,
std::string("ft_match_") + name,
::dsn::apps::filter_type::FT_NO_FILTER);
}
// return true if the data is valid for the filter
inline bool validate_filter(pegasus::pegasus_client::filter_type filter_type,
const std::string &filter_pattern,
const std::string &value)
{
switch (filter_type) {
case pegasus::pegasus_client::FT_NO_FILTER:
return true;
case pegasus::pegasus_client::FT_MATCH_EXACT:
return filter_pattern == value;
case pegasus::pegasus_client::FT_MATCH_ANYWHERE:
case pegasus::pegasus_client::FT_MATCH_PREFIX:
case pegasus::pegasus_client::FT_MATCH_POSTFIX: {
if (filter_pattern.length() == 0)
return true;
if (value.length() < filter_pattern.length())
return false;
if (filter_type == pegasus::pegasus_client::FT_MATCH_ANYWHERE) {
return std::string_view(value).find(filter_pattern) != std::string_view::npos;
} else if (filter_type == pegasus::pegasus_client::FT_MATCH_PREFIX) {
return dsn::utils::mequals(
value.data(), filter_pattern.data(), filter_pattern.length());
} else { // filter_type == pegasus::pegasus_client::FT_MATCH_POSTFIX
return dsn::utils::mequals(value.data() + value.length() - filter_pattern.length(),
filter_pattern.data(),
filter_pattern.length());
}
}
default:
LOG_FATAL("unsupported filter type: {}", filter_type);
}
return false;
}
// return true if the data is valid for the filter
inline bool
validate_filter(scan_data_context *context, const std::string &sort_key, const std::string &value)
{
// for sort key, we only need to check MATCH_EXACT, because it is not supported
// on the server side, but MATCH_PREFIX is already satisified.
if (context->sort_key_filter_type == pegasus::pegasus_client::FT_MATCH_EXACT &&
sort_key.length() > context->sort_key_filter_pattern.length())
return false;
return validate_filter(context->value_filter_type, context->value_filter_pattern, value);
}
inline int compute_ttl_seconds(uint32_t expire_ts_seconds, bool &ts_expired)
{
auto epoch_now = pegasus::utils::epoch_now();
ts_expired = pegasus::check_if_ts_expired(epoch_now, expire_ts_seconds);
if (expire_ts_seconds > 0 && !ts_expired) {
return static_cast<int>(expire_ts_seconds - epoch_now);
}
return 0;
}
inline void batch_execute_multi_set(scan_data_context *context)
{
for (const auto &kv : context->multi_kvs) {
// wait for satisfied with max_multi_set_concurrency
context->sema.wait();
int multi_size = kv.second.size();
context->client->async_multi_set(
kv.first,
kv.second,
[context, multi_size](int err, pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] async_multi_set set failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
context->split_rows += multi_size;
}
context->sema.signal();
},
context->timeout_ms,
context->multi_ttl_seconds);
}
context->multi_kvs.clear();
context->data_count = 0;
}
// copy data by async_multi_set
inline void scan_multi_data_next(scan_data_context *context)
{
if (!context->split_completed.load() && !context->error_occurred->load()) {
context->scanner->async_next([context](int ret,
std::string &&hash_key,
std::string &&sort_key,
std::string &&value,
pegasus::pegasus_client::internal_info &&info,
uint32_t expire_ts_seconds,
uint32_t kv_count) {
if (ret == pegasus::PERR_OK) {
if (validate_filter(context, sort_key, value)) {
bool ts_expired = false;
int ttl_seconds = 0;
ttl_seconds = compute_ttl_seconds(expire_ts_seconds, ts_expired);
if (!ts_expired) {
// empty hashkey should get hashkey by sortkey
if (hash_key == "") {
// wait for satisfied with max_multi_set_concurrency
context->sema.wait();
auto callback = [context](
int err,
pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] async check and set failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
context->split_rows++;
}
context->sema.signal();
};
context->client->async_set(hash_key,
sort_key,
value,
std::move(callback),
context->timeout_ms,
ttl_seconds);
} else {
context->data_count++;
if (context->multi_kvs.find(hash_key) == context->multi_kvs.end()) {
context->multi_kvs.emplace(hash_key,
std::map<std::string, std::string>());
}
if (context->multi_ttl_seconds < ttl_seconds || ttl_seconds == 0) {
context->multi_ttl_seconds = ttl_seconds;
}
context->multi_kvs[hash_key].emplace(std::move(sort_key),
std::move(value));
if (context->data_count >= context->max_batch_count) {
batch_execute_multi_set(context);
}
}
}
}
scan_multi_data_next(context);
} else if (ret == pegasus::PERR_SCAN_COMPLETE) {
batch_execute_multi_set(context);
context->split_completed.store(true);
} else {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] scan next failed: %s\n",
context->split_id,
context->client->get_error_string(ret));
context->error_occurred->store(true);
}
}
});
}
}
inline void scan_data_next(scan_data_context *context)
{
while (!context->split_completed.load() && !context->error_occurred->load() &&
context->split_request_count.load() < context->max_batch_count) {
context->split_request_count++;
context->scanner->async_next([context](int ret,
std::string &&hash_key,
std::string &&sort_key,
std::string &&value,
pegasus::pegasus_client::internal_info &&info,
uint32_t expire_ts_seconds,
int32_t kv_count) {
if (ret == pegasus::PERR_OK) {
if (kv_count != -1 || validate_filter(context, sort_key, value)) {
bool ts_expired = false;
int ttl_seconds = 0;
switch (context->op) {
case SCAN_COPY:
context->split_request_count++;
ttl_seconds = compute_ttl_seconds(expire_ts_seconds, ts_expired);
if (ts_expired) {
scan_data_next(context);
} else if (context->no_overwrite) {
auto callback =
[context](int err,
pegasus::pegasus_client::check_and_set_results &&results,
pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(
stderr,
"ERROR: split[%d] async check and set failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
if (results.set_succeed) {
context->split_rows++;
}
scan_data_next(context);
}
// should put "split_request_count--" at end of the scope,
// to prevent that split_request_count becomes 0 in the middle.
context->split_request_count--;
};
pegasus::pegasus_client::check_and_set_options options;
options.set_value_ttl_seconds = ttl_seconds;
context->client->async_check_and_set(
hash_key,
sort_key,
pegasus::pegasus_client::cas_check_type::CT_VALUE_NOT_EXIST,
"",
sort_key,
value,
options,
std::move(callback),
context->timeout_ms);
} else {
auto callback =
[context](int err, pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] async set failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
context->split_rows++;
scan_data_next(context);
}
// should put "split_request_count--" at end of the scope,
// to prevent that split_request_count becomes 0 in the middle.
context->split_request_count--;
};
context->client->async_set(hash_key,
sort_key,
value,
std::move(callback),
context->timeout_ms,
ttl_seconds);
}
break;
case SCAN_CLEAR:
context->split_request_count++;
context->client->async_del(
hash_key,
sort_key,
[context](int err, pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] async del failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
context->split_rows++;
scan_data_next(context);
}
// should put "split_request_count--" at end of the scope,
// to prevent that split_request_count becomes 0 in the middle.
context->split_request_count--;
},
context->timeout_ms);
break;
case SCAN_COUNT:
if (kv_count != -1) {
context->split_rows += kv_count;
scan_data_next(context);
break;
}
context->split_rows++;
if (context->stat_size && context->statistics) {
long hash_key_size = hash_key.size();
context->statistics->measureTime(
static_cast<uint32_t>(histogram_type::HASH_KEY_SIZE),
hash_key_size);
long sort_key_size = sort_key.size();
context->statistics->measureTime(
static_cast<uint32_t>(histogram_type::SORT_KEY_SIZE),
sort_key_size);
long value_size = value.size();
context->statistics->measureTime(
static_cast<uint32_t>(histogram_type::VALUE_SIZE), value_size);
long row_size = hash_key_size + sort_key_size + value_size;
context->statistics->measureTime(
static_cast<uint32_t>(histogram_type::ROW_SIZE), row_size);
if (context->top_count > 0) {
context->top_rows.push(
std::move(hash_key), std::move(sort_key), row_size);
}
}
if (context->count_hash_key) {
if (hash_key != context->last_hash_key) {
context->split_hash_key_count++;
context->last_hash_key = std::move(hash_key);
}
}
scan_data_next(context);
break;
case SCAN_GEN_GEO:
context->split_request_count++;
ttl_seconds = compute_ttl_seconds(expire_ts_seconds, ts_expired);
if (ts_expired) {
scan_data_next(context);
} else {
context->geoclient->async_set(
hash_key,
sort_key,
value,
[context](int err, pegasus::pegasus_client::internal_info &&info) {
if (err != pegasus::PERR_OK) {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] async set failed: %s\n",
context->split_id,
context->client->get_error_string(err));
context->error_occurred->store(true);
}
} else {
context->split_rows++;
scan_data_next(context);
}
// should put "split_request_count--" at end of the scope,
// to prevent that split_request_count becomes 0 in the middle.
context->split_request_count--;
},
context->timeout_ms,
ttl_seconds);
}
break;
default:
LOG_FATAL("op = {}", context->op);
break;
}
} else {
scan_data_next(context);
}
} else if (ret == pegasus::PERR_SCAN_COMPLETE) {
context->split_completed.store(true);
} else {
if (!context->split_completed.exchange(true)) {
fprintf(stderr,
"ERROR: split[%d] scan next failed: %s\n",
context->split_id,
context->client->get_error_string(ret));
context->error_occurred->store(true);
}
}
// should put "split_request_count--" at end of the scope,
// to prevent that split_request_count becomes 0 in the middle.
context->split_request_count--;
});
if (context->count_hash_key) {
// disable parallel scan if count_hash_key == true
break;
}
}
}
struct node_desc
{
std::string desc;
dsn::host_port hp;
node_desc(const std::string &s, const dsn::host_port &n) : desc(s), hp(n) {}
};
// type: all | replica-server | meta-server
inline bool fill_nodes(shell_context *sc, const std::string &type, std::vector<node_desc> &nodes)
{
if (type == "all" || type == "meta-server") {
for (const auto &hp : sc->meta_list) {
nodes.emplace_back("meta-server", hp);
}
}
if (type == "all" || type == "replica-server") {
std::map<dsn::host_port, dsn::replication::node_status::type> rs_nodes;
::dsn::error_code err =
sc->ddl_client->list_nodes(dsn::replication::node_status::NS_ALIVE, rs_nodes);
if (err != ::dsn::ERR_OK) {
fprintf(stderr, "ERROR: list node failed: %s\n", err.to_string());
return false;
}
for (auto &kv : rs_nodes) {
nodes.emplace_back("replica-server", kv.first);
}
}
return true;
}
// Fetch the metrics according to `query_string` for each target node.
inline std::vector<dsn::http_result> get_metrics(const std::vector<node_desc> &nodes,
const std::string &query_string)
{
std::vector<dsn::http_result> results(nodes.size());
dsn::task_tracker tracker;
for (size_t i = 0; i < nodes.size(); ++i) {
(void)dsn::tasking::enqueue(
LPC_GET_METRICS, &tracker, [&nodes, &query_string, &results, i]() {
dsn::http_url url;
#define SET_RESULT_AND_RETURN_IF_URL_NOT_OK(name, expr) \
do { \
auto err = url.set_##name(expr); \
if (!err) { \
results[i] = dsn::http_result(std::move(err)); \
return; \
} \
} while (0)
SET_RESULT_AND_RETURN_IF_URL_NOT_OK(host, nodes[i].hp.host().c_str());
SET_RESULT_AND_RETURN_IF_URL_NOT_OK(port, nodes[i].hp.port());
SET_RESULT_AND_RETURN_IF_URL_NOT_OK(
path, dsn::metrics_http_service::kMetricsQueryPath.c_str());
SET_RESULT_AND_RETURN_IF_URL_NOT_OK(query, query_string.c_str());
results[i] = dsn::http_get(url);
#undef SET_RESULT_AND_RETURN_IF_URL_NOT_OK
});
}
tracker.wait_outstanding_tasks();
return results;
}
// Adapt the result returned by `get_metrics` into the structure that could be processed by
// `remote_command`.
template <typename... Args>
inline dsn::error_s process_get_metrics_result(const dsn::http_result &result,
const node_desc &node,
const char *what,
Args &&...args)
{
if (dsn_unlikely(!result.error())) {
return FMT_ERR(result.error().code(),
"ERROR: query {} metrics from node {} failed, msg={}",
fmt::format(what, std::forward<Args>(args)...),
node.hp,
result.error());
}
if (dsn_unlikely(result.status() != dsn::http_status_code::kOk)) {
return FMT_ERR(dsn::ERR_HTTP_ERROR,
"ERROR: query {} metrics from node {} failed, http_status={}, msg={}",
fmt::format(what, std::forward<Args>(args)...),
node.hp,
dsn::get_http_status_message(result.status()),
result.body());
}
return dsn::error_s::ok();
}
#define RETURN_SHELL_IF_GET_METRICS_FAILED(result, node, what, ...) \
do { \
const auto &res = process_get_metrics_result(result, node, what, ##__VA_ARGS__); \
if (dsn_unlikely(!res)) { \
fmt::println(res.description()); \
return true; \
} \
} while (0)
// Adapt the result of some parsing operations on the metrics returned by `get_metrics` into the
// structure that could be processed by `remote_command`.
template <typename... Args>
inline dsn::error_s process_parse_metrics_result(const dsn::error_s &result,
const node_desc &node,
const char *what,
Args &&...args)
{
if (dsn_unlikely(!result)) {
return FMT_ERR(result.code(),
"ERROR: {} metrics response from node {} failed, msg={}",
fmt::format(what, std::forward<Args>(args)...),
node.hp,
result);
}
return dsn::error_s::ok();
}
#define RETURN_SHELL_IF_PARSE_METRICS_FAILED(expr, node, what, ...) \
do { \
const auto &res = process_parse_metrics_result(expr, node, what, ##__VA_ARGS__); \
if (dsn_unlikely(!res)) { \
fmt::println(res.description()); \
return true; \
} \
} while (0)
using stat_var_map = std::unordered_map<std::string, double *>;
// Abstract class used to aggregate the stats based on the custom filters while iterating over
// the fetched metrics.
//
// Given the type and attributes of an entity, derived classes need to implement a custom filter
// to return the selected `stat_var_map`, if any. Calculations including addition and subtraction
// are also provided for aggregating the stats. The metric name would be a dimension for the
// aggregation.
class aggregate_stats
{
public:
aggregate_stats() = default;
virtual ~aggregate_stats() = default;
#define CALC_STAT_VARS(entities, op) \
for (const auto &entity : entities) { \
stat_var_map *stat_vars = nullptr; \
RETURN_NOT_OK(get_stat_vars(entity.type, entity.attributes, &stat_vars)); \
\
if (stat_vars == nullptr || stat_vars->empty()) { \
continue; \
} \
\
for (const auto &m : entity.metrics) { \
auto iter = stat_vars->find(m.name); \
if (iter != stat_vars->end()) { \
*iter->second op m.value; \
} \
} \
} \
return dsn::error_s::ok()
// Following interfaces provide calculations over the fetched metrics. They would perform
// each calculation on each metric whose name was found in `stat_var_map` returned by
// `get_stat_vars`.
// Assign the matched metric value directly to the selected member of `stat_var_map` without
// extra calculation.
dsn::error_s assign(const std::vector<dsn::metric_entity_brief_value_snapshot> &entities)
{
CALC_STAT_VARS(entities, =);
}
// Add and assign the matched metric value to the selected member of `stat_var_map`.
dsn::error_s add_assign(const std::vector<dsn::metric_entity_brief_value_snapshot> &entities)
{
CALC_STAT_VARS(entities, +=);
}
// Subtract and assign the matched metric value to the selected member of `stat_var_map`.
dsn::error_s sub_assign(const std::vector<dsn::metric_entity_brief_value_snapshot> &entities)
{
CALC_STAT_VARS(entities, -=);
}
void calc_rates(uint64_t timestamp_ns_start, uint64_t timestamp_ns_end)
{
calc_rates(dsn::calc_metric_sample_duration_s(timestamp_ns_start, timestamp_ns_end));
}
#undef CALC_STAT_VARS
protected:
// Given the type and attributes of an entity, decide which `stat_var_map` is selected, if any.
// Otherwise, `*stat_vars` would be set to nullptr.
virtual dsn::error_s get_stat_vars(const std::string &entity_type,
const dsn::metric_entity::attr_map &entity_attrs,
stat_var_map **stat_vars) = 0;
// Implement self-defined calculation for rates, such as QPS.
virtual void calc_rates(double duration_s) = 0;
};
// Support multiple kinds of aggregations over the fetched metrics, such as sums, increases and
// rates. Users could choose to create aggregations as needed.
class aggregate_stats_calcs
{
public:
aggregate_stats_calcs() noexcept = default;
~aggregate_stats_calcs() = default;
aggregate_stats_calcs(aggregate_stats_calcs &&) noexcept = default;
aggregate_stats_calcs &operator=(aggregate_stats_calcs &&) noexcept = default;
#define DEF_CALC_CREATOR(name) \
template <typename T, typename... Args> \
void create_##name(Args &&...args) \
{ \
_##name = std::make_unique<T>(std::forward<Args>(args)...); \
}
// Create the aggregations as needed.
DEF_CALC_CREATOR(assignments)
DEF_CALC_CREATOR(sums)
DEF_CALC_CREATOR(increases)
DEF_CALC_CREATOR(rates)
#undef DEF_CALC_CREATOR
#define CALC_ASSIGNMENT_STATS(entities) \
do { \
if (_assignments) { \
RETURN_NOT_OK(_assignments->assign(entities)); \
} \
} while (0)
#define CALC_ACCUM_STATS(entities) \
do { \
if (_sums) { \
RETURN_NOT_OK(_sums->add_assign(entities)); \
} \
} while (0)
// Perform the chosen aggregations (both assignment and accum) on the fetched metrics.
dsn::error_s aggregate_metrics(const std::string &json_string)
{
DESERIALIZE_METRIC_QUERY_BRIEF_SNAPSHOT(value, json_string, query_snapshot);
return aggregate_metrics(query_snapshot);
}
dsn::error_s aggregate_metrics(const dsn::metric_query_brief_value_snapshot &query_snapshot)
{
CALC_ASSIGNMENT_STATS(query_snapshot.entities);
CALC_ACCUM_STATS(query_snapshot.entities);
return dsn::error_s::ok();
}
// Perform the chosen aggregations (assignement, accum, delta and rate) on the fetched metrics.
dsn::error_s aggregate_metrics(const std::string &json_string_start,
const std::string &json_string_end)
{
DESERIALIZE_METRIC_QUERY_BRIEF_2_SAMPLES(
json_string_start, json_string_end, query_snapshot_start, query_snapshot_end);
return aggregate_metrics(query_snapshot_start, query_snapshot_end);
}
dsn::error_s
aggregate_metrics(const dsn::metric_query_brief_value_snapshot &query_snapshot_start,
const dsn::metric_query_brief_value_snapshot &query_snapshot_end)
{
// Apply ending sample to the assignment and accum aggregations.
CALC_ASSIGNMENT_STATS(query_snapshot_end.entities);
CALC_ACCUM_STATS(query_snapshot_end.entities);
const std::array deltas_list = {&_increases, &_rates};
for (const auto stats : deltas_list) {
if (!(*stats)) {
continue;
}
RETURN_NOT_OK((*stats)->add_assign(query_snapshot_end.entities));
RETURN_NOT_OK((*stats)->sub_assign(query_snapshot_start.entities));
}
if (_rates) {
_rates->calc_rates(query_snapshot_start.timestamp_ns, query_snapshot_end.timestamp_ns);
}
return dsn::error_s::ok();
}
#undef CALC_ACCUM_STATS
#undef CALC_ASSIGNMENT_STATS
private:
DISALLOW_COPY_AND_ASSIGN(aggregate_stats_calcs);
std::unique_ptr<aggregate_stats> _assignments;
std::unique_ptr<aggregate_stats> _sums;
std::unique_ptr<aggregate_stats> _increases;
std::unique_ptr<aggregate_stats> _rates;
};
// Convenient macros for `get_stat_vars` to set `*stat_vars` to nullptr and return under some
// circumstances.
#define RETURN_NULL_STAT_VARS_IF(expr) \
do { \
if (expr) { \
*stat_vars = nullptr; \
return dsn::error_s::ok(); \
} \
} while (0)
#define RETURN_NULL_STAT_VARS_IF_NOT_OK(expr) \
do { \
const auto &err = (expr); \
if (dsn_unlikely(!err)) { \
*stat_vars = nullptr; \
return err; \
} \
} while (0)
// A helper macro to parse command argument, the result is filled in a string vector variable named
// 'container'.
#define PARSE_STRS(container) \
do { \
const auto param = cmd(param_index++).str(); \
::dsn::utils::split_args(param.c_str(), container, ','); \
if (container.empty()) { \
SHELL_PRINTLN_ERROR( \
"invalid command, '{}' should be in the form of 'val1,val2,val3' and " \
"should not be empty", \
param); \
return false; \
} \
std::set<std::string> str_set(container.begin(), container.end()); \
if (str_set.size() != container.size()) { \
SHELL_PRINTLN_ERROR("invalid command, '{}' has duplicate values", param); \
return false; \
} \
} while (false)