tikv节点突然下线，状态变成Disconnected

a398058068 · 2021 年5 月 8 日 10:54

cat killsnoop-nd.stp
#!/usr/bin/stap
/*
 * killsnoop-nd.stp	Trace process signals.
 *			For Linux, uses SystemTap (non-debuginfo).
 *
 * Copyright (C) 2015 Brendan Gregg.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * 24-Jun-2014	Brendan Gregg	Created this.
 */

global target;
global signal;
global pid_name, fork_by_pid, fork_by_name, fork_count


probe nd_syscall.kill
{
    target[tid()] = uint_arg(1);
    signal[tid()] = uint_arg(2);
}

probe nd_syscall.kill.return
{
    if (target[tid()] != 0) {
        printf("%-6d %-12s %-5d %-6d %6d\
", pid(), execname(),
            signal[tid()], target[tid()], int_arg(1));
        delete target[tid()];
        delete signal[tid()];
    }
}

probe kprocess.create
{
      printf("%-25s: %s (%d) created %d\
",
        ctime(gettimeofday_s()), execname(), pid(), new_pid)
}

probe kprocess.exec
{
      printf("%-25s: %s (%d) is exec'ing %s\
",
        ctime(gettimeofday_s()), execname(), pid(), filename)
}

Sat May  8 10:51:13 2021 : sgagent (3268) is exec'ing 0x7f2ad811bee9
Sat May  8 10:51:13 2021 : sh (3268) is exec'ing "../../monitor/barad/admin/trystart.sh"
Sat May  8 10:51:13 2021 : trystart.sh (3268) created 3269
Sat May  8 10:51:13 2021 : trystart.sh (3269) is exec'ing "/usr/bin/dirname"
Sat May  8 10:51:13 2021 : trystart.sh (3268) created 3270
Sat May  8 10:51:13 2021 : trystart.sh (3270) is exec'ing "/usr/bin/chmod"
Sat May  8 10:51:13 2021 : trystart.sh (3268) created 3271
Sat May  8 10:51:13 2021 : trystart.sh (3271) created 3272
Sat May  8 10:51:13 2021 : trystart.sh (3271) created 3273
Sat May  8 10:51:13 2021 : trystart.sh (3272) is exec'ing "/usr/bin/ps"
Sat May  8 10:51:13 2021 : trystart.sh (3271) created 3274
Sat May  8 10:51:13 2021 : trystart.sh (3271) created 3275
Sat May  8 10:51:13 2021 : trystart.sh (3273) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:13 2021 : trystart.sh (3274) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:13 2021 : trystart.sh (3275) is exec'ing "/usr/bin/wc"
Sat May  8 10:51:13 2021 : trystart.sh (3268) created 3276
Sat May  8 10:51:13 2021 : trystart.sh (3276) created 3277
Sat May  8 10:51:13 2021 : trystart.sh (3276) created 3278
Sat May  8 10:51:13 2021 : trystart.sh (3277) is exec'ing "/usr/bin/ps"
Sat May  8 10:51:13 2021 : trystart.sh (3276) created 3279
Sat May  8 10:51:13 2021 : trystart.sh (3278) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:13 2021 : trystart.sh (3279) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:16 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:16 2021 : barad_agent (1660) created 3281
Sat May  8 10:51:16 2021 : barad_agent (3281) is exec'ing "/bin/sh"
Sat May  8 10:51:16 2021 : sh (3281) created 3282
Sat May  8 10:51:16 2021 : sh (3281) created 3283
Sat May  8 10:51:16 2021 : sh (3282) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:16 2021 : sh (3281) created 3284
Sat May  8 10:51:16 2021 : sh (3283) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:16 2021 : sh (3284) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:18 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:18 2021 : barad_agent (1660) created 3286
Sat May  8 10:51:18 2021 : barad_agent (3286) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:18 2021 : sh (3286) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:18 2021 : barad_agent (1660) created 3287
Sat May  8 10:51:18 2021 : barad_agent (3287) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:18 2021 : sh (3287) created 3288
Sat May  8 10:51:18 2021 : sh (3287) created 3289
Sat May  8 10:51:18 2021 : sh (3288) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:18 2021 : sh (3289) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:19 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:20 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:21 2021 : YDLive (1771) created 3292
Sat May  8 10:51:21 2021 : YDLive (3292) is exec'ing "/usr/local/qcloud/YunJing/YDEyes/YDService"
Sat May  8 10:51:21 2021 : YDService (3292) created 3292
Sat May  8 10:51:22 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:22 2021 : barad_agent (1660) created 3295
Sat May  8 10:51:22 2021 : barad_agent (3295) is exec'ing "/bin/sh"
Sat May  8 10:51:22 2021 : sh (3295) created 3296
Sat May  8 10:51:22 2021 : sh (3295) created 3297
Sat May  8 10:51:22 2021 : sh (3296) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:51:22 2021 : sh (3297) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:26 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:26 2021 : barad_agent (1660) created 3299
Sat May  8 10:51:26 2021 : barad_agent (3299) is exec'ing "/bin/sh"
Sat May  8 10:51:26 2021 : sh (3299) created 3300
Sat May  8 10:51:26 2021 : sh (3299) created 3301
Sat May  8 10:51:26 2021 : sh (3299) created 3302
Sat May  8 10:51:26 2021 : sh (3300) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:26 2021 : sh (3301) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:26 2021 : sh (3302) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:28 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:28 2021 : barad_agent (1660) created 3304
Sat May  8 10:51:28 2021 : barad_agent (3304) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:28 2021 : sh (3304) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:28 2021 : barad_agent (1660) created 3305
Sat May  8 10:51:28 2021 : barad_agent (3305) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:28 2021 : sh (3305) created 3306
Sat May  8 10:51:28 2021 : sh (3305) created 3307
Sat May  8 10:51:28 2021 : sh (3306) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:28 2021 : sh (3307) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:29 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:30 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:32 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:32 2021 : barad_agent (1660) created 3311
Sat May  8 10:51:32 2021 : barad_agent (3311) is exec'ing "/bin/sh"
Sat May  8 10:51:32 2021 : sh (3311) created 3312
Sat May  8 10:51:32 2021 : sh (3311) created 3313
Sat May  8 10:51:32 2021 : sh (3312) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:51:32 2021 : sh (3313) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:35 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:36 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:36 2021 : barad_agent (1660) created 3316
Sat May  8 10:51:36 2021 : barad_agent (3316) is exec'ing "/bin/sh"
Sat May  8 10:51:36 2021 : sh (3316) created 3317
Sat May  8 10:51:36 2021 : sh (3316) created 3318
Sat May  8 10:51:36 2021 : sh (3316) created 3319
Sat May  8 10:51:36 2021 : sh (3317) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:36 2021 : sh (3318) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:36 2021 : sh (3319) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:38 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:38 2021 : barad_agent (1660) created 3321
Sat May  8 10:51:38 2021 : barad_agent (3321) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:38 2021 : sh (3321) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:38 2021 : barad_agent (1660) created 3322
Sat May  8 10:51:38 2021 : barad_agent (3322) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:38 2021 : sh (3322) created 3323
Sat May  8 10:51:38 2021 : sh (3322) created 3324
Sat May  8 10:51:38 2021 : sh (3323) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:38 2021 : sh (3324) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:39 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:40 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:42 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:42 2021 : barad_agent (1660) created 3328
Sat May  8 10:51:42 2021 : barad_agent (3328) is exec'ing "/bin/sh"
Sat May  8 10:51:42 2021 : sh (3328) created 3329
Sat May  8 10:51:42 2021 : sh (3328) created 3330
Sat May  8 10:51:42 2021 : sh (3329) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:51:42 2021 : sh (3330) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:46 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:46 2021 : barad_agent (1660) created 3332
Sat May  8 10:51:46 2021 : barad_agent (3332) is exec'ing "/bin/sh"
Sat May  8 10:51:46 2021 : sh (3332) created 3333
Sat May  8 10:51:46 2021 : sh (3332) created 3334
Sat May  8 10:51:46 2021 : sh (3332) created 3335
Sat May  8 10:51:46 2021 : sh (3333) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:46 2021 : sh (3334) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:46 2021 : sh (3335) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:48 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:48 2021 : barad_agent (1660) created 3337
Sat May  8 10:51:48 2021 : barad_agent (3337) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:48 2021 : sh (3337) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:48 2021 : barad_agent (1660) created 3338
Sat May  8 10:51:48 2021 : barad_agent (3338) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:48 2021 : sh (3338) created 3339
Sat May  8 10:51:48 2021 : sh (3338) created 3340
Sat May  8 10:51:48 2021 : sh (3339) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:48 2021 : sh (3340) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:49 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:50 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:52 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:52 2021 : barad_agent (1660) created 3344
Sat May  8 10:51:52 2021 : barad_agent (3344) is exec'ing "/bin/sh"
Sat May  8 10:51:52 2021 : sh (3344) created 3345
Sat May  8 10:51:52 2021 : sh (3344) created 3346
Sat May  8 10:51:52 2021 : sh (3345) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:51:52 2021 : sh (3346) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:56 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:56 2021 : barad_agent (1660) created 3348
Sat May  8 10:51:56 2021 : barad_agent (3348) is exec'ing "/bin/sh"
Sat May  8 10:51:56 2021 : sh (3348) created 3349
Sat May  8 10:51:56 2021 : sh (3348) created 3350
Sat May  8 10:51:56 2021 : sh (3349) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:56 2021 : sh (3348) created 3351
Sat May  8 10:51:56 2021 : sh (3350) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:56 2021 : sh (3351) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:58 2021 : barad_agent (1660) created 1660
Sat May  8 10:51:58 2021 : barad_agent (1660) created 3353
Sat May  8 10:51:58 2021 : barad_agent (3353) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:58 2021 : sh (3353) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:58 2021 : barad_agent (1660) created 3354
Sat May  8 10:51:58 2021 : barad_agent (3354) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:51:58 2021 : sh (3354) created 3355
Sat May  8 10:51:58 2021 : sh (3354) created 3356
Sat May  8 10:51:58 2021 : sh (3355) is exec'ing "/usr/bin/cat"
Sat May  8 10:51:58 2021 : sh (3356) is exec'ing "/usr/bin/awk"
Sat May  8 10:51:58 2021 : sshd (1106) created 3357
Sat May  8 10:51:58 2021 : sshd (3357) is exec'ing "/usr/sbin/sshd"
Sat May  8 10:51:58 2021 : sshd (3357) created 3358
Sat May  8 10:51:59 2021 : sshd (3357) created 3359
Sat May  8 10:51:59 2021 : sshd (3359) created 3360
Sat May  8 10:51:59 2021 : sshd (3360) is exec'ing "/bin/bash"
Sat May  8 10:51:59 2021 : bash (3360) created 3361
Sat May  8 10:51:59 2021 : bash (3361) created 3362
Sat May  8 10:51:59 2021 : bash (3362) is exec'ing "/usr/bin/id"
Sat May  8 10:51:59 2021 : bash (3360) created 3363
Sat May  8 10:51:59 2021 : bash (3363) created 3364
Sat May  8 10:51:59 2021 : bash (3364) is exec'ing "/usr/bin/hostname"
Sat May  8 10:51:59 2021 : bash (3360) created 3365
Sat May  8 10:51:59 2021 : bash (3365) created 3366
Sat May  8 10:51:59 2021 : bash (3366) is exec'ing "/usr/bin/id"
Sat May  8 10:51:59 2021 : bash (3360) created 3367
Sat May  8 10:51:59 2021 : bash (3367) created 3368
Sat May  8 10:51:59 2021 : bash (3368) is exec'ing "/usr/bin/id"
Sat May  8 10:51:59 2021 : bash (3360) created 3369
Sat May  8 10:51:59 2021 : bash (3369) created 3370
Sat May  8 10:51:59 2021 : bash (3370) is exec'ing "/usr/bin/ls"
Sat May  8 10:51:59 2021 : bash (3360) created 3371
Sat May  8 10:51:59 2021 : bash (3371) created 3372
Sat May  8 10:51:59 2021 : bash (3372) is exec'ing "/usr/bin/uname"
Sat May  8 10:51:59 2021 : bash (3360) created 3373
Sat May  8 10:51:59 2021 : bash (3373) created 3374
Sat May  8 10:51:59 2021 : bash (3374) is exec'ing "/usr/bin/pkg-config"
Sat May  8 10:51:59 2021 : bash (3360) created 3375
Sat May  8 10:51:59 2021 : bash (3375) is exec'ing "/usr/libexec/grepconf.sh"
Sat May  8 10:51:59 2021 : grepconf.sh (3375) created 3376
Sat May  8 10:51:59 2021 : grepconf.sh (3376) is exec'ing "/usr/bin/grep"
Sat May  8 10:51:59 2021 : bash (3360) created 3377
Sat May  8 10:51:59 2021 : bash (3377) created 3378
Sat May  8 10:51:59 2021 : bash (3378) is exec'ing "/usr/bin/tty"
Sat May  8 10:51:59 2021 : bash (3377) created 3379
Sat May  8 10:51:59 2021 : bash (3379) is exec'ing "/usr/bin/tput"
Sat May  8 10:51:59 2021 : bash (3360) created 3380
Sat May  8 10:51:59 2021 : bash (3380) created 3381
Sat May  8 10:51:59 2021 : bash (3381) is exec'ing "/usr/bin/dircolors"
Sat May  8 10:51:59 2021 : bash (3360) created 3382
Sat May  8 10:51:59 2021 : bash (3382) created 3383
Sat May  8 10:51:59 2021 : bash (3383) is exec'ing "/usr/bin/id"
Sat May  8 10:51:59 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:00 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:01 2021 : bash (3360) created 3386
Sat May  8 10:52:01 2021 : bash (3386) is exec'ing "/usr/bin/top"
Sat May  8 10:52:01 2021 : crond (991) created 3387
Sat May  8 10:52:01 2021 : crond (3387) created 3388
Sat May  8 10:52:01 2021 : crond (3388) is exec'ing "/bin/sh"
Sat May  8 10:52:01 2021 : sh (3388) is exec'ing "/usr/bin/flock"
Sat May  8 10:52:01 2021 : flock (3388) created 3389
Sat May  8 10:52:01 2021 : flock (3389) is exec'ing "/bin/sh"
Sat May  8 10:52:01 2021 : sh (3389) created 3390
Sat May  8 10:52:01 2021 : sh (3390) is exec'ing "/usr/local/qcloud/stargate/admin/start.sh"
Sat May  8 10:52:01 2021 : start.sh (3390) created 3391
Sat May  8 10:52:01 2021 : start.sh (3391) is exec'ing "/usr/bin/whoami"
1      systemd      0     3387        0
Sat May  8 10:52:01 2021 : start.sh (3390) created 3392
Sat May  8 10:52:01 2021 : start.sh (3392) created 3393
Sat May  8 10:52:01 2021 : start.sh (3392) created 3394
Sat May  8 10:52:01 2021 : start.sh (3393) is exec'ing "/usr/bin/ps"
Sat May  8 10:52:01 2021 : start.sh (3392) created 3395
Sat May  8 10:52:01 2021 : start.sh (3394) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:01 2021 : start.sh (3392) created 3396
Sat May  8 10:52:01 2021 : start.sh (3395) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:01 2021 : start.sh (3396) is exec'ing "/usr/bin/wc"
Sat May  8 10:52:01 2021 : kworker/u16:0 (5) created 3397
1      systemd      0     3391        0
Sat May  8 10:52:02 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:02 2021 : barad_agent (1660) created 3399
Sat May  8 10:52:02 2021 : barad_agent (3399) is exec'ing "/bin/sh"
Sat May  8 10:52:02 2021 : sh (3399) created 3400
Sat May  8 10:52:02 2021 : sh (3399) created 3401
Sat May  8 10:52:02 2021 : sh (3400) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:02 2021 : sh (3401) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:03 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:03 2021 : barad_agent (1660) created 3403
Sat May  8 10:52:03 2021 : barad_agent (3403) is exec'ing "/bin/sh"
Sat May  8 10:52:03 2021 : sh (3403) created 3404
Sat May  8 10:52:03 2021 : sh (3403) created 3405
Sat May  8 10:52:03 2021 : sh (3403) created 3406
Sat May  8 10:52:03 2021 : sh (3404) is exec'ing "/usr/sbin/ntpdate"
Sat May  8 10:52:03 2021 : sh (3405) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:03 2021 : sh (3406) is exec'ing "/usr/bin/sed"
Sat May  8 10:52:04 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:05 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:06 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:06 2021 : barad_agent (1660) created 3410
Sat May  8 10:52:06 2021 : barad_agent (3410) is exec'ing "/bin/sh"
Sat May  8 10:52:06 2021 : sh (3410) created 3411
Sat May  8 10:52:06 2021 : sh (3410) created 3412
Sat May  8 10:52:06 2021 : sh (3411) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:06 2021 : sh (3410) created 3413
Sat May  8 10:52:06 2021 : sh (3412) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:06 2021 : sh (3413) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:07 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:07 2021 : barad_agent (1660) created 3415
Sat May  8 10:52:07 2021 : barad_agent (3415) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:07 2021 : sh (3415) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:08 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:08 2021 : barad_agent (1660) created 3417
Sat May  8 10:52:08 2021 : barad_agent (3417) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:08 2021 : sh (3417) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:08 2021 : barad_agent (1660) created 3418
Sat May  8 10:52:08 2021 : barad_agent (3418) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:08 2021 : sh (3418) created 3419
Sat May  8 10:52:08 2021 : sh (3418) created 3420
Sat May  8 10:52:08 2021 : sh (3419) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:08 2021 : sh (3420) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:09 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:10 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:12 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:12 2021 : barad_agent (1660) created 3424
Sat May  8 10:52:12 2021 : barad_agent (3424) is exec'ing "/bin/sh"
Sat May  8 10:52:12 2021 : sh (3424) created 3425
Sat May  8 10:52:12 2021 : sh (3424) created 3426
Sat May  8 10:52:12 2021 : sh (3425) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:12 2021 : sh (3426) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:13 2021 : sgagent (1642) created 3427
Sat May  8 10:52:13 2021 : sgagent (3427) is exec'ing 0x7f2ad811bee9
Sat May  8 10:52:13 2021 : sh (3427) is exec'ing "../../monitor/barad/admin/trystart.sh"
Sat May  8 10:52:13 2021 : trystart.sh (3427) created 3428
Sat May  8 10:52:13 2021 : trystart.sh (3428) is exec'ing "/usr/bin/dirname"
Sat May  8 10:52:13 2021 : trystart.sh (3427) created 3429
Sat May  8 10:52:13 2021 : trystart.sh (3429) is exec'ing "/usr/bin/chmod"
Sat May  8 10:52:13 2021 : trystart.sh (3427) created 3430
Sat May  8 10:52:13 2021 : trystart.sh (3430) created 3431
Sat May  8 10:52:13 2021 : trystart.sh (3430) created 3432
Sat May  8 10:52:13 2021 : trystart.sh (3430) created 3433
Sat May  8 10:52:13 2021 : trystart.sh (3431) is exec'ing "/usr/bin/ps"
Sat May  8 10:52:13 2021 : trystart.sh (3430) created 3434
Sat May  8 10:52:13 2021 : trystart.sh (3432) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:13 2021 : trystart.sh (3434) is exec'ing "/usr/bin/wc"
Sat May  8 10:52:13 2021 : trystart.sh (3433) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:13 2021 : trystart.sh (3427) created 3435
Sat May  8 10:52:13 2021 : trystart.sh (3435) created 3436
Sat May  8 10:52:13 2021 : trystart.sh (3435) created 3437
Sat May  8 10:52:13 2021 : trystart.sh (3436) is exec'ing "/usr/bin/ps"
Sat May  8 10:52:13 2021 : trystart.sh (3435) created 3438
Sat May  8 10:52:13 2021 : trystart.sh (3437) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:13 2021 : trystart.sh (3438) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:16 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:16 2021 : barad_agent (1660) created 3440
Sat May  8 10:52:16 2021 : barad_agent (3440) is exec'ing "/bin/sh"
Sat May  8 10:52:16 2021 : sh (3440) created 3441
Sat May  8 10:52:16 2021 : sh (3440) created 3442
Sat May  8 10:52:16 2021 : sh (3441) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:16 2021 : sh (3440) created 3443
Sat May  8 10:52:16 2021 : sh (3442) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:16 2021 : sh (3443) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:18 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:18 2021 : barad_agent (1660) created 3445
Sat May  8 10:52:18 2021 : barad_agent (3445) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:18 2021 : sh (3445) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:18 2021 : barad_agent (1660) created 3446
Sat May  8 10:52:18 2021 : barad_agent (3446) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:18 2021 : sh (3446) created 3447
Sat May  8 10:52:18 2021 : sh (3446) created 3448
Sat May  8 10:52:18 2021 : sh (3447) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:18 2021 : sh (3448) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:19 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:20 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:22 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:22 2021 : barad_agent (1660) created 3452
Sat May  8 10:52:22 2021 : barad_agent (3452) is exec'ing "/bin/sh"
Sat May  8 10:52:22 2021 : sh (3452) created 3453
Sat May  8 10:52:22 2021 : sh (3452) created 3454
Sat May  8 10:52:22 2021 : sh (3453) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:22 2021 : sh (3454) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:23 2021 : kthreadd (2) created 3455
Sat May  8 10:52:23 2021 : kthreadd (2) created 3456
Sat May  8 10:52:26 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:26 2021 : barad_agent (1660) created 3458
Sat May  8 10:52:26 2021 : barad_agent (3458) is exec'ing "/bin/sh"
Sat May  8 10:52:26 2021 : sh (3458) created 3459
Sat May  8 10:52:26 2021 : sh (3458) created 3460
Sat May  8 10:52:26 2021 : sh (3459) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:26 2021 : sh (3458) created 3461
Sat May  8 10:52:26 2021 : sh (3461) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:26 2021 : sh (3460) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:28 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:28 2021 : barad_agent (1660) created 3463
Sat May  8 10:52:28 2021 : barad_agent (3463) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:28 2021 : sh (3463) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:28 2021 : barad_agent (1660) created 3464
Sat May  8 10:52:28 2021 : barad_agent (3464) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:28 2021 : sh (3464) created 3465
Sat May  8 10:52:28 2021 : sh (3464) created 3466
Sat May  8 10:52:28 2021 : sh (3465) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:28 2021 : sh (3466) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:29 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:30 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:32 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:32 2021 : barad_agent (1660) created 3470
Sat May  8 10:52:32 2021 : barad_agent (3470) is exec'ing "/bin/sh"
Sat May  8 10:52:32 2021 : sh (3470) created 3471
Sat May  8 10:52:32 2021 : sh (3470) created 3472
Sat May  8 10:52:32 2021 : sh (3471) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:32 2021 : sh (3472) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:35 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:36 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:36 2021 : barad_agent (1660) created 3475
Sat May  8 10:52:36 2021 : barad_agent (3475) is exec'ing "/bin/sh"
Sat May  8 10:52:36 2021 : sh (3475) created 3476
Sat May  8 10:52:36 2021 : sh (3475) created 3477
Sat May  8 10:52:36 2021 : sh (3475) created 3478
Sat May  8 10:52:36 2021 : sh (3476) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:36 2021 : sh (3477) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:36 2021 : sh (3478) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:38 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:38 2021 : barad_agent (1660) created 3480
Sat May  8 10:52:38 2021 : barad_agent (3480) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:38 2021 : sh (3480) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:38 2021 : barad_agent (1660) created 3481
Sat May  8 10:52:38 2021 : barad_agent (3481) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:38 2021 : sh (3481) created 3482
Sat May  8 10:52:38 2021 : sh (3481) created 3483
Sat May  8 10:52:38 2021 : sh (3482) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:38 2021 : sh (3483) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:39 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:40 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:42 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:42 2021 : barad_agent (1660) created 3487
Sat May  8 10:52:42 2021 : barad_agent (3487) is exec'ing "/bin/sh"
Sat May  8 10:52:42 2021 : sh (3487) created 3488
Sat May  8 10:52:42 2021 : sh (3487) created 3489
Sat May  8 10:52:42 2021 : sh (3488) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:42 2021 : sh (3489) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:46 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:46 2021 : barad_agent (1660) created 3491
Sat May  8 10:52:46 2021 : barad_agent (3491) is exec'ing "/bin/sh"
Sat May  8 10:52:46 2021 : sh (3491) created 3492
Sat May  8 10:52:46 2021 : sh (3491) created 3493
Sat May  8 10:52:46 2021 : sh (3492) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:46 2021 : sh (3491) created 3494
Sat May  8 10:52:46 2021 : sh (3493) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:46 2021 : sh (3494) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:48 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:48 2021 : barad_agent (1660) created 3496
Sat May  8 10:52:48 2021 : barad_agent (3496) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:48 2021 : sh (3496) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:48 2021 : barad_agent (1660) created 3497
Sat May  8 10:52:48 2021 : barad_agent (3497) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:48 2021 : sh (3497) created 3498
Sat May  8 10:52:48 2021 : sh (3497) created 3499
Sat May  8 10:52:48 2021 : sh (3498) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:48 2021 : sh (3499) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:49 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:50 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:52 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:52 2021 : barad_agent (1660) created 3503
Sat May  8 10:52:52 2021 : barad_agent (3503) is exec'ing "/bin/sh"
Sat May  8 10:52:52 2021 : sh (3503) created 3504
Sat May  8 10:52:52 2021 : sh (3503) created 3505
Sat May  8 10:52:52 2021 : sh (3504) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:52:52 2021 : sh (3505) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:56 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:56 2021 : barad_agent (1660) created 3507
Sat May  8 10:52:56 2021 : barad_agent (3507) is exec'ing "/bin/sh"
Sat May  8 10:52:56 2021 : sh (3507) created 3508
Sat May  8 10:52:56 2021 : sh (3507) created 3509
Sat May  8 10:52:56 2021 : sh (3508) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:56 2021 : sh (3507) created 3510
Sat May  8 10:52:56 2021 : sh (3509) is exec'ing "/usr/bin/grep"
Sat May  8 10:52:56 2021 : sh (3510) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:58 2021 : barad_agent (1660) created 1660
Sat May  8 10:52:58 2021 : barad_agent (1660) created 3512
Sat May  8 10:52:58 2021 : barad_agent (3512) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:58 2021 : sh (3512) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:58 2021 : barad_agent (1660) created 3513
Sat May  8 10:52:58 2021 : barad_agent (3513) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:52:58 2021 : sh (3513) created 3514
Sat May  8 10:52:58 2021 : sh (3513) created 3515
Sat May  8 10:52:58 2021 : sh (3514) is exec'ing "/usr/bin/cat"
Sat May  8 10:52:58 2021 : sh (3515) is exec'ing "/usr/bin/awk"
Sat May  8 10:52:59 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:00 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:01 2021 : crond (991) created 3518
Sat May  8 10:53:01 2021 : crond (3518) created 3519
Sat May  8 10:53:01 2021 : crond (3519) is exec'ing "/bin/sh"
Sat May  8 10:53:01 2021 : sh (3519) is exec'ing "/usr/bin/flock"
Sat May  8 10:53:01 2021 : flock (3519) created 3520
Sat May  8 10:53:01 2021 : flock (3520) is exec'ing "/bin/sh"
Sat May  8 10:53:01 2021 : sh (3520) created 3521
Sat May  8 10:53:01 2021 : sh (3521) is exec'ing "/usr/local/qcloud/stargate/admin/start.sh"
Sat May  8 10:53:01 2021 : start.sh (3521) created 3522
Sat May  8 10:53:01 2021 : start.sh (3522) is exec'ing "/usr/bin/whoami"
Sat May  8 10:53:01 2021 : start.sh (3521) created 3523
Sat May  8 10:53:01 2021 : start.sh (3523) created 3524
Sat May  8 10:53:01 2021 : start.sh (3523) created 3525
1      systemd      0     3518        0
Sat May  8 10:53:01 2021 : start.sh (3524) is exec'ing "/usr/bin/ps"
Sat May  8 10:53:01 2021 : start.sh (3523) created 3526
Sat May  8 10:53:01 2021 : start.sh (3523) created 3527
Sat May  8 10:53:01 2021 : start.sh (3525) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:01 2021 : start.sh (3526) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:01 2021 : start.sh (3527) is exec'ing "/usr/bin/wc"
1      systemd      0     3525        0
1      systemd      0     3524        0
1      systemd      0     3523        0
Sat May  8 10:53:01 2021 : kworker/u16:0 (5) created 3528
Sat May  8 10:53:02 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:02 2021 : barad_agent (1660) created 3530
Sat May  8 10:53:02 2021 : barad_agent (3530) is exec'ing "/bin/sh"
Sat May  8 10:53:02 2021 : sh (3530) created 3531
Sat May  8 10:53:02 2021 : sh (3530) created 3532
Sat May  8 10:53:02 2021 : sh (3531) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:53:02 2021 : sh (3532) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:03 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:03 2021 : barad_agent (1660) created 3534
Sat May  8 10:53:03 2021 : barad_agent (3534) is exec'ing "/bin/sh"
Sat May  8 10:53:03 2021 : sh (3534) created 3535
Sat May  8 10:53:03 2021 : sh (3534) created 3536
Sat May  8 10:53:03 2021 : sh (3534) created 3537
Sat May  8 10:53:03 2021 : sh (3535) is exec'ing "/usr/sbin/ntpdate"
Sat May  8 10:53:03 2021 : sh (3536) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:03 2021 : sh (3537) is exec'ing "/usr/bin/sed"
Sat May  8 10:53:04 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:05 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:06 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:06 2021 : barad_agent (1660) created 3541
Sat May  8 10:53:06 2021 : barad_agent (3541) is exec'ing "/bin/sh"
Sat May  8 10:53:06 2021 : sh (3541) created 3542
Sat May  8 10:53:06 2021 : sh (3541) created 3543
Sat May  8 10:53:06 2021 : sh (3541) created 3544
Sat May  8 10:53:06 2021 : sh (3542) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:06 2021 : sh (3543) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:06 2021 : sh (3544) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:07 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:07 2021 : barad_agent (1660) created 3546
Sat May  8 10:53:07 2021 : barad_agent (3546) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:53:07 2021 : sh (3546) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:08 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:08 2021 : barad_agent (1660) created 3548
Sat May  8 10:53:08 2021 : barad_agent (3548) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:53:08 2021 : sh (3548) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:08 2021 : barad_agent (1660) created 3549
Sat May  8 10:53:08 2021 : barad_agent (3549) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:53:08 2021 : sh (3549) created 3550
Sat May  8 10:53:08 2021 : sh (3549) created 3551
Sat May  8 10:53:08 2021 : sh (3550) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:08 2021 : sh (3551) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:09 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:10 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:12 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:12 2021 : barad_agent (1660) created 3555
Sat May  8 10:53:12 2021 : barad_agent (3555) is exec'ing "/bin/sh"
Sat May  8 10:53:12 2021 : sh (3555) created 3556
Sat May  8 10:53:12 2021 : sh (3555) created 3557
Sat May  8 10:53:12 2021 : sh (3556) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:53:12 2021 : sh (3557) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:13 2021 : sgagent (1642) created 3558
Sat May  8 10:53:13 2021 : sgagent (3558) is exec'ing 0x7f2ad811bee9
Sat May  8 10:53:13 2021 : sh (3558) is exec'ing "../../monitor/barad/admin/trystart.sh"
Sat May  8 10:53:13 2021 : trystart.sh (3558) created 3559
Sat May  8 10:53:13 2021 : trystart.sh (3559) is exec'ing "/usr/bin/dirname"
Sat May  8 10:53:13 2021 : trystart.sh (3558) created 3560
Sat May  8 10:53:13 2021 : trystart.sh (3560) is exec'ing "/usr/bin/chmod"
Sat May  8 10:53:13 2021 : trystart.sh (3558) created 3561
Sat May  8 10:53:13 2021 : trystart.sh (3561) created 3562
Sat May  8 10:53:13 2021 : trystart.sh (3561) created 3563
Sat May  8 10:53:13 2021 : trystart.sh (3562) is exec'ing "/usr/bin/ps"
Sat May  8 10:53:13 2021 : trystart.sh (3561) created 3564
Sat May  8 10:53:13 2021 : trystart.sh (3561) created 3565
Sat May  8 10:53:13 2021 : trystart.sh (3563) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:13 2021 : trystart.sh (3564) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:13 2021 : trystart.sh (3565) is exec'ing "/usr/bin/wc"
Sat May  8 10:53:13 2021 : trystart.sh (3558) created 3566
Sat May  8 10:53:13 2021 : trystart.sh (3566) created 3567
Sat May  8 10:53:13 2021 : trystart.sh (3566) created 3568
Sat May  8 10:53:13 2021 : trystart.sh (3567) is exec'ing "/usr/bin/ps"
Sat May  8 10:53:13 2021 : trystart.sh (3566) created 3569
Sat May  8 10:53:13 2021 : trystart.sh (3568) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:13 2021 : trystart.sh (3569) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:16 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:16 2021 : barad_agent (1660) created 3571
Sat May  8 10:53:16 2021 : barad_agent (3571) is exec'ing "/bin/sh"
Sat May  8 10:53:16 2021 : sh (3571) created 3572
Sat May  8 10:53:16 2021 : sh (3571) created 3573
Sat May  8 10:53:16 2021 : sh (3571) created 3574
Sat May  8 10:53:16 2021 : sh (3572) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:16 2021 : sh (3573) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:16 2021 : sh (3574) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:18 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:18 2021 : barad_agent (1660) created 3576
Sat May  8 10:53:18 2021 : barad_agent (3576) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:53:18 2021 : sh (3576) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:18 2021 : barad_agent (1660) created 3577
Sat May  8 10:53:18 2021 : barad_agent (3577) is exec'ing 0x7f8026ca2ee9
Sat May  8 10:53:18 2021 : sh (3577) created 3578
Sat May  8 10:53:18 2021 : sh (3577) created 3579
Sat May  8 10:53:18 2021 : sh (3578) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:18 2021 : sh (3579) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:19 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:20 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:22 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:22 2021 : barad_agent (1660) created 3583
Sat May  8 10:53:22 2021 : barad_agent (3583) is exec'ing "/bin/sh"
Sat May  8 10:53:22 2021 : sh (3583) created 3584
Sat May  8 10:53:22 2021 : sh (3583) created 3585
Sat May  8 10:53:22 2021 : sh (3584) is exec'ing "/usr/bin/lsblk"
Sat May  8 10:53:22 2021 : sh (3585) is exec'ing "/usr/bin/awk"
Sat May  8 10:53:26 2021 : barad_agent (1660) created 1660
Sat May  8 10:53:26 2021 : barad_agent (1660) created 3587
Sat May  8 10:53:26 2021 : barad_agent (3587) is exec'ing "/bin/sh"
Sat May  8 10:53:26 2021 : sh (3587) created 3588
Sat May  8 10:53:26 2021 : sh (3587) created 3589
Sat May  8 10:53:26 2021 : sh (3588) is exec'ing "/usr/bin/cat"
Sat May  8 10:53:26 2021 : sh (3587) created 3590
Sat May  8 10:53:26 2021 : sh (3589) is exec'ing "/usr/bin/grep"
Sat May  8 10:53:26 2021 : sh (3590) is exec'ing "/usr/bin/awk"
^C2803   stap         15    3120      582
WARNING: Number of errors: 0, skipped probes: 48

/var/log/messages

May  8 18:50:01 txy-150-193 systemd: Started Session 326 of user root.
May  8 18:50:01 txy-150-193 systemd: Started Session 327 of user root.
May  8 18:51:01 txy-150-193 systemd: Started Session 329 of user root.
May  8 18:51:59 txy-150-193 systemd-logind: New session 330 of user developer.
May  8 18:51:59 txy-150-193 systemd: Started Session 330 of user developer.
May  8 18:52:01 txy-150-193 systemd: Started Session 331 of user root.
May  8 18:53:01 txy-150-193 systemd: Started Session 332 of user root.
May  8 18:54:01 txy-150-193 systemd: Started Session 333 of user root.
May  8 18:54:24 txy-150-193 su: (to root) developer on pts/1
May  8 18:54:25 txy-150-193 systemd: Stopped Dump dmesg to /var/log/dmesg.
May  8 18:54:25 txy-150-193 systemd: Stopped target Timers.
May  8 18:54:25 txy-150-193 systemd: Stopped Daily Cleanup of Temporary Directories.
May  8 18:54:25 txy-150-193 systemd: Stopped target Cloud-init target.
May  8 18:54:25 txy-150-193 ntpd[767]: ntpd exiting on signal 15
May  8 18:54:25 txy-150-193 systemd: Stopping Session 1 of user root.
May  8 18:54:25 txy-150-193 systemd: Removed slice system-selinux\x2dpolicy\x2dmigrate\x2dlocal\x2dchanges.slice.
May  8 18:54:25 txy-150-193 systemd: Stopping Session 2 of user root.
May  8 18:54:25 txy-150-193 rsyslogd: [origin software="rsyslogd" swVersion="8.24.0-57.el7_9" x-pid="1100" x-info="http://www.rsyslog.com"] exiting on signal 15.
May  8 18:54:25 txy-150-193 systemd: Closed LVM2 poll daemon socket.
May  8 18:54:25 txy-150-193 systemd: Stopping Session 168 of user developer.
May  8 18:54:25 txy-150-193 systemd: Stopping Session 330 of user developer.

这个应该是进程死掉的时间

a398058068 · 2021 年5 月 8 日 11:03

tikv.log

[2021/05/08 18:51:03.388 +08:00] [INFO] [profiler.rs:163] ["starting cpu profiler"]
[2021/05/08 18:51:03.390 +08:00] [INFO] [mod.rs:323] ["start profiling 30 seconds with frequency 99 /s"]
[2021/05/08 18:51:05.457 +08:00] [WARN] [endpoint.rs:639] [error-response] [err="Key is locked (will clean up) primary_lock: 7480000000000004245F7280000000000F35E9 lock_version: 424796766970576918 key: 7480000000000004245F698000000000000001013639353639353430FF3433363235333630FF3637300000000000FA0380000000000F35E9 lock_ttl: 3003 txn_size: 1 lock_for_update_ts: 424796766970576918 min_commit_ts: 424796766970576919"]
[2021/05/08 18:51:12.048 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.16:20170: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.609 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.16:52280: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.641 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.111:58950: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.657 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.34:60502: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.668 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.111:58952: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.692 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.174:39160: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.699 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.34:60506: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:21.762 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.41:33698: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:26.420 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.166:40380: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:28.423 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.111:2379: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:31.470 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.41:20160: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:31.470 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.174:20160: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:31.677 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.34:60508: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:31.694 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.34:60504: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:32.033 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.166:20170: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:34.700 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.34:47044: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:35.710 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.111:58948: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:51:35.714 +08:00] [INFO] [<unknown>] ["ipv4:172.18.150.111:58954: Keepalive watchdog fired. Closing transport."]
[2021/05/08 18:54:25.668 +08:00] [INFO] [signal_handler.rs:20] ["receive signal 15, stopping server..."]
[2021/05/08 18:55:19.889 +08:00] [INFO] [lib.rs:89] ["Welcome to TiKV"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Release Version:   5.0.1"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Edition:           Community"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Git Commit Hash:   e26389a278116b2f61addfa9f15ca25ecf38bc80"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Git Commit Branch: heads/refs/tags/v5.0.1"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["UTC Build Time:    2021-04-23 08:24:13"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Rust Version:      rustc 1.51.0-nightly (bc39d4d9c 2021-01-15)"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Enable Features:   jemalloc mem-profiling portable sse protobuf-codec test-engines-rocksdb cloud-aws cloud-gcp"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [lib.rs:94] ["Profile:           dist_release"]
[2021/05/08 18:55:19.892 +08:00] [INFO] [mod.rs:65] ["memory limit in bytes: 17055675392, cpu cores quota: 8"]
[2021/05/08 18:55:19.895 +08:00] [WARN] [lib.rs:512] ["environment variable `TZ` is missing, using `/etc/localtime`"]
[2021/05/08 18:55:19.895 +08:00] [INFO] [config.rs:590] ["kernel parameters"] [value=0] [param=vm.swappiness]
[2021/05/08 18:55:19.898 +08:00] [WARN] [server.rs:1083] ["check: kernel"] [err="kernel parameters net.core.somaxconn got 128, expect 32768"]
[2021/05/08 18:55:19.898 +08:00] [WARN] [server.rs:1083] ["check: kernel"] [err="kernel parameters net.ipv4.tcp_syncookies got 1, expect 0"]

a398058068 · 2021 年5 月 8 日 11:26

用strace追踪tikv进程好像是被暂停了。从日志也可以看出来。点诊断的那一刻tikv日志就不打印了。然后过了一会进程就消失了
strace -o strace.log -tt -p 1095

19:22:02.859369 rt_sigtimedwait([HUP INT USR1 USR2 TERM], NULL, NULL, 8

完全不知道啥情况还有一个情况是5.0.0我曾经执行过实例性能分析没有这个问题，更新到5.0.1出现的。
另外其他两台kv节点没这个问题。只有这台只要执行了诊断必死。
receive signal 15, stopping server 应该是我reboot系统产生的日志。因为这问题只有重启才能解决，正常启动实例不行。

yilong · 2021 年5 月 10 日 01:25

1.什么虚拟环境？具体版本是什么？
2.能否试试扩容一个新节点，看看是否报错，如果没有问题，可以缩容这个节点。
3. 看起来是被系统终止进程了，可以协调下虚拟机厂商和操作系统厂商一起排查吗？

a398058068 · 2021 年5 月 10 日 02:21

腾讯云 centos7
关键问题是5.0.0没问题。升级5.0.1之后出现的。

yilong · 2021 年5 月 10 日 06:00

麻烦帮忙拿一下卡住之后的栈信息
gdb path/to/tikv pid -batch -ex “info threads” -ex “thread apply all bt”

a398058068 · 2021 年5 月 11 日 09:25

loglog (44.9 KB)

执行了2-3次。自动会断掉不知道为什么。都在日志里了。

YangKeao · 2021 年5 月 11 日 11:48

初步判断可能是 Dwarf 信息不正确，使得在获取 Backtrace 时会进入死循环，这一问题除了出现在 profiler 中也出现在了 gdb 中：尝试使用 gdb 获取 Backtrace，gdb 也会进入死循环，末端 Backtrace 如下：

#0  0x000055f8de280a04 in new_symbol(die_info*, type*, dwarf2_cu*, symbol*) ()
#1  0x000055f8de28254b in process_die(die_info*, dwarf2_cu*) ()
#2  0x000055f8de28646d in inherit_abstract_dies(die_info*, dwarf2_cu*) ()
#3  0x000055f8de286e8b in read_func_scope(die_info*, dwarf2_cu*) ()
#4  0x000055f8de28248d in process_die(die_info*, dwarf2_cu*) ()
#5  0x000055f8de286e13 in read_func_scope(die_info*, dwarf2_cu*) ()
#6  0x000055f8de28248d in process_die(die_info*, dwarf2_cu*) ()
#7  0x000055f8de286e13 in read_func_scope(die_info*, dwarf2_cu*) ()
#8  0x000055f8de28248d in process_die(die_info*, dwarf2_cu*) ()
#9  0x000055f8de286652 in read_lexical_block_scope(die_info*, dwarf2_cu*) ()
#10 0x000055f8de282720 in process_die(die_info*, dwarf2_cu*) ()
#11 0x000055f8de286e13 in read_func_scope(die_info*, dwarf2_cu*) ()
#12 0x000055f8de28248d in process_die(die_info*, dwarf2_cu*) ()
#13 0x000055f8de286e13 in read_func_scope(die_info*, dwarf2_cu*) ()
#14 0x000055f8de2837d6 in process_die(die_info*, dwarf2_cu*) ()

将进一步分析 Dwarf 错误的部分，判断原因在于编译环境还是编译器，以给出解决方案。

YangKeao · 2021 年5 月 12 日 01:41

经调查，原因为在 5.0.1 中 tiup 配置默认启用 jemalloc prof。从而 jemalloc 与 pprof 在拉取 Backtrace 时发生冲突，产生死锁。

Workaround：修改 tikv 启动脚本 run_tikv.sh，去掉其中与 export MALLOC_CONF= 相关的一行。

已开启一个 issue 用于追踪讨论 jemalloc prof 是否应该被打开，以及后续版本是否应当做出调整。 https://github.com/tikv/tikv/issues/10150

a398058068 · 2021 年5 月 20 日 02:23

了解。感谢。

zhenjiaogao · 2021 年5 月 20 日 06:30

system · 2022 年10 月 31 日 19:12

此话题已在最后回复的 1 分钟后被自动关闭。不再允许新回复。