How to track newly created processes in Linux?

I know that with ps I can see the list or tree of the current processes running in the system. But what I want to achieve is to “follow” the new processes that are created when using the computer.

As analogy, when you use tail -f to follow the new contents appended to a file or to any input, then I want to keep a follow list of the process that are currently being created.

Is this even posible?

Answers:

Thank you for visiting the Q&A section on Magenaut. Please note that all the answers may not help you solve the issue immediately. So please treat them as advisements. If you found the post helpful (or not), leave a comment & I’ll get back to you as soon as possible.

Method 1

If kprobes are enabled in the kernel you can use execsnoop from perf-tools:

In first terminal:

% while true; do uptime; sleep 1; done

In another terminal:

% git clone https://github.com/brendangregg/perf-tools.git
% cd perf-tools
% sudo ./execsnoop
Tracing exec()s. Ctrl-C to end.
Instrumenting sys_execve
   PID   PPID ARGS
 83939  83937 cat -v trace_pipe
 83938  83934 gawk -v o=1 -v opt_name=0 -v name= -v opt_duration=0 [...]
 83940  76640 uptime
 83941  76640 sleep 1
 83942  76640 uptime
 83943  76640 sleep 1
 83944  76640 uptime
 83945  76640 sleep 1
^C
Ending tracing...

Method 2

The easiest way is to enable system call auditing

See the following link for details,

Does anyone know a simple way to monitor root process spawn | Server Fault

If you’re monitoring all processes, just remove the -F uid=0 part

Logs are written to /var/log/audit/audit.log

Method 3

Some examples of bpftrace usage to achieve the goal.

  1. The simplest one is tracing all exec calls in the system:
    sudo bpftrace -e 'tracepoint:syscalls:sys_enter_exec*{ printf("pid: %d, comm: %s, args: ", pid, comm); join(args->argv); }'

    There are at least two tracepoints you need to watch sys_enter_execve and sys_enter_execveat. In the example I use the * symbol to match both syscalls (this syntax works since 2019).
  2. One may also want to monitor all threads being created in the system as:
    sudo bpftrace -e 'kprobe:_do_fork{ printf("pid = %d, comm = %sn", pid, comm); }'

    No process arguments for you in this case though, nevertheless it may be useful.

To see the list of all available events execute bpftrace -l.

Method 4

You can use forkstat as stated here: https://stackoverflow.com/a/40532202/781153

Install with: apt-get install forkstat

And just run: forkstat

Method 5

CONFIG_PROC_EVENTS=y

Sample session:

$ su
# ./proc_events.out &
set mcast listen ok
# sleep 2 & sleep 1 &
fork: parent tid=48 pid=48 -> child tid=56 pid=56
fork: parent tid=48 pid=48 -> child tid=57 pid=57
exec: tid=57 pid=57
exec: tid=56 pid=56
exit: tid=57 pid=57 exit_code=0
exit: tid=56 pid=56 exit_code=0

CONFIG_PROC_EVENTS exposes the events to userland via a netlink socket.

proc_events.c

#define _XOPEN_SOURCE 700
#include <sys/socket.h>
#include <linux/netlink.h>
#include <linux/connector.h>
#include <linux/cn_proc.h>
#include <signal.h>
#include <errno.h>
#include <stdbool.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>

static volatile bool need_exit = false;

static int nl_connect()
{
    int rc;
    int nl_sock;
    struct sockaddr_nl sa_nl;

    nl_sock = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR);
    if (nl_sock == -1) {
        perror("socket");
        return -1;
    }
    sa_nl.nl_family = AF_NETLINK;
    sa_nl.nl_groups = CN_IDX_PROC;
    sa_nl.nl_pid = getpid();
    rc = bind(nl_sock, (struct sockaddr *)&sa_nl, sizeof(sa_nl));
    if (rc == -1) {
        perror("bind");
        close(nl_sock);
        return -1;
    }
    return nl_sock;
}

static int set_proc_ev_listen(int nl_sock, bool enable)
{
    int rc;
    struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
        struct nlmsghdr nl_hdr;
        struct __attribute__ ((__packed__)) {
            struct cn_msg cn_msg;
            enum proc_cn_mcast_op cn_mcast;
        };
    } nlcn_msg;

    memset(&nlcn_msg, 0, sizeof(nlcn_msg));
    nlcn_msg.nl_hdr.nlmsg_len = sizeof(nlcn_msg);
    nlcn_msg.nl_hdr.nlmsg_pid = getpid();
    nlcn_msg.nl_hdr.nlmsg_type = NLMSG_DONE;

    nlcn_msg.cn_msg.id.idx = CN_IDX_PROC;
    nlcn_msg.cn_msg.id.val = CN_VAL_PROC;
    nlcn_msg.cn_msg.len = sizeof(enum proc_cn_mcast_op);

    nlcn_msg.cn_mcast = enable ? PROC_CN_MCAST_LISTEN : PROC_CN_MCAST_IGNORE;

    rc = send(nl_sock, &nlcn_msg, sizeof(nlcn_msg), 0);
    if (rc == -1) {
        perror("netlink send");
        return -1;
    }

    return 0;
}

static int handle_proc_ev(int nl_sock)
{
    int rc;
    struct __attribute__ ((aligned(NLMSG_ALIGNTO))) {
        struct nlmsghdr nl_hdr;
        struct __attribute__ ((__packed__)) {
            struct cn_msg cn_msg;
            struct proc_event proc_ev;
        };
    } nlcn_msg;
    while (!need_exit) {
        rc = recv(nl_sock, &nlcn_msg, sizeof(nlcn_msg), 0);
        if (rc == 0) {
            /* shutdown? */
            return 0;
        } else if (rc == -1) {
            if (errno == EINTR) continue;
            perror("netlink recv");
            return -1;
        }
        switch (nlcn_msg.proc_ev.what) {
            case PROC_EVENT_NONE:
                printf("set mcast listen okn");
                break;
            case PROC_EVENT_FORK:
                printf("fork: parent tid=%d pid=%d -> child tid=%d pid=%dn",
                        nlcn_msg.proc_ev.event_data.fork.parent_pid,
                        nlcn_msg.proc_ev.event_data.fork.parent_tgid,
                        nlcn_msg.proc_ev.event_data.fork.child_pid,
                        nlcn_msg.proc_ev.event_data.fork.child_tgid);
                break;
            case PROC_EVENT_EXEC:
                printf("exec: tid=%d pid=%dn",
                        nlcn_msg.proc_ev.event_data.exec.process_pid,
                        nlcn_msg.proc_ev.event_data.exec.process_tgid);
                break;
            case PROC_EVENT_UID:
                printf("uid change: tid=%d pid=%d from %d to %dn",
                        nlcn_msg.proc_ev.event_data.id.process_pid,
                        nlcn_msg.proc_ev.event_data.id.process_tgid,
                        nlcn_msg.proc_ev.event_data.id.r.ruid,
                        nlcn_msg.proc_ev.event_data.id.e.euid);
                break;
            case PROC_EVENT_GID:
                printf("gid change: tid=%d pid=%d from %d to %dn",
                        nlcn_msg.proc_ev.event_data.id.process_pid,
                        nlcn_msg.proc_ev.event_data.id.process_tgid,
                        nlcn_msg.proc_ev.event_data.id.r.rgid,
                        nlcn_msg.proc_ev.event_data.id.e.egid);
                break;
            case PROC_EVENT_EXIT:
                printf("exit: tid=%d pid=%d exit_code=%dn",
                        nlcn_msg.proc_ev.event_data.exit.process_pid,
                        nlcn_msg.proc_ev.event_data.exit.process_tgid,
                        nlcn_msg.proc_ev.event_data.exit.exit_code);
                break;
            default:
                printf("unhandled proc eventn");
                break;
        }
    }

    return 0;
}

static void on_sigint(__attribute__ ((unused)) int unused)
{
    need_exit = true;
}

int main()
{
    int nl_sock;
    int rc = EXIT_SUCCESS;

    signal(SIGINT, &on_sigint);
    siginterrupt(SIGINT, true);
    nl_sock = nl_connect();
    if (nl_sock == -1)
        exit(EXIT_FAILURE);
    rc = set_proc_ev_listen(nl_sock, true);
    if (rc == -1) {
        rc = EXIT_FAILURE;
        goto out;
    }
    rc = handle_proc_ev(nl_sock);
    if (rc == -1) {
        rc = EXIT_FAILURE;
        goto out;
    }
    set_proc_ev_listen(nl_sock, false);
out:
    close(nl_sock);
    exit(rc);
}

GitHub upsatream, code adapted from: https://bewareofgeek.livejournal.com/2945.html

I don’t think however that you can you get process data such as UID and process arguments because exec_proc_event contains so little data: https://github.com/torvalds/linux/blob/v4.16/include/uapi/linux/cn_proc.h#L80 We could try to immediately read it from /proc, but there is a risk that the process finished and another one took its PID, so it wouldn’t be reliable.

Tested on Ubuntu 17.10, which has CONFIG_PROC_EVENTS=y enabled by default.

Method 6

You can apparently follow a process using strace. If you know the PID of the process then you can do:

strace -o strace-<pid>.out -f -p <pid>

Notice the -f switch. It will help you to follow newly created processes that are descendants of the process whose PID was used in the command, above. For information on strace see this question.

Method 7

There are multiple Linux APIs that you can use to do this and multiple usermode tools that use the various APIs. Here are some of the APIs:

  • Netlink process connector – this an API to do precisely what you want but there are issues tracking detailed information short-lived processes
  • audit API – a kernel API enabled by default on most distributions which can send events to usermode for every syscall (if you go this route you need to track exec-like and fork-like syscalls)
  • tracepoints and kprobes – two kernel debugging APIs which can be used to get information on events about process lifecycle
  • ebpf based solutions – can be used in conjunction with tracepoints/kprobes to filter the – events in kernel or run various logic
  • ptrace based solutions – these include simple ptrace debugging as well as the seccomp API

There are really too many details to properly compare these options in a single StackOverflow answer, but I’ve written about this in detail elsewhere.

Method 8

TL;DR: use execnsoop.


  • Check that KPROBES are enabled in the Kernel
    $ zgrep CONFIG_KPROBES= /boot/config-$(uname -r) /proc/config* 2> /dev/null
    /boot/config-4.15.0-64-generic:CONFIG_KPROBES=y
  • Install BPF Compiler Collection (BCC)
    • For Debian/Ubuntu the package linux-headers-* is also needed:
      sudo apt-get install bpfcc-tools linux-headers-$(uname -r)
    • For other distributions see Installing BCC
  • Run execsnoop (examples and options)
    <a href="https://getridbug.com/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="d0a2bfbfa490a5b2a5bea4a5">[email protected]</a> ~ # execsnoop-bpfcc -tx
    TIME(s) PCOMM            PID    PPID   RET ARGS
    5.773   ping             13027  12847    0 /bin/ping hostname

Method 9

You can use SysmonForLinux tool which is originally one of the powerful Windows tools released for Linux by the Sysinternals team for collecting most of the system logs including process creation and termination. Its default configuration logs to all process creations.


All methods was sourced from stackoverflow.com or stackexchange.com, is licensed under cc by-sa 2.5, cc by-sa 3.0 and cc by-sa 4.0

0 0 votes
Article Rating
Subscribe
Notify of
guest

0 Comments
Inline Feedbacks
View all comments
0
Would love your thoughts, please comment.x
()
x