I'm running Ubuntu Desktop and I've noticed that systemd periodically consumes a lot of CPU. This seems to happen periodically and without any obvious triggers. In the past, when I've restarted the machine, the problem disappeared, but I want to understand why this is happening. I used strace to investigate and found that when the CPU usage became high (I know this because of the sound of the CPU fan), different calls of newfstatat were made and some of them with errors ("file not found"). Can anyone provide some guidance on how to diagnose and fix this issue?
Edit:
systemd version:
systemd 251 (251.4-1ubuntu7.3)
+PAM +AUDIT +SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY -P11KIT -QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -BPF_FRAMEWORK -XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified
htop command screenshot
systemd-cgtop screenshot
$ sudo strace -c -p 2268
% time seconds usecs/call calls errors syscall
------ ----------- ----------- --------- --------- ----------------
100,00 0,000001 0 56 34 newfstatat
0,00 0,000000 0 8 close
0,00 0,000000 0 4 recvmsg
0,00 0,000000 0 7 gettid
0,00 0,000000 0 6 epoll_wait
0,00 0,000000 0 4 pipe2
0,00 0,000000 0 40 getrandom
------ ----------- ----------- --------- --------- ----------------
100,00 0,000001 0 125 34 total
$ sudo strace -fvttTyy -p 2268
strace: Process 2268 attached
14:40:14.635234 gettid() = 2268 <0.000017>
14:40:14.635365 epoll_wait(4<anon_inode:[eventpoll]>, [{events=EPOLLIN, data={u32=2321867760, u64=94577501725680}}], 86, -1) = 1 <22.626761>
14:40:37.262325 recvmsg(15<NETLINK:[KOBJECT_UEVENT:2268]>, {msg_name={sa_family=AF_NETLINK, nl_pid=308024, nl_groups=0x000002}, msg_namelen=128 => 12, msg_iov=[{iov_base=[{prefix="libudev", magic=htonl(0xfeedcafe), header_size=40, properties_off=40, properties_len=947, filter_subsystem_hash=htonl(0xf0031db7), filter_devtype_hash=htonl(0x7bcbc5ee), filter_tag_bloom_hi=htonl(0x2000400), filter_tag_bloom_lo=htonl(0x10800000)}, "UDEV_DATABASE_VERSION=1\0ACTION=c"...], iov_len=8192}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_SOCKET, cmsg_type=SCM_CREDENTIALS, cmsg_data={pid=356244, uid=0, gid=0}}], msg_controllen=32, msg_flags=0}, 0) = 987 <0.000064>
14:40:37.263883 getrandom("\x9b\xd1\x64\x37\x24\x2f\x36\xfe\x59\x7b\x32\x6d\x08\x97\x2b\xce", 16, GRND_INSECURE) = 16 <0.000055>
14:40:37.264086 getrandom("\xe6\x88\x2a\xe3\xc8\xe9\x62\x15\x83\xf2\x27\xc9\x70\xf1\x2f\xc3", 16, GRND_INSECURE) = 16 <0.000051>
14:40:37.264248 getrandom("\x63\x14\xa7\xc9\x5f\xe6\x60\xb5\xf2\xcd\xdc\x1b\x22\xe0\x91\xfb", 16, GRND_INSECURE) = 16 <0.000051>
14:40:37.264452 pipe2([22<pipe:[3287284]>, 23<pipe:[3287284]>], O_NONBLOCK|O_CLOEXEC) = 0 <0.000073>
14:40:37.264657 pipe2([24<pipe:[3287286]>, 25<pipe:[3287286]>], O_NONBLOCK|O_CLOEXEC) = 0 <0.000057>
14:40:37.264860 getrandom("\x78\xc5\xf6\xfd\x29\xa3\x15\x9a\x7f\x53\xec\x31\xad\x9e\x7c\xc5", 16, GRND_INSECURE) = 16 <0.000052>
14:40:37.265038 getrandom("\xed\x43\xba\x70\xef\x28\x9a\x3f\x01\x1c\x39\xcc\x51\x4e\x89\xa3", 16, GRND_INSECURE) = 16 <0.000052>
14:40:37.265277 getrandom("\x96\x15\xc3\xb0\x46\x67\xef\x77\x4a\x9c\xe3\x18\xb8\xf5\xb4\x33", 16, GRND_INSECURE) = 16 <0.000049>
14:40:37.265591 getrandom("\x59\x7f\x23\xeb\x76\xea\x81\x82\xed\x4b\x01\x8e\xe5\xfa\x7d\x5c", 16, GRND_INSECURE) = 16 <0.000063>
14:40:37.266157 getrandom("\x7b\xae\xc6\x5c\x39\x01\x37\x03\x35\xc1\xbf\x89\xd8\xf4\x33\x5d", 16, GRND_INSECURE) = 16 <0.000055>
14:40:37.267407 getrandom("\x8b\xf7\x9d\x77\xe7\xfc\x49\x29\xbc\x92\xad\xac\xfb\x80\xe3\x8f", 16, GRND_INSECURE) = 16 <0.000067>
14:40:37.269564 getrandom("\x01\x44\x07\xa8\x8e\xf6\xc5\xe1\xd3\xd4\xf6\x46\x37\x2e\xe8\x44", 16, GRND_INSECURE) = 16 <0.000029>
14:40:37.277204 getrandom("\x89\xa6\xdc\x1f\x5e\xd6\x74\xab\x3b\x3b\x3a\x48\xcb\x6c\xcc\x5f", 16, GRND_INSECURE) = 16 <0.000021>
14:40:37.310595 getrandom("\x38\x40\x5e\x95\xfc\xff\x5e\x76\xda\xc0\x24\x13\x58\xcf\x04\xdb", 16, GRND_INSECURE) = 16 <0.000027>
14:40:37.402512 getrandom("\x70\x25\x9d\x31\x15\xa2\xbe\x60\x35\x96\x5f\x64\x3f\xd8\xd1\x53", 16, GRND_INSECURE) = 16 <0.000056>
14:40:37.748203 getrandom("\x26\x6d\xb2\x58\x07\xe4\x11\x14\xd1\x10\xe3\x11\xbd\x23\x8d\xce", 16, GRND_INSECURE) = 16 <0.000179>
14:40:39.005297 getrandom("\xb5\xf0\xee\x04\x5f\x6f\x84\x7b\x57\x69\x9e\x81\xb9\xf2\x04\xb4", 16, GRND_INSECURE) = 16 <0.000138>
14:40:41.050125 newfstatat(AT_FDCWD</>, "/dev/disk/by-path/pci-0000:00:1d.0-usb-0:1.6:1.0-scsi-0:0:0:0", {st_dev=makedev(0, 0x5), st_ino=363, st_mode=S_IFBLK|0660, st_nlink=1, st_uid=0, st_gid=6, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x8, 0x10), st_atime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_atime_nsec=277691097, st_mtime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_mtime_nsec=277691097, st_ctime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_ctime_nsec=277691097}, 0) = 0 <0.000221>
14:40:41.050959 newfstatat(AT_FDCWD</>, "/dev/disk/by-id/usb-Generic-_Multi-Card_20090516388200000-0:0", {st_dev=makedev(0, 0x5), st_ino=363, st_mode=S_IFBLK|0660, st_nlink=1, st_uid=0, st_gid=6, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x8, 0x10), st_atime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_atime_nsec=277691097, st_mtime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_mtime_nsec=277691097, st_ctime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_ctime_nsec=277691097}, 0) = 0 <0.000044>
14:40:41.051148 newfstatat(AT_FDCWD</>, "/dev/disk/by-diskseq/18086", 0x7fff5c2fe760, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000061>
14:40:41.051926 newfstatat(AT_FDCWD</>, "/home/ubuntu/.config/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000168>
14:40:41.052538 newfstatat(AT_FDCWD</>, "/etc/xdg/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=18352495, st_mode=S_IFDIR|0755, st_nlink=7, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_mtime_nsec=407382145, st_ctime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_ctime_nsec=407382145}, 0) = 0 <0.000122>
14:40:41.052968 newfstatat(AT_FDCWD</>, "/etc/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=18352495, st_mode=S_IFDIR|0755, st_nlink=7, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_mtime_nsec=407382145, st_ctime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_ctime_nsec=407382145}, 0) = 0 <0.000023>
14:40:41.053169 newfstatat(AT_FDCWD</>, "/run/user/1000/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000039>
14:40:41.053280 newfstatat(AT_FDCWD</>, "/run/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000038>
14:40:41.053397 newfstatat(AT_FDCWD</>, "/home/ubuntu/.local/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000033>
14:40:41.053505 newfstatat(AT_FDCWD</>, "/var/lib/flatpak/exports/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000020>
14:40:41.053650 newfstatat(AT_FDCWD</>, "/usr/local/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000015>
14:40:41.053729 newfstatat(AT_FDCWD</>, "/usr/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000018>
14:40:41.053819 newfstatat(AT_FDCWD</>, "/usr/local/lib/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000031>
14:40:41.053912 newfstatat(AT_FDCWD</>, "/usr/lib/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=8259220, st_mode=S_IFDIR|0755, st_nlink=14, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=48, st_size=20480, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1681357099 /* 2023-04-13T04:38:19.600985446+0100 */, st_mtime_nsec=600985446, st_ctime=1681357099 /* 2023-04-13T04:38:19.600985446+0100 */, st_ctime_nsec=600985446}, 0) = 0 <0.000021>
14:40:41.054513 getrandom("\x7d\xe1\x15\xfa\x80\xf6\xc4\xeb\x30\xff\xe1\x1a\x32\x1e\xd8\x1c", 16, GRND_INSECURE) = 16 <0.000013>
14:40:41.058269 gettid() = 2268 <0.000016>
14:40:41.058360 epoll_wait(4<anon_inode:[eventpoll]>, [{events=EPOLLIN, data={u32=2321867760, u64=94577501725680}}], 86, 0) = 1 <0.000041>
14:40:41.058501 recvmsg(15<NETLINK:[KOBJECT_UEVENT:2268]>, {msg_name={sa_family=AF_NETLINK, nl_pid=308024, nl_groups=0x000002}, msg_namelen=128 => 12, msg_iov=[{iov_base=[{prefix="libudev", magic=htonl(0xfeedcafe), header_size=40, properties_off=40, properties_len=947, filter_subsystem_hash=htonl(0xf0031db7), filter_devtype_hash=htonl(0x7bcbc5ee), filter_tag_bloom_hi=htonl(0x2000400), filter_tag_bloom_lo=htonl(0x10800000)}, "UDEV_DATABASE_VERSION=1\0ACTION=c"...], iov_len=8192}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_SOCKET, cmsg_type=SCM_CREDENTIALS, cmsg_data={pid=356244, uid=0, gid=0}}], msg_controllen=32, msg_flags=0}, 0) = 987 <0.000022>
14:40:41.058701 getrandom("\x80\x1d\x29\x7d\x8b\x2e\x72\xd8\x5d\x7b\xe6\x05\xd5\xb9\x02\xcc", 16, GRND_INSECURE) = 16 <0.000016>
14:40:41.058790 getrandom("\x23\xff\x23\x09\x75\xf1\x58\x97\xf7\xe3\xbf\x89\x57\x27\x65\x44", 16, GRND_INSECURE) = 16 <0.000011>
14:40:41.058905 getrandom("\x89\xe9\x02\x45\x62\x60\x97\x26\x1c\x1a\x20\x9a\xe0\xd5\x18\xc3", 16, GRND_INSECURE) = 16 <0.000012>
14:40:48.480425 newfstatat(AT_FDCWD</>, "/dev/disk/by-path/pci-0000:00:1d.0-usb-0:1.6:1.0-scsi-0:0:0:0", {st_dev=makedev(0, 0x5), st_ino=363, st_mode=S_IFBLK|0660, st_nlink=1, st_uid=0, st_gid=6, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x8, 0x10), st_atime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_atime_nsec=277691097, st_mtime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_mtime_nsec=277691097, st_ctime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_ctime_nsec=277691097}, 0) = 0 <0.000216>
14:40:48.481153 newfstatat(AT_FDCWD</>, "/dev/disk/by-id/usb-Generic-_Multi-Card_20090516388200000-0:0", {st_dev=makedev(0, 0x5), st_ino=363, st_mode=S_IFBLK|0660, st_nlink=1, st_uid=0, st_gid=6, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x8, 0x10), st_atime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_atime_nsec=277691097, st_mtime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_mtime_nsec=277691097, st_ctime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_ctime_nsec=277691097}, 0) = 0 <0.000053>
14:40:48.481384 newfstatat(AT_FDCWD</>, "/dev/disk/by-diskseq/18087", {st_dev=makedev(0, 0x5), st_ino=363, st_mode=S_IFBLK|0660, st_nlink=1, st_uid=0, st_gid=6, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x8, 0x10), st_atime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_atime_nsec=277691097, st_mtime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_mtime_nsec=277691097, st_ctime=1681479637 /* 2023-04-14T14:40:37.277691097+0100 */, st_ctime_nsec=277691097}, 0) = 0 <0.000065>
14:40:48.481599 newfstatat(AT_FDCWD</>, "/home/ubuntu/.config/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000043>
14:40:48.481703 newfstatat(AT_FDCWD</>, "/etc/xdg/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=18352495, st_mode=S_IFDIR|0755, st_nlink=7, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_mtime_nsec=407382145, st_ctime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_ctime_nsec=407382145}, 0) = 0 <0.000047>
14:40:48.481829 newfstatat(AT_FDCWD</>, "/etc/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=18352495, st_mode=S_IFDIR|0755, st_nlink=7, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=8, st_size=4096, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_mtime_nsec=407382145, st_ctime=1677139713 /* 2023-02-23T09:08:33.407382145+0100 */, st_ctime_nsec=407382145}, 0) = 0 <0.000041>
14:40:48.481936 newfstatat(AT_FDCWD</>, "/run/user/1000/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000060>
14:40:48.482063 newfstatat(AT_FDCWD</>, "/run/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000023>
14:40:48.482138 newfstatat(AT_FDCWD</>, "/home/ubuntu/.local/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000072>
14:40:48.482267 newfstatat(AT_FDCWD</>, "/var/lib/flatpak/exports/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000036>
14:40:48.482363 newfstatat(AT_FDCWD</>, "/usr/local/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000019>
14:40:48.482453 newfstatat(AT_FDCWD</>, "/usr/share/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000013>
14:40:48.482529 newfstatat(AT_FDCWD</>, "/usr/local/lib/systemd/user", 0x7fff5c2fe3a0, 0) = -1 ENOENT (Aucun fichier ou dossier de ce type) <0.000019>
14:40:48.482615 newfstatat(AT_FDCWD</>, "/usr/lib/systemd/user", {st_dev=makedev(0x8, 0x1), st_ino=8259220, st_mode=S_IFDIR|0755, st_nlink=14, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=48, st_size=20480, st_atime=1681357107 /* 2023-04-13T04:38:27.861624256+0100 */, st_atime_nsec=861624256, st_mtime=1681357099 /* 2023-04-13T04:38:19.600985446+0100 */, st_mtime_nsec=600985446, st_ctime=1681357099 /* 2023-04-13T04:38:19.600985446+0100 */, st_ctime_nsec=600985446}, 0) = 0 <0.000017>
14:40:48.483185 getrandom("\x93\xf8\x80\xf8\xaa\xc4\x0a\x50\x17\x69\xb1\x12\xd1\x21\x5d\xb0", 16, GRND_INSECURE) = 16 <0.000034>
14:40:48.487005 gettid() = 2268 <0.000018>
14:40:48.487095 epoll_wait(4<anon_inode:[eventpoll]>, [], 86, 0) = 0 <0.000069>
14:40:48.630122 close(22<pipe:[3287284]>) = 0 <0.000142>
14:40:48.630574 close(23<pipe:[3287284]>) = 0 <0.000055>
14:40:48.630738 close(24<pipe:[3287286]>) = 0 <0.000019>
14:40:48.630818 close(25<pipe:[3287286]>) = 0 <0.000022>
14:40:48.630906 gettid() = 2268 <0.000013>
14:40:48.630973 epoll_wait(4<anon_inode:[eventpoll]>, ^Cstrace: Process 2268 detached
<detached ...>
Thanks.
