kernel: rc rc0: receive overflow

My Ubuntu server 20.04 keeps freezing after a few hours of operation to the point where the fan is running hot and I can't even establish a connection to ssh to it.

After rebooting it behaves normally with no CPU load spikes or memory issues. The server is a fresh install and only really running an empty MongoDB database.

I looked at the kern.log and found it flooded with these logs at the time before the reboot.

Dec 10 21:41:05 backend kernel: [102373.982171] rc rc0: receive overflow
Dec 10 21:41:05 backend kernel: [102373.986010] rc rc0: receive overflow

It just started happening again and I got a chance to see which processes are running hot.

  /usr/sbin/rsyslogd -n -iNONE
  /lib/systemd/systemd-journald
  /usr/lib/accountsservice/accounts-daemon

Do you have any idea why this keeps happening?

I've had the same problem on an Intel NUC and strace-ing the process revealed a couple of interesting files:

openat(AT_FDCWD, "/run/log/journal/0322fcd160934520a68be3469a358ed3/system.journal", O_RDWR|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)
gettid()                                = 393
timerfd_settime(17, TFD_TIMER_ABSTIME, {it_interval={tv_sec=0, tv_nsec=0}, it_value={tv_sec=13887, tv_nsec=175660000}}, NULL) = 0
epoll_wait(8, [], 62, 0)                = 0
ftruncate(37, 16777216)                 = 0
gettid()                                = 393
epoll_wait(8, [{events=EPOLLIN, data={u32=3603220400, u64=94565898176432}}], 62, -1) = 1
read(9, "4,1594792,13834610419,-;rc rc0: "..., 8192) = 79
access("/sys/subsystem/rc/devices/rc0", F_OK) = -1 ENOENT (No such file or directory)
access("/sys/bus/rc/devices/rc0", F_OK) = -1 ENOENT (No such file or directory)
access("/sys/class/rc/rc0", F_OK)       = 0
openat(AT_FDCWD, "/", O_RDONLY|O_CLOEXEC|O_PATH|O_DIRECTORY) = 23
openat(23, "sys", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0555, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "class", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "rc", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "rc0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFLNK|0777, st_size=0, ...}, AT_EMPTY_PATH) = 0
readlinkat(36, "rc0", "../../devices/pnp0/00:01/rc/rc0", 4096) = 31
close(23)                               = 0
openat(36, "..", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
close(36)                               = 0
openat(23, "..", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
close(23)                               = 0
openat(36, "devices", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "pnp0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "00:01", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
openat(23, "rc", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 36
newfstatat(36, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(23)                               = 0
openat(36, "rc0", O_RDONLY|O_NOFOLLOW|O_CLOEXEC|O_PATH) = 23
newfstatat(23, "", {st_mode=S_IFDIR|0755, st_size=0, ...}, AT_EMPTY_PATH) = 0
close(36)                               = 0
close(23)                               = 0
access("/sys/devices/pnp0/00:01/rc/rc0/uevent", F_OK) = 0
openat(AT_FDCWD, "/sys/devices/pnp0/00:01/rc/rc0/uevent", O_RDONLY|O_CLOEXEC) = 23
newfstatat(23, "", {st_mode=S_IFREG|0644, st_size=4096, ...}, AT_EMPTY_PATH) = 0
read(23, "NAME=rc-rc6-mce\nDRV_NAME=ite-cir"..., 4104) = 66
close(23)                               = 0
readlinkat(AT_FDCWD, "/sys/devices/pnp0/00:01/rc/rc0/subsystem", "../../../../../class/rc", 4096) = 23
openat(AT_FDCWD, "/run/udev/data/+rc:rc0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/run/log/journal/0322fcd160934520a68be3469a358ed3/system.journal", O_RDWR|O_NONBLOCK|O_CLOEXEC) = -1 ENOENT (No such file or directory)

And looking at these files gave me a way to reproduce it:

marius@nuc:~$ cat /sys/devices/pnp0/00\:01/rc/rc0/input4/name
ITE8708 CIR transceiver
marius@nuc:~$ cat /sys/devices/pnp0/00\:01/rc/rc0/protocols
rc-5 nec [rc-6] jvc sony rc-5-sz sanyo sharp mce_kbd xmp imon rc-mm [lirc]

The NUC sits beneath a TV, it has an IR receiver(had no idea about that) and it captures the signal from the TV remote. Pressing any button on the remote would trigger this problem.

I found the kernel module responsible for this using lsmod and disabled it with sudo modprobe -r ite_cir. This fix is temporary, I'll have to look into blacklisting the module.

Some methods I used for high CPU load issues on the NUC which worked.

#SOLUTION 1
sudo vim /etc/default/grub
# Add the following parameter to the GRUB_CMDLINE_LINUX_DEFAULT property
pci=nomsi
# And try also addin this parameter to the GRUB_CMDLINE_LINUX_DEFAULT and GRUB_CMDLINE_LINUX properties
pcie_aspm=off 
#
sudo update-grub
sudo reboot

#SOLUTION 2 - Update the bios firmware
wget https://downloadmirror.intel.com/758735/BN0089.bio 
sudo cp BN0089.bio /boot/
sudo cp BN0089.bio /root/
sudo reboot

#SOLUTION 3 (Usually this fixes the problem)
# Disable IR kernel module
sudo modprobe -r ite_cir
# Check journal to make sure 'kernel: rc rc0: receive overflow' message is gone
sudo journalctl -f
# If disabling the IR module resolved your issue, you have to blacklist it to make it permanent.  
# Edit this file: /etc/modprobe.d/blacklist.conf
sudo vi /etc/modprobe.d/blacklist.conf
# Add to it the following line:
blacklist ite_cir
# Update initramfs
sudo update-initramfs -u