I have a AWS t3.micro 2GB 20.04 machine, that i want to use for Netbox software. I created a RDS instance in AWS, so no DB problem on computing, back-end side.
When i complete all steps in the Netbox guide with Nginx and Python3, i'm unable to use VM in no way, no SSH, console etc.
I connected AWS VM serial console port with Putty, i can send break command and sysrq keys, no problem until here.
Both while rebooting and sysrq Out Of Memory task kill, i can see python3 in Out of memory. First thing i don't understand how can be a lightweight software like Netbox consume high memory? It's not normal i guess.
Second, even sysrq kills OOM python3, i can't get list of CPU traceback with sysrq. Waited for almost 10 minutes, no way.
Where we can start to troubleshoot?
Thanks
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# ps -o pid,user,%mem,command ax | sort -b -k3 -r
[38185.422338] sysrq: HELP : loglevel(0-9) reboot(b) crash(c) terminate-all-task s(e) memory-full-oom-kill(f) kill-all-tasks(i) thaw-filesystems(j) sak(k) show-b acktrace-all-active-cpus(l) show-memory-usage(m) nice-all-RT-tasks(n) poweroff(o ) show-registers(p) show-all-timers(q) unraw(r) sync(s) show-task-states(t) unmo unt(u) show-blocked-tasks(w) dump-ftrace-buffer(z)
[38271.650186] sysrq: Show Memory
330 root 2.3 /sbin/multipathd -d -s
1452 netbox 15.4 /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker high default low
1336 netbox 10.8 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
1335 netbox 10.8 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
862 netbox 10.8 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
863 netbox 10.7 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
860 netbox 10.7 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
639 root 1.9 /usr/lib/snapd/snapd
584 netbox 1.9 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/ gunicorn.py netbox.wsgi
1718 root 1.6 apt-get check -qq
767 root 1.3 /usr/bin/python3 /usr/share/unattended-upgrades/unattended -upgrade-shutdown --wait-for-signal
585 root 1.3 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startu p-triggers
865 root 0.9 /snap/amazon-ssm-agent/6312/ssm-agent-worker
456 systemd+ 0.8 /lib/systemd/systemd-resolved
611 root 0.6 /snap/amazon-ssm-agent/6312/amazon-ssm-agent
1779 root 0.6 /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py h ousekeeping
1 root 0.5 /sbin/init
791 root 0.4 sshd: /usr/sbin/sshd -D -o AuthorizedKeysCommand /usr/shar e/ec2-instance-connect/eic_run_authorized_keys %u %f -o AuthorizedKeysCommandUse r ec2-instance-connect [listener] 0 of 10-100 startups
723 www-data 0.4 nginx: worker process
722 www-data 0.4 nginx: worker process
764 root 0.4 /usr/sbin/ModemManager
203 root 0.4 /lib/systemd/systemd-udevd
453 systemd+ 0.4 /lib/systemd/systemd-networkd
165 root 0.4 /lib/systemd/systemd-journald
1000 root 0.4 /lib/systemd/systemd --user
1006 root 0.4 -bash
1001 root 0.4 (sd-pam)
1861 root 0.3 ps -o pid,user,%mem,command ax
499 root 0.3 /usr/sbin/cron -f
642 root 0.3 /usr/lib/udisks2/udisksd
737 redis 0.3 /usr/bin/redis-server 127.0.0.1:6379
500 message+ 0.3 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
420 systemd+ 0.3 /lib/systemd/systemd-timesyncd
640 root 0.3 /lib/systemd/systemd-logind
610 syslog 0.2 /usr/sbin/rsyslogd -n -iNONE
521 root 0.2 /usr/sbin/irqbalance --foreground
671 daemon 0.2 /usr/sbin/atd -f
1723 root 0.2 /usr/sbin/CRON -f
604 root 0.2 /usr/lib/policykit-1/polkitd --no-debug
837 root 0.2 /sbin/agetty -o -p -- \u --noclear tty1 linux
1700 root 0.2 /bin/sh /usr/lib/apt/apt.systemd.daily lock_is_held instal l
1696 root 0.2 /bin/sh /usr/lib/apt/apt.systemd.daily install
816 root 0.2 /bin/login -p --
1862 root 0.1 sort -b -k3 -r
1725 root 0.1 run-parts --report /etc/cron.daily
711 root 0.1 nginx: master process /usr/sbin/nginx -g daemon on; master _process on;
491 root 0.1 /usr/lib/accountsservice/accounts-daemon
116 root 0.0 [zswap-shrink]
30 root 0.0 [writeback]
87 root 0.0 [watchdogd]
101 root 0.0 [vfio-irqfd-clea]
82 root 0.0 [tpm_dev_wq]
11 root 0.0 [rcu_tasks_trace]
10 root 0.0 [rcu_tasks_rude_]
13 root 0.0 [rcu_sched]
4 root 0.0 [rcu_par_gp]
3 root 0.0 [rcu_gp]
29 root 0.0 [oom_reaper]
97 root 0.0 [nvme-wq]
98 root 0.0 [nvme-reset-wq]
99 root 0.0 [nvme-delete-wq]
5 root 0.0 [netns]
9 root 0.0 [mm_percpu_wq]
103 root 0.0 [mld]
20 root 0.0 [migration/1]
14 root 0.0 [migration/0]
84 root 0.0 [md]
117 root 0.0 [kworker/u5:0]
1858 root 0.0 [kworker/u4:3-writeback]
1839 root 0.0 [kworker/u4:1+loop1]
1852 root 0.0 [kworker/u4:0-writeback]
693 root 0.0 [kworker/1:4-mm_percpu_wq]
100 root 0.0 [kworker/1:1H-events_highpri]
1333 root 0.0 [kworker/1:1-events]
23 root 0.0 [kworker/1:0H-events_highpri]
126 root 0.0 [kworker/0:2-events]
89 root 0.0 [kworker/0:1H-kblockd]
1417 root 0.0 [kworker/0:1-events]
7 root 0.0 [kworker/0:0H-events_highpri]
95 root 0.0 [kthrotld]
2 root 0.0 [kthreadd]
92 root 0.0 [kswapd0]
113 root 0.0 [kstrp]
21 root 0.0 [ksoftirqd/1]
12 root 0.0 [ksoftirqd/0]
32 root 0.0 [ksmd]
328 root 0.0 [kmpathd]
327 root 0.0 [kmpath_rdacd]
329 root 0.0 [kmpath_handlerd]
79 root 0.0 [kintegrityd]
28 root 0.0 [khungtaskd]
33 root 0.0 [khugepaged]
24 root 0.0 [kdevtmpfs]
31 root 0.0 [kcompactd0]
80 root 0.0 [kblockd]
26 root 0.0 [kauditd]
326 root 0.0 [kaluad]
123 root 0.0 [jbd2/nvme0n1p1-]
104 root 0.0 [ipv6_addrconf]
25 root 0.0 [inet_frag_wq]
19 root 0.0 [idle_inject/1]
15 root 0.0 [idle_inject/0]
124 root 0.0 [ext4-rsv-conver]
214 root 0.0 [ena]
85 root 0.0 [edac-poller]
93 root 0.0 [ecryptfs-kthrea]
86 root 0.0 [devfreq_wq]
283 root 0.0 [cryptd]
18 root 0.0 [cpuhp/1]
17 root 0.0 [cpuhp/0]
122 root 0.0 [charger_manager]
81 root 0.0 [blkcg_punt_bio]
83 root 0.0 [ata_sff]
96 root 0.0 [acpi_thermal_pm]
492 root 0.0 /usr/sbin/acpid
1778 root 0.0 /bin/sh /etc/cron.daily/netbox-housekeeping
1724 root 0.0 /bin/sh -c test -x /usr/sbin/anacron || ( cd / && run-part s --report /etc/cron.daily )
PID USER %MEM COMMAND
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# [38805.151817] sysrq: Show backtrace of all active CPUs
root@ip-172-31-45-233:~# [40221.971222] sysrq: Manual OOM execution
[40221.975154] Out of memory: Killed process 1452 (python3) total-vm:243236kB, anon-rss:117172kB, file-rss:1196kB, shmem-rss:0kB, UID:115 pgtables:364kB oom_score_adj:0
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# ^C
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# [40380.324029] sysrq: Show Memory
root@ip-172-31-45-233:~# ps -o pid,user,%mem,command ax | sort -b -k3 -r
1913 root 3.2 /usr/bin/python3 /usr/bin/unattended-upgrade
1922 netbox 3.0 /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker high default low
330 root 2.3 /sbin/multipathd -d -s
639 root 2.0 /usr/lib/snapd/snapd
584 netbox 2.0 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
1336 netbox 10.9 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
863 netbox 10.9 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
862 netbox 10.9 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
1779 root 10.9 /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py housekeeping
1335 netbox 10.8 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
860 netbox 10.8 /opt/netbox/venv/bin/python3 /opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi
767 root 1.3 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal
585 root 1.3 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
865 root 0.9 /snap/amazon-ssm-agent/6312/ssm-agent-worker
456 systemd+ 0.8 /lib/systemd/systemd-resolved
611 root 0.6 /snap/amazon-ssm-agent/6312/amazon-ssm-agent
1 root 0.6 /sbin/init
453 systemd+ 0.5 /lib/systemd/systemd-networkd
791 root 0.4 sshd: /usr/sbin/sshd -D -o AuthorizedKeysCommand /usr/share/ec2-instance-connect/eic_run_authorized_keys %u %f -o AuthorizedKeysCommandUser ec2-instance-connect [listener] 0 of 10-100 startups
723 www-data 0.4 nginx: worker process
722 www-data 0.4 nginx: worker process
764 root 0.4 /usr/sbin/ModemManager
737 redis 0.4 /usr/bin/redis-server 127.0.0.1:6379
500 message+ 0.4 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
165 root 0.4 /lib/systemd/systemd-journald
1000 root 0.4 /lib/systemd/systemd --user
1006 root 0.4 -bash
1001 root 0.4 (sd-pam)
1931 root 0.3 ps -o pid,user,%mem,command ax
491 root 0.3 /usr/lib/accountsservice/accounts-daemon
203 root 0.3 /lib/systemd/systemd-udevd
420 systemd+ 0.3 /lib/systemd/systemd-timesyncd
640 root 0.3 /lib/systemd/systemd-logind
610 syslog 0.2 /usr/sbin/rsyslogd -n -iNONE
521 root 0.2 /usr/sbin/irqbalance --foreground
499 root 0.2 /usr/sbin/cron -f
671 daemon 0.2 /usr/sbin/atd -f
1723 root 0.2 /usr/sbin/CRON -f
642 root 0.2 /usr/lib/udisks2/udisksd
837 root 0.2 /sbin/agetty -o -p -- \u --noclear tty1 linux
1700 root 0.2 /bin/sh /usr/lib/apt/apt.systemd.daily lock_is_held install
1696 root 0.2 /bin/sh /usr/lib/apt/apt.systemd.daily install
816 root 0.2 /bin/login -p --
1932 root 0.1 sort -b -k3 -r
1725 root 0.1 run-parts --report /etc/cron.daily
711 root 0.1 nginx: master process /usr/sbin/nginx -g daemon on; master_process on;
604 root 0.1 /usr/lib/policykit-1/polkitd --no-debug
116 root 0.0 [zswap-shrink]
30 root 0.0 [writeback]
87 root 0.0 [watchdogd]
101 root 0.0 [vfio-irqfd-clea]
82 root 0.0 [tpm_dev_wq]
11 root 0.0 [rcu_tasks_trace]
10 root 0.0 [rcu_tasks_rude_]
13 root 0.0 [rcu_sched]
4 root 0.0 [rcu_par_gp]
3 root 0.0 [rcu_gp]
29 root 0.0 [oom_reaper]
97 root 0.0 [nvme-wq]
98 root 0.0 [nvme-reset-wq]
99 root 0.0 [nvme-delete-wq]
5 root 0.0 [netns]
9 root 0.0 [mm_percpu_wq]
103 root 0.0 [mld]
20 root 0.0 [migration/1]
14 root 0.0 [migration/0]
84 root 0.0 [md]
117 root 0.0 [kworker/u5:0]
1880 root 0.0 [kworker/u4:2+loop1]
1876 root 0.0 [kworker/u4:1-events_unbound]
1926 root 0.0 [kworker/u4:0-events_power_efficient]
693 root 0.0 [kworker/1:4-events]
100 root 0.0 [kworker/1:1H-events_highpri]
23 root 0.0 [kworker/1:0H-events_highpri]
1885 root 0.0 [kworker/1:0-events]
89 root 0.0 [kworker/0:1H-kblockd]
1896 root 0.0 [kworker/0:1-events]
7 root 0.0 [kworker/0:0H-events_highpri]
1872 root 0.0 [kworker/0:0-events]
95 root 0.0 [kthrotld]
2 root 0.0 [kthreadd]
92 root 0.0 [kswapd0]
113 root 0.0 [kstrp]
21 root 0.0 [ksoftirqd/1]
12 root 0.0 [ksoftirqd/0]
32 root 0.0 [ksmd]
328 root 0.0 [kmpathd]
327 root 0.0 [kmpath_rdacd]
329 root 0.0 [kmpath_handlerd]
79 root 0.0 [kintegrityd]
28 root 0.0 [khungtaskd]
33 root 0.0 [khugepaged]
24 root 0.0 [kdevtmpfs]
31 root 0.0 [kcompactd0]
80 root 0.0 [kblockd]
26 root 0.0 [kauditd]
326 root 0.0 [kaluad]
123 root 0.0 [jbd2/nvme0n1p1-]
104 root 0.0 [ipv6_addrconf]
25 root 0.0 [inet_frag_wq]
19 root 0.0 [idle_inject/1]
15 root 0.0 [idle_inject/0]
124 root 0.0 [ext4-rsv-conver]
214 root 0.0 [ena]
85 root 0.0 [edac-poller]
93 root 0.0 [ecryptfs-kthrea]
86 root 0.0 [devfreq_wq]
283 root 0.0 [cryptd]
18 root 0.0 [cpuhp/1]
17 root 0.0 [cpuhp/0]
122 root 0.0 [charger_manager]
81 root 0.0 [blkcg_punt_bio]
83 root 0.0 [ata_sff]
96 root 0.0 [acpi_thermal_pm]
492 root 0.0 /usr/sbin/acpid
1778 root 0.0 /bin/sh /etc/cron.daily/netbox-housekeeping
1724 root 0.0 /bin/sh -c test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )
PID USER %MEM COMMAND
root@ip-172-31-45-233:~# [41594.823220] sysrq: Show backtrace of all active CPUs
root@ip-172-31-45-233:~# # ps -ef | egrep -v "STIME|$LOGNAME" | sort +3 -r | head -n 15
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# # ps -ef | egrep -v "STIME|$LOGNAME" | sort +3 -r | head -n 15
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# ps
PID TTY TIME CMD
816 ttyS0 00:00:00 login
1006 ttyS0 00:00:01 bash
1934 ttyS0 00:00:00 ps
root@ip-172-31-45-233:~#
troot@ip-172-31-45-233:~#
root@ip-172-31-45-233:~# top
top - 12:07:47 up 11:38, 1 user, load average: 20.88, 20.10, 19.60
Tasks: 117 total, 1 running, 116 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.1 us, 4.5 sy, 0.0 ni, 0.0 id, 91.5 wa, 0.0 hi, 0.0 si, 3.8 st
MiB Mem : 749.2 total, 40.5 free, 667.7 used, 41.0 buff/cache
MiB Swap: 0.0 total, 0.0 free, 0.0 used. 8.5 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
92 root 20 0 0 0 0 S 11.6 0.0 65:08.23 kswapd0
865 root 20 0 1314548 7468 0 S 1.4 1.0 84:33.11 ssm-age+
611 root 20 0 1305364 5360 0 S 1.1 0.7 79:24.68 amazon-+
639 root 20 0 874880 16000 0 S 0.5 2.1 29:55.16 snapd
1935 root 20 0 11000 3384 2724 R 0.3 0.4 0:00.26 top
1 root 20 0 168656 6452 3056 D 0.2 0.8 1:05.95 systemd
165 root 19 -1 43964 4580 3140 D 0.2 0.6 0:07.11 systemd+
491 root 20 0 241060 1084 8 S 0.2 0.1 0:27.40 account+
737 redis 20 0 51708 2180 612 D 0.2 0.3 1:41.60 redis-s+
862 netbox 20 0 126588 83756 1992 D 0.2 10.9 0:24.72 gunicorn
1335 netbox 20 0 126588 82736 956 D 0.2 10.8 0:16.25 gunicorn
1336 netbox 20 0 126596 82988 1200 D 0.2 10.8 0:16.89 gunicorn
1779 root 20 0 131980 83100 808 D 0.2 10.8 0:28.10 python
1876 root 20 0 0 0 0 D 0.2 0.0 0:01.27 kworker+
1913 root 20 0 245280 25216 2412 D 0.2 3.3 0:03.28 unatten+
1922 netbox 20 0 63296 23324 2168 D 0.2 3.0 0:02.65 python3
2 root 20 0 0 0 0 S 0.0 0.0 0:00.06 kthreadd
