Nginx cache for frequent and different requests

I have a service. This service has a GET handle(/api/v1/retrieve_blocks). This handle's requests and responces are small, no more than 1024 characters.

regular situation:

• this handle is constantly under 500rps.
• Almost all requests are different by url's parameters. for example:
  host/api/v1/retrieve_blocks?entity_id=1111
  host/api/v1/retrieve_blocks?entity_id=2222
  host/api/v1/retrieve_blocks?entity_id=3333
  

During incidents:

• this handle could be under 4000rps.
• almost all aditional requests are repeated.

We decided to establish the nginx cache for stable operation during the incident. But even at 500 rps, in a normal situation, the disk I/O in our service is overloaded, which leads to an increase in handle timings.

The full nginx configs is following:

nginx: [warn] could not build optimal variables_hash, you should increase either variables_hash_max_size: 1024 or variables_hash_bucket_size: 64; ignoring variables_hash_bucket_size
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
user www-data;
worker_processes  4;
worker_rlimit_nofile 16384;
daemon off;

error_log  /var/log/nginx/error.log;
pid        /var/run/nginx.pid;

events {
    worker_connections  8192;
    use epoll;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    sendfile        on;
    tcp_nopush      on;

    keepalive_timeout  65;
    tcp_nodelay        on;
    
    
    server_names_hash_bucket_size 256;
    types_hash_bucket_size 64;
    map_hash_bucket_size 128;

    #Enable gzip
    gzip  on;
    gzip_disable    msie6;
    gzip_vary       on;
    gzip_proxied    any;
    gzip_min_length     1100;
    gzip_http_version   1.0;
    gzip_buffers        4 8k;
    gzip_comp_level     5;
    gzip_types          text/plain text/css application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json application/x-protobuf;

    #Increase fasctgi-buffers
    fastcgi_buffer_size 8192k;
    fastcgi_busy_buffers_size 16384k;
    fastcgi_buffers 8 8192k;
        

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

# configuration file /etc/nginx/mime.types:

types {
    text/html                             html htm shtml;
    text/css                              css;
    text/xml                              xml xsl;
    image/gif                             gif;
    image/jpeg                            jpeg jpg;
    application/javascript                js yate;
    application/atom+xml                  atom;
    application/rss+xml                   rss;

    text/mathml                           mml;
    text/plain                            txt;
    text/vnd.sun.j2me.app-descriptor      jad;
    text/vnd.wap.wml                      wml;
    text/x-component                      htc;

    image/png                             png;
    image/tiff                            tif tiff;
    image/vnd.wap.wbmp                    wbmp;
    image/x-icon                          ico;
    image/x-jng                           jng;
    image/x-ms-bmp                        bmp;
    image/svg+xml                         svg svgz;
    image/webp                            webp;

    application/font-woff                 woff;
    application/java-archive              jar war ear;
    application/json                      json jsx;
    application/mac-binhex40              hqx;
    application/msword                    doc;
    application/pdf                       pdf;
    application/postscript                ps eps ai;
    application/rtf                       rtf;
    application/vnd.apple.mpegurl         m3u8;
    application/vnd.ms-excel              xls;
    application/vnd.ms-fontobject         eot;
    application/vnd.ms-powerpoint         ppt;
    application/vnd.wap.wmlc              wmlc;
    application/vnd.google-earth.kml+xml  kml;
    application/vnd.google-earth.kmz      kmz;
    application/x-7z-compressed           7z;
    application/x-cocoa                   cco;
    application/x-java-archive-diff       jardiff;
    application/x-java-jnlp-file          jnlp;
    application/x-makeself                run;
    application/x-perl                    pl pm;
    application/x-pilot                   prc pdb;
    application/x-rar-compressed          rar;
    application/x-redhat-package-manager  rpm;
    application/x-sea                     sea;
    application/x-shockwave-flash         swf;
    application/x-stuffit                 sit;
    application/x-tcl                     tcl tk;
    application/x-x509-ca-cert            der pem crt;
    application/x-xpinstall               xpi;
    application/xhtml+xml                 xhtml;
    application/xspf+xml                  xspf;
    application/zip                       zip;

    application/octet-stream              bin exe dll;
    application/octet-stream              deb;
    application/octet-stream              dmg;
    application/octet-stream              iso img;
    application/octet-stream              msi msp msm;

    application/vnd.openxmlformats-officedocument.wordprocessingml.document    docx;
    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet          xlsx;
    application/vnd.openxmlformats-officedocument.presentationml.presentation  pptx;

    audio/midi                            mid midi kar;
    audio/mpeg                            mp3;
    audio/ogg                             ogg;
    audio/x-m4a                           m4a;
    audio/x-realaudio                     ra;

    video/3gpp                            3gpp 3gp;
    video/mp2t                            ts;
    video/mp4                             mp4;
    video/mpeg                            mpeg mpg;
    video/quicktime                       mov;
    video/webm                            webm;
    video/x-flv                           flv;
    video/x-m4v                           m4v;
    video/x-mng                           mng;
    video/x-ms-asf                        asx asf;
    video/x-ms-wmv                        wmv;
    video/x-msvideo                       avi;
}

# configuration file /etc/nginx/conf.d/01-accesslog-tskv.conf:
tskv_log_format access_log_tskv $http_x_remote_ip $http_x_yarequestid $upstream_http_x_yarequestid $http_host $remote_addr $request_time $upstream_response_time $upstream_cache_status $upstream_status $scheme $bytes_sent $args $ssl_session_id $ssl_protocol $ssl_cipher $ssl_handshake_time $upstream_http_x_yataxi_api_operationid $request_body;
tskv_log_format access_log_tskv_no_body $http_x_remote_ip $http_x_yarequestid $upstream_http_x_yarequestid $http_host $remote_addr $request_time $upstream_response_time $upstream_cache_status $upstream_status $scheme $bytes_sent $args $ssl_session_id $ssl_protocol $ssl_cipher $ssl_handshake_time $upstream_http_x_yataxi_api_operationid;
tskv_log_format access_log_tskv_no_body_ja3 $http_x_remote_ip $http_x_yarequestid $upstream_http_x_yarequestid $http_host $remote_addr $request_time $upstream_response_time $upstream_cache_status $upstream_status $scheme $bytes_sent $args $ssl_session_id $ssl_protocol $ssl_cipher $ssl_handshake_time $upstream_http_x_yataxi_api_operationid $http_x_riotech_ja3;
tskv_log syslog:server=127.0.0.1 access_log_tskv_no_body;

# configuration file /etc/nginx/conf.d/01-accesslog.conf:
log_format defaultformat '[$time_local] $http_host $remote_addr "$request" $status "$http_referer" "$http_user_agent" "$http_cookie" $request_time $upstream_cache_status $bytes_sent "$upstream_response_time" $request_length $msec "$upstream_http_x_yareqfinish" "$sent_http_x_yauuid" "$upstream_http_x_yarequestid" "$upstream_http_x_yamisc" "$http_X_MSISDN_h121ZY615d623L631O2L3;$http_HW_3GPP_RAT_Type;$http_X_SGSN_IP;$http_X_MegaFon_IMSI" $http_x_remote_ip $http_x_real_ip "$http_x_forwarded_for" "$upstream_http_x_yataxi_api_operationid"';
access_log      syslog:server=127.0.0.1  defaultformat;
error_log       syslog:server=127.0.0.1;

# configuration file /etc/nginx/conf.d/02-client-body-buffer-size.conf:
client_body_buffer_size 1m;

# configuration file /etc/nginx/sites-enabled/riotech-geo-dorblu-agent.conf:
server {
    listen [::]:5033 ipv6only=off default_server;

    location /dorblu-unistat {
        root /var/lib/riotech/dorblu-agent;
        try_files /yasm_stats.json =404;
    }
}

# configuration file /etc/nginx/sites-enabled/riotech-taxi-service-name:
upstream taxi_service_name_upstream {
    server unix:/var/lib/riotech/taxi-service-name/server.socket;
    keepalive 600;
}

proxy_cache_path /var/cache/riotech/taxi-service-name levels=1:2 keys_zone=qc_api_v1_blocks_zone:150m max_size=6g use_temp_path=off;

server {
    include listen;

    server_name service-name.taxi.dev.riotech.net;
    server_name service-name.taxi.tst.riotech.net;
    server_name service-name.taxi.riotech.net;

    client_max_body_size 3M;

    location / {
        proxy_http_version 1.1;
        proxy_pass http://taxi_service_name_upstream/;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header Connection "";
        proxy_ignore_client_abort on;
    }

location /api/v1/blocks {
        proxy_http_version 1.1;
        proxy_pass http://taxi_service_name_upstream/api/v1/blocks;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header Connection "";
        proxy_ignore_client_abort on;

        sendfile on;

        proxy_cache qc_api_v1_blocks_zone;
        proxy_cache_key $scheme$request_method$proxy_host$request_uri;
        proxy_cache_valid 200 1m;
        add_header X-Cache-Status $upstream_cache_status;
    }
}


# configuration file /etc/nginx/listen:
listen [::]:80;
listen [::]:8179;
listen [::]:8180;
include locations/404-any;
include locations/500-html;

# configuration file /etc/nginx/locations/404-any:
location @404 {
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header Host $host;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-riotech-NotFound-Project "autodetect";
    if ( $http_accept = application/json ) {
    set $custom_error_page "on";
    }
    if ( $server_port ~* '^(8079|8179)$' ) {
    set $custom_error_page "on";
    }
    if ( $host ~* "riotech.net" ) {
    set $custom_error_page "on";
    }
    if ( $custom_error_page !~ "on" ) {
    proxy_pass http://any.riotech.ru;
    }
}
error_page 404 @404;

# configuration file /etc/nginx/locations/500-html:
error_page 500 502 503 /500.html;
location = /500.html {
    root /usr/local/www/common/error-handlers;
}

Server's OS is Ubuntu 16.04.1 LTS (Xenial Xerus) It has about 20 GB of free RAM; And only one disk which is ssd.

I will be very grateful for any advice and any information about the problem. Concrete questions:

1. Globally, what am I doing wrong?
2. Is nginx suitable for such purposes?
3. Can disabling nginx logs help?
4. Now I'm thinking about mounting the cache folder to RAM. Norm idea?

Thank you for your attention)

One thing that can be optimized is to change gzip compression level to 1. It doesn't change disk I/O, but it decreases CPU usage.

The tskv logging could be one reason for bad performance. I have no experience with that module, so I don't know how well-written it is.

For nginx normal logging, it is useful either to disable it, or make nginx write it to disk in batches:

access_log /path/to/access.log combined buffer=8k flush=1m;

This tells nginx to collect logs in memory for up to 1 megabyte, before it is written to disk.

Also, since your log target seems to be syslog, you might want to look into how syslog flushes its log files.