How do I allow git HTTPS cloning without a password but require one for pushing?

I am using nginx, and my current server configuration is as follows:

server {
    listen 443 ssl;
    listen [::]:443 ssl;

    server_name  git.wozzes.me;

    ssl_certificate           /etc/letsencrypt/live/wozzes.me/fullchain.pem;
    ssl_certificate_key       /etc/letsencrypt/live/wozzes.me/privkey.pem;
    ssl_protocols             TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
    ssl_prefer_server_ciphers on;
    ssl_ciphers               'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';

    # static repo files for cloning over https
    location ~ ^.*\.git/objects/([0-9a-f]+/[0-9a-f]+|pack/pack-[0-9a-f]+.(pack|idx))$ {
        root /var/lib/git/repositories/;
    }

    # requests that need to go to git-http-backend
    location ~ ^.*\.git/(HEAD|info/refs|objects/info/.*|git-(upload|receive)-pack)$ {
        root /var/lib/git/repositories/;

        fastcgi_pass  unix:/var/run/fcgiwrap/fcgiwrap.sock;
        fastcgi_param SCRIPT_FILENAME   /usr/libexec/git-core/git-http-backend;
        fastcgi_param PATH_INFO         $uri;
        fastcgi_param GIT_PROJECT_ROOT  $document_root;
        fastcgi_param GIT_HTTP_EXPORT_ALL "";
        fastcgi_param REMOTE_USER $remote_user;
        include fastcgi_params;
    }

    try_files $uri @cgit;
    location @cgit {
        fastcgi_pass  unix:/var/run/fcgiwrap/fcgiwrap.sock;
        fastcgi_param SCRIPT_FILENAME   /usr/share/webapps/cgit/cgit.cgi;
        fastcgi_param PATH_INFO         $uri;
        include fastcgi_params;
   }
}

This configuration allows me to clone repositories without using a password, but it also allows users to push to my repositories freely, which is not what I want. If I add an auth section to the git-http-backend part it also starts requiring a password for regular cloning.

How do I allow unauthenticated cloning but require a password for pushing?