How to whitelist IPs in NGINX configuration?

if ($remote_addr = "14.51.20.3") {
  set $block_me_now "B";
}
 
 if ($remote_addr = "104.1.20.29") {
  set $block_me_now "B";
}

if ($block_me_now != "B") {
    return 403;
    break;
}