Hello and thank you for your answer.
I've adjusted my settings according to your advice, but now it looks like
it just directly whitelist every connection without greylisting.
smtp$ sudo spamdb | sort
WHITE|104.47.1.210|||1528919648|1528919648|1532030048|1|0
WHITE|104.47.6.201|||1528919611|1528919611|1532030011|1|0
WHITE|185.234.216.189|||1528917936|1528917936|1532029991|1|3
WHITE|185.234.216.204|||1528919598|1528919598|1532029998|1|0
WHITE|209.85.213.46|||1528918933|1528918933|1532029333|1|0
WHITE|209.85.213.53|||1528918873|1528918873|1532029273|1|0
WHITE|40.92.67.106|||1528918696|1528918696|1532029096|1|0
WHITE|40.92.68.98|||1528918725|1528918725|1532029125|1|0
WHITE|59.70.207.21|||1528918455|1528918455|1532028855|1|0
WHITE|91.121.119.198|||1528919326|1528919326|1532029726|1|0
WHITE|91.136.10.81|||1528919583|1528919583|1532029983|1|0
This is how my files look like now. spamd.conf is the original one.
smtp$ sudo cat /etc/rc.conf.local
httpd_flags=
pkg_scripts=postfix dovecot saslauthd dbus_daemon avahi_daemon messagebus
mysqld php70_fpm
smtpd_flags=NO
unbound_flags=
spamd_flags="-v -G 2:4:864"
spamd_grey=YES
spamlogd_flags="-I"
-----------------------------
smtp$ sudo cat /etc/pf.conf
ext_if = "em0"
int_if = "fxp0"
localnet = $int_if:network
tcp_services = "{ domain, ntp, imap, imaps, pop3, pop3s }"
mail_services = "{ smtp, smtps, submission }"
udp_services = "{ domain, ntp }"
icmp_types = "echoreq"
table <martians> { 0.0.0.0/8 10.0.0.0/8 127.0.0.0/8 169.254.0.0/16 \
172.16.0.0/12 192.0.0.0/24 192.0.2.0/24 224.0.0.0/3 \
192.168.0.0/16 198.18.0.0/15 198.51.100.0/24 \
203.0.113.0/24 }
table <bruteforce> persist
table <abusers> persist file "/etc/abusers"
table <spamd-white> persist
table <nospamd> persist file "/etc/mail/nospamd"
set block-policy drop
set loginterface egress
set skip on lo0
match in all scrub (no-df random-id max-mss 1440)
match out on egress inet from !(egress:network) to any nat-to (egress:0)
antispoof quick for { egress $ext_if int_if }
block in quick on egress from <martians> to any
block return out quick on egress from any to <martians>
block in quick log on egress from <abusers> to any label "abusers"
block all
pass out quick inet
pass in on egress inet proto tcp from any to any port smtp \
divert-to 127.0.0.1 port spamd
pass in on egress proto tcp from <nospamd> to any port smtp
pass in log on egress proto tcp from <spamd-white> to any port smtp
pass out log on egress proto tcp to any port smtp
pass in on { $ext_if } inet
pass log quick proto tcp from any to (egress) port ssh flags S/SA keep state \
(max-src-conn 15, max-src-conn-rate 5/3, overload <bruteforce> flush
global)
pass log quick proto tcp from any to (egress) port $tcp_services flags S/SA
keep state \
(max-src-conn 50, max-src-conn-rate 15/5, overload <bruteforce> flush
global)
pass log quick proto tcp from any to (egress) port $mail_services flags S/SA
keep state \
(max-src-conn 50, max-src-conn-rate 25/5, overload <bruteforce> flush
global)
pass in on egress inet proto tcp from any to (egress) port { 80 443 }
pass inet proto tcp from { self, $localnet }
pass quick inet proto tcp to port $tcp_services keep state
pass quick inet proto tcp to port $mail_services keep state
pass quick inet proto udp to port $udp_services keep state
pass out on $ext_if inet proto udp to port 33433 >< 33626
pass inet proto icmp all icmp-type $icmp_types
