This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-jspf.git
commit 336b2eea77f375236aee7f54c01abb674ae98322 Author: Benoit TELLIER <[email protected]> AuthorDate: Thu Nov 19 17:41:39 2026 +0100 [PERF] SPF: keep the parsed records instead of parsing them on every check Every check parsed again the text of each record it retrieved, the domain's and each include's or redirect's: term regex, reflective creation and wiring of each term, parse of each ip4/ip6 network. Up to 77% of the CPU of a check, 50-59% of the others. A provider include tree, the same for most of the mail a server receives, was parsed again for each MAIL FROM. CachingSPFRecordParser keeps the parsed records, keyed by their text: a changed DNS record is a new key, so no expiry is needed. The SPF constructors that build their parser use it. - The terms of a kept record are shared by concurrent checks: they keep only their configuration, set when parsed, and the state of a check lives in its SPFSession or in the checkers created per check. A test runs concurrent checks of the same records, synchronous and asynchronous, against their sequential results. - Each call returns a new SPF1Record holding the shared terms: the policies add modifiers to the record they get. - The records are the sender's: at most 1000 are kept, the least recently used dropped, and neither those longer than 2048 characters nor the invalid ones, which are parsed as before. The SPF constructor taking a parser keeps it as given. --- .../main/java/org/apache/james/jspf/impl/SPF.java | 4 +- .../james/jspf/parser/CachingSPFRecordParser.java | 123 ++++++++++++ .../jspf/parser/CachingSPFRecordParserTest.java | 208 +++++++++++++++++++++ 3 files changed, 334 insertions(+), 1 deletion(-) diff --git a/resolver/src/main/java/org/apache/james/jspf/impl/SPF.java b/resolver/src/main/java/org/apache/james/jspf/impl/SPF.java index aae949a..6e2db81 100644 --- a/resolver/src/main/java/org/apache/james/jspf/impl/SPF.java +++ b/resolver/src/main/java/org/apache/james/jspf/impl/SPF.java @@ -46,6 +46,7 @@ import org.apache.james.jspf.executor.FutureSPFResult; import org.apache.james.jspf.executor.SPFExecutor; import org.apache.james.jspf.executor.SPFResult; import org.apache.james.jspf.executor.SynchronousSPFExecutor; +import org.apache.james.jspf.parser.CachingSPFRecordParser; import org.apache.james.jspf.parser.RFC4408SPF1Parser; import org.apache.james.jspf.policies.InitialChecksPolicy; import org.apache.james.jspf.policies.NeutralIfNotMatchPolicy; @@ -258,7 +259,8 @@ public class SPF implements SPFChecker { wiringService.put(DNSServiceEnabled.class, this.dnsProbe); this.macroExpand = new MacroExpand(this.dnsProbe); wiringService.put(MacroExpandEnabled.class, this.macroExpand); - this.parser = new RFC4408SPF1Parser(new DefaultTermsFactory(wiringService)); + // the terms of jSPF can be shared by concurrent checks: the parsed records are kept + this.parser = new CachingSPFRecordParser(new RFC4408SPF1Parser(new DefaultTermsFactory(wiringService))); // We add this after the parser creation because services cannot be null wiringService.put(SPFCheckEnabled.class, this); } diff --git a/resolver/src/main/java/org/apache/james/jspf/parser/CachingSPFRecordParser.java b/resolver/src/main/java/org/apache/james/jspf/parser/CachingSPFRecordParser.java new file mode 100644 index 0000000..0b7b998 --- /dev/null +++ b/resolver/src/main/java/org/apache/james/jspf/parser/CachingSPFRecordParser.java @@ -0,0 +1,123 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.jspf.parser; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.james.jspf.core.SPF1Record; +import org.apache.james.jspf.core.SPFRecordParser; +import org.apache.james.jspf.core.exceptions.NeutralException; +import org.apache.james.jspf.core.exceptions.NoneException; +import org.apache.james.jspf.core.exceptions.PermErrorException; + +/** + * Keeps the records parsed by a parser, keyed by their text, so that every check does not parse + * again the records it retrieves: a provider include tree, the same for most of the mail a server + * receives, is parsed once. + * + * The terms of a cached record are shared by the checks: the terms of jSPF keep only their + * configuration, the state of a check lives in its SPFSession. Each call returns a new SPF1Record + * holding these terms, as the policies add modifiers to the record they get. + * + * The records are the sender's: the cache keeps at most a given number of them, the least + * recently used ones being dropped, and not the longer ones nor the invalid ones. + */ +public class CachingSPFRecordParser implements SPFRecordParser { + + /** + * The number of records kept by default + */ + public static final int DEFAULT_MAX_RECORDS = 1000; + + /** + * The length of the longest record kept by default: more than most of the legitimate records, + * which are made to fit in a DNS answer over UDP + */ + public static final int DEFAULT_MAX_RECORD_LENGTH = 2048; + + private final SPFRecordParser parser; + + private final int maxRecordLength; + + private final Map<String, SPF1Record> records; + + /** + * @param parser the parser of the records not cached + */ + public CachingSPFRecordParser(SPFRecordParser parser) { + this(parser, DEFAULT_MAX_RECORDS, DEFAULT_MAX_RECORD_LENGTH); + } + + /** + * @param parser the parser of the records not cached + * @param maxRecords the number of records kept at most + * @param maxRecordLength the length of the longest record kept + */ + public CachingSPFRecordParser(SPFRecordParser parser, final int maxRecords, int maxRecordLength) { + this.parser = parser; + this.maxRecordLength = maxRecordLength; + this.records = new LinkedHashMap<String, SPF1Record>(16, 0.75f, true) { + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(Map.Entry<String, SPF1Record> eldest) { + return size() > maxRecords; + } + }; + } + + /** + * @see org.apache.james.jspf.core.SPFRecordParser#parse(java.lang.String) + */ + public SPF1Record parse(String spfRecord) throws PermErrorException, NoneException, NeutralException { + if (spfRecord == null || spfRecord.length() > maxRecordLength) { + return parser.parse(spfRecord); + } + SPF1Record parsed; + synchronized (records) { + parsed = records.get(spfRecord); + } + if (parsed == null) { + // an invalid record throws: it is not kept + parsed = parser.parse(spfRecord); + synchronized (records) { + records.put(spfRecord, parsed); + } + } + return copy(parsed); + } + + /** + * @return the number of records kept + */ + public int size() { + synchronized (records) { + return records.size(); + } + } + + private static SPF1Record copy(SPF1Record parsed) { + SPF1Record copy = new SPF1Record(parsed.getRecord()); + copy.getDirectives().addAll(parsed.getDirectives()); + copy.getModifiers().addAll(parsed.getModifiers()); + return copy; + } +} diff --git a/resolver/src/test/java/org/apache/james/jspf/parser/CachingSPFRecordParserTest.java b/resolver/src/test/java/org/apache/james/jspf/parser/CachingSPFRecordParserTest.java new file mode 100644 index 0000000..9a444b9 --- /dev/null +++ b/resolver/src/test/java/org/apache/james/jspf/parser/CachingSPFRecordParserTest.java @@ -0,0 +1,208 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.jspf.parser; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.james.jspf.core.SPF1Record; +import org.apache.james.jspf.core.SPFRecordParser; +import org.apache.james.jspf.core.exceptions.NeutralException; +import org.apache.james.jspf.core.exceptions.NoneException; +import org.apache.james.jspf.core.exceptions.PermErrorException; +import org.apache.james.jspf.executor.AsynchronousSPFExecutor; +import org.apache.james.jspf.executor.SPFResult; +import org.apache.james.jspf.executor.SynchronousSPFExecutor; +import org.apache.james.jspf.helpers.FakeResolver; +import org.apache.james.jspf.impl.DNSServiceXBillImpl; +import org.apache.james.jspf.impl.DefaultTermsFactory; +import org.apache.james.jspf.impl.SPF; +import org.apache.james.jspf.wiring.WiringServiceTable; +import org.junit.Before; +import org.junit.Test; +import org.xbill.DNS.AAAARecord; +import org.xbill.DNS.ARecord; +import org.xbill.DNS.DClass; +import org.xbill.DNS.Lookup; +import org.xbill.DNS.MXRecord; +import org.xbill.DNS.Name; +import org.xbill.DNS.TXTRecord; + +public class CachingSPFRecordParserTest { + + private static final class CountingParser implements SPFRecordParser { + private final SPFRecordParser parser = new RFC4408SPF1Parser(new DefaultTermsFactory(new WiringServiceTable())); + private final AtomicInteger parsed = new AtomicInteger(); + + public SPF1Record parse(String spfRecord) throws PermErrorException, NoneException, NeutralException { + parsed.incrementAndGet(); + return parser.parse(spfRecord); + } + } + + private CountingParser counting; + + @Before + public void setUp() { + counting = new CountingParser(); + Lookup.getDefaultCache(DClass.IN).clearCache(); + } + + @Test + public void shouldParseARecordOnce() throws Exception { + CachingSPFRecordParser parser = new CachingSPFRecordParser(counting); + + parser.parse("v=spf1 ip4:192.0.2.0/24 -all"); + parser.parse("v=spf1 ip4:192.0.2.0/24 -all"); + + assertEquals(1, counting.parsed.get()); + } + + @Test + public void shouldGiveANewRecordSharingTheTerms() throws Exception { + CachingSPFRecordParser parser = new CachingSPFRecordParser(counting); + + SPF1Record first = parser.parse("v=spf1 ip4:192.0.2.0/24 -all"); + first.getModifiers().add(null); + SPF1Record second = parser.parse("v=spf1 ip4:192.0.2.0/24 -all"); + + assertNotSame(first, second); + assertEquals(0, second.getModifiers().size()); + assertEquals(2, second.getDirectives().size()); + assertSame(first.getDirectives().get(0), second.getDirectives().get(0)); + } + + @Test + public void shouldNotKeepAnInvalidRecord() throws Exception { + CachingSPFRecordParser parser = new CachingSPFRecordParser(counting); + + for (int i = 0; i < 2; i++) { + try { + parser.parse("v=spf1 xip4:192.0.2.1"); + fail("expected a permerror"); + } catch (PermErrorException e) { + // expected + } + } + + assertEquals(2, counting.parsed.get()); + assertEquals(0, parser.size()); + } + + @Test + public void shouldNotKeepALongRecord() throws Exception { + CachingSPFRecordParser parser = new CachingSPFRecordParser(counting, 10, 30); + + parser.parse("v=spf1 ip4:192.0.2.0/24 ip4:198.51.100.0/24 -all"); + parser.parse("v=spf1 ip4:192.0.2.0/24 ip4:198.51.100.0/24 -all"); + + assertEquals(2, counting.parsed.get()); + assertEquals(0, parser.size()); + } + + @Test + public void shouldKeepAtMostTheGivenNumberOfRecords() throws Exception { + CachingSPFRecordParser parser = new CachingSPFRecordParser(counting, 2, 2048); + + parser.parse("v=spf1 ip4:192.0.2.1 -all"); + parser.parse("v=spf1 ip4:192.0.2.2 -all"); + parser.parse("v=spf1 ip4:192.0.2.1 -all"); + parser.parse("v=spf1 ip4:192.0.2.3 -all"); + // the least recently used one was dropped + parser.parse("v=spf1 ip4:192.0.2.1 -all"); + parser.parse("v=spf1 ip4:192.0.2.2 -all"); + + assertEquals(2, parser.size()); + assertEquals(4, counting.parsed.get()); + } + + private static FakeResolver includeTree() throws Exception { + FakeResolver resolver = new FakeResolver(); + resolver.addRecord(new TXTRecord(Name.fromString("example.com."), DClass.IN, 30L, + "v=spf1 ip4:192.0.2.0/24 include:inc.example.com a:a.example.com mx exists:%{i}.ex.example.com -all exp=explain.example.com")); + resolver.addRecord(new TXTRecord(Name.fromString("inc.example.com."), DClass.IN, 30L, "v=spf1 ip6:2001:db8::/32 ~all")); + resolver.addRecord(new TXTRecord(Name.fromString("explain.example.com."), DClass.IN, 30L, "%{i} is not allowed for %{d}")); + resolver.addRecord(new ARecord(Name.fromString("a.example.com."), DClass.IN, 30L, InetAddress.getByName("198.51.100.1"))); + resolver.addRecord(new MXRecord(Name.fromString("example.com."), DClass.IN, 30L, 10, Name.fromString("mx.example.com."))); + resolver.addRecord(new ARecord(Name.fromString("mx.example.com."), DClass.IN, 30L, InetAddress.getByName("198.51.100.2"))); + resolver.addRecord(new AAAARecord(Name.fromString("mx.example.com."), DClass.IN, 30L, InetAddress.getByName("2001:db9::2"))); + resolver.addRecord(new ARecord(Name.fromString("203.0.113.7.ex.example.com."), DClass.IN, 30L, InetAddress.getByName("127.0.0.2"))); + return resolver; + } + + private static final String[] CLIENTS = {"192.0.2.5", "2001:db8::1", "198.51.100.1", "198.51.100.2", "203.0.113.7", + "203.0.113.8", "10.0.0.1", "2001:db9::2", "2001:db9::3"}; + + private static String check(SPF spf, String client) { + SPFResult result = spf.checkSPF(client, "[email protected]", "example.com"); + return result.getResult() + " " + result.getExplanation(); + } + + private void checkConcurrently(SPF spf) throws Exception { + Map<String, String> expected = new LinkedHashMap<>(); + for (String client : CLIENTS) { + expected.put(client, check(spf, client)); + } + assertEquals("pass ", expected.get("192.0.2.5")); + assertEquals("pass ", expected.get("2001:db8::1")); + assertEquals("pass ", expected.get("203.0.113.7")); + assertEquals("fail 203.0.113.8 is not allowed for example.com", expected.get("203.0.113.8")); + assertEquals("pass ", expected.get("2001:db9::2")); + + ExecutorService threads = Executors.newFixedThreadPool(8); + try { + List<Future<?>> checks = new ArrayList<>(); + for (int n = 0; n < 400; n++) { + String client = CLIENTS[n % CLIENTS.length]; + checks.add(threads.submit(() -> assertEquals(client, expected.get(client), check(spf, client)))); + } + for (Future<?> check : checks) { + check.get(60, TimeUnit.SECONDS); + } + } finally { + threads.shutdownNow(); + } + } + + @Test + public void concurrentSynchronousChecksShouldShareTheParsedRecords() throws Exception { + DNSServiceXBillImpl dns = new DNSServiceXBillImpl(includeTree()); + checkConcurrently(new SPF(dns, new SynchronousSPFExecutor(dns))); + } + + @Test + public void concurrentAsynchronousChecksShouldShareTheParsedRecords() throws Exception { + DNSServiceXBillImpl dns = new DNSServiceXBillImpl(includeTree()); + checkConcurrently(new SPF(dns, new AsynchronousSPFExecutor(dns))); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
