This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-mime4j.git
commit b256a924f9e824b5ef9ccb9dd04f90e2c4568bdf Author: Benoit TELLIER <[email protected]> AuthorDate: Tue Nov 10 10:46:39 2026 +0100 [FIX] JavaCC token managers: the comment-nesting counter is per parser AddressListParser, ContentTypeParser, ContentDispositionParser and DateTimeParser set STATIC=false, and callers build a new parser for each parse, but their TOKEN_MGR_DECLS declared "static int commentNest". So the JVM had one counter per grammar, shared without synchronisation by every thread. A nested comment such as "(x (y) z)" leaves NESTED_COMMENT only when that counter comes back to exactly 0: a parse running on another thread at the same time made it skip 0, and the lexer swallowed the rest of the field and failed at EOF, or reach 0 too early, and the lexer read comment text as tokens. Any sender could thus make the strict parse of other messages, often other users' mail, fail or return wrong mailboxes, MIME type, parameters or date, and the same happened under ordinary concurrent load. The counter is now an instance field of the token manager, as StructuredFieldParser, ContentLanguageParser and MimeVersionParser already declared it. A test parses nested comments with the four parsers on concurrent threads, and another fails if a grammar with STATIC=false declares a static field in TOKEN_MGR_DECLS. --- .../contentdisposition/ContentDispositionParser.jj | 2 +- .../mime4j/field/contenttype/ContentTypeParser.jj | 2 +- .../james/mime4j/field/datetime/DateTimeParser.jj | 2 +- .../mime4j/field/address/AddressListParser.jjt | 2 +- .../mime4j/field/CommentNestThreadSafetyTest.java | 175 +++++++++++++++++++++ 5 files changed, 179 insertions(+), 4 deletions(-) diff --git a/dom/src/main/javacc/org/apache/james/mime4j/field/contentdisposition/ContentDispositionParser.jj b/dom/src/main/javacc/org/apache/james/mime4j/field/contentdisposition/ContentDispositionParser.jj index 50231994..4be05a38 100644 --- a/dom/src/main/javacc/org/apache/james/mime4j/field/contentdisposition/ContentDispositionParser.jj +++ b/dom/src/main/javacc/org/apache/james/mime4j/field/contentdisposition/ContentDispositionParser.jj @@ -157,7 +157,7 @@ TOKEN_MGR_DECLS : // This is because the outermost level must be treated // specially anyway, because the outermost ")" has a // different token type than inner ")" instances. - static int commentNest; + int commentNest; } MORE : diff --git a/dom/src/main/javacc/org/apache/james/mime4j/field/contenttype/ContentTypeParser.jj b/dom/src/main/javacc/org/apache/james/mime4j/field/contenttype/ContentTypeParser.jj index 0c40237a..46bcbdff 100644 --- a/dom/src/main/javacc/org/apache/james/mime4j/field/contenttype/ContentTypeParser.jj +++ b/dom/src/main/javacc/org/apache/james/mime4j/field/contenttype/ContentTypeParser.jj @@ -160,7 +160,7 @@ TOKEN_MGR_DECLS : // This is because the outermost level must be treated // specially anyway, because the outermost ")" has a // different token type than inner ")" instances. - static int commentNest; + int commentNest; } MORE : diff --git a/dom/src/main/javacc/org/apache/james/mime4j/field/datetime/DateTimeParser.jj b/dom/src/main/javacc/org/apache/james/mime4j/field/datetime/DateTimeParser.jj index 46522ba3..0c6ee9b5 100644 --- a/dom/src/main/javacc/org/apache/james/mime4j/field/datetime/DateTimeParser.jj +++ b/dom/src/main/javacc/org/apache/james/mime4j/field/datetime/DateTimeParser.jj @@ -335,7 +335,7 @@ TOKEN_MGR_DECLS : // This is because the outermost level must be treated // specially anyway, because the outermost ")" has a // different token type than inner ")" instances. - static int commentNest; + int commentNest; } MORE : diff --git a/dom/src/main/jjtree/org/apache/james/mime4j/field/address/AddressListParser.jjt b/dom/src/main/jjtree/org/apache/james/mime4j/field/address/AddressListParser.jjt index a6a1924c..78bcf0a0 100644 --- a/dom/src/main/jjtree/org/apache/james/mime4j/field/address/AddressListParser.jjt +++ b/dom/src/main/jjtree/org/apache/james/mime4j/field/address/AddressListParser.jjt @@ -264,7 +264,7 @@ TOKEN_MGR_DECLS : // This is because the outermost level must be treated // specially anyway, because the outermost ")" has a // different token type than inner ")" instances. - static int commentNest; + int commentNest; } MORE : diff --git a/dom/src/test/java/org/apache/james/mime4j/field/CommentNestThreadSafetyTest.java b/dom/src/test/java/org/apache/james/mime4j/field/CommentNestThreadSafetyTest.java new file mode 100644 index 00000000..396af33a --- /dev/null +++ b/dom/src/test/java/org/apache/james/mime4j/field/CommentNestThreadSafetyTest.java @@ -0,0 +1,175 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.mime4j.field; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.james.mime4j.dom.address.AddressList; +import org.apache.james.mime4j.dom.address.Mailbox; +import org.apache.james.mime4j.dom.datetime.DateTime; +import org.apache.james.mime4j.field.address.DefaultAddressParser; +import org.apache.james.mime4j.field.contentdisposition.parser.ContentDispositionParser; +import org.apache.james.mime4j.field.contenttype.parser.ContentTypeParser; +import org.apache.james.mime4j.field.datetime.parser.DateTimeParser; +import org.junit.Test; + +/** + * The comment-nesting counter of the JavaCC token managers belongs to one parser: + * parsers running on other threads must not change how a nested comment ends. + */ +public class CommentNestThreadSafetyTest { + + private static final int THREADS = 8; + private static final int ITERATIONS = 3000; + + private static String deepComment() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 200; i++) { + sb.append('('); + } + sb.append('x'); + for (int i = 0; i < 200; i++) { + sb.append(')'); + } + return sb.toString(); + } + + private interface Check { + void run(String comment) throws Exception; + } + + private static final Check ADDRESS = comment -> { + AddressList list = DefaultAddressParser.DEFAULT.parseAddressList("a@b " + comment + ", c@d"); + assertEquals(2, list.size()); + Mailbox first = (Mailbox) list.get(0); + assertEquals("a@b", first.getAddress()); + assertEquals("c@d", ((Mailbox) list.get(1)).getAddress()); + }; + + private static final Check CONTENT_TYPE = comment -> { + ContentTypeParser parser = new ContentTypeParser( + new StringReader("text/plain " + comment + "; charset=us-ascii")); + parser.parseAll(); + assertEquals("text", parser.getType()); + assertEquals("plain", parser.getSubType()); + assertEquals("charset", parser.getParamNames().get(0)); + assertEquals("us-ascii", parser.getParamValues().get(0)); + }; + + private static final Check CONTENT_DISPOSITION = comment -> { + ContentDispositionParser parser = new ContentDispositionParser( + new StringReader("attachment " + comment + "; filename=f")); + parser.parseAll(); + assertEquals("attachment", parser.getDispositionType()); + assertEquals("f", parser.getParameters().get("filename")); + }; + + private static final Check DATE_TIME = comment -> { + DateTime dt = new DateTimeParser( + new StringReader("Thu, 1 Jan 2026 10:20:30 +0100 " + comment)).parseAll(); + assertEquals(2026, dt.getYear()); + assertEquals(30, dt.getSecond()); + assertEquals(100, dt.getTimeZone()); + }; + + @Test + public void nestedCommentsParseTheSameOnConcurrentThreads() throws Exception { + final Check[] checks = {ADDRESS, CONTENT_TYPE, CONTENT_DISPOSITION, DATE_TIME}; + final String[] comments = {"(x (y) z)", deepComment()}; + ExecutorService executor = Executors.newFixedThreadPool(THREADS); + try { + final CountDownLatch start = new CountDownLatch(1); + List<Future<Void>> futures = new ArrayList<>(); + for (int t = 0; t < THREADS; t++) { + final int thread = t; + futures.add(executor.submit(new Callable<Void>() { + @Override + public Void call() throws Exception { + start.await(); + for (int i = 0; i < ITERATIONS; i++) { + Check check = checks[(thread + i) % checks.length]; + check.run(comments[thread % comments.length]); + } + return null; + } + })); + } + start.countDown(); + for (Future<Void> future : futures) { + future.get(2, TimeUnit.MINUTES); + } + } finally { + executor.shutdownNow(); + } + } + + private static final Pattern TOKEN_MGR_DECLS = Pattern.compile( + "TOKEN_MGR_DECLS\\s*:\\s*\\{(.*?)\\n\\}", Pattern.DOTALL); + private static final Pattern STATIC_FIELD = Pattern.compile("(?m)^\\s*static\\s"); + + @Test + public void grammarsDeclareNoStaticTokenManagerField() throws IOException { + List<File> grammars = new ArrayList<>(); + collectGrammars(new File("src/main"), grammars); + assertTrue("no grammar found under " + new File("src/main").getAbsolutePath(), grammars.size() >= 7); + for (File grammar : grammars) { + String content = new String(Files.readAllBytes(grammar.toPath()), StandardCharsets.UTF_8); + if (!content.contains("STATIC=false") && !content.contains("STATIC = false")) { + continue; + } + Matcher decls = TOKEN_MGR_DECLS.matcher(content); + while (decls.find()) { + assertTrue(grammar + " declares a static field in TOKEN_MGR_DECLS", + !STATIC_FIELD.matcher(decls.group(1)).find()); + } + } + } + + private static void collectGrammars(File dir, List<File> out) { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (File child : children) { + if (child.isDirectory()) { + collectGrammars(child, out); + } else if (child.getName().endsWith(".jj") || child.getName().endsWith(".jjt")) { + out.add(child); + } + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
