rzo1 commented on code in PR #1110: URL: https://github.com/apache/opennlp/pull/1110#discussion_r3496549376
########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/ExtendedPictographic.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize.uax29; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.BitSet; + +/** + * Tests the Unicode {@code Extended_Pictographic} property of a code point. + * + * <p>This is the one extra property the word boundary algorithm needs (rule WB3c), to keep emoji + * zero-width-joiner sequences together. The data is loaded once from the {@code emoji-data.txt} + * derived resource of the Unicode Character Database and stored in a {@link BitSet}, so membership + * is an O(1) bit test.</p> + */ +public final class ExtendedPictographic { + + private static final String RESOURCE = "ExtendedPictographic.txt"; + + // Loaded lazily on first use (see members()) so a missing or unreadable resource surfaces as a Review Comment: We can drop that comment. It is an answer to my first PR review. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordBreakProperty.java: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize.uax29; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Looks up the Unicode {@link WordBreak Word_Break} property of a code point. + * + * <p>The data is loaded once from the {@code WordBreakProperty.txt} resource of the Unicode + * Character Database (parsed with simple cursor scanning, no regular expression). Lookup is O(1) + * for the Basic Multilingual Plane (a direct array index) and O(log n) for supplementary code + * points (a binary search over a small sorted range table), so it imposes no per-character + * allocation on the word boundary algorithm.</p> + */ +public final class WordBreakProperty { + + private static final String RESOURCE = "WordBreakProperty.txt"; + + private static final WordBreak[] VALUES = WordBreak.values(); + + // Loaded lazily on first use (see data()) so a missing or unreadable resource surfaces as a + // catchable exception at call time rather than an ExceptionInInitializerError that permanently + // poisons the class -- a real risk in container, OSGi, shaded, or modular setups. + private static volatile Data data; + + private WordBreakProperty() { + } + + // Immutable Word_Break tables: ordinal per BMP code point, plus supplementary ranges sorted by + // start for binary search. + private static final class Data { + final byte[] bmp; + final int[] supplementaryStart; + final int[] supplementaryEnd; + final byte[] supplementaryValue; + + Data(byte[] bmp, int[] start, int[] end, byte[] value) { + this.bmp = bmp; + this.supplementaryStart = start; + this.supplementaryEnd = end; + this.supplementaryValue = value; + } + } + + // Double-checked lazy initialization: load() runs once on first use, and a failure leaves the + // field null so a later call retries instead of the class being permanently unusable. Review Comment: sane nit as before. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/ExtendedPictographic.java: ########## @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize.uax29; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.BitSet; + +/** + * Tests the Unicode {@code Extended_Pictographic} property of a code point. + * + * <p>This is the one extra property the word boundary algorithm needs (rule WB3c), to keep emoji + * zero-width-joiner sequences together. The data is loaded once from the {@code emoji-data.txt} + * derived resource of the Unicode Character Database and stored in a {@link BitSet}, so membership + * is an O(1) bit test.</p> + */ +public final class ExtendedPictographic { + + private static final String RESOURCE = "ExtendedPictographic.txt"; + + // Loaded lazily on first use (see members()) so a missing or unreadable resource surfaces as a + // catchable exception at call time rather than an ExceptionInInitializerError that permanently + // poisons the class -- a real risk in container, OSGi, shaded, or modular setups. + private static volatile BitSet members; + + private ExtendedPictographic() { + } + + // Double-checked lazy initialization: load() runs once on first use, and a failure leaves the Review Comment: Think this comment is also obvious. ########## opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/uax29/WordBreakProperty.java: ########## @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize.uax29; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Looks up the Unicode {@link WordBreak Word_Break} property of a code point. + * + * <p>The data is loaded once from the {@code WordBreakProperty.txt} resource of the Unicode + * Character Database (parsed with simple cursor scanning, no regular expression). Lookup is O(1) + * for the Basic Multilingual Plane (a direct array index) and O(log n) for supplementary code + * points (a binary search over a small sorted range table), so it imposes no per-character + * allocation on the word boundary algorithm.</p> + */ +public final class WordBreakProperty { + + private static final String RESOURCE = "WordBreakProperty.txt"; + + private static final WordBreak[] VALUES = WordBreak.values(); + + // Loaded lazily on first use (see data()) so a missing or unreadable resource surfaces as a Review Comment: sane nit as before. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
