[
https://issues.apache.org/jira/browse/OPENNLP-1991?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18126309#comment-18126309
]
ASF GitHub Bot commented on OPENNLP-1991:
-----------------------------------------
krickert commented on code in PR #1323:
URL: https://github.com/apache/opennlp/pull/1323#discussion_r4247897270
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java:
##########
@@ -0,0 +1,942 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.ParamChecks;
+import opennlp.tools.util.StringUtil;
+
+/**
+ * The weights of the feedforward transition parser: embeddings for words,
tags, and arc
+ * labels, one hidden layer with cube activation, and a transition output
layer, stored
+ * in a versioned binary format. The architecture follows
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>The same class scores configurations during training and decoding.
Unknown words
+ * use a learned fallback row; words are matched case-insensitively after
+ * {@link #normalize(String) normalization}.</p>
+ *
+ * <p>Instances returned by {@link FeedforwardDependencyTrainer} are immutable
and safe
+ * to share between threads. {@link FeedforwardDependencyTrainer#refine
refine} updates
+ * an independent copy.</p>
+ *
+ * @see FeedforwardDependencyParser
+ * @see FeedforwardDependencyTrainer
+ * @since 3.0.0
+ */
+@ThreadSafe
+public class FeedforwardDependencyModel {
+
+ /** The format header written before every serialized model; it carries the
format version. */
+ static final String MAGIC = "ONLP-FFDP-1";
+
+ /** The maximum number of (template position, embedding row) pairs the
scoring cache holds. */
+ static final int CACHE_ENTRY_LIMIT = 32_768;
+
+ /** The vocabulary label used in load errors for the word map. */
+ private static final String WORD_VOCABULARY = "word vocabulary";
+
+ /** The vocabulary label used in load errors for the tag map. */
+ private static final String TAG_VOCABULARY = "tag vocabulary";
+
+ /** The vocabulary label used in load errors for the label map. */
+ private static final String LABEL_VOCABULARY = "label vocabulary";
+
+ /** Maximum combined entries across the word, tag, and label maps. */
+ private static final int MAX_VOCABULARY_ENTRIES = 2_000_000;
+
+ /** Maximum transitions accepted from a serialized model. */
+ private static final int MAX_TRANSITIONS = 100_000;
+
+ /** Maximum embedding width accepted from a serialized model. */
+ static final int MAX_EMBEDDING_SIZE = 4_096;
+
+ /** Maximum hidden-layer width accepted from a serialized model. */
+ static final int MAX_HIDDEN_SIZE = 65_536;
+
+ /** Maximum float values allocated while loading a serialized model. */
+ static final long MAX_MODEL_FLOAT_VALUES = 100_000_000L;
+
+ /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual
lowering. */
+ private static final int GREEK_CAPITAL_SIGMA = 0x03A3;
+
+ /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the
capital. */
+ private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2';
+
+ /** The vocabulary key of every word, tag, or label the model has no
embedding row for. */
+ static final String UNKNOWN = "*UNK*";
+
+ /** The lazy scoring cache; {@code null} until {@link
#enableScoringCache()}. */
+ private volatile ContributionCache cache;
+
+ private final Map<String, Integer> wordIds;
+ private final Map<String, Integer> tagIds;
+ private final Map<String, Integer> labelIds;
+ private final String[] transitions;
+
+ private final int embeddingSize;
+ private final float[][] embeddings;
+ private final float[][] hiddenWeights;
+ private final float[] hiddenBias;
+ private final float[][] outputWeights;
+ private final float[] outputBias;
+
+ /**
+ * Assembles a model from its vocabularies and weights, copying the maps so
later
+ * changes by the caller do not reach the model.
+ *
+ * @param wordIds The word vocabulary, mapping each normalized word to its
row.
+ * @param tagIds The tag vocabulary.
+ * @param labelIds The dependency label vocabulary.
+ * @param transitions The transition inventory, indexed by output row.
+ * @param embeddingSize The embedding dimensionality.
+ * @param embeddings The embedding rows for words, tags, and labels.
+ * @param hiddenWeights The hidden layer weights.
+ * @param hiddenBias The hidden layer bias.
+ * @param outputWeights The output layer weights.
+ * @param outputBias The output layer bias.
+ */
+ FeedforwardDependencyModel(Map<String, Integer> wordIds, Map<String,
Integer> tagIds,
+ Map<String, Integer> labelIds, String[] transitions, int embeddingSize,
+ float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias,
+ float[][] outputWeights, float[] outputBias) {
+ this.wordIds = Map.copyOf(wordIds);
+ this.tagIds = Map.copyOf(tagIds);
+ this.labelIds = Map.copyOf(labelIds);
+ this.transitions = transitions;
+ this.embeddingSize = embeddingSize;
+ this.embeddings = embeddings;
+ this.hiddenWeights = hiddenWeights;
+ this.hiddenBias = hiddenBias;
+ this.outputWeights = outputWeights;
+ this.outputBias = outputBias;
+ }
+
+ /**
+ * Scores every transition for a configuration described by embedding row
indices.
+ *
+ * @param features The embedding rows of the configuration, as produced by
+ * {@link #featureIds(String[])}. Must not be {@code null}.
+ * @return One unnormalized score per transition, indexed like
+ * {@link #transitions()}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code features} is {@code
null}, does not
+ * have the required length, or contains an invalid embedding index.
+ * @throws IllegalStateException Thrown if the model produces a non-finite
transition score.
+ */
+ double[] score(int[] features) {
+ ParamChecks.requireNonNullArg(features, "features");
+ if (features.length != FeedforwardContext.FEATURE_COUNT) {
+ throw new IllegalArgumentException("features must contain "
+ + FeedforwardContext.FEATURE_COUNT + " embedding indices");
+ }
+ for (int i = 0; i < features.length; i++) {
+ if (features[i] < 0 || features[i] >= embeddings.length) {
+ throw new IllegalArgumentException(
+ "feature embedding out of range at index " + i + ": " +
features[i]);
+ }
+ }
+ final int hidden = hiddenBias.length;
+ final double[] h = new double[hidden];
+ for (int j = 0; j < hidden; j++) {
+ h[j] = hiddenBias[j];
+ }
+ final ContributionCache cache = this.cache;
+ for (int f = 0; f < features.length; f++) {
+ final int row = features[f];
+ final double[] contribution = cache == null ? null :
cache.contribution(this, f, row);
+ if (contribution != null) {
+ for (int j = 0; j < hidden; j++) {
+ h[j] += contribution[j];
+ }
+ } else {
+ final float[] embedding = embeddings[row];
Review Comment:
One `FeedforwardNetwork.addContribution`, used by direct scoring and the
cache.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java:
##########
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ParamChecks;
+
+/**
+ * Trains the {@link FeedforwardDependencyModel}: oracle-derived transition
examples,
+ * minibatch AdaGrad over a softmax cross-entropy loss, cube activation, and
inverted
+ * dropout on the hidden layer, the training recipe of
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Words below the frequency cutoff share a learned unknown embedding;
absent
+ * template positions share a learned padding embedding. Non-projective
samples have no
+ * arc-standard derivation and are skipped. Training is deterministic for a
fixed
+ * {@link Settings#seed()}.</p>
+ *
+ * @since 3.0.0
+ */
+public final class FeedforwardDependencyTrainer {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(FeedforwardDependencyTrainer.class);
+
+ /** The AdaGrad denominator offset that keeps the first steps finite. */
+ private static final double ADAGRAD_EPSILON = 1e-6;
+
+ /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */
+ private static final double CUBE_DERIVATIVE_FACTOR = 3.0;
+
+ /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn +
fanOut))}. */
+ private static final double GLOROT_NUMERATOR = 6.0;
+
+ /** The half-width of the uniform range embeddings are drawn from. */
+ private static final double EMBEDDING_INIT_SCALE = 0.01;
+
+ /** The smallest probability the reported training loss takes the log of. */
+ private static final double LOSS_PROBABILITY_FLOOR = 1e-12;
+
+ /** The sentence loss reported when refinement made no update for a
sentence. */
+ private static final double NO_UPDATE = -1.0;
+
+ /** Special-symbol rows present before any training vocabulary is added. */
+ private static final int MIN_VOCABULARY_ROWS = 8;
+
+ /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */
+ private static final int MIN_TRANSITIONS = 2;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardDependencyTrainer() {
+ }
+
+ /**
+ * The training hyperparameters.
+ *
+ * @param embeddingSize The embedding dimensionality. Must be positive.
+ * @param hiddenSize The hidden layer width. Must be positive.
+ * @param epochs The number of passes over the examples. Must be positive.
+ * @param batchSize The minibatch size. Must be positive.
+ * @param learningRate The AdaGrad step size. Must be positive.
+ * @param l2 The L2 penalty applied to the dense weights. Must not be
negative.
+ * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}.
+ * @param wordCutoff The minimum frequency for a word to get its own
embedding. Must
+ * not be negative.
+ * @param seed The random seed making a run reproducible.
+ */
+ public record Settings(int embeddingSize, int hiddenSize, int epochs, int
batchSize,
+ double learningRate, double l2, double dropout, int wordCutoff, long
seed) {
+
+ /** The default embedding dimensionality. */
+ private static final int DEFAULT_EMBEDDING_SIZE = 50;
+
+ /** The default hidden layer width. */
+ private static final int DEFAULT_HIDDEN_SIZE = 200;
+
+ /** The default number of epochs. */
+ private static final int DEFAULT_EPOCHS = 10;
+
+ /** The default minibatch size. */
+ private static final int DEFAULT_BATCH_SIZE = 256;
+
+ /** The default AdaGrad step size. */
+ private static final double DEFAULT_LEARNING_RATE = 0.02;
+
+ /** The default L2 penalty. */
+ private static final double DEFAULT_L2 = 1e-8;
+
+ /** The default hidden dropout probability. */
+ private static final double DEFAULT_DROPOUT = 0.5;
+
+ /** The default minimum word frequency for an embedding row. */
+ private static final int DEFAULT_WORD_CUTOFF = 2;
+
+ /** The default random seed. */
+ static final long DEFAULT_SEED = 17L;
+
+ /**
+ * Validates the hyperparameters.
+ *
+ * @throws IllegalArgumentException Thrown if a value is outside its
documented
+ * range.
+ */
+ public Settings {
+ requirePositive(embeddingSize, "embeddingSize");
+ requirePositive(hiddenSize, "hiddenSize");
+ requirePositive(epochs, "epochs");
+ requirePositive(batchSize, "batchSize");
+ if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) {
+ throw new IllegalArgumentException("embeddingSize exceeds the model
format limit: "
+ + embeddingSize);
+ }
+ if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) {
+ throw new IllegalArgumentException("hiddenSize exceeds the model
format limit: "
+ + hiddenSize);
+ }
+ final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS,
+ MIN_TRANSITIONS, embeddingSize, hiddenSize);
+ if (minimumModelValues >
FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("embeddingSize and hiddenSize
exceed the model "
+ + "format allocation limit");
+ }
+ if (ParamChecks.requireFinite(learningRate, "learningRate") <= 0.0) {
+ throw new IllegalArgumentException("learningRate must be positive: " +
learningRate);
+ }
+ ParamChecks.requireNonNegative(ParamChecks.requireFinite(l2, "l2"),
"l2");
+ if (!(dropout >= 0.0 && dropout < 1.0)) {
+ throw new IllegalArgumentException("dropout must be in [0, 1): " +
dropout);
+ }
+ ParamChecks.requireNonNegative(wordCutoff, "wordCutoff");
+ }
+
+ /**
+ * Rejects a hyperparameter that is not positive.
+ *
+ * @param value The value to check.
+ * @param name The hyperparameter name used in the exception message.
+ * @throws IllegalArgumentException Thrown if {@code value} is not
positive.
+ */
+ private static void requirePositive(int value, String name) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(name + " must be positive: " +
value);
+ }
+ }
+
+ /**
+ * @return The default hyperparameters. Never {@code null}.
+ */
+ public static Settings defaults() {
+ return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE,
DEFAULT_EPOCHS,
+ DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2,
DEFAULT_DROPOUT,
+ DEFAULT_WORD_CUTOFF, DEFAULT_SEED);
+ }
+ }
+
+ /**
+ * Trains a model from dependency samples.
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null} or
no
+ * trainable example can be derived from the samples.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings) throws IOException {
+ return train(samples, settings, null);
+ }
+
+ /**
+ * Trains a model from dependency samples, seeding word embeddings from a
pretrained
+ * source.
+ *
+ * <p>The provider is consulted once per vocabulary word during
initialization; words
+ * it returns {@code null} for keep their random initialization, and all
embeddings
+ * remain trainable afterwards. The pretrained source is a training-time
ingredient
+ * only: the learned embeddings ship inside the model, so parsing carries no
+ * dependency on the source.</p>
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @param pretrained Maps a normalized word to its pretrained vector of
exactly
+ * {@link Settings#embeddingSize()} dimensions, or {@code
null} for
+ * unknown words. May be {@code null} to disable seeding.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if {@code samples} or {@code
settings} is
+ * {@code null}, no trainable example can be derived, or a
pretrained vector
+ * has the wrong dimensionality.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings, Function<String, float[]> pretrained)
+ throws IOException {
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ final List<DependencySample> corpus = readAll(samples);
+ final FeedforwardDependencyModel model = initialize(corpus, settings);
+ if (pretrained != null) {
+ seed(model, pretrained, settings);
+ }
+ final List<int[]> featureList = new ArrayList<>();
+ final List<Integer> goldList = new ArrayList<>();
+ collectExamples(corpus, model, featureList, goldList);
+ if (featureList.isEmpty()) {
+ throw new IllegalArgumentException("no trainable examples in the
samples");
+ }
+ optimize(model, featureList, goldList, settings);
+ return model;
+ }
+
+ /**
+ * Refines a locally trained model with beam search and the early-update
method of
+ * <a href="https://aclanthology.org/P04-1015/";>Collins and Roark
(2004)</a>. The loss
+ * is conditional likelihood over candidate paths scored by summed
log-probabilities.
+ *
+ * <p>Refinement updates a copy with per-sentence AdaGrad steps and no
dropout.
+ * {@link Settings#epochs()} sets the number of refinement passes. Use the
same beam
+ * size for parsing the result.</p>
+ *
+ * <p>The transition inventory comes from {@code model} and is not extended,
because
+ * its size is the width of the trained output layer. A refinement corpus
using a
+ * relation label the original training set lacked is therefore rejected
rather than
+ * silently ignored.</p>
+ *
+ * @param model The locally trained model to refine. Left untouched. Must
not be
+ * {@code null}.
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters; {@code epochs}, {@code learningRate},
+ * {@code l2}, and {@code seed} apply. Must not be {@code
null}.
+ * @param beamSize The beam width to track the gold derivation in. Must be
at least 2.
+ * @return A new refined model, distinct from {@code model}. Never {@code
null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null},
+ * {@code beamSize} is below 2, no trainable sample can be derived,
or a sample
+ * requires a transition {@code model} does not know.
+ */
+ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel
model,
+ ObjectStream<DependencySample> samples, Settings settings, int beamSize)
+ throws IOException {
+ ParamChecks.requireNonNullArg(model, "model");
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ if (beamSize < 2) {
+ throw new IllegalArgumentException("beamSize must be at least 2: " +
beamSize);
+ }
+ final List<DependencySample> corpus = readAll(samples);
+ final String[] outcomes = model.transitions();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ final Transition[] transitions = new Transition[outcomes.length];
+ for (int i = 0; i < transitions.length; i++) {
+ transitionIds.put(outcomes[i], i);
+ transitions[i] = Transition.decode(outcomes[i]);
+ }
+
+ final List<DependencySample> trainable = new ArrayList<>();
+ final List<int[]> oracles = new ArrayList<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ final int[] encoded = new int[oracle.size()];
+ for (int i = 0; i < encoded.length; i++) {
+ final String outcome = oracle.get(i).encode();
+ final Integer id = transitionIds.get(outcome);
+ if (id == null) {
+ // The outcome space was fixed by the original training set, so a
relation
+ // label it never saw has no output unit to push probability onto.
+ throw new IllegalArgumentException(
+ "unknown transition in the refinement samples: " + outcome);
+ }
+ encoded[i] = id;
+ }
+ trainable.add(s);
+ oracles.add(encoded);
+ }
+ if (trainable.isEmpty()) {
+ throw new IllegalArgumentException("no trainable samples for
refinement");
+ }
+
+ final FeedforwardDependencyModel refined = model.copy();
+ final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings);
+ final Random random = new Random(settings.seed());
+ final int[] order = new int[trainable.size()];
+ for (int i = 0; i < order.length; i++) {
+ order[i] = i;
+ }
+ for (int epoch = 1; epoch <= settings.epochs(); epoch++) {
+ final long epochStart = System.currentTimeMillis();
+ shuffle(order, random);
+ double loss = 0.0;
+ int updates = 0;
+ for (final int index : order) {
+ final double sentenceLoss =
optimizer.refineSentence(trainable.get(index),
+ oracles.get(index), transitions, beamSize);
+ if (sentenceLoss >= 0.0) {
+ loss += sentenceLoss;
+ updates++;
+ }
+ }
+ checkFinite(refined);
+ logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch,
+ loss / Math.max(updates, 1), updates, System.currentTimeMillis() -
epochStart);
+ }
+ return refined;
+ }
+
+ /** One candidate path in the refinement beam: the parent link forms the
history. */
+ private static final class BeamNode {
+ private final BeamNode parent;
+ private final int[] features;
+ private final int transition;
+ private final double score;
+ private final boolean gold;
+ private ArcStandardState state;
+
+ /**
+ * Extends {@code parent} by one transition; the start node passes {@code
null}.
+ *
+ * @param parent The node this one extends, or {@code null} for the start
node.
+ * @param features The embedding rows the transition was scored on.
+ * @param transition The transition index taken from {@code parent}.
+ * @param score The summed log-probability of the path.
+ * @param gold Whether the path is the gold derivation so far.
+ */
+ private BeamNode(BeamNode parent, int[] features, int transition, double
score,
+ boolean gold) {
+ this.parent = parent;
+ this.features = features;
+ this.transition = transition;
+ this.score = score;
+ this.gold = gold;
+ }
+ }
+
+ /** The forward, backward, and AdaGrad state for global refinement. */
+ private static final class GlobalOptimizer {
+ private final FeedforwardDependencyModel model;
+ private final Settings settings;
+ private final int embeddingSize;
+ private final int hiddenSize;
+ private final int outputSize;
+ private final int inputSize;
+
+ private final double[][] embeddingAccumulator;
+ private final double[][] hiddenAccumulator;
+ private final double[] hiddenBiasAccumulator;
+ private final double[][] outputAccumulator;
+ private final double[] outputBiasAccumulator;
+
+ private final double[][] hiddenGradient;
+ private final double[] hiddenBiasGradient;
+ private final double[][] outputGradient;
+ private final double[] outputBiasGradient;
+ private final Map<Integer, double[]> embeddingGradients = new HashMap<>();
+
+ private final double[] x;
+ private final double[] pre;
+ private final double[] hidden;
+ private final double[] probabilities;
+ private final double[] hiddenDelta;
+ private final double[] inputDelta;
+
+ /**
+ * Sizes the accumulators and scratch buffers for one refinement run.
+ *
+ * @param model The model copy the run updates in place.
+ * @param settings The hyperparameters; {@code learningRate} and {@code
l2} apply.
+ */
+ private GlobalOptimizer(FeedforwardDependencyModel model, Settings
settings) {
+ this.model = model;
+ this.settings = settings;
+ this.embeddingSize = model.embeddings()[0].length;
+ this.hiddenSize = model.hiddenBias().length;
+ this.outputSize = model.outputBias().length;
+ this.inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize;
+ this.embeddingAccumulator =
+ new double[model.embeddings().length][embeddingSize];
+ this.hiddenAccumulator = new double[hiddenSize][inputSize];
+ this.hiddenBiasAccumulator = new double[hiddenSize];
+ this.outputAccumulator = new double[outputSize][hiddenSize];
+ this.outputBiasAccumulator = new double[outputSize];
+ this.hiddenGradient = new double[hiddenSize][inputSize];
+ this.hiddenBiasGradient = new double[hiddenSize];
+ this.outputGradient = new double[outputSize][hiddenSize];
+ this.outputBiasGradient = new double[outputSize];
+ this.x = new double[inputSize];
+ this.pre = new double[hiddenSize];
+ this.hidden = new double[hiddenSize];
+ this.probabilities = new double[outputSize];
+ this.hiddenDelta = new double[hiddenSize];
+ this.inputDelta = new double[inputSize];
+ }
+
+ /**
+ * Decodes one sentence with the beam, updating on the early-update point
or the
+ * final beam.
+ *
+ * @param sample The sentence.
+ * @param oracle The gold transition indexes.
+ * @param transitions The decoded transition inventory.
+ * @param beamSize The beam width.
+ * @return The sentence loss, or {@link
FeedforwardDependencyTrainer#NO_UPDATE} when the
+ * sentence produced no update.
+ */
+ private double refineSentence(DependencySample sample, int[] oracle,
+ Transition[] transitions, int beamSize) {
+ final String[] tokens = sample.getTokens();
+ final String[] tags = sample.getTags();
+ final BeamNode root = new BeamNode(null, null, -1, 0.0, true);
+ root.state = new ArcStandardState(tokens.length);
+ List<BeamNode> beam = List.of(root);
+
+ for (int step = 0; step < oracle.length; step++) {
+ final List<BeamNode> expansions = new ArrayList<>();
+ BeamNode goldChild = null;
+ for (final BeamNode node : beam) {
+ final int[] features =
+ model.featureIds(FeedforwardContext.extract(node.state, tokens,
tags));
+ forward(features);
+ logSoftmaxInPlace(probabilities);
+ for (int i = 0; i < outputSize; i++) {
+ if (node.state.canApply(transitions[i])) {
+ final boolean goldNext = node.gold && i == oracle[step];
+ final BeamNode child = new BeamNode(node, features, i,
+ node.score + probabilities[i], goldNext);
+ expansions.add(child);
+ if (goldNext) {
+ goldChild = child;
+ }
+ }
+ }
+ }
+ expansions.sort((a, b) -> Double.compare(b.score, a.score));
+ final List<BeamNode> survivors =
+ new ArrayList<>(expansions.subList(0, Math.min(beamSize,
expansions.size())));
+ boolean goldRetained = false;
+ for (final BeamNode survivor : survivors) {
+ if (survivor.gold) {
+ goldRetained = true;
+ break;
+ }
+ }
+ if (!goldRetained) {
+ if (goldChild == null) {
+ // The gold transition was not applicable in the gold
configuration, so
+ // this sentence yields no update.
+ return NO_UPDATE;
+ }
+ survivors.add(goldChild);
+ return updateFromCandidates(survivors);
+ }
+ if (step == oracle.length - 1) {
+ return updateFromCandidates(survivors);
+ }
+ for (final BeamNode survivor : survivors) {
+ survivor.state = survivor.parent.state.copy();
+ survivor.state.apply(transitions[survivor.transition]);
+ }
+ beam = survivors;
+ }
+ return NO_UPDATE;
+ }
+
+ /**
+ * Applies the conditional-likelihood update over the candidate paths.
+ *
+ * @param candidates The surviving paths plus, on early update, the gold
path.
+ * @return The negative log-probability of the gold path under the
candidates.
+ */
+ private double updateFromCandidates(List<BeamNode> candidates) {
+ double max = Double.NEGATIVE_INFINITY;
+ double goldScore = Double.NEGATIVE_INFINITY;
+ for (final BeamNode candidate : candidates) {
+ max = Math.max(max, candidate.score);
+ if (candidate.gold) {
+ goldScore = candidate.score;
+ }
+ }
+ double normalizer = 0.0;
+ for (final BeamNode candidate : candidates) {
+ normalizer += Math.exp(candidate.score - max);
+ }
+ final double logNormalizer = max + Math.log(normalizer);
+
+ zero(hiddenGradient);
+ Arrays.fill(hiddenBiasGradient, 0.0);
+ zero(outputGradient);
+ Arrays.fill(outputBiasGradient, 0.0);
+ embeddingGradients.clear();
+ for (final BeamNode candidate : candidates) {
+ final double weight = Math.exp(candidate.score - logNormalizer)
+ - (candidate.gold ? 1.0 : 0.0);
+ if (weight == 0.0) {
+ continue;
+ }
+ for (BeamNode node = candidate; node.parent != null; node =
node.parent) {
+ backward(node.features, node.transition, weight);
+ }
+ }
+ update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1,
settings);
+ updateVector(model.hiddenBias(), hiddenBiasGradient,
hiddenBiasAccumulator, 1,
+ settings);
+ update(model.outputWeights(), outputGradient, outputAccumulator, 1,
settings);
+ updateVector(model.outputBias(), outputBiasGradient,
outputBiasAccumulator, 1,
+ settings);
+ for (final Map.Entry<Integer, double[]> entry :
embeddingGradients.entrySet()) {
+ final float[] embeddingRow = model.embeddings()[entry.getKey()];
+ final double[] accumulatorRow = embeddingAccumulator[entry.getKey()];
+ final double[] gradientRow = entry.getValue();
+ for (int d = 0; d < embeddingSize; d++) {
+ final double gradient = gradientRow[d];
+ accumulatorRow[d] += gradient * gradient;
+ embeddingRow[d] -= settings.learningRate() * gradient
+ / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON);
+ }
+ }
+ return logNormalizer - goldScore;
+ }
+
+ /**
+ * Computes hidden activations and raw output scores for one feature
vector into
+ * the scratch buffers.
+ *
+ * @param features The embedding rows of the configuration.
+ */
+ private void forward(int[] features) {
+ final float[][] embeddings = model.embeddings();
+ for (int f = 0; f < features.length; f++) {
+ final float[] embedding = embeddings[features[f]];
+ final int offset = f * embeddingSize;
+ for (int d = 0; d < embeddingSize; d++) {
+ x[offset + d] = embedding[d];
+ }
+ }
+ final float[][] hiddenWeights = model.hiddenWeights();
+ final float[] hiddenBias = model.hiddenBias();
+ for (int j = 0; j < hiddenSize; j++) {
+ final float[] weightRow = hiddenWeights[j];
+ double sum = hiddenBias[j];
+ for (int k = 0; k < inputSize; k++) {
+ sum += weightRow[k] * x[k];
+ }
+ pre[j] = sum;
+ hidden[j] = sum * sum * sum;
+ }
+ final float[][] outputWeights = model.outputWeights();
+ final float[] outputBias = model.outputBias();
+ for (int o = 0; o < outputSize; o++) {
+ final float[] weightRow = outputWeights[o];
+ double sum = outputBias[o];
+ for (int j = 0; j < hiddenSize; j++) {
+ sum += weightRow[j] * hidden[j];
+ }
+ probabilities[o] = sum;
+ }
+ }
+
+ /**
+ * Accumulates gradients for one decoded step: the weighted difference
between the
+ * step's softmax and its chosen transition.
+ *
+ * @param features The step's input features.
+ * @param chosen The transition the path took at this step.
+ * @param weight The path's weight in the candidate distribution.
+ */
+ private void backward(int[] features, int chosen, double weight) {
+ forward(features);
+ double max = Double.NEGATIVE_INFINITY;
+ for (int o = 0; o < outputSize; o++) {
+ max = Math.max(max, probabilities[o]);
+ }
+ double normalizer = 0.0;
+ for (int o = 0; o < outputSize; o++) {
+ probabilities[o] = Math.exp(probabilities[o] - max);
+ normalizer += probabilities[o];
+ }
+ Arrays.fill(hiddenDelta, 0.0);
+ Arrays.fill(inputDelta, 0.0);
+ final float[][] outputWeights = model.outputWeights();
+ for (int o = 0; o < outputSize; o++) {
+ // dL/dlogit for a path's step under the conditional likelihood: the
path weight
+ // times how the step's log-probability responds to this logit
+ final double delta =
+ weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] /
normalizer);
+ outputBiasGradient[o] += delta;
+ final double[] gradientRow = outputGradient[o];
+ final float[] weightRow = outputWeights[o];
+ for (int j = 0; j < hiddenSize; j++) {
+ gradientRow[j] += delta * hidden[j];
+ hiddenDelta[j] += delta * weightRow[j];
+ }
+ }
+ final float[][] hiddenWeights = model.hiddenWeights();
+ for (int j = 0; j < hiddenSize; j++) {
+ final double preDelta = hiddenDelta[j] * CUBE_DERIVATIVE_FACTOR *
pre[j] * pre[j];
+ hiddenBiasGradient[j] += preDelta;
+ final double[] gradientRow = hiddenGradient[j];
+ final float[] weightRow = hiddenWeights[j];
+ for (int k = 0; k < inputSize; k++) {
+ gradientRow[k] += preDelta * x[k];
+ inputDelta[k] += preDelta * weightRow[k];
+ }
+ }
+ for (int f = 0; f < features.length; f++) {
+ final double[] embeddingGradient = embeddingGradients
+ .computeIfAbsent(features[f], key -> new double[embeddingSize]);
+ final int offset = f * embeddingSize;
+ for (int d = 0; d < embeddingSize; d++) {
+ embeddingGradient[d] += inputDelta[offset + d];
+ }
+ }
+ }
+
+ /**
+ * Turns raw scores into log-probabilities in place.
+ *
+ * @param scores The raw scores, overwritten with their log-softmax.
+ */
+ private void logSoftmaxInPlace(double[] scores) {
+ double max = Double.NEGATIVE_INFINITY;
+ for (final double score : scores) {
+ max = Math.max(max, score);
+ }
+ double sum = 0.0;
+ for (final double score : scores) {
+ sum += Math.exp(score - max);
+ }
+ final double logSum = Math.log(sum);
+ for (int i = 0; i < scores.length; i++) {
+ scores[i] = (scores[i] - max) - logSum;
+ }
+ }
+ }
+
+ /**
+ * Reads a sample stream into memory; both trainers pass over the corpus
repeatedly.
+ *
+ * @param samples The stream to drain.
+ * @return All samples in stream order. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ */
+ private static List<DependencySample> readAll(ObjectStream<DependencySample>
samples)
+ throws IOException {
+ final List<DependencySample> corpus = new ArrayList<>();
+ DependencySample sample;
+ while ((sample = samples.read()) != null) {
+ corpus.add(sample);
+ }
+ return corpus;
+ }
+
+ /**
+ * Overwrites the random word rows with pretrained vectors where available.
+ *
+ * @param model The freshly initialized model whose word rows are seeded.
+ * @param pretrained The pretrained vector source, returning {@code null}
for a word it
+ * does not cover.
+ * @param settings The hyperparameters, which fix the expected vector width.
+ * @throws IllegalArgumentException Thrown if a pretrained vector has a
different width
+ * than the embedding size or contains a non-finite value.
+ */
+ private static void seed(FeedforwardDependencyModel model,
+ Function<String, float[]> pretrained, Settings settings) {
+ int seeded = 0;
+ for (final Map.Entry<String, Integer> entry : model.wordIds().entrySet()) {
+ if (FeedforwardDependencyModel.isSpecialSymbol(entry.getKey())) {
+ // The special unknown, padding, and root symbols have no pretrained
+ // counterpart, so they keep their random initialization.
+ continue;
+ }
+ final float[] vector = pretrained.apply(entry.getKey());
+ if (vector == null) {
+ continue;
+ }
+ if (vector.length != settings.embeddingSize()) {
+ throw new IllegalArgumentException("pretrained vector for '" +
entry.getKey()
+ + "' has " + vector.length + " dimensions, expected "
+ + settings.embeddingSize());
+ }
+ for (final float value : vector) {
+ if (!Float.isFinite(value)) {
+ throw new IllegalArgumentException(
+ "pretrained vector for '" + entry.getKey() + "' contains a
non-finite value");
+ }
+ }
+ System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0,
vector.length);
+ seeded++;
+ }
+ logger.info("seeded {} of {} word embeddings from the pretrained source",
seeded,
+ model.wordIds().size());
+ }
+
+ /**
+ * Builds the vocabularies and randomly initialized weights. A word, tag, or
label in
+ * the training data that spells a reserved symbol shares the reserved row,
as an
+ * unknown symbol does, so the reserved rows are never displaced.
+ *
+ * @param corpus The training samples.
+ * @param settings The hyperparameters.
+ * @return The untrained model. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the vocabularies and settings
would
+ * produce more model values than the model format can store.
+ */
+ private static FeedforwardDependencyModel initialize(List<DependencySample>
corpus,
+ Settings settings) {
+ final Map<String, Integer> wordCounts = new HashMap<>();
+ final Map<String, Integer> tagIds = new HashMap<>();
+ final Map<String, Integer> labelIds = new HashMap<>();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ for (final String token : s.getTokens()) {
+ wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1,
Integer::sum);
+ }
+ for (final String tag : s.getTags()) {
+ tagIds.putIfAbsent(tag, 0);
+ }
+ final DependencyTree tree = s.getTree();
+ for (int i = 0; i < tree.size(); i++) {
+ labelIds.putIfAbsent(tree.relationOf(i), 0);
+ }
+ for (final Transition transition : oracle) {
+ transitionIds.putIfAbsent(transition.encode(), 0);
+ }
+ }
+
+ int row = 0;
+ final Map<String, Integer> wordIds = new HashMap<>();
+ row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
DependencyContextGenerator.ROOT_VALUE);
+ for (final Map.Entry<String, Integer> entry : wordCounts.entrySet()) {
+ if (entry.getValue() >= settings.wordCutoff() &&
!wordIds.containsKey(entry.getKey())) {
+ wordIds.put(entry.getKey(), row++);
+ }
+ }
+ final Map<String, Integer> tags = new HashMap<>();
+ row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
DependencyContextGenerator.ROOT_VALUE);
+ for (final String tag : tagIds.keySet()) {
+ if (!tags.containsKey(tag)) {
+ tags.put(tag, row++);
+ }
+ }
+ final Map<String, Integer> labels = new HashMap<>();
+ row = addSpecialSymbols(labels, row, FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE);
+ for (final String label : labelIds.keySet()) {
+ if (!labels.containsKey(label)) {
+ labels.put(label, row++);
+ }
+ }
+
+ final String[] transitions = transitionIds.keySet().toArray(String[]::new);
+ Arrays.sort(transitions);
+
+ final long modelValues = modelFloatValues(row, transitions.length,
+ settings.embeddingSize(), settings.hiddenSize());
+ if (modelValues > FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("training vocabulary and settings
require "
+ + modelValues + " model values, limit is "
+ + FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES);
+ }
+
+ final Random random = new Random(settings.seed());
+ final int inputSize = FeedforwardContext.FEATURE_COUNT *
settings.embeddingSize();
+ final float[][] embeddings = uniform(random, row, settings.embeddingSize(),
+ EMBEDDING_INIT_SCALE);
+ final float[][] hiddenWeights = uniform(random, settings.hiddenSize(),
inputSize,
+ Math.sqrt(GLOROT_NUMERATOR / (inputSize + settings.hiddenSize())));
+ final float[][] outputWeights = uniform(random, transitions.length,
+ settings.hiddenSize(),
+ Math.sqrt(GLOROT_NUMERATOR / (settings.hiddenSize() +
transitions.length)));
+ return new FeedforwardDependencyModel(wordIds, tags, labels, transitions,
+ settings.embeddingSize(), embeddings, hiddenWeights,
+ new float[settings.hiddenSize()], outputWeights, new
float[transitions.length]);
+ }
+
+ /**
+ * Assigns the next embedding rows to the special symbols of one vocabulary.
+ *
+ * @param ids The vocabulary to fill.
+ * @param row The next free embedding row.
+ * @param symbols The special symbols, in the order they take their rows.
+ * @return The next free embedding row after the symbols.
+ */
+ private static int addSpecialSymbols(Map<String, Integer> ids, int row,
String... symbols) {
+ int next = row;
+ for (final String symbol : symbols) {
+ ids.put(symbol, next);
+ next++;
+ }
+ return next;
+ }
+
+ /**
+ * @param vocabularyRows The combined word, tag, and label row count.
+ * @param transitionCount The output layer width.
+ * @param embeddingSize The embedding dimensionality.
+ * @param hiddenSize The hidden layer width.
+ * @return The number of float values a model of that shape stores.
+ */
+ private static long modelFloatValues(int vocabularyRows, int transitionCount,
+ int embeddingSize, int hiddenSize) {
+ final long inputSize = (long) FeedforwardContext.FEATURE_COUNT *
embeddingSize;
+ return (long) vocabularyRows * embeddingSize
+ + (long) hiddenSize * inputSize
+ + hiddenSize
+ + (long) transitionCount * hiddenSize
+ + transitionCount;
+ }
+
+ /**
+ * Replays the oracle over every projective sample, emitting one example per
step.
+ *
+ * @param corpus The training samples.
+ * @param model The initialized model whose vocabularies map the features.
+ * @param featureList Receives the embedding rows of each configuration.
+ * @param goldList Receives the gold transition index of each configuration.
+ */
+ private static void collectExamples(List<DependencySample> corpus,
+ FeedforwardDependencyModel model, List<int[]> featureList, List<Integer>
goldList) {
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ final String[] transitions = model.transitions();
+ for (int i = 0; i < transitions.length; i++) {
+ transitionIds.put(transitions[i], i);
+ }
+ int skipped = 0;
+ for (final DependencySample sample : corpus) {
+ if (!ArcStandardOracle.isProjective(sample.getTree())) {
Review Comment:
The projectivity filter and oracle run once per sample in `derive`; decoded
transitions and the index map are built once by the network and exposed from
there.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java:
##########
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ParamChecks;
+
+/**
+ * Trains the {@link FeedforwardDependencyModel}: oracle-derived transition
examples,
+ * minibatch AdaGrad over a softmax cross-entropy loss, cube activation, and
inverted
+ * dropout on the hidden layer, the training recipe of
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Words below the frequency cutoff share a learned unknown embedding;
absent
+ * template positions share a learned padding embedding. Non-projective
samples have no
+ * arc-standard derivation and are skipped. Training is deterministic for a
fixed
+ * {@link Settings#seed()}.</p>
+ *
+ * @since 3.0.0
+ */
+public final class FeedforwardDependencyTrainer {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(FeedforwardDependencyTrainer.class);
+
+ /** The AdaGrad denominator offset that keeps the first steps finite. */
+ private static final double ADAGRAD_EPSILON = 1e-6;
+
+ /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */
+ private static final double CUBE_DERIVATIVE_FACTOR = 3.0;
+
+ /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn +
fanOut))}. */
+ private static final double GLOROT_NUMERATOR = 6.0;
+
+ /** The half-width of the uniform range embeddings are drawn from. */
+ private static final double EMBEDDING_INIT_SCALE = 0.01;
+
+ /** The smallest probability the reported training loss takes the log of. */
+ private static final double LOSS_PROBABILITY_FLOOR = 1e-12;
+
+ /** The sentence loss reported when refinement made no update for a
sentence. */
+ private static final double NO_UPDATE = -1.0;
+
+ /** Special-symbol rows present before any training vocabulary is added. */
+ private static final int MIN_VOCABULARY_ROWS = 8;
+
+ /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */
+ private static final int MIN_TRANSITIONS = 2;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardDependencyTrainer() {
+ }
+
+ /**
+ * The training hyperparameters.
+ *
+ * @param embeddingSize The embedding dimensionality. Must be positive.
+ * @param hiddenSize The hidden layer width. Must be positive.
+ * @param epochs The number of passes over the examples. Must be positive.
+ * @param batchSize The minibatch size. Must be positive.
+ * @param learningRate The AdaGrad step size. Must be positive.
+ * @param l2 The L2 penalty applied to the dense weights. Must not be
negative.
+ * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}.
+ * @param wordCutoff The minimum frequency for a word to get its own
embedding. Must
+ * not be negative.
+ * @param seed The random seed making a run reproducible.
+ */
+ public record Settings(int embeddingSize, int hiddenSize, int epochs, int
batchSize,
+ double learningRate, double l2, double dropout, int wordCutoff, long
seed) {
+
+ /** The default embedding dimensionality. */
+ private static final int DEFAULT_EMBEDDING_SIZE = 50;
+
+ /** The default hidden layer width. */
+ private static final int DEFAULT_HIDDEN_SIZE = 200;
+
+ /** The default number of epochs. */
+ private static final int DEFAULT_EPOCHS = 10;
+
+ /** The default minibatch size. */
+ private static final int DEFAULT_BATCH_SIZE = 256;
+
+ /** The default AdaGrad step size. */
+ private static final double DEFAULT_LEARNING_RATE = 0.02;
+
+ /** The default L2 penalty. */
+ private static final double DEFAULT_L2 = 1e-8;
+
+ /** The default hidden dropout probability. */
+ private static final double DEFAULT_DROPOUT = 0.5;
+
+ /** The default minimum word frequency for an embedding row. */
+ private static final int DEFAULT_WORD_CUTOFF = 2;
+
+ /** The default random seed. */
+ static final long DEFAULT_SEED = 17L;
+
+ /**
+ * Validates the hyperparameters.
+ *
+ * @throws IllegalArgumentException Thrown if a value is outside its
documented
+ * range.
+ */
+ public Settings {
+ requirePositive(embeddingSize, "embeddingSize");
+ requirePositive(hiddenSize, "hiddenSize");
+ requirePositive(epochs, "epochs");
+ requirePositive(batchSize, "batchSize");
+ if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) {
+ throw new IllegalArgumentException("embeddingSize exceeds the model
format limit: "
+ + embeddingSize);
+ }
+ if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) {
+ throw new IllegalArgumentException("hiddenSize exceeds the model
format limit: "
+ + hiddenSize);
+ }
+ final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS,
+ MIN_TRANSITIONS, embeddingSize, hiddenSize);
+ if (minimumModelValues >
FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("embeddingSize and hiddenSize
exceed the model "
+ + "format allocation limit");
+ }
+ if (ParamChecks.requireFinite(learningRate, "learningRate") <= 0.0) {
+ throw new IllegalArgumentException("learningRate must be positive: " +
learningRate);
+ }
+ ParamChecks.requireNonNegative(ParamChecks.requireFinite(l2, "l2"),
"l2");
+ if (!(dropout >= 0.0 && dropout < 1.0)) {
+ throw new IllegalArgumentException("dropout must be in [0, 1): " +
dropout);
+ }
+ ParamChecks.requireNonNegative(wordCutoff, "wordCutoff");
+ }
+
+ /**
+ * Rejects a hyperparameter that is not positive.
+ *
+ * @param value The value to check.
+ * @param name The hyperparameter name used in the exception message.
+ * @throws IllegalArgumentException Thrown if {@code value} is not
positive.
+ */
+ private static void requirePositive(int value, String name) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(name + " must be positive: " +
value);
+ }
+ }
+
+ /**
+ * @return The default hyperparameters. Never {@code null}.
+ */
+ public static Settings defaults() {
+ return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE,
DEFAULT_EPOCHS,
+ DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2,
DEFAULT_DROPOUT,
+ DEFAULT_WORD_CUTOFF, DEFAULT_SEED);
+ }
+ }
+
+ /**
+ * Trains a model from dependency samples.
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null} or
no
+ * trainable example can be derived from the samples.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings) throws IOException {
+ return train(samples, settings, null);
+ }
+
+ /**
+ * Trains a model from dependency samples, seeding word embeddings from a
pretrained
+ * source.
+ *
+ * <p>The provider is consulted once per vocabulary word during
initialization; words
+ * it returns {@code null} for keep their random initialization, and all
embeddings
+ * remain trainable afterwards. The pretrained source is a training-time
ingredient
+ * only: the learned embeddings ship inside the model, so parsing carries no
+ * dependency on the source.</p>
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @param pretrained Maps a normalized word to its pretrained vector of
exactly
+ * {@link Settings#embeddingSize()} dimensions, or {@code
null} for
+ * unknown words. May be {@code null} to disable seeding.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if {@code samples} or {@code
settings} is
+ * {@code null}, no trainable example can be derived, or a
pretrained vector
+ * has the wrong dimensionality.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings, Function<String, float[]> pretrained)
+ throws IOException {
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ final List<DependencySample> corpus = readAll(samples);
+ final FeedforwardDependencyModel model = initialize(corpus, settings);
+ if (pretrained != null) {
+ seed(model, pretrained, settings);
+ }
+ final List<int[]> featureList = new ArrayList<>();
+ final List<Integer> goldList = new ArrayList<>();
+ collectExamples(corpus, model, featureList, goldList);
+ if (featureList.isEmpty()) {
+ throw new IllegalArgumentException("no trainable examples in the
samples");
+ }
+ optimize(model, featureList, goldList, settings);
+ return model;
+ }
+
+ /**
+ * Refines a locally trained model with beam search and the early-update
method of
+ * <a href="https://aclanthology.org/P04-1015/";>Collins and Roark
(2004)</a>. The loss
+ * is conditional likelihood over candidate paths scored by summed
log-probabilities.
+ *
+ * <p>Refinement updates a copy with per-sentence AdaGrad steps and no
dropout.
+ * {@link Settings#epochs()} sets the number of refinement passes. Use the
same beam
+ * size for parsing the result.</p>
+ *
+ * <p>The transition inventory comes from {@code model} and is not extended,
because
+ * its size is the width of the trained output layer. A refinement corpus
using a
+ * relation label the original training set lacked is therefore rejected
rather than
+ * silently ignored.</p>
+ *
+ * @param model The locally trained model to refine. Left untouched. Must
not be
+ * {@code null}.
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters; {@code epochs}, {@code learningRate},
+ * {@code l2}, and {@code seed} apply. Must not be {@code
null}.
+ * @param beamSize The beam width to track the gold derivation in. Must be
at least 2.
+ * @return A new refined model, distinct from {@code model}. Never {@code
null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null},
+ * {@code beamSize} is below 2, no trainable sample can be derived,
or a sample
+ * requires a transition {@code model} does not know.
+ */
+ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel
model,
+ ObjectStream<DependencySample> samples, Settings settings, int beamSize)
+ throws IOException {
+ ParamChecks.requireNonNullArg(model, "model");
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ if (beamSize < 2) {
+ throw new IllegalArgumentException("beamSize must be at least 2: " +
beamSize);
+ }
+ final List<DependencySample> corpus = readAll(samples);
+ final String[] outcomes = model.transitions();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ final Transition[] transitions = new Transition[outcomes.length];
+ for (int i = 0; i < transitions.length; i++) {
+ transitionIds.put(outcomes[i], i);
+ transitions[i] = Transition.decode(outcomes[i]);
+ }
+
+ final List<DependencySample> trainable = new ArrayList<>();
+ final List<int[]> oracles = new ArrayList<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ final int[] encoded = new int[oracle.size()];
+ for (int i = 0; i < encoded.length; i++) {
+ final String outcome = oracle.get(i).encode();
+ final Integer id = transitionIds.get(outcome);
+ if (id == null) {
+ // The outcome space was fixed by the original training set, so a
relation
+ // label it never saw has no output unit to push probability onto.
+ throw new IllegalArgumentException(
+ "unknown transition in the refinement samples: " + outcome);
+ }
+ encoded[i] = id;
+ }
+ trainable.add(s);
+ oracles.add(encoded);
+ }
+ if (trainable.isEmpty()) {
+ throw new IllegalArgumentException("no trainable samples for
refinement");
+ }
+
+ final FeedforwardDependencyModel refined = model.copy();
+ final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings);
+ final Random random = new Random(settings.seed());
+ final int[] order = new int[trainable.size()];
+ for (int i = 0; i < order.length; i++) {
+ order[i] = i;
+ }
+ for (int epoch = 1; epoch <= settings.epochs(); epoch++) {
+ final long epochStart = System.currentTimeMillis();
+ shuffle(order, random);
+ double loss = 0.0;
+ int updates = 0;
+ for (final int index : order) {
+ final double sentenceLoss =
optimizer.refineSentence(trainable.get(index),
+ oracles.get(index), transitions, beamSize);
+ if (sentenceLoss >= 0.0) {
+ loss += sentenceLoss;
+ updates++;
+ }
+ }
+ checkFinite(refined);
+ logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch,
+ loss / Math.max(updates, 1), updates, System.currentTimeMillis() -
epochStart);
+ }
+ return refined;
+ }
+
+ /** One candidate path in the refinement beam: the parent link forms the
history. */
+ private static final class BeamNode {
+ private final BeamNode parent;
+ private final int[] features;
+ private final int transition;
+ private final double score;
+ private final boolean gold;
+ private ArcStandardState state;
+
+ /**
+ * Extends {@code parent} by one transition; the start node passes {@code
null}.
+ *
+ * @param parent The node this one extends, or {@code null} for the start
node.
+ * @param features The embedding rows the transition was scored on.
+ * @param transition The transition index taken from {@code parent}.
+ * @param score The summed log-probability of the path.
+ * @param gold Whether the path is the gold derivation so far.
+ */
+ private BeamNode(BeamNode parent, int[] features, int transition, double
score,
+ boolean gold) {
+ this.parent = parent;
+ this.features = features;
+ this.transition = transition;
+ this.score = score;
+ this.gold = gold;
+ }
+ }
+
+ /** The forward, backward, and AdaGrad state for global refinement. */
+ private static final class GlobalOptimizer {
+ private final FeedforwardDependencyModel model;
+ private final Settings settings;
+ private final int embeddingSize;
+ private final int hiddenSize;
+ private final int outputSize;
+ private final int inputSize;
+
+ private final double[][] embeddingAccumulator;
+ private final double[][] hiddenAccumulator;
+ private final double[] hiddenBiasAccumulator;
+ private final double[][] outputAccumulator;
+ private final double[] outputBiasAccumulator;
+
+ private final double[][] hiddenGradient;
+ private final double[] hiddenBiasGradient;
+ private final double[][] outputGradient;
+ private final double[] outputBiasGradient;
+ private final Map<Integer, double[]> embeddingGradients = new HashMap<>();
+
+ private final double[] x;
+ private final double[] pre;
+ private final double[] hidden;
+ private final double[] probabilities;
+ private final double[] hiddenDelta;
+ private final double[] inputDelta;
+
+ /**
+ * Sizes the accumulators and scratch buffers for one refinement run.
+ *
+ * @param model The model copy the run updates in place.
+ * @param settings The hyperparameters; {@code learningRate} and {@code
l2} apply.
+ */
+ private GlobalOptimizer(FeedforwardDependencyModel model, Settings
settings) {
+ this.model = model;
+ this.settings = settings;
+ this.embeddingSize = model.embeddings()[0].length;
+ this.hiddenSize = model.hiddenBias().length;
+ this.outputSize = model.outputBias().length;
+ this.inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize;
+ this.embeddingAccumulator =
+ new double[model.embeddings().length][embeddingSize];
+ this.hiddenAccumulator = new double[hiddenSize][inputSize];
+ this.hiddenBiasAccumulator = new double[hiddenSize];
+ this.outputAccumulator = new double[outputSize][hiddenSize];
+ this.outputBiasAccumulator = new double[outputSize];
+ this.hiddenGradient = new double[hiddenSize][inputSize];
+ this.hiddenBiasGradient = new double[hiddenSize];
+ this.outputGradient = new double[outputSize][hiddenSize];
+ this.outputBiasGradient = new double[outputSize];
+ this.x = new double[inputSize];
+ this.pre = new double[hiddenSize];
+ this.hidden = new double[hiddenSize];
+ this.probabilities = new double[outputSize];
+ this.hiddenDelta = new double[hiddenSize];
+ this.inputDelta = new double[inputSize];
+ }
+
+ /**
+ * Decodes one sentence with the beam, updating on the early-update point
or the
+ * final beam.
+ *
+ * @param sample The sentence.
+ * @param oracle The gold transition indexes.
+ * @param transitions The decoded transition inventory.
+ * @param beamSize The beam width.
+ * @return The sentence loss, or {@link
FeedforwardDependencyTrainer#NO_UPDATE} when the
+ * sentence produced no update.
+ */
+ private double refineSentence(DependencySample sample, int[] oracle,
Review Comment:
`ArcStandardBeam` is the one beam step for decoding and refinement.
##########
opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java:
##########
@@ -0,0 +1,1176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.Function;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.ParamChecks;
+
+/**
+ * Trains the {@link FeedforwardDependencyModel}: oracle-derived transition
examples,
+ * minibatch AdaGrad over a softmax cross-entropy loss, cube activation, and
inverted
+ * dropout on the hidden layer, the training recipe of
+ * <a href="https://aclanthology.org/D14-1082/";>Chen and Manning (2014)</a>.
+ *
+ * <p>Words below the frequency cutoff share a learned unknown embedding;
absent
+ * template positions share a learned padding embedding. Non-projective
samples have no
+ * arc-standard derivation and are skipped. Training is deterministic for a
fixed
+ * {@link Settings#seed()}.</p>
+ *
+ * @since 3.0.0
+ */
+public final class FeedforwardDependencyTrainer {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(FeedforwardDependencyTrainer.class);
+
+ /** The AdaGrad denominator offset that keeps the first steps finite. */
+ private static final double ADAGRAD_EPSILON = 1e-6;
+
+ /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */
+ private static final double CUBE_DERIVATIVE_FACTOR = 3.0;
+
+ /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn +
fanOut))}. */
+ private static final double GLOROT_NUMERATOR = 6.0;
+
+ /** The half-width of the uniform range embeddings are drawn from. */
+ private static final double EMBEDDING_INIT_SCALE = 0.01;
+
+ /** The smallest probability the reported training loss takes the log of. */
+ private static final double LOSS_PROBABILITY_FLOOR = 1e-12;
+
+ /** The sentence loss reported when refinement made no update for a
sentence. */
+ private static final double NO_UPDATE = -1.0;
+
+ /** Special-symbol rows present before any training vocabulary is added. */
+ private static final int MIN_VOCABULARY_ROWS = 8;
+
+ /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */
+ private static final int MIN_TRANSITIONS = 2;
+
+ /** Prevents construction of this utility class. */
+ private FeedforwardDependencyTrainer() {
+ }
+
+ /**
+ * The training hyperparameters.
+ *
+ * @param embeddingSize The embedding dimensionality. Must be positive.
+ * @param hiddenSize The hidden layer width. Must be positive.
+ * @param epochs The number of passes over the examples. Must be positive.
+ * @param batchSize The minibatch size. Must be positive.
+ * @param learningRate The AdaGrad step size. Must be positive.
+ * @param l2 The L2 penalty applied to the dense weights. Must not be
negative.
+ * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}.
+ * @param wordCutoff The minimum frequency for a word to get its own
embedding. Must
+ * not be negative.
+ * @param seed The random seed making a run reproducible.
+ */
+ public record Settings(int embeddingSize, int hiddenSize, int epochs, int
batchSize,
+ double learningRate, double l2, double dropout, int wordCutoff, long
seed) {
+
+ /** The default embedding dimensionality. */
+ private static final int DEFAULT_EMBEDDING_SIZE = 50;
+
+ /** The default hidden layer width. */
+ private static final int DEFAULT_HIDDEN_SIZE = 200;
+
+ /** The default number of epochs. */
+ private static final int DEFAULT_EPOCHS = 10;
+
+ /** The default minibatch size. */
+ private static final int DEFAULT_BATCH_SIZE = 256;
+
+ /** The default AdaGrad step size. */
+ private static final double DEFAULT_LEARNING_RATE = 0.02;
+
+ /** The default L2 penalty. */
+ private static final double DEFAULT_L2 = 1e-8;
+
+ /** The default hidden dropout probability. */
+ private static final double DEFAULT_DROPOUT = 0.5;
+
+ /** The default minimum word frequency for an embedding row. */
+ private static final int DEFAULT_WORD_CUTOFF = 2;
+
+ /** The default random seed. */
+ static final long DEFAULT_SEED = 17L;
+
+ /**
+ * Validates the hyperparameters.
+ *
+ * @throws IllegalArgumentException Thrown if a value is outside its
documented
+ * range.
+ */
+ public Settings {
+ requirePositive(embeddingSize, "embeddingSize");
+ requirePositive(hiddenSize, "hiddenSize");
+ requirePositive(epochs, "epochs");
+ requirePositive(batchSize, "batchSize");
+ if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) {
+ throw new IllegalArgumentException("embeddingSize exceeds the model
format limit: "
+ + embeddingSize);
+ }
+ if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) {
+ throw new IllegalArgumentException("hiddenSize exceeds the model
format limit: "
+ + hiddenSize);
+ }
+ final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS,
+ MIN_TRANSITIONS, embeddingSize, hiddenSize);
+ if (minimumModelValues >
FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) {
+ throw new IllegalArgumentException("embeddingSize and hiddenSize
exceed the model "
+ + "format allocation limit");
+ }
+ if (ParamChecks.requireFinite(learningRate, "learningRate") <= 0.0) {
+ throw new IllegalArgumentException("learningRate must be positive: " +
learningRate);
+ }
+ ParamChecks.requireNonNegative(ParamChecks.requireFinite(l2, "l2"),
"l2");
+ if (!(dropout >= 0.0 && dropout < 1.0)) {
+ throw new IllegalArgumentException("dropout must be in [0, 1): " +
dropout);
+ }
+ ParamChecks.requireNonNegative(wordCutoff, "wordCutoff");
+ }
+
+ /**
+ * Rejects a hyperparameter that is not positive.
+ *
+ * @param value The value to check.
+ * @param name The hyperparameter name used in the exception message.
+ * @throws IllegalArgumentException Thrown if {@code value} is not
positive.
+ */
+ private static void requirePositive(int value, String name) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(name + " must be positive: " +
value);
+ }
+ }
+
+ /**
+ * @return The default hyperparameters. Never {@code null}.
+ */
+ public static Settings defaults() {
+ return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE,
DEFAULT_EPOCHS,
+ DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2,
DEFAULT_DROPOUT,
+ DEFAULT_WORD_CUTOFF, DEFAULT_SEED);
+ }
+ }
+
+ /**
+ * Trains a model from dependency samples.
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null} or
no
+ * trainable example can be derived from the samples.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings) throws IOException {
+ return train(samples, settings, null);
+ }
+
+ /**
+ * Trains a model from dependency samples, seeding word embeddings from a
pretrained
+ * source.
+ *
+ * <p>The provider is consulted once per vocabulary word during
initialization; words
+ * it returns {@code null} for keep their random initialization, and all
embeddings
+ * remain trainable afterwards. The pretrained source is a training-time
ingredient
+ * only: the learned embeddings ship inside the model, so parsing carries no
+ * dependency on the source.</p>
+ *
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters. Must not be {@code null}.
+ * @param pretrained Maps a normalized word to its pretrained vector of
exactly
+ * {@link Settings#embeddingSize()} dimensions, or {@code
null} for
+ * unknown words. May be {@code null} to disable seeding.
+ * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if {@code samples} or {@code
settings} is
+ * {@code null}, no trainable example can be derived, or a
pretrained vector
+ * has the wrong dimensionality.
+ */
+ public static FeedforwardDependencyModel
train(ObjectStream<DependencySample> samples,
+ Settings settings, Function<String, float[]> pretrained)
+ throws IOException {
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ final List<DependencySample> corpus = readAll(samples);
+ final FeedforwardDependencyModel model = initialize(corpus, settings);
+ if (pretrained != null) {
+ seed(model, pretrained, settings);
+ }
+ final List<int[]> featureList = new ArrayList<>();
+ final List<Integer> goldList = new ArrayList<>();
+ collectExamples(corpus, model, featureList, goldList);
+ if (featureList.isEmpty()) {
+ throw new IllegalArgumentException("no trainable examples in the
samples");
+ }
+ optimize(model, featureList, goldList, settings);
+ return model;
+ }
+
+ /**
+ * Refines a locally trained model with beam search and the early-update
method of
+ * <a href="https://aclanthology.org/P04-1015/";>Collins and Roark
(2004)</a>. The loss
+ * is conditional likelihood over candidate paths scored by summed
log-probabilities.
+ *
+ * <p>Refinement updates a copy with per-sentence AdaGrad steps and no
dropout.
+ * {@link Settings#epochs()} sets the number of refinement passes. Use the
same beam
+ * size for parsing the result.</p>
+ *
+ * <p>The transition inventory comes from {@code model} and is not extended,
because
+ * its size is the width of the trained output layer. A refinement corpus
using a
+ * relation label the original training set lacked is therefore rejected
rather than
+ * silently ignored.</p>
+ *
+ * @param model The locally trained model to refine. Left untouched. Must
not be
+ * {@code null}.
+ * @param samples The training samples. Must not be {@code null}.
+ * @param settings The hyperparameters; {@code epochs}, {@code learningRate},
+ * {@code l2}, and {@code seed} apply. Must not be {@code
null}.
+ * @param beamSize The beam width to track the gold derivation in. Must be
at least 2.
+ * @return A new refined model, distinct from {@code model}. Never {@code
null}.
+ * @throws IOException Thrown if reading the samples fails.
+ * @throws IllegalArgumentException Thrown if a parameter is {@code null},
+ * {@code beamSize} is below 2, no trainable sample can be derived,
or a sample
+ * requires a transition {@code model} does not know.
+ */
+ public static FeedforwardDependencyModel refine(FeedforwardDependencyModel
model,
+ ObjectStream<DependencySample> samples, Settings settings, int beamSize)
+ throws IOException {
+ ParamChecks.requireNonNullArg(model, "model");
+ ParamChecks.requireNonNullArg(samples, "samples");
+ ParamChecks.requireNonNullArg(settings, "settings");
+ if (beamSize < 2) {
+ throw new IllegalArgumentException("beamSize must be at least 2: " +
beamSize);
+ }
+ final List<DependencySample> corpus = readAll(samples);
+ final String[] outcomes = model.transitions();
+ final Map<String, Integer> transitionIds = new HashMap<>();
+ final Transition[] transitions = new Transition[outcomes.length];
+ for (int i = 0; i < transitions.length; i++) {
+ transitionIds.put(outcomes[i], i);
+ transitions[i] = Transition.decode(outcomes[i]);
+ }
+
+ final List<DependencySample> trainable = new ArrayList<>();
+ final List<int[]> oracles = new ArrayList<>();
+ for (final DependencySample s : corpus) {
+ if (!ArcStandardOracle.isProjective(s.getTree())) {
+ continue;
+ }
+ final List<Transition> oracle =
ArcStandardOracle.transitions(s.getTree());
+ final int[] encoded = new int[oracle.size()];
+ for (int i = 0; i < encoded.length; i++) {
+ final String outcome = oracle.get(i).encode();
+ final Integer id = transitionIds.get(outcome);
+ if (id == null) {
+ // The outcome space was fixed by the original training set, so a
relation
+ // label it never saw has no output unit to push probability onto.
+ throw new IllegalArgumentException(
+ "unknown transition in the refinement samples: " + outcome);
+ }
+ encoded[i] = id;
+ }
+ trainable.add(s);
+ oracles.add(encoded);
+ }
+ if (trainable.isEmpty()) {
+ throw new IllegalArgumentException("no trainable samples for
refinement");
+ }
+
+ final FeedforwardDependencyModel refined = model.copy();
+ final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings);
+ final Random random = new Random(settings.seed());
+ final int[] order = new int[trainable.size()];
+ for (int i = 0; i < order.length; i++) {
+ order[i] = i;
+ }
+ for (int epoch = 1; epoch <= settings.epochs(); epoch++) {
+ final long epochStart = System.currentTimeMillis();
+ shuffle(order, random);
+ double loss = 0.0;
+ int updates = 0;
+ for (final int index : order) {
+ final double sentenceLoss =
optimizer.refineSentence(trainable.get(index),
+ oracles.get(index), transitions, beamSize);
+ if (sentenceLoss >= 0.0) {
+ loss += sentenceLoss;
+ updates++;
+ }
+ }
+ checkFinite(refined);
+ logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch,
+ loss / Math.max(updates, 1), updates, System.currentTimeMillis() -
epochStart);
+ }
+ return refined;
+ }
+
+ /** One candidate path in the refinement beam: the parent link forms the
history. */
+ private static final class BeamNode {
+ private final BeamNode parent;
+ private final int[] features;
+ private final int transition;
+ private final double score;
+ private final boolean gold;
+ private ArcStandardState state;
+
+ /**
+ * Extends {@code parent} by one transition; the start node passes {@code
null}.
+ *
+ * @param parent The node this one extends, or {@code null} for the start
node.
+ * @param features The embedding rows the transition was scored on.
+ * @param transition The transition index taken from {@code parent}.
+ * @param score The summed log-probability of the path.
+ * @param gold Whether the path is the gold derivation so far.
+ */
+ private BeamNode(BeamNode parent, int[] features, int transition, double
score,
+ boolean gold) {
+ this.parent = parent;
+ this.features = features;
+ this.transition = transition;
+ this.score = score;
+ this.gold = gold;
+ }
+ }
+
+ /** The forward, backward, and AdaGrad state for global refinement. */
+ private static final class GlobalOptimizer {
+ private final FeedforwardDependencyModel model;
+ private final Settings settings;
+ private final int embeddingSize;
+ private final int hiddenSize;
+ private final int outputSize;
+ private final int inputSize;
+
+ private final double[][] embeddingAccumulator;
+ private final double[][] hiddenAccumulator;
+ private final double[] hiddenBiasAccumulator;
+ private final double[][] outputAccumulator;
+ private final double[] outputBiasAccumulator;
+
+ private final double[][] hiddenGradient;
+ private final double[] hiddenBiasGradient;
+ private final double[][] outputGradient;
+ private final double[] outputBiasGradient;
+ private final Map<Integer, double[]> embeddingGradients = new HashMap<>();
+
+ private final double[] x;
+ private final double[] pre;
+ private final double[] hidden;
+ private final double[] probabilities;
+ private final double[] hiddenDelta;
+ private final double[] inputDelta;
+
+ /**
+ * Sizes the accumulators and scratch buffers for one refinement run.
+ *
+ * @param model The model copy the run updates in place.
+ * @param settings The hyperparameters; {@code learningRate} and {@code
l2} apply.
+ */
+ private GlobalOptimizer(FeedforwardDependencyModel model, Settings
settings) {
+ this.model = model;
+ this.settings = settings;
+ this.embeddingSize = model.embeddings()[0].length;
+ this.hiddenSize = model.hiddenBias().length;
+ this.outputSize = model.outputBias().length;
+ this.inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize;
+ this.embeddingAccumulator =
+ new double[model.embeddings().length][embeddingSize];
+ this.hiddenAccumulator = new double[hiddenSize][inputSize];
+ this.hiddenBiasAccumulator = new double[hiddenSize];
+ this.outputAccumulator = new double[outputSize][hiddenSize];
+ this.outputBiasAccumulator = new double[outputSize];
+ this.hiddenGradient = new double[hiddenSize][inputSize];
+ this.hiddenBiasGradient = new double[hiddenSize];
+ this.outputGradient = new double[outputSize][hiddenSize];
+ this.outputBiasGradient = new double[outputSize];
+ this.x = new double[inputSize];
+ this.pre = new double[hiddenSize];
+ this.hidden = new double[hiddenSize];
+ this.probabilities = new double[outputSize];
+ this.hiddenDelta = new double[hiddenSize];
+ this.inputDelta = new double[inputSize];
+ }
+
+ /**
+ * Decodes one sentence with the beam, updating on the early-update point
or the
+ * final beam.
+ *
+ * @param sample The sentence.
+ * @param oracle The gold transition indexes.
+ * @param transitions The decoded transition inventory.
+ * @param beamSize The beam width.
+ * @return The sentence loss, or {@link
FeedforwardDependencyTrainer#NO_UPDATE} when the
+ * sentence produced no update.
+ */
+ private double refineSentence(DependencySample sample, int[] oracle,
+ Transition[] transitions, int beamSize) {
+ final String[] tokens = sample.getTokens();
+ final String[] tags = sample.getTags();
+ final BeamNode root = new BeamNode(null, null, -1, 0.0, true);
+ root.state = new ArcStandardState(tokens.length);
+ List<BeamNode> beam = List.of(root);
+
+ for (int step = 0; step < oracle.length; step++) {
+ final List<BeamNode> expansions = new ArrayList<>();
+ BeamNode goldChild = null;
+ for (final BeamNode node : beam) {
+ final int[] features =
+ model.featureIds(FeedforwardContext.extract(node.state, tokens,
tags));
+ forward(features);
+ logSoftmaxInPlace(probabilities);
+ for (int i = 0; i < outputSize; i++) {
+ if (node.state.canApply(transitions[i])) {
+ final boolean goldNext = node.gold && i == oracle[step];
+ final BeamNode child = new BeamNode(node, features, i,
+ node.score + probabilities[i], goldNext);
+ expansions.add(child);
+ if (goldNext) {
+ goldChild = child;
+ }
+ }
+ }
+ }
+ expansions.sort((a, b) -> Double.compare(b.score, a.score));
+ final List<BeamNode> survivors =
+ new ArrayList<>(expansions.subList(0, Math.min(beamSize,
expansions.size())));
+ boolean goldRetained = false;
+ for (final BeamNode survivor : survivors) {
+ if (survivor.gold) {
+ goldRetained = true;
+ break;
+ }
+ }
+ if (!goldRetained) {
+ if (goldChild == null) {
+ // The gold transition was not applicable in the gold
configuration, so
+ // this sentence yields no update.
+ return NO_UPDATE;
+ }
+ survivors.add(goldChild);
+ return updateFromCandidates(survivors);
+ }
+ if (step == oracle.length - 1) {
+ return updateFromCandidates(survivors);
+ }
+ for (final BeamNode survivor : survivors) {
+ survivor.state = survivor.parent.state.copy();
+ survivor.state.apply(transitions[survivor.transition]);
+ }
+ beam = survivors;
+ }
+ return NO_UPDATE;
+ }
+
+ /**
+ * Applies the conditional-likelihood update over the candidate paths.
+ *
+ * @param candidates The surviving paths plus, on early update, the gold
path.
+ * @return The negative log-probability of the gold path under the
candidates.
+ */
+ private double updateFromCandidates(List<BeamNode> candidates) {
Review Comment:
Done. Weights are summed per beam node and each node with a nonzero weight
is backpropagated once.
##########
opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java:
##########
@@ -0,0 +1,760 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.depparse;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.tools.util.StringUtil;
+
+import static opennlp.tools.depparse.DependencyTestSamples.SHE_EATS_FISH_TAGS;
+import static
opennlp.tools.depparse.DependencyTestSamples.SHE_EATS_FISH_TOKENS;
+import static opennlp.tools.depparse.DependencyTestSamples.SHE_EATS_FISH_TREE;
+import static opennlp.tools.depparse.DependencyTestSamples.THE_DOG_BARKS_TAGS;
+import static
opennlp.tools.depparse.DependencyTestSamples.THE_DOG_BARKS_TOKENS;
+import static opennlp.tools.depparse.DependencyTestSamples.THE_DOG_BARKS_TREE;
+import static opennlp.tools.depparse.DependencyTestSamples.corpus;
+import static opennlp.tools.depparse.DependencyTestSamples.nonProjectiveSample;
+import static opennlp.tools.depparse.DependencyTestSamples.sample;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests feedforward training, parsing, refinement, and model serialization.
+ */
+public class FeedforwardDependencyParserTest {
+
+ /** The random seed shared by every training run in this class. */
+ private static final long SEED =
FeedforwardDependencyTrainer.Settings.DEFAULT_SEED;
+
+ /** The header every serialized model starts with. */
+ private static final String MAGIC = FeedforwardDependencyModel.MAGIC;
+
+ /** The embedding width of the test networks. */
+ private static final int EMBEDDING_SIZE = 16;
+
+ /** The hidden width of the test networks. */
+ private static final int HIDDEN_SIZE = 32;
+
+ /** The minibatch size of the test networks. */
+ private static final int BATCH_SIZE = 32;
+
+ /** The word cutoff that gives every corpus word its own embedding row. */
+ private static final int WORD_CUTOFF = 1;
+
+ /** The number of vocabularies a serialized model starts with. */
+ private static final int VOCABULARY_COUNT = 3;
+
+ /** The first tag row in the hand-written minimal model, after three word
rows. */
+ private static final int FIRST_TAG_ID = 3;
+
+ /** The first label row in the hand-written minimal model, after three tag
rows. */
+ private static final int FIRST_LABEL_ID = 6;
+
+ /** The total embedding rows of the hand-written minimal model. */
+ private static final int MINIMAL_ROWS = 8;
+
+ private static FeedforwardDependencyModel model;
+ private static FeedforwardDependencyParser parser;
+
+ /**
+ * Builds the shared test hyperparameters: no dropout, no L2, and a fixed
seed.
+ *
+ * @param epochs The number of passes over the corpus.
+ * @param learningRate The AdaGrad step size.
+ * @return The settings. Never {@code null}.
+ */
+ private static FeedforwardDependencyTrainer.Settings settings(int epochs,
+ double learningRate) {
+ return new FeedforwardDependencyTrainer.Settings(EMBEDDING_SIZE,
HIDDEN_SIZE, epochs,
+ BATCH_SIZE, learningRate, 0.0, 0.0, WORD_CUTOFF, SEED);
+ }
+
+ /**
+ * @return The distinct training sentences, one per parameterized
invocation. Never
+ * {@code null}.
+ */
+ private static List<DependencySample> trainingSamples() {
+ return DependencyTestSamples.sentences();
+ }
+
+ /**
+ * Trains the shared model once for all tests, with dropout off and a fixed
seed so
+ * the test network memorizes the corpus deterministically.
+ *
+ * @throws IOException Thrown if reading the in-memory samples fails.
+ */
+ @BeforeAll
+ static void trainParser() throws IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(120, 0.05);
+ model = FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings);
+ parser = new FeedforwardDependencyParser(model);
+ }
+
+ /** Checks that the greedy parser reproduces a training sentence. */
+ @Test
+ void testMemorizesTrainingSentences() {
+ assertEquals(THE_DOG_BARKS_TREE, parser.parse(THE_DOG_BARKS_TOKENS,
THE_DOG_BARKS_TAGS));
+ }
+
+ /** Checks UAS and LAS of 1.0 on the training corpus. */
+ @Test
+ void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException {
+ final DependencyEvaluator evaluator = new DependencyEvaluator(parser);
+ evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus()));
+ assertEquals(1.0d, evaluator.getUas());
+ assertEquals(1.0d, evaluator.getLas());
+ }
+
+ /** Checks that unseen words still parse to a single-rooted tree. */
+ @Test
+ void testUnknownWordsStillYieldASingleRootedTree() {
+ final DependencyTree parsed = parser.parse(new String[] {"unseen",
"words"},
+ new String[] {"JJ", "NNS"});
+ assertEquals(2, parsed.size());
+ parsed.root();
+ }
+
+ /** Checks that the transition inventory holds only actions the oracle
produced. */
+ @Test
+ void testTransitionInventoryContainsOnlyObservedActions() {
+ assertFalse(List.of(model.transitions()).contains("LEFT_ARC:root"));
+ }
+
+ /**
+ * Checks that a beam of one yields the greedy parse for every training
sample.
+ *
+ * @param sample The training sample to parse both ways.
+ */
+ @ParameterizedTest(name = "beam of one matches greedy for {0}")
+ @MethodSource("trainingSamples")
+ void testBeamOfOneMatchesGreedy(DependencySample sample) {
+ final FeedforwardDependencyParser beamed = new
FeedforwardDependencyParser(model, 1);
+ assertEquals(parser.parse(sample.getTokens(), sample.getTags()),
+ beamed.parse(sample.getTokens(), sample.getTags()));
+ }
+
+ /** Checks that beam search reproduces a training sentence. */
+ @Test
+ void testBeamedParserReproducesTrainingSentences() {
+ final FeedforwardDependencyParser beamed = new
FeedforwardDependencyParser(model, 4);
+ assertEquals(THE_DOG_BARKS_TREE, beamed.parse(THE_DOG_BARKS_TOKENS,
THE_DOG_BARKS_TAGS));
+ }
+
+ /** Checks that beam search is deterministic and single-rooted on unseen
words. */
+ @Test
+ void testBeamedParseIsDeterministicAndSingleRooted() {
+ final FeedforwardDependencyParser beamed = new
FeedforwardDependencyParser(model, 8);
+ final String[] tokens = {"unseen", "words", "everywhere"};
+ final String[] tags = {"JJ", "NNS", "RB"};
+ final DependencyTree first = beamed.parse(tokens, tags);
+ assertEquals(first, beamed.parse(tokens, tags));
+ assertEquals(3, first.size());
+ first.root();
+ }
+
+ /** Checks that refinement keeps perfect scores on the training corpus. */
+ @Test
+ void testRefinementKeepsToyPerformance() throws IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(60, 0.05);
+ final FeedforwardDependencyModel local =
FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings);
+ final FeedforwardDependencyTrainer.Settings refineSettings = settings(2,
0.01);
+ final FeedforwardDependencyModel refined =
FeedforwardDependencyTrainer.refine(
+ local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings,
2);
+
+ final DependencyEvaluator evaluator =
+ new DependencyEvaluator(new FeedforwardDependencyParser(refined, 2));
+ evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus()));
+ assertEquals(1.0d, evaluator.getUas());
+ assertEquals(1.0d, evaluator.getLas());
+ }
+
+ /** Checks that refinement fails loudly on a relation label the model never
saw. */
+ @Test
+ void testRefineRejectsAnUnknownRelation() throws IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01);
+ final List<DependencySample> unseenRelation = List.of(
+ sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN",
"VBZ"},
+ new int[] {1, 2, -1}, new String[] {"det", "dislocated", "root"}));
+ final IllegalArgumentException e =
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.refine(model,
+ ObjectStreamUtils.createObjectStream(unseenRelation), settings,
2));
+ assertEquals("unknown transition in the refinement samples:
LEFT_ARC:dislocated",
+ e.getMessage());
+ }
+
+ /** Checks that refinement returns a distinct model and leaves the input
scores unchanged. */
+ @Test
+ void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws
IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(60, 0.05);
+ final FeedforwardDependencyModel local =
FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings);
+ final String[] tokens = THE_DOG_BARKS_TOKENS;
+ final String[] tags = THE_DOG_BARKS_TAGS;
+ final int[] features = local.featureIds(
+ FeedforwardContext.extract(new ArcStandardState(tokens.length),
tokens, tags));
+ final double[] before = local.score(features);
+
+ final FeedforwardDependencyTrainer.Settings refineSettings = settings(2,
0.01);
+ final FeedforwardDependencyModel refined =
FeedforwardDependencyTrainer.refine(
+ local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings,
2);
+
+ assertNotSame(local, refined);
+ assertArrayEquals(before, local.score(features));
+ assertFalse(Arrays.equals(before, refined.score(features)));
+ }
+
+ /**
+ * Checks the final-sigma rule applied after code-point case mapping.
+ *
+ * @param input The capitalized word.
+ * @param expected The normalized word.
+ */
+ @ParameterizedTest(name = "normalize({0}) is {1}")
+ @CsvSource({
+ // ODOS, road, all caps: the trailing sigma position lowers to U+03C2
+ "\u039F\u0394\u039F\u03A3, \u03BF\u03B4\u03BF\u03C2",
+ // SOFIA: the word-initial sigma is not final and lowers to the medial
U+03C3
+ "\u03A3\u039F\u03A6\u0399\u0391, \u03C3\u03BF\u03C6\u03B9\u03B1",
+ // a lone capital sigma has no preceding letter, so the rule does not
fire
+ "\u03A3, \u03C3",
+ // case-ignorable combining marks and apostrophes do not change the
cased-letter test
+ "\u039F\u0301\u03A3, \u03BF\u0301\u03C2",
+ "\u039F\u03A3\u0301\u0391, \u03BF\u03C3\u0301\u03B1",
+ "\u039F\u2019\u03A3, \u03BF\u2019\u03C2",
+ "\u039F\u03A3\u2019\u0391, \u03BF\u03C3\u2019\u03B1"
+ })
+ void testNormalizeAppliesTheFinalSigmaRule(String input, String expected) {
+ assertEquals(expected, FeedforwardDependencyModel.normalize(input));
+ }
+
+ /** Checks that an uncased predecessor does not trigger the final-sigma
rule. */
+ @Test
+ void testFinalSigmaRequiresCasedLetterContext() {
+ assertEquals("\u4E2D\u03C3",
FeedforwardDependencyModel.normalize("\u4E2D\u03A3"));
+ }
+
+ /**
+ * Checks that normalization reuses a string when no case mapping is needed.
+ */
+ @Test
+ void testNormalizeReturnsTheSameInstanceForLowercaseWords() {
+ final String plain = "barks";
+ assertSame(plain, FeedforwardDependencyModel.normalize(plain));
+ // lowercase Greek with its native final sigma is already normalized
+ final String greek = "\u03BF\u03B4\u03BF\u03C2";
+ assertSame(greek, FeedforwardDependencyModel.normalize(greek));
+ }
+
+ /**
+ * Checks that non-projective samples are skipped before transition
validation.
+ */
+ @Test
+ void testNonProjectiveSampleWithUnknownRelationIsSkippedNotFatal() throws
IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01);
+ // heads {2, 3, -1, 2}: the arcs from 2 to 0 and from 3 to 1 cross, so the
tree
+ // is non-projective, and "dislocated" is a relation the model was never
trained on
+ final List<DependencySample> mixed = List.of(
+ nonProjectiveSample(new String[] {"a", "b", "c", "d"},
+ new String[] {"DT", "NN", "VBZ", "NN"},
+ new String[] {"det", "dislocated", "root", "obj"}),
+ sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN",
"VBZ"},
+ new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}));
+
+ final FeedforwardDependencyModel refined =
FeedforwardDependencyTrainer.refine(
+ model, ObjectStreamUtils.createObjectStream(mixed), settings, 2);
+ assertNotSame(model, refined);
+ }
+
+ /**
+ * Checks that serialized vocabulary entries follow their numeric ids.
+ */
+ @Test
+ void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws
IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ model.serialize(out);
+ try (DataInputStream data = new DataInputStream(
+ new ByteArrayInputStream(out.toByteArray()))) {
+ data.readUTF();
+ for (int vocab = 0; vocab < VOCABULARY_COUNT; vocab++) {
+ final int size = data.readInt();
+ int previous = Integer.MIN_VALUE;
+ for (int entry = 0; entry < size; entry++) {
+ data.readUTF();
+ final int id = data.readInt();
+ assertTrue(id > previous,
+ "vocabulary " + vocab + " must be written in ascending id
order");
+ previous = id;
+ }
+ }
+ }
+ }
+
+ /**
+ * Compares cached and direct scoring, including repeated cache reads.
+ */
+ @Test
+ void testScoringCacheMatchesTheDirectPath() {
+ final FeedforwardDependencyModel uncached = model.copy();
+ final String[] tokens = THE_DOG_BARKS_TOKENS;
+ final String[] tags = THE_DOG_BARKS_TAGS;
+ final int[] features = model.featureIds(
+ FeedforwardContext.extract(new ArcStandardState(tokens.length),
tokens, tags));
+
+ for (int round = 0; round < 3; round++) {
+ final double[] cached = model.score(features);
+ final double[] direct = uncached.score(features);
+ assertArrayEquals(direct, cached);
+ }
+ }
+
+ /** Checks per-code-point lowering and reserved-symbol pass-through. */
+ @Test
+ void testNormalizeUsesTheUnicodeDataCaseMapping() {
+ // StringUtil maps per code point via UnicodeData, so no character
expands; the JDK's
+ // String.toLowerCase would render this word as "i" + COMBINING DOT ABOVE
instead.
+ assertEquals(StringUtil.toLowerCase("\u0130STANBUL"),
+ FeedforwardDependencyModel.normalize("\u0130STANBUL"));
+ assertEquals("istanbul",
FeedforwardDependencyModel.normalize("\u0130STANBUL"));
+ assertEquals("*hello", FeedforwardDependencyModel.normalize("*HELLO"));
+ // reserved symbols still pass through untouched
+ assertEquals(FeedforwardDependencyModel.UNKNOWN,
+
FeedforwardDependencyModel.normalize(FeedforwardDependencyModel.UNKNOWN));
+ assertNull(FeedforwardDependencyModel.normalize(null));
+ }
+
+ /** Checks the null and beam-size guards of refinement. */
+ @Test
+ void testRefineValidation() {
+ final FeedforwardDependencyTrainer.Settings settings =
+ FeedforwardDependencyTrainer.Settings.defaults();
+ assertEquals("model must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.refine(null,
+ ObjectStreamUtils.createObjectStream(corpus()), settings,
4)).getMessage());
+ assertThrows(IllegalArgumentException.class, () ->
FeedforwardDependencyTrainer
+ .refine(model, ObjectStreamUtils.createObjectStream(corpus()),
settings, 1));
+ }
+
+ /** Checks the parser constructor guards. */
+ @Test
+ void testBeamSizeValidation() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new FeedforwardDependencyParser(model, 0));
+ assertThrows(IllegalArgumentException.class,
+ () -> new FeedforwardDependencyParser(null, 4));
+ }
+
+ /** Checks that a serialized and reloaded model parses like the original. */
+ @Test
+ void testModelRoundTripThroughSerialization() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ model.serialize(out);
+ final FeedforwardDependencyModel reloaded =
+ FeedforwardDependencyModel.load(new
ByteArrayInputStream(out.toByteArray()));
+ final DependencyTree parsed = new FeedforwardDependencyParser(reloaded)
+ .parse(SHE_EATS_FISH_TOKENS, SHE_EATS_FISH_TAGS);
+ assertEquals(SHE_EATS_FISH_TREE, parsed);
+ }
+
+ /** Checks that arbitrary bytes are rejected with an IOException. */
+ @Test
+ void testCorruptModelIsRejected() {
+ assertThrows(IOException.class, () -> FeedforwardDependencyModel.load(
+ new ByteArrayInputStream("not a
model".getBytes(StandardCharsets.UTF_8))));
+ }
+
+ /** Checks that a negative vocabulary size is rejected with an
InvalidFormatException. */
+ @Test
+ void testNegativeVocabularyCountFailsWithInvalidFormat() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ data.writeUTF(MAGIC);
+ data.writeInt(-1);
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that a hidden matrix of the wrong width is rejected. */
+ @Test
+ void testInconsistentModelDimensionsFailDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ writeMinimalModel(data, FIRST_TAG_ID, 1, 0.0f);
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that overlapping vocabulary ids are rejected. */
+ @Test
+ void testDuplicateVocabularyIdsFailDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ writeMinimalModel(data, 0, FeedforwardContext.FEATURE_COUNT, 0.0f);
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that a NaN weight is rejected. */
+ @Test
+ void testNonFiniteWeightFailsDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ writeMinimalModel(data, FIRST_TAG_ID, FeedforwardContext.FEATURE_COUNT,
Float.NaN);
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that bytes after the model are rejected. */
+ @Test
+ void testTrailingModelDataFailsDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ model.serialize(out);
+ out.write(1);
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that an inventory without SHIFT is rejected. */
+ @Test
+ void testModelWithoutShiftFailsDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ writeModelWithTransitions(data, Transition.rightArc("root").encode());
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /** Checks that an inventory without RIGHT_ARC is rejected. */
+ @Test
+ void testModelWithoutRootArcFailsDuringLoading() throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (DataOutputStream data = new DataOutputStream(out)) {
+ writeModelWithTransitions(data, Transition.SHIFT.encode());
+ }
+ assertThrows(InvalidFormatException.class, () ->
FeedforwardDependencyModel.load(
+ new ByteArrayInputStream(out.toByteArray())));
+ }
+
+ /**
+ * Writes a test vocabulary with consecutive embedding indices.
+ *
+ * @param data The output to write to.
+ * @param first The embedding row of the first symbol.
+ * @param symbols The symbols, in row order.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static void writeVocabulary(DataOutputStream data, int first,
String... symbols)
+ throws IOException {
+ data.writeInt(symbols.length);
+ for (int i = 0; i < symbols.length; i++) {
+ data.writeUTF(symbols[i]);
+ data.writeInt(first + i);
+ }
+ }
+
+ /**
+ * Writes the smallest complete model, with selected fields exposed for
corruption.
+ *
+ * @param data The output to write to.
+ * @param firstTagId The embedding row of the first tag symbol.
+ * @param hiddenColumns The hidden matrix width.
+ * @param outputBias The single output bias value.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static void writeMinimalModel(DataOutputStream data, int firstTagId,
+ int hiddenColumns, float outputBias) throws IOException {
+ data.writeUTF(MAGIC);
+ writeVocabulary(data, 0,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
+ DependencyContextGenerator.ROOT_VALUE);
+ writeVocabulary(data, firstTagId,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
+ DependencyContextGenerator.ROOT_VALUE);
+ writeVocabulary(data, FIRST_LABEL_ID,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE);
+ data.writeInt(1);
+ data.writeUTF(Transition.SHIFT.encode());
+ data.writeInt(1);
+ writeMatrix(data, MINIMAL_ROWS, 1);
+ writeMatrix(data, 1, hiddenColumns);
+ writeVector(data, 1);
+ writeMatrix(data, 1, 1);
+ data.writeInt(1);
+ data.writeFloat(outputBias);
+ }
+
+ /**
+ * Writes a structurally complete model with the selected transition
inventory.
+ *
+ * @param data The output to write to.
+ * @param transitions The encoded transitions, in output order.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static void writeModelWithTransitions(DataOutputStream data,
+ String... transitions) throws IOException {
+ data.writeUTF(MAGIC);
+ writeVocabulary(data, 0,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
+ DependencyContextGenerator.ROOT_VALUE);
+ writeVocabulary(data, FIRST_TAG_ID,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE,
+ DependencyContextGenerator.ROOT_VALUE);
+ writeVocabulary(data, FIRST_LABEL_ID,
+ FeedforwardDependencyModel.UNKNOWN,
+ DependencyContextGenerator.NONE_VALUE);
+ data.writeInt(transitions.length);
+ for (final String transition : transitions) {
+ data.writeUTF(transition);
+ }
+ data.writeInt(1);
+ writeMatrix(data, MINIMAL_ROWS, 1);
+ writeMatrix(data, 1, FeedforwardContext.FEATURE_COUNT);
+ writeVector(data, 1);
+ writeMatrix(data, transitions.length, 1);
+ writeVector(data, transitions.length);
+ }
+
+ /**
+ * Writes a zero-filled matrix in the model format.
+ *
+ * @param data The output to write to.
+ * @param rows The row count.
+ * @param columns The column count.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static void writeMatrix(DataOutputStream data, int rows, int columns)
+ throws IOException {
+ data.writeInt(rows);
+ data.writeInt(columns);
+ for (int i = 0; i < rows * columns; i++) {
+ data.writeFloat(0.0f);
+ }
+ }
+
+ /**
+ * Writes a zero-filled vector in the model format.
+ *
+ * @param data The output to write to.
+ * @param length The vector length.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static void writeVector(DataOutputStream data, int length) throws
IOException {
+ data.writeInt(length);
+ for (int i = 0; i < length; i++) {
+ data.writeFloat(0.0f);
+ }
+ }
+
+ /**
+ * Checks that each out-of-range hyperparameter is rejected.
+ *
+ * @param embeddingSize The embedding width.
+ * @param hiddenSize The hidden width.
+ * @param learningRate The step size.
+ * @param l2 The L2 penalty.
+ * @param dropout The dropout probability.
+ */
+ @ParameterizedTest(name = "embedding {0}, hidden {1}, rate {2}, l2 {3},
dropout {4}")
+ @CsvSource({
+ "0, 32, 0.05, 0.0, 0.0",
+ "16, 32, -1.0, 0.0, 0.0",
+ "16, 32, 0.05, 0.0, 1.0",
+ "16, 32, NaN, 0.0, 0.0",
+ "16, 32, 0.05, Infinity, 0.0",
+ "4097, 32, 0.05, 0.0, 0.0",
+ "16, 65537, 0.05, 0.0, 0.0",
+ "1000, 3000, 0.05, 0.0, 0.0"
+ })
+ void testSettingsValidation(int embeddingSize, int hiddenSize, double
learningRate,
+ double l2, double dropout) {
+ assertThrows(IllegalArgumentException.class, () -> new
FeedforwardDependencyTrainer
+ .Settings(embeddingSize, hiddenSize, 10, BATCH_SIZE, learningRate, l2,
dropout,
+ WORD_CUTOFF, SEED));
+ }
+
+ /** Checks that a diverging run fails with an IllegalStateException. */
+ @Test
+ void testTrainingRejectsNonFiniteWeights() {
+ final FeedforwardDependencyTrainer.Settings settings =
+ new FeedforwardDependencyTrainer.Settings(
+ 4, 4, 2, 1, Double.MAX_VALUE, 0.0, 0.0, WORD_CUTOFF, SEED);
+ assertThrows(IllegalStateException.class,
+ () -> FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings));
+ }
+
+ /** Checks the feature length guards of the model. */
+ @Test
+ void testModelRejectsInvalidFeatureArrays() {
+ assertThrows(IllegalArgumentException.class, () -> model.featureIds(new
String[0]));
+ assertThrows(IllegalArgumentException.class, () -> model.score(new
int[0]));
+ }
+
+ /** Checks that pretrained vectors seed rows and malformed vectors are
rejected. */
+ @Test
+ void testPretrainedSeedingAppliesAndValidates() throws IOException {
+ // near-zero learning keeps the seeded row observable after one epoch
+ final FeedforwardDependencyTrainer.Settings settings =
+ new FeedforwardDependencyTrainer.Settings(4, 8, 1, BATCH_SIZE, 1e-9,
0.0, 0.0,
+ WORD_CUTOFF, SEED);
+ final float[] vector = {0.25f, -0.5f, 0.75f, -1.0f};
+ final FeedforwardDependencyModel seeded =
FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings,
+ word -> "dog".equals(word) ? vector.clone() : null);
+ final int row = seeded.wordIds().get("dog");
+ for (int d = 0; d < vector.length; d++) {
+ assertEquals(vector[d], seeded.embeddings()[row][d], 1e-4);
+ }
+ assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings,
+ word -> new float[] {1.0f}));
+ assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()), settings,
+ word -> new float[] {Float.NaN, 0.0f, 0.0f, 0.0f}));
+ }
+
+ /** Checks the null and length guards of the parser and trainer. */
+ @Test
+ void testArgumentValidation() {
+ assertEquals("model must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> new FeedforwardDependencyParser(null)).getMessage());
+ assertEquals("samples must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.train(null,
+ FeedforwardDependencyTrainer.Settings.defaults())).getMessage());
+ assertEquals("settings must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(corpus()),
null)).getMessage());
+ assertEquals("tokens must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> parser.parse(null, new String[] {"DT"})).getMessage());
+ assertEquals("symbols must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> model.featureIds(null)).getMessage());
+ assertEquals("features must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> model.score(null)).getMessage());
+ assertEquals("out must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> model.serialize(null)).getMessage());
+ assertEquals("in must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyModel.load((InputStream)
null)).getMessage());
+ assertEquals("path must not be null",
assertThrows(IllegalArgumentException.class,
+ () -> FeedforwardDependencyModel.load((Path) null)).getMessage());
+ assertThrows(IllegalArgumentException.class,
+ () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"}));
+ }
+
+ /**
+ * Verifies that training data using the model's reserved symbols as
ordinary tags,
+ * labels, or tokens does not displace the reserved rows: the model reloads
and the
+ * colliding symbols share the reserved rows the way unknown symbols do.
+ *
+ * @throws IOException Thrown if serialization or loading fails.
+ */
+ @Test
+ void testReservedSymbolsInTrainingDataSurviveReload() throws IOException {
+ final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01);
+ final List<DependencySample> colliding = List.of(
+ sample(new String[] {"*ROOT*", "*UNK*", "barks"}, new String[]
{"*UNK*", "NN", "VBZ"},
+ new int[] {1, 2, -1}, new String[] {"*NULL*", "nsubj", "root"}),
+ sample(new String[] {"dogs", "bark"}, new String[] {"*NULL*",
"*ROOT*"},
+ new int[] {1, -1}, new String[] {"*UNK*", "root"}));
+ final FeedforwardDependencyModel trained =
FeedforwardDependencyTrainer.train(
+ ObjectStreamUtils.createObjectStream(colliding), settings);
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ trained.serialize(out);
+ final FeedforwardDependencyModel reloaded =
+ FeedforwardDependencyModel.load(new
ByteArrayInputStream(out.toByteArray()));
+ final String[] tokens = {"*ROOT*", "*UNK*", "barks"};
+ final String[] tags = {"*UNK*", "NN", "VBZ"};
+ assertEquals(new FeedforwardDependencyParser(trained).parse(tokens, tags),
+ new FeedforwardDependencyParser(reloaded).parse(tokens, tags));
+ }
+ /**
+ * Checks that training on samples without an arc-standard derivation fails
with the
+ * documented message instead of producing an empty model.
+ */
+ @Test
+ void testTrainingWithOnlyNonProjectiveSamplesFailsLoud() {
Review Comment:
The EdgeCase class is merged into `DependencyParserEdgeCaseTest`; `SEED`,
the parameter factory and the shared trainers are in `DependencyTestSamples`.
> Add a feedforward neural dependency parser
> ------------------------------------------
>
> Key: OPENNLP-1991
> URL: https://issues.apache.org/jira/browse/OPENNLP-1991
> Project: OpenNLP
> Issue Type: Improvement
> Reporter: Kristian Rickert
> Assignee: Kristian Rickert
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Add a feedforward neural implementation of the dependency parser API
> introduced by OPENNLP-547. Use OpenNLP’s standard model container, artifact
> serializers, TrainingParameters, factory, and CLI conventions. Include
> deterministic training, greedy and beam decoding, model round-trip tests, and
> held-out Universal Dependencies evaluation. Address the deferred review
> feedback on shared scoring/training logic and parser-owned caches. Implement
> this without new dependencies.
> The list above may still get this ticket to further split. For now this is a
> placeholder to capture the code that was moved out of OPENNLP-547. I'll edit
> after 547 is merged.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
