This is an automated email from the ASF dual-hosted git repository.
bodewig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ant.git
The following commit(s) were added to refs/heads/master by this push:
new 3ec6100 #64855 Add filterbeforeconcat option to Concat
new a29c130 Merge pull request #141 from
TheConstructor/input-file-tokenizer
3ec6100 is described below
commit 3ec610046f33d540c3d533ad34529c7ee869a215
Author: Matthias Vill <git...@cconstruct.de>
AuthorDate: Thu Oct 29 17:53:57 2020 +0100
#64855 Add filterbeforeconcat option to Concat
---
manual/Tasks/concat.html | 9 +
src/main/org/apache/tools/ant/taskdefs/Concat.java | 216 +++++++++++++++------
src/tests/antunit/taskdefs/concat-test.xml | 44 +++++
3 files changed, 209 insertions(+), 60 deletions(-)
diff --git a/manual/Tasks/concat.html b/manual/Tasks/concat.html
index 513797c..71aba61 100644
--- a/manual/Tasks/concat.html
+++ b/manual/Tasks/concat.html
@@ -125,6 +125,15 @@
<td>No; default is <q>false</q></td>
</tr>
<tr>
+ <td>filterbeforeconcat</td>
+ <td>If this attribute is set to <q>true</q>, the task applies the
filterchain to each
+ input after applying <code>fixlastline</code>. If this attribute is
<q>false</q>, concat
+ will apply the filterchain only once to the already concatenated
inputs. Filtering of
+ <code>header</code> and <code>footer</code> is not affected by this
setting.
+ <em>Since Ant 1.10.10</em></td>
+ <td>No; default is <q>false</q></td>
+ </tr>
+ <tr>
<td>ignoreempty</td>
<td><em>Since Ant 1.8.0</em> Specifies whether or not the file
specified
by <var>destfile</var> should be created if the source resource list
is empty.
diff --git a/src/main/org/apache/tools/ant/taskdefs/Concat.java
b/src/main/org/apache/tools/ant/taskdefs/Concat.java
index 333bad9..8a56046 100644
--- a/src/main/org/apache/tools/ant/taskdefs/Concat.java
+++ b/src/main/org/apache/tools/ant/taskdefs/Concat.java
@@ -212,34 +212,17 @@ public class Concat extends Task implements
ResourceCollection {
}
/**
- * This class reads from each of the source files in turn.
- * The concatenated result can then be filtered as
- * a single stream.
+ * This class reads a Reader and ensures it ends with the desired linebreak
+ * @since Ant 1.10.10
*/
- private final class MultiReader<S> extends Reader {
- private Reader reader = null;
- private int lastPos = 0;
- private char[] lastChars = new char[eolString.length()];
+ private final class LastLineFixingReader extends Reader {
+ private final Reader reader;
+ private int lastPos = 0;
+ private final char[] lastChars = new char[eolString.length()];
private boolean needAddSeparator = false;
- private Iterator<S> readerSources;
- private ReaderFactory<S> factory;
-
- private MultiReader(Iterator<S> readerSources, ReaderFactory<S>
factory) {
- this.readerSources = readerSources;
- this.factory = factory;
- }
-
- private Reader getReader() throws IOException {
- if (reader == null && readerSources.hasNext()) {
- reader = factory.getReader(readerSources.next());
- Arrays.fill(lastChars, (char) 0);
- }
- return reader;
- }
- private void nextReader() throws IOException {
- close();
- reader = null;
+ private LastLineFixingReader(Reader reader) {
+ this.reader = reader;
}
/**
@@ -253,25 +236,21 @@ public class Concat extends Task implements
ResourceCollection {
public int read() throws IOException {
if (needAddSeparator) {
if (lastPos >= eolString.length()) {
- lastPos = 0;
- needAddSeparator = false;
+ return -1;
} else {
return eolString.charAt(lastPos++);
}
}
- while (getReader() != null) {
- int ch = getReader().read();
- if (ch == -1) {
- nextReader();
- if (isFixLastLine() && isMissingEndOfLine()) {
- needAddSeparator = true;
- lastPos = 1;
- return eolString.charAt(0);
- }
- } else {
- addLastChar((char) ch);
- return ch;
+ int ch = reader.read();
+ if (ch == -1) {
+ if (isMissingEndOfLine()) {
+ needAddSeparator = true;
+ lastPos = 1;
+ return eolString.charAt(0);
}
+ } else {
+ addLastChar((char) ch);
+ return ch;
}
return -1;
}
@@ -289,13 +268,12 @@ public class Concat extends Task implements
ResourceCollection {
throws IOException {
int amountRead = 0;
- while (getReader() != null || needAddSeparator) {
+ while (true) {
if (needAddSeparator) {
- cbuf[off] = eolString.charAt(lastPos++);
if (lastPos >= eolString.length()) {
- lastPos = 0;
- needAddSeparator = false;
+ break;
}
+ cbuf[off] = eolString.charAt(lastPos++);
len--;
off++;
amountRead++;
@@ -304,23 +282,22 @@ public class Concat extends Task implements
ResourceCollection {
}
continue;
}
- int nRead = getReader().read(cbuf, off, len);
+ int nRead = reader.read(cbuf, off, len);
if (nRead == -1 || nRead == 0) {
- nextReader();
- if (isFixLastLine() && isMissingEndOfLine()) {
+ if (isMissingEndOfLine()) {
needAddSeparator = true;
lastPos = 0;
+ } else {
+ break;
}
} else {
- if (isFixLastLine()) {
- for (int i = nRead;
- i > (nRead - lastChars.length);
- --i) {
- if (i <= 0) {
- break;
- }
- addLastChar(cbuf[off + i - 1]);
+ for (int i = nRead;
+ i > (nRead - lastChars.length);
+ --i) {
+ if (i <= 0) {
+ break;
}
+ addLastChar(cbuf[off + i - 1]);
}
len -= nRead;
off += nRead;
@@ -341,9 +318,7 @@ public class Concat extends Task implements
ResourceCollection {
*/
@Override
public void close() throws IOException {
- if (reader != null) {
- reader.close();
- }
+ reader.close();
}
/**
@@ -367,6 +342,107 @@ public class Concat extends Task implements
ResourceCollection {
}
return false;
}
+ }
+
+ /**
+ * This class reads from each of the source files in turn.
+ * The concatenated result can then be filtered as
+ * a single stream.
+ */
+ private final class MultiReader<S> extends Reader {
+ private Reader reader = null;
+ private Iterator<S> readerSources;
+ private ReaderFactory<S> factory;
+ private final boolean filterBeforeConcat;
+
+ private MultiReader(Iterator<S> readerSources, ReaderFactory<S>
factory,
+ boolean filterBeforeConcat) {
+ this.readerSources = readerSources;
+ this.factory = factory;
+ this.filterBeforeConcat = filterBeforeConcat;
+ }
+
+ private Reader getReader() throws IOException {
+ if (reader == null && readerSources.hasNext()) {
+ reader = factory.getReader(readerSources.next());
+ if(isFixLastLine())
+ {
+ reader = new LastLineFixingReader(reader);
+ }
+ if(filterBeforeConcat)
+ {
+ reader = getFilteredReader(reader);
+ }
+ }
+ return reader;
+ }
+
+ private void nextReader() throws IOException {
+ close();
+ reader = null;
+ }
+
+ /**
+ * Read a character from the current reader object. Advance
+ * to the next if the reader is finished.
+ * @return the character read, -1 for EOF on the last reader.
+ * @exception IOException - possibly thrown by the read for a reader
+ * object.
+ */
+ @Override
+ public int read() throws IOException {
+ while (getReader() != null) {
+ int ch = getReader().read();
+ if (ch == -1) {
+ nextReader();
+ } else {
+ return ch;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Read into the buffer <code>cbuf</code>.
+ * @param cbuf The array to be read into.
+ * @param off The offset.
+ * @param len The length to read.
+ * @exception IOException - possibly thrown by the reads to the
+ * reader objects.
+ */
+ @Override
+ public int read(char[] cbuf, int off, int len)
+ throws IOException {
+
+ int amountRead = 0;
+ while (getReader() != null) {
+ int nRead = getReader().read(cbuf, off, len);
+ if (nRead == -1 || nRead == 0) {
+ nextReader();
+ } else {
+ len -= nRead;
+ off += nRead;
+ amountRead += nRead;
+ if (len == 0) {
+ return amountRead;
+ }
+ }
+ }
+ if (amountRead == 0) {
+ return -1;
+ }
+ return amountRead;
+ }
+
+ /**
+ * Close the current reader
+ */
+ @Override
+ public void close() throws IOException {
+ if (reader != null) {
+ reader.close();
+ }
+ }
private boolean isFixLastLine() {
return fixLastLine && textBuffer == null;
@@ -386,8 +462,14 @@ public class Concat extends Task implements
ResourceCollection {
result.setManagingComponent(this);
return result;
}
- Reader resourceReader = getFilteredReader(
- new MultiReader<>(c.iterator(), resourceReaderFactory));
+ Reader resourceReader;
+ if(filterBeforeConcat) {
+ resourceReader = new MultiReader<>(c.iterator(),
+ resourceReaderFactory, true);
+ } else {
+ resourceReader = getFilteredReader(
+ new MultiReader<>(c.iterator(), resourceReaderFactory,
false));
+ }
Reader rdr;
if (header == null && footer == null) {
rdr = resourceReader;
@@ -416,7 +498,7 @@ public class Concat extends Task implements
ResourceCollection {
}
}
rdr = new MultiReader<>(Arrays.asList(readers).iterator(),
- identityReaderFactory);
+ identityReaderFactory, false);
}
return outputEncoding == null ? new ReaderInputStream(rdr)
: new ReaderInputStream(rdr, outputEncoding);
@@ -454,6 +536,9 @@ public class Concat extends Task implements
ResourceCollection {
/** Stores the binary attribute */
private boolean binary;
+ /** Stores the filterBeforeConcat attribute */
+ private boolean filterBeforeConcat;
+
// Child elements.
/**
@@ -780,6 +865,17 @@ public class Concat extends Task implements
ResourceCollection {
}
/**
+ * Set the filterBeforeConcat attribute. If true, concat will filter each
+ * input through the filterchain before concatenating the results. This
+ * allows to e.g. use the FileTokenizer to tokenize each input.
+ * @param filterBeforeConcat if true, filter each input before
concatenation
+ * @since Ant 1.10.10
+ */
+ public void setFilterBeforeConcat(final boolean filterBeforeConcat) {
+ this.filterBeforeConcat = filterBeforeConcat;
+ }
+
+ /**
* Execute the concat task.
*/
@Override
diff --git a/src/tests/antunit/taskdefs/concat-test.xml
b/src/tests/antunit/taskdefs/concat-test.xml
index c596d90..82182ff 100644
--- a/src/tests/antunit/taskdefs/concat-test.xml
+++ b/src/tests/antunit/taskdefs/concat-test.xml
@@ -118,6 +118,50 @@
</au:assertTrue>
</target>
+ <target name="testFilterIsAppliedToConcatenation"
+ depends="-fixlastline-setup"
+ description="https://bz.apache.org/bugzilla/show_bug.cgi?id=64855";>
+ <au:assertTrue>
+ <resourcesmatch>
+
<string>1${line.separator}2${line.separator}1${line.separator}2${line.separator}</string>
+ <concat fixlastline="true">
+ <filelist dir="${input}">
+ <file name="1"/>
+ <file name="2"/>
+ </filelist>
+ <filterchain>
+ <tokenfilter>
+ <filetokenizer/>
+ <replaceregex pattern="(.*)" flags="s" replace="\1\1"/>
+ </tokenfilter>
+ </filterchain>
+ </concat>
+ </resourcesmatch>
+ </au:assertTrue>
+ </target>
+
+ <target name="testFilterBeforeConcatActuallyAppliesFilterToEachInput"
+ depends="-fixlastline-setup"
+ description="https://bz.apache.org/bugzilla/show_bug.cgi?id=64855";>
+ <au:assertTrue>
+ <resourcesmatch>
+
<string>1${line.separator}1${line.separator}2${line.separator}2${line.separator}</string>
+ <concat fixlastline="true" filterbeforeconcat="true">
+ <filelist dir="${input}">
+ <file name="1"/>
+ <file name="2"/>
+ </filelist>
+ <filterchain>
+ <tokenfilter>
+ <filetokenizer/>
+ <replaceregex pattern="(.*)" flags="s" replace="\1\1"/>
+ </tokenfilter>
+ </filterchain>
+ </concat>
+ </resourcesmatch>
+ </au:assertTrue>
+ </target>
+
<target name="testIgnoreEmptyFalseFileIsCreated">
<mkdir dir="${input}" />
<mkdir dir="${output}" />
