This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4837-ooxml-contents in repository https://gitbox.apache.org/repos/asf/tika.git
commit e5b6466ee891fd7cdc9da99f35b433cb1a7d27c3 Author: tallison <[email protected]> AuthorDate: Mon Aug 24 12:59:25 2026 -0400 TIKA-4837 - flag extra contents in ooxml files --- .../main/java/org/apache/tika/metadata/Office.java | 23 +++ .../apache/tika/metadata/metadata-key-fields.json | 2 + .../org/apache/tika/metadata/metadata-keys.json | 2 + .../microsoft/ooxml/OOXMLExtractorFactory.java | 65 ++++++++ .../microsoft/ooxml/MacroPartResolutionTest.java | 145 ++++++++++++++++++ .../microsoft/ooxml/ZipStructureCoverageTest.java | 169 +++++++++++++++++++++ 6 files changed, 406 insertions(+) diff --git a/tika-core/src/main/java/org/apache/tika/metadata/Office.java b/tika-core/src/main/java/org/apache/tika/metadata/Office.java index f2d7ccc686..6cbe8811d4 100644 --- a/tika-core/src/main/java/org/apache/tika/metadata/Office.java +++ b/tika-core/src/main/java/org/apache/tika/metadata/Office.java @@ -234,6 +234,29 @@ public interface Office { // Security-relevant: Framesets can load external URLs Property HAS_FRAMESETS = Property.internalBoolean("msoffice:doc:has-framesets"); + /** + * Security-relevant, best-effort: true when the OOXML package contains one or more + * declared parts that are NOT reachable from the package root through the OPC + * relationship graph. Office and Tika load content by following relationships, so an + * unreferenced part is carried in the file but never parsed -- a place to hide bytes + * that a raw-ZIP reader (AV/DLP/CDR) can still see. (Note: a part with no declared + * content type is a different case -- POI rejects the whole package at open time, so + * it cannot appear on a successfully parsed file.) + * <p>This is an informational signal, NOT a guarantee, and it is evadable (a payload + * referenced by a relationship type Tika ignores is still reachable): per + * <a href="https://tika.apache.org/security-model.html";>Tika's security model</a>, + * Tika is not a security boundary and does not attempt to detect parser differentials. + * See {@link #UNREFERENCED_PART_NAMES} for the part names. + */ + Property HAS_UNREFERENCED_PARTS = + Property.internalBoolean("msoffice:has-unreferenced-parts"); + + /** + * The names of the parts flagged by {@link #HAS_UNREFERENCED_PARTS}. + */ + Property UNREFERENCED_PART_NAMES = + Property.internalTextBag("msoffice:unreferenced-part-names"); + /** * 1-based sheet number for a resource (e.g. an embedded image) * anchored to exactly one sheet of a workbook. For resources diff --git a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json index 3392c0e3be..3e22709d9b 100644 --- a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json +++ b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json @@ -317,6 +317,7 @@ {"class":"org.apache.tika.metadata.Office","field":"HAS_POWER_QUERY","key":"msoffice:excel:has-power-query"}, {"class":"org.apache.tika.metadata.Office","field":"HAS_SUBDOCUMENTS","key":"msoffice:doc:has-subdocuments"}, {"class":"org.apache.tika.metadata.Office","field":"HAS_TRACK_CHANGES","key":"msoffice:has-track-changes"}, + {"class":"org.apache.tika.metadata.Office","field":"HAS_UNREFERENCED_PARTS","key":"msoffice:has-unreferenced-parts"}, {"class":"org.apache.tika.metadata.Office","field":"HAS_VERY_HIDDEN_SHEETS","key":"msoffice:excel:has-very-hidden-sheets"}, {"class":"org.apache.tika.metadata.Office","field":"HAS_VML_HYPERLINKS","key":"msoffice:has-vml-hyperlinks"}, {"class":"org.apache.tika.metadata.Office","field":"HAS_WEB_QUERIES","key":"msoffice:excel:has-web-queries"}, @@ -341,6 +342,7 @@ {"class":"org.apache.tika.metadata.Office","field":"SLIDE_COUNT","key":"meta:slide-count"}, {"class":"org.apache.tika.metadata.Office","field":"TABLE_COUNT","key":"meta:table-count"}, {"class":"org.apache.tika.metadata.Office","field":"UNLISTED_SLIDE_NAMES","key":"msoffice:ppt:unlisted-slide-names"}, + {"class":"org.apache.tika.metadata.Office","field":"UNREFERENCED_PART_NAMES","key":"msoffice:unreferenced-part-names"}, {"class":"org.apache.tika.metadata.Office","field":"VERY_HIDDEN_SHEET_NAMES","key":"msoffice:excel:very-hidden-sheet-names"}, {"class":"org.apache.tika.metadata.Office","field":"WORD_COUNT","key":"meta:word-count"}, {"class":"org.apache.tika.metadata.Office","field":"WORKBOOK_CODENAME","key":"msoffice:excel:workbook-codename"}, diff --git a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json index 817b002655..2faedf37f4 100644 --- a/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json +++ b/tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json @@ -366,6 +366,7 @@ {"key":"msoffice:has-hover-hyperlinks","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:has-linked-ole-objects","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:has-track-changes","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, + {"key":"msoffice:has-unreferenced-parts","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:has-vml-hyperlinks","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:ocx-name","namespace":"msoffice","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:ppt:has-animations","namespace":"msoffice","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"}, @@ -374,6 +375,7 @@ {"key":"msoffice:ppt:num-unlisted-slides","namespace":"msoffice","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"}, {"key":"msoffice:ppt:unlisted-slide-names","namespace":"msoffice","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, {"key":"msoffice:prog-id","namespace":"msoffice","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"}, + {"key":"msoffice:unreferenced-part-names","namespace":"msoffice","valueType":"TEXT","cardinality":"BAG","module":"tika-core"}, {"key":"multipart:boundary","namespace":"multipart","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"}, {"key":"multipart:subtype","namespace":"multipart","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"}, {"key":"netcdf:file-type-description","namespace":"netcdf","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-parser-scientific-module"}, diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java index 3447e6fa44..f39770c780 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLExtractorFactory.java @@ -17,6 +17,9 @@ package org.apache.tika.parser.microsoft.ooxml; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashSet; import java.util.Locale; import java.util.Set; @@ -24,7 +27,9 @@ import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackageAccess; import org.apache.poi.openxml4j.opc.PackagePart; +import org.apache.poi.openxml4j.opc.PackageRelationship; import org.apache.poi.openxml4j.opc.PackageRelationshipCollection; +import org.apache.poi.openxml4j.opc.TargetMode; import org.apache.poi.util.LocaleUtil; import org.apache.poi.xslf.usermodel.XSLFRelation; import org.apache.poi.xssf.usermodel.XSSFRelation; @@ -39,6 +44,7 @@ import org.apache.tika.exception.TikaException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.HttpHeaders; import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.EmptyParser; @@ -96,6 +102,7 @@ public class OOXMLExtractorFactory { if (prc != null && prc.size() > 0) { metadata.set(TikaCoreProperties.HAS_SIGNATURE, "true"); } + markUnreferencedParts(pkg, metadata); } MediaType type = null; @@ -216,6 +223,64 @@ public class OOXMLExtractorFactory { return type != null && VISIO_SUBTYPES.contains(type.getSubtype()); } + private static final String RELATIONSHIPS_CONTENT_TYPE = + "application/vnd.openxmlformats-package.relationships+xml"; + + /** + * Best-effort, security-relevant signal: flags declared parts that are not reachable + * from the package root via the OPC relationship graph -- content carried in the file + * but outside the structure Office loads by following relationships. Reuses the + * already-open package (an in-memory walk of relationships POI loads anyway). Never + * throws: a failure here must not break the parse. + * + * @see Office#HAS_UNREFERENCED_PARTS + */ + private static void markUnreferencedParts(OPCPackage pkg, Metadata metadata) { + try { + Set<String> reachable = new HashSet<>(); + Deque<PackagePart> queue = new ArrayDeque<>(); + addRelatedParts(pkg.getRelationships(), rel -> pkg.getPart(rel), reachable, queue); + while (!queue.isEmpty()) { + PackagePart part = queue.poll(); + addRelatedParts(part.getRelationships(), part::getRelatedPart, reachable, queue); + } + for (PackagePart part : pkg.getParts()) { + // relationship parts (_rels/*.rels) are never relationship targets; skip them + if (RELATIONSHIPS_CONTENT_TYPE.equals(part.getContentType())) { + continue; + } + if (!reachable.contains(part.getPartName().getName())) { + metadata.set(Office.HAS_UNREFERENCED_PARTS, true); + metadata.add(Office.UNREFERENCED_PART_NAMES, part.getPartName().getName()); + } + } + } catch (Exception e) { + // best-effort only; never fail the parse over this signal + } + } + + @FunctionalInterface + private interface PartResolver { + PackagePart resolve(PackageRelationship rel) throws Exception; + } + + private static void addRelatedParts(PackageRelationshipCollection rels, PartResolver resolver, + Set<String> reachable, Deque<PackagePart> queue) { + for (PackageRelationship rel : rels) { + if (rel.getTargetMode() != TargetMode.INTERNAL) { + continue; + } + try { + PackagePart part = resolver.resolve(rel); + if (part != null && reachable.add(part.getPartName().getName())) { + queue.add(part); + } + } catch (Exception e) { + // unresolved/broken relationship target -- ignore for this best-effort signal + } + } + } + private static String getCorePartContentType(OPCPackage pkg) { try { PackageRelationshipCollection rels = diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/MacroPartResolutionTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/MacroPartResolutionTest.java new file mode 100644 index 0000000000..8bdd565d8b --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/MacroPartResolutionTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.microsoft.OfficeParserConfig; + +/** + * Guards that Tika resolves the VBA project through the OPC relationship graph, + * NOT by raw-zip filename-suffix / stream order. + * + * <p>POI's {@code VBAMacroReader.openOOXML} selects the VBA project as the first + * zip entry (in stream order) whose name ends with "vbaProject.bin", bypassing + * OPC. A tool built directly on that (an AV/DLP/CDR macro scanner) can be fooled + * by a decoy vbaProject.bin ordered before the real one, while Office executes + * the relationship-referenced part (CWE-436 parser differential; reported against + * POI by n0mi1k, fixed there as a plain bug). + * + * <p>Tika does not use that path: it constructs {@code VBAMacroReader} only from a + * {@code POIFSFileSystem} built from the OPC-resolved {@code vbaProject} part. This + * test crafts an .xlsm with a decoy vbaProject.bin (a DIFFERENT, valid VBA project + * lifted from testWORD_macros.docm) placed first in stream order, and asserts Tika + * still extracts the real /xl/vbaProject.bin macro. If a future change routes macro + * extraction through the raw-zip path, this fails. + */ +public class MacroPartResolutionTest extends TikaTest { + + private static final String REAL_MACRO = "Sub Dirty()"; // testEXCEL_macro.xlsm + private static final String DECOY_MACRO = "Sub Embolden()"; // testWORD_macros.docm + + private byte[] resourceBytes(String name) throws Exception { + try (TikaInputStream tis = getResourceAsStream("/test-documents/" + name)) { + return tis.readAllBytes(); + } + } + + private byte[] entryEndingWith(byte[] zip, String suffix) throws Exception { + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(zip))) { + ZipEntry e; + byte[] buf = new byte[8192]; + while ((e = zis.getNextEntry()) != null) { + if (e.getName().toLowerCase(java.util.Locale.ROOT).endsWith(suffix)) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + int n; + while ((n = zis.read(buf)) > 0) { + bos.write(buf, 0, n); + } + return bos.toByteArray(); + } + } + } + throw new IllegalStateException("no entry ending with " + suffix); + } + + private String allContent(List<Metadata> list) { + StringBuilder sb = new StringBuilder(); + for (Metadata m : list) { + String c = m.get(TikaCoreProperties.TIKA_CONTENT); + if (c != null) { + sb.append(c).append('\n'); + } + } + return sb.toString(); + } + + private List<Metadata> parseWithMacros(byte[] bytes) throws Exception { + ParseContext context = new ParseContext(); + OfficeParserConfig config = new OfficeParserConfig(); + config.setExtractMacros(true); + context.set(OfficeParserConfig.class, config); + return getRecursiveMetadata(TikaInputStream.get(bytes), new Metadata(), context, false); + } + + @Test + public void testVbaProjectResolvedViaOpcNotStreamOrder() throws Exception { + byte[] xlsm = resourceBytes("testEXCEL_macro.xlsm"); + // A different, valid VBA project to act as the decoy. + byte[] decoyVba = entryEndingWith(resourceBytes("testWORD_macros.docm"), "vbaproject.bin"); + + // Sanity: the two projects have distinct, non-overlapping macro signatures. + String realOnly = allContent(parseWithMacros(xlsm)); + assertTrue(realOnly.contains(REAL_MACRO), "baseline xlsm should expose real macro"); + assertFalse(realOnly.contains(DECOY_MACRO), "baseline xlsm must not contain decoy macro"); + + // Craft an .xlsm whose FIRST *vbaProject.bin entry (stream order) is the + // undeclared decoy, with the real, relationship-referenced /xl/vbaProject.bin + // kept intact later in the stream. + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(bos)) { + zos.putNextEntry(new ZipEntry("xl/decoy/vbaProject.bin")); // undeclared, first in stream + zos.write(decoyVba); + zos.closeEntry(); + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(xlsm))) { + ZipEntry e; + byte[] buf = new byte[8192]; + while ((e = zis.getNextEntry()) != null) { + zos.putNextEntry(new ZipEntry(e.getName())); + int n; + while ((n = zis.read(buf)) > 0) { + zos.write(buf, 0, n); + } + zos.closeEntry(); + } + } + } + + String crafted = allContent(parseWithMacros(bos.toByteArray())); + // Tika must extract the OPC-resolved real macro, and must NOT have been + // steered to the stream-order decoy. + assertTrue(crafted.contains(REAL_MACRO), + "Tika must extract the OPC-resolved /xl/vbaProject.bin macro"); + assertFalse(crafted.contains(DECOY_MACRO), + "Tika must not extract the stream-order decoy vbaProject.bin macro"); + } +} diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/ZipStructureCoverageTest.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/ZipStructureCoverageTest.java new file mode 100644 index 0000000000..3a95816e46 --- /dev/null +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/ZipStructureCoverageTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.microsoft.ooxml; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.TikaTest; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.Office; +import org.apache.tika.metadata.TikaCoreProperties; + +/** + * Characterizes the boundary of Tika's OOXML parsing and the + * {@link Office#HAS_UNREFERENCED_PARTS} signal. Tika (like Office) loads content by + * following the OPC relationship graph, so a declared part that nothing references is + * carried in the file but sits outside the parsed structure -- a place to hide bytes a + * raw-ZIP reader (AV/DLP/CDR) can still see. + * + * <p>Two related facts pinned here: + * <ul> + * <li>A part with NO content type is not a silent case at all: POI rejects the whole + * package at open time (OPC rule M.1.14), so it never reaches a successful parse.</li> + * <li>A part WITH a content type but no referencing relationship opens fine and is + * flagged. Whether its bytes are also read depends on type: a {@code application/zip} + * part is ignored, while a {@code wordprocessingml} part is still read by POI's + * content-type-based enumeration -- so "unreferenced" is not the same as "unparsed".</li> + * </ul> + * + * <p>Consistent with Tika's <a href="https://tika.apache.org/security-model.html";>security + * model</a>: Tika is not a security boundary and this signal is best-effort and evadable. + */ +public class ZipStructureCoverageTest extends TikaTest { + + private static final String VISIBLE = "VISIBLE_BODY_TEXT_MARKER"; + private static final String ORPHAN = "ORPHAN_DECLARED_PART_MARKER"; + private static final String SMUGGLED = "SMUGGLED_UNREFERENCED_ZIP_MARKER"; + + private static final String CONTENT_TYPES = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" + + "<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\";>" + + "<Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/>" + + "<Default Extension=\"xml\" ContentType=\"application/xml\"/>" + + "<Default Extension=\"zip\" ContentType=\"application/zip\"/>" + + "<Override PartName=\"/word/document.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml\"/>" + + "<Override PartName=\"/word/orphan.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml\"/>" + + "</Types>"; + + private static final String RELS = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" + + "<Relationships xmlns=\"http://schemas.openxmlformats.org/package/2006/relationships\";>" + + "<Relationship Id=\"rId1\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument\"; Target=\"word/document.xml\"/>" + + "</Relationships>"; + + private static final String DOCUMENT = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" + + "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\";>" + + "<w:body><w:p><w:r><w:t>" + VISIBLE + "</w:t></w:r></w:p></w:body>" + + "</w:document>"; + + private static final String ORPHAN_DOC = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" + + "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\";>" + + "<w:body><w:p><w:r><w:t>" + ORPHAN + "</w:t></w:r></w:p></w:body>" + + "</w:document>"; + + private static final String SMUGGLED_PAYLOAD = "not-really-a-zip " + SMUGGLED; + + private void put(ZipOutputStream zos, String name, String content) throws Exception { + zos.putNextEntry(new ZipEntry(name)); + zos.write(content.getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + + private String allContent(List<Metadata> list) { + StringBuilder sb = new StringBuilder(); + for (Metadata m : list) { + String c = m.get(TikaCoreProperties.TIKA_CONTENT); + if (c != null) { + sb.append(c).append('\n'); + } + } + return sb.toString(); + } + + @Test + public void testUnreferencedPartsAreFlagged() throws Exception { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(bos)) { + put(zos, "[Content_Types].xml", CONTENT_TYPES); + put(zos, "_rels/.rels", RELS); + put(zos, "word/document.xml", DOCUMENT); + // declared (wordprocessingml) but referenced by nothing + put(zos, "word/orphan.xml", ORPHAN_DOC); + // declared (application/zip) but referenced by nothing + put(zos, "my-stuff/please-ignore.zip", SMUGGLED_PAYLOAD); + } + + List<Metadata> metadataList = getRecursiveMetadata(TikaInputStream.get(bos.toByteArray()), true); + Metadata container = metadataList.get(0); + String content = allContent(metadataList); + + // Sanity: the referenced body is extracted. + assertTrue(content.contains(VISIBLE), "referenced body text should be extracted"); + // "unreferenced" is not "unparsed": the wordprocessingml orphan is still read by + // POI's content-type enumeration, but the application/zip part is not. + assertTrue(content.contains(ORPHAN), "unreferenced wordprocessingml part is still read"); + assertFalse(content.contains(SMUGGLED), "unreferenced application/zip part is not parsed"); + + // The signal fires and names both unreferenced parts. + assertEquals("true", container.get(Office.HAS_UNREFERENCED_PARTS)); + List<String> names = Arrays.asList(container.getValues(Office.UNREFERENCED_PART_NAMES)); + assertTrue(names.contains("/my-stuff/please-ignore.zip"), + "should list the unreferenced zip; got: " + names); + assertTrue(names.contains("/word/orphan.xml"), + "should list the unreferenced orphan; got: " + names); + } + + @Test + public void testCleanPackageHasNoUnreferencedFlag() throws Exception { + // Every declared part is referenced from the root relationship graph. + String cleanContentTypes = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" + + "<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\";>" + + "<Default Extension=\"rels\" ContentType=\"application/vnd.openxmlformats-package.relationships+xml\"/>" + + "<Default Extension=\"xml\" ContentType=\"application/xml\"/>" + + "<Override PartName=\"/word/document.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml\"/>" + + "</Types>"; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(bos)) { + put(zos, "[Content_Types].xml", cleanContentTypes); + put(zos, "_rels/.rels", RELS); + put(zos, "word/document.xml", DOCUMENT); + } + List<Metadata> metadataList = + getRecursiveMetadata(TikaInputStream.get(bos.toByteArray()), true); + Metadata container = metadataList.get(0); + assertTrue(allContent(metadataList).contains(VISIBLE)); + assertNull(container.get(Office.HAS_UNREFERENCED_PARTS), + "clean package must not set HAS_UNREFERENCED_PARTS; listed: " + + Arrays.toString(container.getValues(Office.UNREFERENCED_PART_NAMES))); + } +}
