dschmidt commented on code in PR #3037: URL: https://github.com/apache/tika/pull/3037#discussion_r3810092421
########## tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/RawTiffParser.java: ########## @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.parser.image; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BoundedInputStream; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import org.apache.tika.annotation.TikaComponent; +import org.apache.tika.config.ConfigDeserializer; +import org.apache.tika.config.JsonConfig; +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.EmbeddedDocumentExtractor; +import org.apache.tika.extractor.EmbeddedDocumentUtil; +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.HttpHeaders; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.metadata.TikaCoreProperties; +import org.apache.tika.mime.MediaType; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.XHTMLContentHandler; + +/** + * Parser for TIFF-based camera raw images: Nikon NEF/NRW, Sony ARW/SRF/SR2, + * Pentax PEF/PTX, Adobe DNG and Canon CR2. + * <p> + * These formats are TIFF containers: metadata extraction is inherited from + * {@link TiffParser}. In addition, this parser extracts the camera-generated + * JPEG preview images embedded in the raw file and hands them to the + * {@link EmbeddedDocumentExtractor}. Previews are referenced from the IFD + * chain or from SubIFDs, either via the JPEGInterchangeFormat/ + * JPEGInterchangeFormatLength tags or as a single JPEG-compressed strip + * (DNG, CR2). Strips holding raw sensor data are also JPEG-encoded in some + * formats (lossless JPEG in CR2 and DNG), so strip candidates are only + * accepted for displayable images: PhotometricInterpretation RGB or YCbCr, + * or 8 bits per sample when PhotometricInterpretation is absent (CR2). + */ +@TikaComponent +public class RawTiffParser extends TiffParser { + + /** + * Serial version UID + */ + private static final long serialVersionUID = 5385105345533384662L; + + private static final Set<MediaType> SUPPORTED_TYPES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + MediaType.image("x-raw-nikon"), + MediaType.image("x-raw-sony"), + MediaType.image("x-raw-pentax"), + MediaType.image("x-raw-adobe"), + MediaType.image("x-canon-cr2")))); + + private static final String JPEG_MIME = "image/jpeg"; + + private static final int TAG_BITS_PER_SAMPLE = 0x0102; + private static final int TAG_COMPRESSION = 0x0103; + private static final int TAG_PHOTOMETRIC_INTERPRETATION = 0x0106; + private static final int TAG_STRIP_OFFSETS = 0x0111; + private static final int TAG_STRIP_BYTE_COUNTS = 0x0117; + private static final int TAG_SUB_IFDS = 0x014A; + private static final int TAG_JPEG_INTERCHANGE_FORMAT = 0x0201; + private static final int TAG_JPEG_INTERCHANGE_FORMAT_LENGTH = 0x0202; + + private static final int COMPRESSION_OLD_JPEG = 6; + private static final int COMPRESSION_JPEG = 7; + private static final int PHOTOMETRIC_RGB = 2; + private static final int PHOTOMETRIC_YCBCR = 6; + + private static final int MAX_IFDS = 32; + private static final int MAX_ENTRIES_PER_IFD = 1024; + // at most MAX_IFDS are ever processed; cap the pending queue so a crafted + // file packed with SubIFD pointers cannot grow it without bound + private static final int MAX_PENDING_IFDS = 1024; + //previews are camera-generated JPEGs, tens of MB is already generous + private static final long DEFAULT_MAX_PREVIEW_LENGTH_BYTES = 100 * 1024 * 1024; + + private final RawTiffParserConfig defaultConfig; + + public RawTiffParser() { + this(new RawTiffParserConfig()); + } + + public RawTiffParser(RawTiffParserConfig config) { + this.defaultConfig = config; + } + + public RawTiffParser(JsonConfig jsonConfig) { + this(ConfigDeserializer.buildConfig(jsonConfig, RawTiffParserConfig.class)); + } + + @Override + public Set<MediaType> getSupportedTypes(ParseContext context) { + return SUPPORTED_TYPES; + } + + @Override + public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata, + ParseContext context) throws IOException, SAXException, TikaException { + extractMetadata(tis, handler, metadata, context); + XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata, context); + xhtml.startDocument(); + if (defaultConfig.isExtractPreviews()) { + extractPreviews(tis, xhtml, metadata, context); + } + xhtml.endDocument(); + } + + private void extractPreviews(TikaInputStream tis, XHTMLContentHandler xhtml, Metadata metadata, + ParseContext context) throws IOException, SAXException { + List<Preview> previews; + try (RandomAccessFile raf = new RandomAccessFile(tis.getFile(), "r")) { + previews = locateJpegPreviews(raf); + } catch (TiffStructureException e) { + EmbeddedDocumentUtil.recordException(e, metadata); + return; + } + if (previews.isEmpty()) { + return; + } + EmbeddedDocumentExtractor extractor = + EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context); + int count = 0; + for (Preview preview : previews) { + Metadata previewMetadata = Metadata.newInstance(context); + previewMetadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, + TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString()); + previewMetadata.set(HttpHeaders.CONTENT_TYPE, JPEG_MIME); + EmbeddedDocumentUtil.setGeneratedResourceName(previewMetadata, + EmbeddedDocumentUtil.EmbeddedResourcePrefix.THUMBNAIL, count, JPEG_MIME); + count++; + if (!extractor.shouldParseEmbedded(previewMetadata, context)) { + continue; + } + //stream the preview region instead of loading it onto the heap + try (InputStream fileStream = Files.newInputStream(tis.getPath())) { + IOUtils.skipFully(fileStream, preview.offset()); + BoundedInputStream bounded = BoundedInputStream.builder() + .setInputStream(fileStream) + .setMaxCount(preview.length()) + .get(); + try (TikaInputStream previewStream = TikaInputStream.get(bounded)) { + extractor.parseEmbedded(previewStream, xhtml, previewMetadata, context, true); + } + } + } + } + + /** + * Walks the TIFF IFD chain and any SubIFDs (traversal bounded by + * {@link #MAX_IFDS}) and returns the embedded JPEG previews. + */ + private List<Preview> locateJpegPreviews(RandomAccessFile raf) + throws IOException, TiffStructureException { + long fileLength = raf.length(); + if (fileLength < 8) { + throw new TiffStructureException("file too short for a TIFF header"); + } + raf.seek(0); + int b0 = raf.read(); + int b1 = raf.read(); + boolean bigEndian; + if (b0 == 'M' && b1 == 'M') { + bigEndian = true; + } else if (b0 == 'I' && b1 == 'I') { + bigEndian = false; + } else { + throw new TiffStructureException("not a TIFF byte order marker"); + } + if (readUInt16(raf, bigEndian) != 42) { + throw new TiffStructureException("bad TIFF magic number"); + } Review Comment: Great catch!, Copilot :) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
