bamaer commented on code in PR #8270: URL: https://github.com/apache/hop/pull/8270#discussion_r3943976560
########## ui/src/main/java/org/apache/hop/ui/hopgui/notifications/providers/RssNotificationProvider.java: ########## @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.hopgui.notifications.providers; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import javax.xml.parsers.DocumentBuilder; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.notifications.INotificationProvider; +import org.apache.hop.core.notifications.Notification; +import org.apache.hop.core.notifications.NotificationCategory; +import org.apache.hop.core.notifications.NotificationPriority; +import org.apache.hop.core.xml.XmlParserFactoryProducer; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * RSS/Atom feed notification provider. Supports both RSS 2.0 and Atom 1.0 feeds. Can be configured + * with any feed URL. + */ +public class RssNotificationProvider implements INotificationProvider { + private String feedUrl; + private String providerId; + private String providerName; + private boolean enabled = true; + private long pollInterval = 3600000; // 1 hour default + private String username; + private String password; + + /** What the feed last answered, so a poll that changes nothing costs a 304. */ + private final NotificationHttp.Conditional conditional = new NotificationHttp.Conditional(); + + /** The entries of the last answer, replayed while the feed keeps saying "not modified". */ + private List<Notification> lastFetched = new ArrayList<>(); + + /** + * Create a new RSS notification provider + * + * @param feedUrl The URL of the RSS/Atom feed + * @param providerId Unique identifier for this provider instance + * @param providerName Human-readable name for this provider + */ + public RssNotificationProvider(String feedUrl, String providerId, String providerName) { + this.feedUrl = feedUrl; + this.providerId = providerId; + this.providerName = providerName; + } + + @Override + public String getId() { + return providerId; + } + + @Override + public String getName() { + return providerName; + } + + @Override + public String getDescription() { + return "RSS/Atom feed provider for: " + feedUrl; + } + + @Override + public List<Notification> fetchNotifications() throws HopException { + List<Notification> notifications = new ArrayList<>(); + + if (feedUrl == null || feedUrl.isEmpty()) { + return notifications; + } + + try { + CloseableHttpClient client = NotificationHttp.newClient(username, password); + HttpGet request = new HttpGet(feedUrl); + request.addHeader( + "Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml"); + conditional.applyTo(request); + + try (ClassicHttpResponse response = (ClassicHttpResponse) client.execute(request)) { + // Check HTTP status code + int statusCode = response.getCode(); + if (statusCode == 304) { + // Unchanged since the last poll. + return new ArrayList<>(lastFetched); + } + if (statusCode < 200 || statusCode >= 300) { + throw new HopException("The feed at " + feedUrl + " returned HTTP " + statusCode + "."); + } + + HttpEntity entity = response.getEntity(); + if (entity == null) { + throw new HopException("The feed at " + feedUrl + " returned an empty response."); + } + + try (InputStream rawInputStream = entity.getContent(); + BufferedInputStream inputStream = new BufferedInputStream(rawInputStream, 8192)) { + // Read first few bytes to check for BOM or non-XML content + inputStream.mark(1024); + byte[] buffer = new byte[1024]; + int bytesRead = inputStream.read(buffer); + inputStream.reset(); + + if (bytesRead > 0) { + String contentStart = + stripByteOrderMark( + new String( + buffer, + 0, + Math.min(bytesRead, 100), + java.nio.charset.StandardCharsets.UTF_8)); + // Check if it looks like HTML (common error response) + if (contentStart.trim().startsWith("<html") + || contentStart.trim().startsWith("<!DOCTYPE html")) { + throw new HopException( + "The feed at " + + feedUrl + + " returned an HTML page instead of a feed. Check the URL, or whether a" + + " proxy or sign-in page is answering for it."); + } + // Check if it starts with XML declaration or valid XML tag + String trimmed = contentStart.trim(); + if (!trimmed.startsWith("<?xml") + && !trimmed.startsWith("<feed") + && !trimmed.startsWith("<rss") + && !trimmed.startsWith("<rdf:RDF")) { + throw new HopException( + "The response from " + feedUrl + " is not an RSS or Atom feed."); + } + } + + DocumentBuilder builder = + XmlParserFactoryProducer.createSecureDocBuilderFactory().newDocumentBuilder(); + Document document = builder.parse(inputStream); + + // Check if it's Atom or RSS + Element root = document.getDocumentElement(); + if (root == null) { + throw new HopException("The feed at " + feedUrl + " is an empty XML document."); + } + String rootName = root.getNodeName(); + + if ("feed".equals(rootName) || rootName.contains("atom")) { + // Atom feed + notifications.addAll(parseAtomFeed(document)); + } else if ("rss".equals(rootName) || "rdf:RDF".equals(rootName)) { + // RSS feed + notifications.addAll(parseRssFeed(document)); + } else { + throw new HopException( + "The feed at " + feedUrl + " is in an unsupported format: <" + rootName + ">."); + } + } + + // Only now that the feed has been read and understood. Remembering the validator any + // earlier means a parse that fails still arms the next request's If-None-Match: the feed + // would answer 304 forever, this method would return the entries it never managed to read + // (none), and because that is not a failure the error banner would clear itself. + conditional.remember(response); + } + lastFetched = new ArrayList<>(notifications); + } catch (HopException e) { + throw e; + } catch (Exception e) { + // Reported to the user through the panel's error banner. NotificationService catches this + // per provider, so one unreachable feed does not stop the others. + throw new HopException("Could not read the feed at " + feedUrl + ": " + e.getMessage(), e); + } + + return notifications; + } + + /** + * Drop a leading byte order mark. + * + * <p>Feeds written by Windows tooling are commonly served with one. Decoded from UTF-8 it is + * U+FEFF, which {@link String#trim()} leaves alone - it only strips characters up to U+0020 - so + * the sniff below would find the feed starting with an invisible character rather than with + * {@code <?xml}, and reject a feed the XML parser handles perfectly well. + * + * @param text The decoded start of the response + * @return The text without its byte order mark + */ + static String stripByteOrderMark(String text) { + if (!text.isEmpty() && text.charAt(0) == '\uFEFF') { + return text.substring(1); + } + return text; + } + + /** Parse Atom 1.0 feed */ + List<Notification> parseAtomFeed(Document document) { + List<Notification> notifications = new ArrayList<>(); + NodeList entries = document.getElementsByTagName("entry"); + + for (int i = 0; i < entries.getLength(); i++) { + Element entry = (Element) entries.item(i); + try { + String id = getElementText(entry, "id"); + String title = getElementText(entry, "title"); + String summary = getElementText(entry, "summary"); + if (summary == null || summary.isEmpty()) { + summary = getElementText(entry, "content"); + } + String link = getElementLink(entry); + String publishedText = getElementText(entry, "published"); + if (publishedText == null || publishedText.isEmpty()) { + publishedText = getElementText(entry, "updated"); + } + Date published = parseAtomDate(publishedText); + if (published == null) { + published = new Date(); + } + + String localId = entryId(id, link, title, publishedText); + if (localId == null) { + LogChannel.UI.logDetailed( + "Skipping an entry in the feed at " + feedUrl + " that cannot be identified"); + continue; + } + + Notification notification = + new Notification( + localId, + title != null ? title : "Untitled", + summary != null ? summary : "", Review Comment: Added `FeedText.plainText()` — strips tags, drops `<script>`/`<style>` content, decodes named/decimal/hex entities including double-escaped HTML, collapses whitespace. Ampersands are doubled at `setText` time in the panel. Three things worth recording from doing it. The stripping has to happen in the provider, not at display: the message is truncated to 250 characters first, so stripping afterwards would cut an entry opening with a long `<div class="…">` down to markup and then to nothing. `CLabel` needed the ampersand escaping too — it paints with `SWT.DRAW_MNEMONIC` on the desktop and RAP renders it through a `CLabelLCA` that calls `MnemonicUtil`, so the notification title was losing ampersands on both platforms until it was escaped as well. And testing against LWN's feed found a bug in the fix itself: ` ` decodes to U+00A0, which Java's `\s` doesn't match, so numeric non-breaking spaces survived collapsing where the named ` ` didn't. Verified in the desktop GUI and in Hop Web against a feed with ampersands in both titles and bodies. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
