bamaer commented on code in PR #9533:
URL: https://github.com/apache/hop/pull/9533#discussion_r4596819462
##########
assemblies/plugins/pom.xml:
##########
@@ -1839,6 +1839,12 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-tech-microsoft365</artifactId>
Review Comment:
**Blocker.** This adds the plugin to the `full` profile only. The release
client doesn't ship it, and it has no entry in
`plugins/misc/marketplace/src/main/resources/org/apache/hop/marketplace/optional-plugins.yaml`,
so the marketplace doesn't offer it either. It's the only plugin in `full`
that is in neither. Suggest adding an entry there (`hop-tech-microsoft365`,
`installPath`/`modulePath: plugins/tech/microsoft365`, category Technology),
and "(marketplace plugin `hop-tech-microsoft365`)" in the `vfs.adoc` table, as
for HDFS and Git.
##########
plugins/tech/microsoft365/pom.xml:
##########
@@ -0,0 +1,263 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-plugins-tech</artifactId>
+ <version>2.21.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>hop-tech-microsoft365</artifactId>
+ <packaging>jar</packaging>
+ <name>Hop Plugins Technology Microsoft 365</name>
+
+ <properties>
+ <azure-eventhubs.version>3.3.0</azure-eventhubs.version>
+ <azure-graph.version>6.58.0</azure-graph.version>
+ </properties>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-bom</artifactId>
+ <version>${netty.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-sdk-bom</artifactId>
+ <version>${azure-sdk-bom.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-libs</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.microsoft.graph</groupId>
+ <artifactId>microsoft-graph</artifactId>
+ <version>${azure-graph.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.graph</groupId>
+ <artifactId>microsoft-graph-core</artifactId>
+ <version>3.6.5</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-abstractions</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-authentication-azure</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-http-okHttp</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-serialization-form</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-serialization-json</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-serialization-multipart</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.kiota</groupId>
+ <artifactId>microsoft-kiota-serialization-text</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.github.std-uritemplate</groupId>
+ <artifactId>std-uritemplate</artifactId>
+ <version>2.0.0</version>
+ </dependency>
+ <!-- Shared with the other plugins through lib/core -->
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-core</artifactId>
Review Comment:
**Should fix.** `azure-core` pulls in `jackson-datatype-jsr310` 2.13.5, and
the zip puts it in `lib/core`. The client already has 2.18.3 there, so after
installing the plugin, `lib/core` holds both versions. The Azure plugin
excludes `com.fasterxml.jackson.datatype:*` from `azure-core` for this reason.
With the same exclusion here, the zip no longer ships the jar, and the plugin
still loads and passes the classpath check.
##########
plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java:
##########
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.sharepoint.metadatatype;
+
+import com.azure.core.credential.TokenCredential;
+import com.azure.identity.ClientCertificateCredentialBuilder;
+import com.azure.identity.ClientSecretCredentialBuilder;
+import com.azure.identity.DefaultAzureCredentialBuilder;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.IVfsBrowseLocation;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataCategory;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IEnumHasCode;
+import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+
+@Getter
+@Setter
+@NullMarked
+@GuiPlugin(classLoaderGroup = "vfs-graph")
+@HopMetadata(
+ key = "sharepoint",
Review Comment:
**Blocker.** The metadata key is also the folder name
(`metadata/sharepoint/`) and goes into exports, so it can't change after a
release without `legacyKeys`. The other connection types use
`<name>-connection` (`smb-connection`, `webdav-connection`, `hdfs-connection`).
Suggest `key = "sharepoint-connection"` while it's still free to change.
##########
plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java:
##########
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.vfs.sharepoint;
+
+import com.microsoft.graph.core.models.UploadResult;
+import com.microsoft.graph.core.tasks.LargeFileUploadTask;
+import com.microsoft.graph.drives.item.items.item.DriveItemItemRequestBuilder;
+import
com.microsoft.graph.drives.item.items.item.children.ChildrenRequestBuilder;
+import
com.microsoft.graph.drives.item.items.item.createuploadsession.CreateUploadSessionPostRequestBody;
+import com.microsoft.graph.models.BaseCollectionPaginationCountResponse;
+import com.microsoft.graph.models.DriveCollectionResponse;
+import com.microsoft.graph.models.DriveItem;
+import com.microsoft.graph.models.DriveItemCollectionResponse;
+import com.microsoft.graph.models.DriveItemUploadableProperties;
+import com.microsoft.graph.models.Folder;
+import com.microsoft.graph.models.ItemReference;
+import com.microsoft.graph.models.Site;
+import com.microsoft.graph.models.SiteCollectionResponse;
+import com.microsoft.graph.models.UploadSession;
+import com.microsoft.graph.models.odataerrors.ODataError;
+import com.microsoft.graph.serviceclient.GraphServiceClient;
+import com.microsoft.graph.sites.item.drives.DrivesRequestBuilder;
+import com.microsoft.kiota.ApiException;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.apache.commons.vfs2.Capability;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.AbstractFileSystem;
+import org.apache.hop.core.logging.LogChannel;
+import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * One named Microsoft 365 connection. The sites are the folders of the root,
the document libraries
+ * (drives) of a site are the folders of that site.
+ */
+@NullMarked
+public class SharePointFileSystem extends AbstractFileSystem {
+
+ public static final List<Capability> CAPABILITIES =
+ List.of(
+ Capability.CREATE,
+ Capability.DELETE,
+ Capability.RENAME,
+ Capability.GET_TYPE,
+ Capability.GET_LAST_MODIFIED,
+ Capability.LIST_CHILDREN,
+ Capability.READ_CONTENT,
+ Capability.URI,
+ Capability.WRITE_CONTENT);
+
+ /** Above this size the content is sent in slices through an upload session.
*/
+ static final long SIMPLE_UPLOAD_LIMIT = 4L * 1024 * 1024;
+
+ /** The id that addresses the root folder of a document library, and the
root site of a tenant. */
+ static final String ROOT_ITEM_ID = "root";
+
+ /** A site created by a Microsoft Teams team may live under /teams. */
+ private static final List<String> MANAGED_PATHS = List.of("sites", "teams");
+
+ /** The only properties of a file or folder that are used, the others are
not requested. */
+ private static final String[] ITEM_PROPERTIES = {
+ "id", "name", "size", "lastModifiedDateTime", "file", "folder", "package"
+ };
+
+ private static final String CONFLICT_BEHAVIOR =
"@microsoft.graph.conflictBehavior";
+ private static final int HTTP_NOT_FOUND = 404;
+ private static final int HTTP_CONFLICT = 409;
+
+ private final GraphServiceClient client;
+
+ /** Site name to site id, whatever the case of the name, null until the
sites are listed. */
+ private volatile @Nullable Map<String, String> siteIds;
+
+ /** The SharePoint host of the tenant, null until it is needed. */
+ private volatile @Nullable String hostname;
+
+ /** False once Microsoft Graph refused to list the sites. */
+ private volatile boolean sitesListable = true;
+
+ /** Site id to (drive name to drive id, whatever the case of the name). */
+ private final Map<String, Map<String, String>> driveIds = new
ConcurrentHashMap<>();
+
+ public SharePointFileSystem(
+ FileName rootName,
+ GraphServiceClient client,
+ @Nullable String hostname,
+ @Nullable FileSystemOptions fileSystemOptions) {
+ super(rootName, null, fileSystemOptions);
+ this.client = client;
+ this.hostname = hostname == null || hostname.isBlank() ? null :
hostname.trim();
+ }
+
+ @Override
+ protected void addCapabilities(Collection<Capability> capabilities) {
+ capabilities.addAll(CAPABILITIES);
+ }
+
+ @Override
+ protected FileObject createFile(AbstractFileName name) {
+ return new SharePointFileObject(name, this);
+ }
+
+ /**
+ * Lists the sites of the tenant and refreshes the cache.
+ *
+ * @return the site ids by site name
+ */
+ Map<String, String> listSites() throws FileSystemException {
+ Map<String, String> sites = newNameMap();
+ try {
+ addSites(sites, client.sites().get());
+ } catch (ApiException e) {
+ throw new FileSystemException(
+ "Microsoft Graph refused to list the SharePoint sites: " +
describe(e), e);
+ }
+ if (sites.isEmpty()) {
+ // Depending on the tenant the plain listing is empty and only the
search returns the sites
+ try {
+ String url = client.getRequestAdapter().getBaseUrl() +
"/sites?search=*";
+ addSites(sites, client.sites().withUrl(url).get());
+ } catch (ApiException e) {
+ LogChannel.GENERAL.logBasic(
+ "Microsoft Graph refused to search the SharePoint sites: " +
describe(e));
+ }
+ }
+ if (sites.isEmpty()) {
+ LogChannel.GENERAL.logBasic(
+ "Microsoft Graph returned no SharePoint site. Listing the sites
needs the application"
+ + " permission Sites.Read.All with admin consent. With
Sites.Selected the sites"
+ + " can not be listed, use the full path of a site the
application was granted.");
+ }
+ siteIds = sites;
+ driveIds.clear();
+ return sites;
+ }
+
+ /**
+ * @return the HTTP status, the error code and the message of Microsoft Graph
+ */
+ static String describe(ApiException e) {
+ StringBuilder text = new StringBuilder("HTTP
").append(e.getResponseStatusCode());
+ if (e instanceof ODataError error && error.getError() != null) {
+ text.append(' ').append(error.getError().getCode());
+ text.append(": ").append(error.getError().getMessage());
+ } else if (e.getMessage() != null) {
+ text.append(": ").append(e.getMessage());
+ }
+ return text.toString();
+ }
+
+ private void addSites(Map<String, String> sites, @Nullable
SiteCollectionResponse firstPage) {
+ forEachPaged(
+ firstPage,
+ SiteCollectionResponse::getValue,
+ nextLink -> client.sites().withUrl(nextLink).get(),
+ site -> addSite(sites, site));
+ }
+
+ /** Two sites can have the same name, only the first one is kept. */
+ private static void addSite(Map<String, String> sites, Site site) {
+ String name = site.getName() != null ? site.getName() :
site.getDisplayName();
+ if (name == null || site.getId() == null) {
+ return;
+ }
+ String namesake = sites.putIfAbsent(name, site.getId());
+ if (namesake != null && !namesake.equals(site.getId())) {
+ LogChannel.GENERAL.logBasic(
+ "The SharePoint site "
+ + site.getWebUrl()
+ + " is ignored: another site is already named '"
+ + name
+ + "'");
+ }
+ }
+
+ /**
+ * Reads a site by its address, {@code /sites/name} of the SharePoint host
of the tenant. This
+ * also works when the application can not list the sites (Sites.Selected).
+ *
+ * @return the id of the site or null if it does not exist or can not be read
+ */
+ private @Nullable String findSiteIdByPath(String siteName) {
+ String host;
+ try {
+ host = hostname();
+ } catch (ApiException e) {
+ LogChannel.GENERAL.logBasic(
+ "The SharePoint host of the tenant can not be read, set it on the
connection: "
+ + describe(e));
+ return null;
+ }
+ if (host == null) {
+ return null;
+ }
+ String encodedSiteName = SharePointFileName.encodeSegment(siteName);
+ for (String managedPath : MANAGED_PATHS) {
+ // The address of the site relative to the host, such as
contoso.sharepoint.com:/sites/Finance
+ String address = host + ":/" + managedPath + "/";
+ String url = client.getRequestAdapter().getBaseUrl() + "/sites/" +
address + encodedSiteName;
+ try {
+ Site site = client.sites().bySiteId(ROOT_ITEM_ID).withUrl(url).get();
+ if (site != null && site.getId() != null) {
+ return site.getId();
+ }
+ } catch (ApiException e) {
+ LogChannel.GENERAL.logBasic(
+ "The SharePoint site "
+ + host
+ + "/"
+ + managedPath
+ + "/"
+ + siteName
+ + " can not be read: "
+ + describe(e));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Lists the sites, an application that is only granted some sites
(Sites.Selected) can not list
+ * them and reaches its sites by their address.
+ */
+ private Map<String, String> listSitesIfAllowed() {
+ try {
+ return listSites();
+ } catch (FileSystemException e) {
+ LogChannel.GENERAL.logDetailed(e.getMessage());
+ Map<String, String> sites = newNameMap();
+ sitesListable = false;
+ siteIds = sites;
+ return sites;
+ }
+ }
+
+ /**
+ * @return the SharePoint host of the tenant, such as
contoso.sharepoint.com, or null if the root
+ * site can not be read
+ */
+ private @Nullable String hostname() {
+ if (hostname == null) {
+ Site root = client.sites().bySiteId(ROOT_ITEM_ID).get();
+ if (root != null && root.getSiteCollection() != null) {
+ hostname = root.getSiteCollection().getHostname();
+ }
+ if (hostname == null && root != null && root.getWebUrl() != null) {
+ hostname = URI.create(root.getWebUrl()).getHost();
+ }
+ }
+ return hostname;
+ }
+
+ /**
+ * @return the id of the site or null if there is no site with this name
+ */
+ @Nullable String findSiteId(String siteName) throws FileSystemException {
Review Comment:
**Should fix.** Any path lookup first lists every site in the tenant,
following `@odata.nextLink` through all pages. That happens once per process
(each `hop-run`) and again after a connection is saved. A lookup of a site that
doesn't exist lists them all again, plus the two path probes. With 452 sites in
a Graph stub, reading one file took 8 requests, 3 of them `/sites` pages. On a
large tenant, where every Teams team has its own site, that is a lot of paging
before the first byte. Suggest trying `findSiteIdByPath` first, and keeping the
listing for browsing `name:///` and as the fallback.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]