vvysotskyi commented on a change in pull request #2401: URL: https://github.com/apache/drill/pull/2401#discussion_r767162796
########## File path: contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/oauth/TestOAuthAccessTokenRepository.java ########## @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.http.oauth; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.util.DrillFileUtils; +import org.apache.drill.exec.store.http.HttpOAuthConfig; +import org.apache.drill.exec.store.http.TestHttpPlugin; +import org.apache.drill.shaded.guava.com.google.common.base.Charsets; +import org.apache.drill.shaded.guava.com.google.common.io.Files; + +import org.junit.BeforeClass; +import org.junit.Test; + + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +public class TestOAuthAccessTokenRepository { Review comment: Please extend the test class from `BaseTest`. ########## File path: exec/java-exec/src/main/java/org/apache/drill/exec/store/http/HttpOAuthConfig.java ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.store.http; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.drill.common.PlanStringBuilder; + +import java.util.Map; + +@Slf4j +@Builder +@Getter +@Setter +@Accessors(fluent = true) +@EqualsAndHashCode +@JsonInclude(JsonInclude.Include.NON_DEFAULT) +@JsonDeserialize(builder = HttpOAuthConfig.HttpOAuthConfigBuilder.class) +public class HttpOAuthConfig { + + @JsonProperty("baseURL") + private final String baseURL; + + @JsonProperty("clientID") + private final String clientID; + + @JsonProperty("clientSecret") + private final String clientSecret; + + @JsonProperty("callbackURL") + private final String callbackURL; + + @JsonProperty("authorizationURL") + private final String authorizationURL; + + @JsonProperty("authorizationPath") + private final String authorizationPath; + + @JsonProperty("authorizationParams") + private final Map<String, String> authorizationParams; + + @JsonProperty("accessTokenPath") + private final String accessTokenPath; + + @JsonProperty("generateCSRFToken") + private final boolean generateCSRFToken; + + @JsonProperty("scope") + private final String scope; + + @JsonProperty("tokens") + private final Map<String, String> tokens; + + /** + * Clone constructor used for updating tokens + * @param that The original oAuth configs + * @param tokens The updated tokens + */ + public HttpOAuthConfig(HttpOAuthConfig that, Map<String, String> tokens) { + this.baseURL = that.baseURL; + this.clientID = that.clientID; + this.clientSecret = that.clientSecret; + this.callbackURL = that.callbackURL; + this.authorizationURL = that.authorizationURL; + this.authorizationPath = that.authorizationPath; + this.authorizationParams = that.authorizationParams; + this.accessTokenPath = that.accessTokenPath; + this.generateCSRFToken = that.generateCSRFToken; + this.scope = that.scope; + this.tokens = tokens; + } + + private HttpOAuthConfig(HttpOAuthConfig.HttpOAuthConfigBuilder builder) { + this.baseURL = builder.baseURL; + this.clientID = builder.clientID; + this.clientSecret = builder.clientSecret; + this.callbackURL = builder.callbackURL; + this.authorizationURL = builder.authorizationURL; + this.authorizationPath = builder.authorizationPath; + this.authorizationParams = builder.authorizationParams; + this.accessTokenPath = builder.accessTokenPath; + this.generateCSRFToken = builder.generateCSRFToken; + this.scope = builder.scope; + this.tokens = builder.tokens; + } + + @Override + public String toString() { + return new PlanStringBuilder(this) + .field("baseURL", baseURL) + .field("clientID", clientID) + .maskedField("clientSecret", clientSecret) + .field("callbackURL", callbackURL) + .field("authorizationURL", authorizationURL) + .field("authorizationParams", authorizationParams) + .field("authorizationPath", authorizationPath) + .field("accessTokenPath", accessTokenPath) + .field("generateCSRFToken", generateCSRFToken) + .field("tokens", tokens.keySet()) + .toString(); + } + + @JsonPOJOBuilder(withPrefix = "") + public static class HttpOAuthConfigBuilder { + @Getter + @Setter Review comment: Instead of adding annotations for every field, you can add them to the class and they will be applied to every field. ########## File path: exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java ########## @@ -180,6 +193,48 @@ public Response enablePlugin(@PathParam("name") String name, @PathParam("val") B } } + @GET + @Path("/storage/{name}/update_oath2_authtoken") + @Produces(MediaType.TEXT_HTML) + public Response updateAuthToken(@PathParam("name") String name, @QueryParam("code") String code) { + try { + if (storage.getPlugin(name).getConfig().getClass().getSimpleName().equalsIgnoreCase("HttpStoragePluginConfig")) { + HttpStoragePluginConfig config = (HttpStoragePluginConfig)storage.getPlugin(name).getConfig(); Review comment: I don't think adding http plugin related code to the exec module is a good idea. ########## File path: exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StorageResources.java ########## @@ -180,6 +193,48 @@ public Response enablePlugin(@PathParam("name") String name, @PathParam("val") B } } + @GET + @Path("/storage/{name}/update_oath2_authtoken") + @Produces(MediaType.TEXT_HTML) + public Response updateAuthToken(@PathParam("name") String name, @QueryParam("code") String code) { + try { + if (storage.getPlugin(name).getConfig().getClass().getSimpleName().equalsIgnoreCase("HttpStoragePluginConfig")) { + HttpStoragePluginConfig config = (HttpStoragePluginConfig)storage.getPlugin(name).getConfig(); Review comment: Can we do all these actions when creating the HTTP plugin? ########## File path: contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/oauth/AccessTokenRepository.java ########## @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.store.http.oauth; + +import okhttp3.OkHttpClient.Builder; +import okhttp3.OkHttpClient; +import okhttp3.Request; + +import org.apache.commons.lang3.StringUtils; +import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.exec.store.StoragePluginRegistry; +import org.apache.drill.exec.store.http.HttpOAuthConfig; +import org.apache.drill.exec.store.http.HttpStoragePluginConfig; +import org.apache.drill.exec.store.http.util.HttpProxyConfig; +import org.apache.drill.exec.store.http.util.SimpleHttp; +import org.apache.parquet.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + + +public class AccessTokenRepository { + + private static final Logger logger = LoggerFactory.getLogger(AccessTokenRepository.class); + private String accessToken; + private final OkHttpClient client; + private final HttpOAuthConfig oAuthConfig; + private final StoragePluginRegistry registry; + private final HttpStoragePluginConfig pluginConfig; + + public AccessTokenRepository(HttpOAuthConfig oAuthConfig, + HttpProxyConfig proxyConfig, + HttpStoragePluginConfig pluginConfig, + StoragePluginRegistry registry) { + Builder builder = new OkHttpClient.Builder(); + this.oAuthConfig = oAuthConfig; + this.registry = registry; + this.pluginConfig = pluginConfig; + + if (oAuthConfig.tokens() != null && oAuthConfig.tokens().containsKey("accessToken")) { + accessToken = oAuthConfig.tokens().get("accessToken"); + } + + // Add proxy info + SimpleHttp.addProxyInfo(builder, proxyConfig); + client = builder.build(); + } + + /** + * Returns the current access token. Does not perform an HTTP request. + * @return The current access token. + */ + public String getAccessToken() { + logger.debug("Getting Access token"); + if (accessToken == null) { + return refreshAccessToken(); + } + return accessToken; + } + + /** + * Refreshes the access token using the code and other information from the HTTP OAuthConfig. + * This executes a POST request. This method will throw exceptions if any of the required fields + * are empty. This plugin also updates the configuration in the storage plugin registry. + * + * In the event that a user submits a request and the access token is expired, the API will + * return a 401 non-authorized response. In the event of a 401 response, the AccessTokenAuthenticator will + * create additional calls to obtain an updated token. This process should be transparent to the user. + * + * @return String of the new access token. + */ + public String refreshAccessToken() { + Request request; + logger.debug("Refreshing Access Token."); + validateKeys(); + + // If the refresh token is present process with that + if (oAuthConfig.tokens().containsKey("refreshToken") && + StringUtils.isNotEmpty(oAuthConfig.tokens().get("refreshToken"))) { + request = OAuthUtils.getAccessTokenRequestFromRefreshToken(oAuthConfig); + } else { + request = OAuthUtils.getAccessTokenRequest(oAuthConfig); + } + + // Update/Refresh the tokens + Map<String, String> tokens = OAuthUtils.getOAuthTokens(client, request); + HttpOAuthConfig updatedConfig = new HttpOAuthConfig(oAuthConfig, tokens); + + if (tokens.containsKey("accessToken")) { + accessToken = tokens.get("accessToken"); + } + + // This null check is here for testing only. In actual Drill, the registry will not be null. + if (registry != null) { + OAuthUtils.updateOAuthTokens(registry, updatedConfig, pluginConfig); Review comment: Please see the previous comment regarding storing access tokens in ZK... ########## File path: exec/jdbc-all/pom.xml ########## @@ -562,7 +562,7 @@ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users. </message> - <maxsize>46600000</maxsize> + <maxsize>50000000</maxsize> Review comment: Is there any reason for updating this limit? The functionality is only for HTTP plugin, so it shouldn't affect jdbc driver size. ########## File path: contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/oauth/AccessTokenAuthenticator.java ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.drill.exec.store.http.oauth; + +import lombok.NonNull; +import okhttp3.Authenticator; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.Route; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AccessTokenAuthenticator implements Authenticator { + private final static Logger logger = LoggerFactory.getLogger(AccessTokenAuthenticator.class); + + private final AccessTokenRepository accessTokenRepository; + + public AccessTokenAuthenticator(AccessTokenRepository accessTokenRepository) { + this.accessTokenRepository = accessTokenRepository; + } + + @Override + public Request authenticate(Route route, @NotNull Response response) { Review comment: According to this logic, the access token will be regenerated for every `authenticate` call. So do we actually need to store it in the storage config and persist it to the zookeeper instead of holding it for a specific HTTP client? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: dev-unsubscr...@drill.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
