Apache9 commented on code in PR #5488: URL: https://github.com/apache/hbase/pull/5488#discussion_r1383316307
########## hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectedFunctionCache.java: ########## @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.reflect.ClassPath; + +/** + * Cache to hold resolved Functions generated through reflection. These can be costly to create, but + * then are much faster than typical Method.invoke calls when executing. Upon construction, finds + * all subclasses in the same package of the passed baseClass. For each found class, creates a + * lambda using + * {@link ReflectionUtils#getOneArgStaticMethodAsFunction(Class, String, Class, Class)}. These are + * added to a hashmap for fast lookup by name later. + * @param <I> the input argument type for the resolved functions + * @param <R> the return type for the resolved functions + */ +@InterfaceAudience.Private +final public class ReflectedFunctionCache<I, R> { + + private static final Logger LOG = LoggerFactory.getLogger(ReflectedFunctionCache.class); + + private final Map<String, Function<I, ? extends R>> lambdasByClass; + + private ReflectedFunctionCache(Map<String, Function<I, ? extends R>> lambdasByClass) { + this.lambdasByClass = lambdasByClass; + } + + /** + * Create a cache of reflected functions using the provided classloader and baseClass. Will find + * all subclasses of the provided baseClass (in the same package), and then foreach look for a + * static one-arg method with the methodName and argClass. The expectation is that the method + * returns a value whose class extends the baseClass. This was primarily designed for use by our + * Filter and Comparator parseFrom methods. + */ + public static <I, R> ReflectedFunctionCache<I, R> create(ClassLoader classLoader, + Class<R> baseClass, Class<I> argClass, String methodName) { + Map<String, Function<I, ? extends R>> lambdasByClass = new HashMap<>(); + Set<? extends Class<? extends R>> classes = getSubclassesInPackage(classLoader, baseClass); + for (Class<? extends R> clazz : classes) { + Function<I, ? extends R> func = createFunction(clazz, methodName, argClass, clazz); + if (func != null) { + lambdasByClass.put(clazz.getName(), func); + } + } + return new ReflectedFunctionCache<>(lambdasByClass); + } + + /** + * Get and execute the Function for the given className, passing the argument to the function and + * returning the result. + * @param className the full name of the class to lookup + * @param argument the argument to pass to the function, if found. + * @return null if a function is not found for classname, otherwise the result of the function. + */ + public R getAndCallByName(String className, I argument) { + Function<I, ? extends R> lambda = lambdasByClass.get(className); + + // todo: if we ever make java9+ our lowest supported jdk version, we can + // handle generating these for newly loaded classes from our DynamicClassLoader using + // MethodHandles.privateLookupIn(). For now this is not possible, because we can't easily + // create a privileged lookup in a non-default ClassLoader. + if (lambda == null) { + return null; + } + + return lambda.apply(argument); + } + + private static <R> Set<Class<? extends R>> getSubclassesInPackage(ClassLoader classLoader, + Class<R> baseClass) { + try { + return ClassPath.from(classLoader).getAllClasses().stream() Review Comment: Will this cause we load all classes even if it is not used by now? I used to use guava's ClassPath in a project but it performed differently when executing in IDE and in command line, finally I chose to use ClassPathScanningCandidateComponentProvider in spring for scanning classes... ########## hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectedFunctionCache.java: ########## @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.reflect.ClassPath; + +/** + * Cache to hold resolved Functions generated through reflection. These can be costly to create, but + * then are much faster than typical Method.invoke calls when executing. Upon construction, finds + * all subclasses in the same package of the passed baseClass. For each found class, creates a + * lambda using + * {@link ReflectionUtils#getOneArgStaticMethodAsFunction(Class, String, Class, Class)}. These are + * added to a hashmap for fast lookup by name later. + * @param <I> the input argument type for the resolved functions + * @param <R> the return type for the resolved functions + */ +@InterfaceAudience.Private +final public class ReflectedFunctionCache<I, R> { Review Comment: Use 'public final'. And is it better to call it 'ReflectionFunctionCache'? Anyway, not a native English speaker, feel free to choose the one you like. ########## hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java: ########## @@ -208,6 +214,30 @@ private static String getTaskName(long id, String name) { return id + " (" + name + ")"; } + /** + * Creates a Function which can be called to performantly execute a reflected static method. The + * creation of the Function itself may not be fast, but executing that method thereafter should be + * much faster than {@link #invokeMethod(Object, String, Object...)}. + * @param lookupClazz the class to find the static method in + * @param methodName the method name + * @param argumentClazz the type of the argument + * @param returnValueClass the type of the return value + * @return a function which when called executes the requested static method. + * @throws Throwable exception types from the underlying reflection + */ + public static <I, R> Function<I, R> getOneArgStaticMethodAsFunction(Class<?> lookupClazz, + String methodName, Class<I> argumentClazz, Class<R> returnValueClass) throws Throwable { + MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle methodHandle = lookup.findStatic(lookupClazz, methodName, + MethodType.methodType(returnValueClass, argumentClazz)); + CallSite site = + LambdaMetafactory.metafactory(lookup, "apply", MethodType.methodType(Function.class), + methodHandle.type().generic(), methodHandle, methodHandle.type()); + + return (Function<I, R>) site.getTarget().invokeExact(); Review Comment: So this is the magic here? Somehow we can convert a Method to a Function, so the invocation will be faster? ########## hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java: ########## @@ -99,4 +129,51 @@ public void testBigDecimalComparator() throws Exception { ProtobufUtil.toComparator(ProtobufUtil.toComparator(bigDecimalComparator)))); } + /** + * Test that we can load and deserialize custom comparators. Good to have generally, but also + * proves that this still works after HBASE-27276 despite not going through our fast function + * caches. + */ + @Test + public void testCustomComparator() throws Exception { + ByteArrayComparable baseFilter = new BinaryComparator("foo".getBytes()); + ComparatorProtos.Comparator proto = ProtobufUtil.toComparator(baseFilter); + String className = "CustomLoadedComparator" + allowFastReflectionFallthrough; + proto = proto.toBuilder().setName(className).build(); + + Configuration conf = HBaseConfiguration.create(); + HBaseTestingUtil testUtil = new HBaseTestingUtil(); + String dataTestDir = testUtil.getDataTestDir().toString(); + + // First make sure the test bed is clean, delete any pre-existing class. + // Below toComparator call is expected to fail because the comparator is not loaded now + ClassLoaderTestHelper.deleteClass(className, dataTestDir, conf); + try { + ProtobufUtil.toComparator(proto); + fail("expected to fail"); + } catch (IOException e) { + // do nothing, this is expected + } + + // Write a jar to be loaded into the classloader + String code = StringSubstitutor.replace( + IOUtils.toString(getClass().getResourceAsStream("/CustomLoadedComparator.java"), + Charset.defaultCharset()), + Collections.singletonMap("suffix", allowFastReflectionFallthrough)); + ClassLoaderTestHelper.buildJar(dataTestDir, className, code, + ClassLoaderTestHelper.localDirPath(conf)); + + // Disallow fallthrough at first, we expect below to fail + ProtobufUtil.setAllowFastReflectionFallthrough(false); Review Comment: Why disable fallthrough will lead to a failure here? It is because the class is not loaded when initializing, so it is not in the ReflectedFunctionCache? So we do not add special test for testing the scenario where ReflectedFunctionCache can work? As it is enabled by default so testing filter usage is enough? ########## hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java: ########## @@ -99,4 +129,51 @@ public void testBigDecimalComparator() throws Exception { ProtobufUtil.toComparator(ProtobufUtil.toComparator(bigDecimalComparator)))); } + /** + * Test that we can load and deserialize custom comparators. Good to have generally, but also + * proves that this still works after HBASE-27276 despite not going through our fast function + * caches. + */ + @Test + public void testCustomComparator() throws Exception { + ByteArrayComparable baseFilter = new BinaryComparator("foo".getBytes()); + ComparatorProtos.Comparator proto = ProtobufUtil.toComparator(baseFilter); + String className = "CustomLoadedComparator" + allowFastReflectionFallthrough; + proto = proto.toBuilder().setName(className).build(); + + Configuration conf = HBaseConfiguration.create(); + HBaseTestingUtil testUtil = new HBaseTestingUtil(); + String dataTestDir = testUtil.getDataTestDir().toString(); + + // First make sure the test bed is clean, delete any pre-existing class. + // Below toComparator call is expected to fail because the comparator is not loaded now + ClassLoaderTestHelper.deleteClass(className, dataTestDir, conf); + try { + ProtobufUtil.toComparator(proto); + fail("expected to fail"); + } catch (IOException e) { + // do nothing, this is expected + } + + // Write a jar to be loaded into the classloader + String code = StringSubstitutor.replace( + IOUtils.toString(getClass().getResourceAsStream("/CustomLoadedComparator.java"), Review Comment: Better append a suffix like '.template' to the file name -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@hbase.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
