madrob commented on a change in pull request #254: URL: https://github.com/apache/solr/pull/254#discussion_r698717978
########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/StringsDSL.java ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import static org.apache.solr.bench.generators.SourceDSL.checkArguments; +import static org.apache.solr.bench.generators.SourceDSL.integers; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.SplittableRandom; +import org.eclipse.jetty.io.RuntimeIOException; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.generators.Generate; +import org.quicktheories.impl.BenchmarkRandomSource; +import org.quicktheories.impl.SplittableRandomSource; + +public class StringsDSL { + + private static final int BASIC_LATIN_LAST_CODEPOINT = 0x007E; + private static final int BASIC_LATIN_FIRST_CODEPOINT = 0x0020; + private static final int ASCII_LAST_CODEPOINT = 0x007F; + private static final int LARGEST_DEFINED_BMP_CODEPOINT = 65533; + + private static final List<String> words; + + private static final int wordsSize; + + static { + try { + words = + Files.readAllLines( + Paths.get( + "src", + "resources", + "words.txt")); // english word list via https://github.com/dwyl/english-words + } catch (IOException e) { + throw new RuntimeIOException(e); + } + wordsSize = words.size(); + } + + public WordListGeneratorBuilder wordList() { + return new WordListGeneratorBuilder( + new SolrGen( Review comment: rawtypes ########## File path: solr/benchmark/src/java/org/apache/solr/bench/Docs.java ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench; + +import static org.apache.solr.bench.BaseBenchState.log; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.commons.math3.random.RandomGenerator; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.solr.bench.generators.SolrGen; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.SuppressForbidden; +import org.quicktheories.core.Gen; +import org.quicktheories.impl.BenchmarkRandomSource; + +/** + * A tool to generate controlled random data for a benchmark. {@link SolrInputDocument}s are created + * based on supplied FieldDef definitions. + * + * <p>You can call getDocument to build and retrieve one {@link SolrInputDocument} at a time, or you + * can call {@link #preGenerate} to generate the given number of documents in RAM, and then retrieve + * them via {@link #generatedDocsIterator}. + */ +public class Docs { + + private final ThreadLocal<RandomGenerator> random; + private final RandomGenerator randomParent; + private Queue<SolrInputDocument> docs = new ConcurrentLinkedQueue<>(); + + private final Map<String, Gen<?>> fields = new HashMap<>(); + + private static final AtomicInteger ID = new AtomicInteger(); + + private ExecutorService executorService; + private int stringFields; + private int multiStringFields; + private int integerFields; + private int longFields; + + public static Docs docs() { + return new Docs(); + } + + public static Docs docs(RandomGenerator random) { + return new Docs(random); + } + + private Docs(RandomGenerator random) { + this.randomParent = random; + this.random = + ThreadLocal.withInitial( + () -> + new SplittableRandomGenerator( + random.nextLong())); // TODO: pluggable RandomGenerator + } + + private Docs() { + this(new SplittableRandomGenerator(Long.getLong("randomSeed"))); + } + + @SuppressForbidden(reason = "This module does not need to deal with logging context") + public Iterator<SolrInputDocument> preGenerate(int numDocs) throws InterruptedException { + log("preGenerate docs " + numDocs + " ..."); + docs.clear(); + executorService = + Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors() + 1, + new SolrNamedThreadFactory("SolrJMH DocMaker")); + + for (int i = 0; i < numDocs; i++) { + executorService.submit( + () -> { + try { + SolrInputDocument doc = Docs.this.inputDocument(); + docs.add(doc); + } catch (Exception e) { + executorService.shutdownNow(); + throw new RuntimeException(e); + } + }); + } + + executorService.shutdown(); + boolean result = executorService.awaitTermination(10, TimeUnit.MINUTES); + if (!result) { + throw new RuntimeException("Timeout waiting for doc adds to finish"); + } + log( + "done preGenerateDocs docs=" + + docs.size() + + " ram=" + + RamUsageEstimator.humanReadableUnits(RamUsageEstimator.sizeOfObject(docs))); + + if (numDocs != docs.size()) { + throw new IllegalStateException("numDocs != " + docs.size()); + } + + return docs.iterator(); + } + + public Iterator<SolrInputDocument> generatedDocsIterator() { + return docs.iterator(); + } + + public SolrInputDocument inputDocument() { + SolrInputDocument doc = new SolrInputDocument(); + + for (Map.Entry<String, Gen<?>> entry : fields.entrySet()) { + doc.addField( + entry.getKey(), entry.getValue().generate(new BenchmarkRandomSource(random.get()))); + } + + return doc; + } + + public SolrDocument document() { + SolrDocument doc = new SolrDocument(); + + for (Map.Entry<String, Gen<?>> entry : fields.entrySet()) { + doc.addField( + entry.getKey(), entry.getValue().generate(new BenchmarkRandomSource(random.get()))); + } + + return doc; + } + + public Docs field(String name, SolrGen<?> generator) { Review comment: ```suggestion public Docs field(String name, Gen<?> generator) { ``` ########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/ArraysDSL.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import static org.apache.solr.bench.generators.SourceDSL.checkArguments; + +import java.lang.reflect.Array; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.quicktheories.api.AsString; +import org.quicktheories.core.Gen; +import org.quicktheories.generators.Generate; + +/** + * A Class for creating Sources of type T[], that will generate and shrink array objects of either + * fixed or bounded size. If fixed size, an array will be shrunk by reducing the individual items in + * tandem. If the array is of bounded size, shrinking will begin by removing random items until the + * array is of minimum size, at which point it will continue to shrink as if an array of fixed + * length. + * + * <p>There is a possibility that the shrinker will get stuck at a local minima when one value in + * the array shrinks to it's "smallest" value without falsifying + */ +public class ArraysDSL { + + /** + * Creates an ArrayGeneratorBuilder of Integers that can be used to create an array Source + * + * @param source a Source of type Integer + * @return an ArrayGeneratorBuilder of type Integer + */ + public ArrayGeneratorBuilder<Integer> ofIntegers(Gen<Integer> source) { + return new ArrayGeneratorBuilder<>(source, Integer.class); + } + + /** + * Creates an ArrayGeneratorBuilder of Characters that can be used to create an array Source + * + * @param source a Source of type Character + * @return an ArrayGeneratorBuilder of type Character + */ + public ArrayGeneratorBuilder<Character> ofCharacters(Gen<Character> source) { + return new ArrayGeneratorBuilder<>(source, Character.class); + } + + /** + * Creates an ArrayGeneratorBuilder of Strings that can be used to create an array Source + * + * @param source a Source of type String + * @return an ArrayGeneratorBuilder of type String + */ + public ArrayGeneratorBuilder<String> ofStrings(Gen<String> source) { + return new ArrayGeneratorBuilder<>(source, String.class); + } + + /** + * Creates an ArrayGeneratorBuilder of the given class that can be used to create an array Source + * + * @param <T> type of value to generate + * @param source a Source of type T + * @param c a Class of type T + * @return an ArrayGeneratorBuilder of type T + */ + public <T> ArrayGeneratorBuilder<T> ofClass(Gen<T> source, Class<T> c) { + return new ArrayGeneratorBuilder<>(source, c); + } + + public static class ArrayGeneratorBuilder<T> { + + private final Gen<T> source; + private final Class<T> c; + + ArrayGeneratorBuilder(Gen<T> source, Class<T> c) { + this.source = source; + this.c = c; + } + + /** + * Generates arrays of specified type T of fixed length + * + * @param length - fixed length + * @return a Source of type T[] + */ + public Gen<T[]> withLength(int length) { + return withLengthBetween(length, length); + } + + public Gen<T[]> withLengths(Gen<Integer> lengths) { + return Lists.listsOf(source, Lists.arrayList(), lengths) + .map( + l -> l.toArray((T[]) Array.newInstance(c, 0)) // will generate + // correct size if + // zero is less than + // the length of the + // array + ) Review comment: I think this is a weird thing that spotless apply did, can you put the comment all on one line? ########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/SolrGen.java ########## @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import org.quicktheories.core.Gen; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.impl.BenchmarkRandomSource; + +public class SolrGen<T> implements Gen<T> { + + protected final Gen<T> child; + private final Type type; + private Distribution distribution = Distribution.Uniform; + + SolrGen() { + this(Type.Unknown); + } + + SolrGen(Gen<T> child, Type type) { + this.child = child; + if (child instanceof SolrGen) { + ((SolrGen<Object>) child).distribution = distribution; + } + this.type = type; + } + + SolrGen(Type type) { + child = null; + this.type = type; + } + + SolrGen(Gen<T> child) { + this(child, Type.Unknown); + } + + @Override + public T generate(RandomnessSource in) { + // System.out.println("gen : " + toString() + " child: " + child.toString()); + if (in instanceof BenchmarkRandomSource) { + return child.generate(((BenchmarkRandomSource) in).withDistribution(distribution)); + } + return child.generate(in); + } + + public Type type() { + return type; + } + + public SolrGen<T> tracked(RandomDataHistogram.Counts collector) { + return new TrackingGenerator<>(this, collector); + } + + public SolrGen<T> withDistribution(Distribution distribution) { + // System.out.println("set dist gen : " + toString() + " child: " + child.toString()); + this.distribution = distribution; + if (this.child instanceof SolrGen) { + ((SolrGen<Object>) this.child).distribution = distribution; + } + return this; + } + + @Override + public String toString() { + return "SolrGen{" + + "child=" + + child + + ", type=" + + type + + ", distribution=" + + distribution + + '}'; + } + + protected Distribution getDistribution() { + return this.distribution; + } + + public class TrackingGenerator<T> extends SolrGen<T> { Review comment: This class doesn't make sense to me, it both extends and wraps `SolrGen` when it feels like maybe we only need one or the other. Should probably be a static inner class, regardless. ########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/SolrGen.java ########## @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import org.quicktheories.core.Gen; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.impl.BenchmarkRandomSource; + +public class SolrGen<T> implements Gen<T> { + + protected final Gen<T> child; + private final Type type; + private Distribution distribution = Distribution.Uniform; + + SolrGen() { + this(Type.Unknown); + } + + SolrGen(Gen<T> child, Type type) { + this.child = child; + if (child instanceof SolrGen) { + ((SolrGen<Object>) child).distribution = distribution; + } + this.type = type; + } + + SolrGen(Type type) { + child = null; + this.type = type; + } + + SolrGen(Gen<T> child) { + this(child, Type.Unknown); + } + + @Override + public T generate(RandomnessSource in) { + // System.out.println("gen : " + toString() + " child: " + child.toString()); + if (in instanceof BenchmarkRandomSource) { + return child.generate(((BenchmarkRandomSource) in).withDistribution(distribution)); + } + return child.generate(in); + } + + public Type type() { + return type; + } + + public SolrGen<T> tracked(RandomDataHistogram.Counts collector) { + return new TrackingGenerator<>(this, collector); + } + + public SolrGen<T> withDistribution(Distribution distribution) { + // System.out.println("set dist gen : " + toString() + " child: " + child.toString()); + this.distribution = distribution; + if (this.child instanceof SolrGen) { + ((SolrGen<Object>) this.child).distribution = distribution; Review comment: ```suggestion ((SolrGen<?>) this.child).distribution = distribution; ``` ########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/SolrGen.java ########## @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import org.quicktheories.core.Gen; +import org.quicktheories.core.RandomnessSource; +import org.quicktheories.impl.BenchmarkRandomSource; + +public class SolrGen<T> implements Gen<T> { + + protected final Gen<T> child; + private final Type type; + private Distribution distribution = Distribution.Uniform; + + SolrGen() { + this(Type.Unknown); + } + + SolrGen(Gen<T> child, Type type) { + this.child = child; + if (child instanceof SolrGen) { + ((SolrGen<Object>) child).distribution = distribution; Review comment: ```suggestion ((SolrGen<?>) child).distribution = distribution; ``` ########## File path: solr/benchmark/src/java/org/apache/solr/bench/generators/ArraysDSL.java ########## @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.generators; + +import static org.apache.solr.bench.generators.SourceDSL.checkArguments; + +import java.lang.reflect.Array; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.quicktheories.api.AsString; +import org.quicktheories.core.Gen; +import org.quicktheories.generators.Generate; + +/** + * A Class for creating Sources of type T[], that will generate and shrink array objects of either + * fixed or bounded size. If fixed size, an array will be shrunk by reducing the individual items in + * tandem. If the array is of bounded size, shrinking will begin by removing random items until the + * array is of minimum size, at which point it will continue to shrink as if an array of fixed + * length. + * + * <p>There is a possibility that the shrinker will get stuck at a local minima when one value in + * the array shrinks to it's "smallest" value without falsifying + */ +public class ArraysDSL { + + /** + * Creates an ArrayGeneratorBuilder of Integers that can be used to create an array Source + * + * @param source a Source of type Integer + * @return an ArrayGeneratorBuilder of type Integer + */ + public ArrayGeneratorBuilder<Integer> ofIntegers(Gen<Integer> source) { + return new ArrayGeneratorBuilder<>(source, Integer.class); + } + + /** + * Creates an ArrayGeneratorBuilder of Characters that can be used to create an array Source + * + * @param source a Source of type Character + * @return an ArrayGeneratorBuilder of type Character + */ + public ArrayGeneratorBuilder<Character> ofCharacters(Gen<Character> source) { + return new ArrayGeneratorBuilder<>(source, Character.class); + } + + /** + * Creates an ArrayGeneratorBuilder of Strings that can be used to create an array Source + * + * @param source a Source of type String + * @return an ArrayGeneratorBuilder of type String + */ + public ArrayGeneratorBuilder<String> ofStrings(Gen<String> source) { + return new ArrayGeneratorBuilder<>(source, String.class); + } + + /** + * Creates an ArrayGeneratorBuilder of the given class that can be used to create an array Source + * + * @param <T> type of value to generate + * @param source a Source of type T + * @param c a Class of type T + * @return an ArrayGeneratorBuilder of type T + */ + public <T> ArrayGeneratorBuilder<T> ofClass(Gen<T> source, Class<T> c) { + return new ArrayGeneratorBuilder<>(source, c); + } + + public static class ArrayGeneratorBuilder<T> { + + private final Gen<T> source; + private final Class<T> c; + + ArrayGeneratorBuilder(Gen<T> source, Class<T> c) { + this.source = source; + this.c = c; + } + + /** + * Generates arrays of specified type T of fixed length + * + * @param length - fixed length + * @return a Source of type T[] + */ + public Gen<T[]> withLength(int length) { + return withLengthBetween(length, length); + } + + public Gen<T[]> withLengths(Gen<Integer> lengths) { + return Lists.listsOf(source, Lists.arrayList(), lengths) + .map( + l -> l.toArray((T[]) Array.newInstance(c, 0)) // will generate Review comment: Suppress the unchecked warning here please. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org For additional commands, e-mail: issues-h...@solr.apache.org
