robertwb commented on a change in pull request #10621: [BEAM-9056] Staging artifacts from environment URL: https://github.com/apache/beam/pull/10621#discussion_r384849942
########## File path: runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java ########## @@ -175,6 +197,90 @@ public static Environment createProcessEnvironment( } } + public static Collection<ArtifactInformation> getArtifacts(PipelineOptions options) { + Set<String> pathsToStage = Sets.newHashSet(); + // TODO(heejong): remove jar_packages experimental flag when cross-language dependency + // management is implemented for all runners. + List<String> experiments = options.as(ExperimentalOptions.class).getExperiments(); + if (experiments != null) { + Optional<String> jarPackages = + experiments.stream() + .filter((String flag) -> flag.startsWith("jar_packages=")) + .findFirst(); + jarPackages.ifPresent( + s -> pathsToStage.addAll(Arrays.asList(s.replaceFirst("jar_packages=", "").split(",")))); + } + List<String> stagingFiles = options.as(PortablePipelineOptions.class).getFilesToStage(); + if (stagingFiles == null) { + pathsToStage.addAll( + detectClassPathResourcesToStage(Environments.class.getClassLoader(), options)); + if (pathsToStage.isEmpty()) { + throw new IllegalArgumentException("No classpath elements found."); + } + LOG.debug( + "PortablePipelineOptions.filesToStage was not specified. " + + "Defaulting to files from the classpath: {}", + pathsToStage.size()); + } else { + pathsToStage.addAll(stagingFiles); + } + + ImmutableList.Builder<ArtifactInformation> filesToStage = ImmutableList.builder(); + for (String path : pathsToStage) { + File file = new File(path); + if (new File(path).exists()) { + // Spurious items get added to the classpath. Filter by just those that exist. + if (file.isDirectory()) { + // Zip up directories so we can upload them to the artifact service. Review comment: Looks like this was the previous behavior. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services