xiangfu0 commented on code in PR #20149:
URL: https://github.com/apache/pinot/pull/20149#discussion_r4556996025


##########
pinot-core/src/main/java/org/apache/pinot/core/util/trace/ContinuousJfrStarter.java:
##########
@@ -153,490 +131,417 @@ public class ContinuousJfrStarter implements 
PinotClusterConfigChangeListener {
   public static final String MAX_AGE = "maxAge";
   public static final String DEFAULT_MAX_AGE = "P7D";
 
-  /// A flag to track whether the JFR recording has been started.
-  /// This is specially useful for testing and quickstarts, where servers, 
brokers and other components are executed
-  /// in the same JVM.
-  public static final ContinuousJfrStarter INSTANCE = new 
ContinuousJfrStarter();
+  /// Keys that older versions read. The JVM owns these options now, so this 
class ignores them and logs the
+  /// replacement for each one.
+  private static final Map<String, String> REMOVED_KEYS = Map.of(
+      "directory", "-XX:FlightRecorderOptions:repository=<path>",
+      "dumpPath", "-XX:FlightRecorderOptions:dumppath=<path>",
+      "maxDumps", "a cleanup of the repository directory outside the JVM",
+      "preserveRepository", 
"-XX:FlightRecorderOptions:preserve-repository=true",
+      "repositoryMaxTotalSize", "a cleanup of the repository directory outside 
the JVM");
+
   private static final Object GLOBAL_RECORDING_LOCK = new Object();
+  /// The recordings that this class runs, by recording name. Several 
instances in the same JVM share an entry.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private static final Map<String, RecordingState> GLOBAL_RECORDINGS = new 
HashMap<>();
+  private static final Map<String, SharedRecording> GLOBAL_RECORDINGS = new 
HashMap<>();
+  /// The last stopped recording for each recording name.
+  ///
+  /// `Recording.stop()` keeps the recorded chunks in the repository, but 
`Recording.close()` deletes them. This class
+  /// keeps the last stopped recording open, so its data survives a disable or 
a restart of the recording. It closes
+  /// the recording before it, so the disk usage stays bounded.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean _running = false;
+  private static final Map<String, Recording> STOPPED_RECORDINGS = new 
HashMap<>();
+
+  private final String _instanceId;
+  private final boolean _recordingStartedByJvmArgument;
   @GuardedBy("GLOBAL_RECORDING_LOCK")
+  @Nullable
   private Map<String, Object> _currentConfig;
+  /// The name of the recording this instance holds a reference to, or null if 
it does not record.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private String _recordingName;
-  private final MBeanServer _mBeanServer;
   @Nullable
-  private final ObjectName _diagnosticCommandObjectName;
+  private String _recordingName;
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean _jvmArgumentWarningLogged;
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean _closed;
 
-  @VisibleForTesting
-  protected ContinuousJfrStarter() {
-    this(ManagementFactory.getPlatformMBeanServer(), 
createDiagnosticCommandObjectName());
+  public ContinuousJfrStarter(String instanceId) {
+    this(instanceId, ManagementFactory.getRuntimeMXBean().getInputArguments());
   }
 
   @VisibleForTesting
-  protected ContinuousJfrStarter(MBeanServer mBeanServer, @Nullable ObjectName 
diagnosticCommandObjectName) {
-    _mBeanServer = mBeanServer;
-    _diagnosticCommandObjectName = diagnosticCommandObjectName;
+  ContinuousJfrStarter(String instanceId, List<String> jvmArguments) {
+    _instanceId = instanceId;
+    _recordingStartedByJvmArgument = jvmArguments.stream()
+        .anyMatch(argument -> 
argument.startsWith(START_FLIGHT_RECORDING_ARGUMENT));
   }
 
   @Override
   public void onChange(Set<String> changedConfigs, Map<String, String> 
clusterConfigs) {
     boolean jfrChanged = changedConfigs.stream()
         .anyMatch(changedConfig -> 
changedConfig.startsWith(CommonConstants.JFR));
-    if (!jfrChanged && _currentConfig != null) {
-      LOGGER.debug("ChangedConfigs: {} does not contain any JFR config. 
Skipping updates", changedConfigs);
-      return;
-    }
-    PinotConfiguration config = new PinotConfiguration(clusterConfigs);
-    PinotConfiguration subset = config.subset(CommonConstants.JFR);
+    PinotConfiguration subset = new 
PinotConfiguration(clusterConfigs).subset(CommonConstants.JFR);
 
     synchronized (GLOBAL_RECORDING_LOCK) {
+      if (_closed) {
+        return;
+      }
+      if (!jfrChanged && _currentConfig != null) {
+        LOGGER.debug("ChangedConfigs: {} does not contain any JFR config. 
Skipping updates", changedConfigs);
+        return;
+      }
       Map<String, Object> newSubsetMap = subset.toMap();
-
-      if (_currentConfig != null && _currentConfig.equals(newSubsetMap)) {
-        // No change
+      if (newSubsetMap.equals(_currentConfig)) {
         LOGGER.debug("JFR config change detected, but no actual change in 
config");
         return;
       }
-
-      if (!applyConfig(subset, newSubsetMap)) {
-        LOGGER.warn("Failed to apply continuous JFR config update");
+      if (applyConfig(subset)) {
+        _currentConfig = newSubsetMap;
+      } else {
+        LOGGER.warn("Failed to apply continuous JFR config update on instance 
{}", _instanceId);
       }
     }
   }
 
+  /// Returns whether this instance holds a reference to a recording that is 
running.
   public boolean isRunning() {
     synchronized (GLOBAL_RECORDING_LOCK) {
-      return _running;
+      return _recordingName != null
+          && GLOBAL_RECORDINGS.get(_recordingName)._recording.getState() == 
RecordingState.RUNNING;
     }
   }
 
-  @VisibleForTesting
-  protected static void resetGlobalStateForTesting() {
+  /// Gives back the reference of this instance to its recording, and ignores 
later config changes. The component that
+  /// registered this instance calls this method when it stops. Without it, a 
component that restarts in the same JVM
+  /// (for example in tests) keeps a reference to the recording, and 
`enabled=false` no longer stops it.
+  public void close() {
     synchronized (GLOBAL_RECORDING_LOCK) {
-      GLOBAL_RECORDINGS.clear();
+      if (_closed) {
+        return;
+      }
+      _closed = true;
+      releaseRecordingReference();
+      _currentConfig = null;
     }
   }
 
-  @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean applyConfig(PinotConfiguration subset, Map<String, Object> 
newSubsetMap) {
-    boolean enabled = subset.getProperty(ENABLED, DEFAULT_ENABLED);
-    if (!enabled) {
-      if (!releaseRecordingReference()) {
-        return false;
-      }
-      _currentConfig = newSubsetMap;
-      return true;
-    }
-    String targetRecordingName = subset.getProperty(NAME, DEFAULT_NAME);
-    boolean alreadyAttachedToTarget = _running && 
targetRecordingName.equals(_recordingName);
-    if (!alreadyAttachedToTarget && !releaseRecordingReference()) {
-      return false;
-    }
-    if (!acquireRecordingReference(subset, newSubsetMap, targetRecordingName, 
alreadyAttachedToTarget)) {
-      return false;
+  /// Closes all the recordings that this class started, in all instances, and 
forgets them.
+  @VisibleForTesting
+  static void resetGlobalStateForTesting() {
+    synchronized (GLOBAL_RECORDING_LOCK) {
+      GLOBAL_RECORDINGS.values().forEach(sharedRecording -> 
sharedRecording._recording.close());
+      GLOBAL_RECORDINGS.clear();
+      STOPPED_RECORDINGS.values().forEach(Recording::close);
+      STOPPED_RECORDINGS.clear();
     }
-    _currentConfig = newSubsetMap;
-    return true;
   }
 
-  @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean acquireRecordingReference(PinotConfiguration subset, 
Map<String, Object> configMap,
-      String recordingName, boolean alreadyAttachedToTarget) {
-    RecordingState recordingState = GLOBAL_RECORDINGS.get(recordingName);
-    if (recordingState == null) {
-      if (!startRecording(subset, recordingName)) {
-        return false;
-      }
-      recordingState = new RecordingState(new HashMap<>(configMap), 1);
-      updateRepositoryCleanupTask(subset, recordingName, recordingState);
-      GLOBAL_RECORDINGS.put(recordingName, recordingState);
-      _recordingName = recordingName;
-      _running = true;
-      return true;
-    }
-    if (!recordingState._config.equals(configMap)) {
-      if (!restartRecording(subset, recordingName)) {
-        LOGGER.warn("Failed to restart continuous JFR recording '{}' for 
config update", recordingName);
-        return false;
-      }
-      recordingState._config = new HashMap<>(configMap);
-      updateRepositoryCleanupTask(subset, recordingName, recordingState);
-    }
-    if (!alreadyAttachedToTarget) {
-      recordingState._referenceCount++;
+  /// Returns the recording that this instance holds a reference to, or null 
if it does not record.
+  @VisibleForTesting
+  @Nullable
+  Recording getRecording() {
+    synchronized (GLOBAL_RECORDING_LOCK) {
+      return _recordingName != null ? 
GLOBAL_RECORDINGS.get(_recordingName)._recording : null;
     }
-    _recordingName = recordingName;
-    _running = true;
-    return true;
   }
 
-  @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean restartRecording(PinotConfiguration subset, String 
recordingName) {
-    if (!stopRecording(recordingName)) {
-      LOGGER.warn("Failed to stop continuous JFR recording '{}' before 
restart", recordingName);
-      return false;
+  /// Returns the last stopped recording with the given name, or null if there 
is none.
+  @VisibleForTesting
+  @Nullable
+  static Recording getStoppedRecording(String recordingName) {
+    synchronized (GLOBAL_RECORDING_LOCK) {
+      return STOPPED_RECORDINGS.get(recordingName);
     }
-    return startRecording(subset, recordingName);
   }
 
+  /// Applies the config to this instance. Returns false and keeps the current 
state if the config is not valid.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean releaseRecordingReference() {
-    if (!_running) {
-      return true;
+  private boolean applyConfig(PinotConfiguration subset) {
+    for (Map.Entry<String, String> removedKey : REMOVED_KEYS.entrySet()) {
+      if (subset.containsKey(removedKey.getKey())) {
+        LOGGER.warn("Ignoring continuous JFR config '{}.{}'. Use {} instead", 
CommonConstants.JFR,
+            removedKey.getKey(), removedKey.getValue());
+      }
     }
-    assert _recordingName != null;
-    RecordingState recordingState = GLOBAL_RECORDINGS.get(_recordingName);
-    if (recordingState == null) {
-      LOGGER.warn("No global state found for running continuous JFR recording 
'{}'", _recordingName);
-      _running = false;
-      _recordingName = null;
-      return true;
+    Boolean shouldRecord = shouldRecord(subset);
+    if (shouldRecord == null) {
+      return false;
     }
-    if (recordingState._referenceCount > 1) {
-      recordingState._referenceCount--;
-      _running = false;
-      _recordingName = null;
+    if (!shouldRecord) {
+      releaseRecordingReference();
       return true;
     }
-    if (!stopRecording(_recordingName)) {
-      return false;
-    }
-    cancelRepositoryCleanupTask(recordingState);
-    GLOBAL_RECORDINGS.remove(_recordingName);
-    _running = false;
-    _recordingName = null;
-    return true;
-  }
-
-  @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean stopRecording(String recordingName) {
-    if (!isDiagnosticCommandAvailable()) {
-      LOGGER.warn("JFR DiagnosticCommand MBean is unavailable. Cannot stop 
continuous JFR recording '{}'",
-          recordingName);
+    RecordingSpec spec;
+    try {
+      spec = RecordingSpec.from(subset);
+    } catch (IllegalArgumentException e) {
+      LOGGER.error("Invalid continuous JFR config on instance {}", 
_instanceId, e);
       return false;
     }
-    if (!executeDiagnosticCommand(JFR_STOP_COMMAND, "name=" + recordingName)) {
-      LOGGER.warn("Failed to stop continuous JFR recording '{}'", 
recordingName);
-      return false;
+    if (_recordingName != null && !_recordingName.equals(spec._name)) {
+      releaseRecordingReference();
     }
-    LOGGER.info("Stopped continuous JFR recording {}", recordingName);
-    return true;
+    return acquireRecordingReference(spec);
   }
 
+  /// Returns whether this instance must record, or null if the instance regex 
is not valid.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean startRecording(PinotConfiguration subset, String 
recordingName) {
+  @Nullable
+  private Boolean shouldRecord(PinotConfiguration subset) {
     if (!subset.getProperty(ENABLED, DEFAULT_ENABLED)) {
-      LOGGER.info("Continuous JFR recording is disabled");
-      return true;
-    }
-    if (!isDiagnosticCommandAvailable()) {
-      LOGGER.warn("JFR DiagnosticCommand MBean is unavailable. Cannot start 
continuous JFR recording");
       return false;
     }
-    if (!applyRuntimeOptions(subset)) {
-      LOGGER.warn("Failed to apply JFR runtime options for recording '{}'", 
recordingName);
+    if (_recordingStartedByJvmArgument) {
+      if (!_jvmArgumentWarningLogged) {
+        LOGGER.warn("Ignoring continuous JFR cluster configs on instance {}, 
because the JVM starts a recording with "
+            + "{}", _instanceId, START_FLIGHT_RECORDING_ARGUMENT);
+        _jvmArgumentWarningLogged = true;
+      }
       return false;
     }
-
-    String maxAge = subset.getProperty(MAX_AGE, DEFAULT_MAX_AGE);
-    try {
-      List<String> startArguments = new ArrayList<>();
-      startArguments.add("name=" + recordingName);
-      startArguments.add("settings=" + subset.getProperty(CONFIGURATION, 
DEFAULT_CONFIGURATION));
-      startArguments.add("dumponexit=" + subset.getProperty(DUMP_ON_EXIT, 
DEFAULT_DUMP_ON_EXIT));
-      boolean toDisk = subset.getProperty(TO_DISK, DEFAULT_TO_DISK);
-      startArguments.add("disk=" + toDisk);
-      if (toDisk) {
-        try {
-          long maxSize = DataSizeUtils.toBytes(subset.getProperty(MAX_SIZE, 
DEFAULT_MAX_SIZE));
-          startArguments.add("maxsize=" + maxSize);
-          startArguments.add("maxage=" + toJfrTimeArgument(maxAge));
-        } catch (IllegalArgumentException | ClassCastException e) {
-          throw new RuntimeException("Failed to parse maxSize configuration 
for continuous JFR recording '"
-              + recordingName + "'", e);
-        }
-      }
-      if (!executeDiagnosticCommand(JFR_START_COMMAND, 
startArguments.toArray(new String[0]))) {
-        LOGGER.warn("Failed to start continuous JFR recording '{}'", 
recordingName);
-        return false;
-      }
-    } catch (DateTimeParseException e) {
-      throw new RuntimeException("Failed to parse duration '" + maxAge + "'", 
e);
+    String instanceRegex = subset.getProperty(INSTANCE_REGEX);
+    if (instanceRegex == null || instanceRegex.isEmpty()) {
+      return true;
     }
-
-    LOGGER.info("Started continuous JFR recording {} with configuration: {}", 
recordingName, subset);
-    return true;
-  }
-
-  @VisibleForTesting
-  protected boolean executeDiagnosticCommand(String operationName, String... 
arguments) {
     try {
-      _mBeanServer.invoke(_diagnosticCommandObjectName, operationName, new 
Object[]{arguments},
-          new String[]{String[].class.getName()});
-      return true;
-    } catch (Exception e) {
-      LOGGER.warn("Failed to execute JFR command '{}' with arguments {}", 
operationName, Arrays.toString(arguments), e);
-      return false;
+      return Pattern.compile(instanceRegex).matcher(_instanceId).matches();
+    } catch (PatternSyntaxException e) {
+      LOGGER.error("Invalid continuous JFR config '{}.{}': '{}'. Keeping the 
current recording state on instance {}",
+          CommonConstants.JFR, INSTANCE_REGEX, instanceRegex, _instanceId, e);
+      return null;
     }
   }
 
-  private boolean applyRuntimeOptions(PinotConfiguration subset) {
-    List<String> configureArguments = new ArrayList<>();
-    String repositoryPath = subset.getProperty(DIRECTORY, (String) null);
-    if (repositoryPath != null && !repositoryPath.isEmpty()) {
-      configureArguments.add("repositorypath=" + repositoryPath);
-    }
-    String dumpPath = subset.getProperty(DUMP_PATH, (String) null);
-    if (dumpPath != null && !dumpPath.isEmpty()) {
-      configureArguments.add("dumppath=" + dumpPath);
-    }
-    if (configureArguments.isEmpty()) {
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean acquireRecordingReference(RecordingSpec spec) {
+    SharedRecording sharedRecording = GLOBAL_RECORDINGS.get(spec._name);
+    if (sharedRecording == null) {
+      Recording recording = startRecording(spec);
+      if (recording == null) {
+        return false;
+      }
+      GLOBAL_RECORDINGS.put(spec._name, new SharedRecording(recording, spec));
+      _recordingName = spec._name;
       return true;
     }
-    boolean preserveRepository = subset.getProperty(PRESERVE_REPOSITORY, 
DEFAULT_PRESERVE_REPOSITORY);
-    if (preserveRepository) {
-      List<String> configureArgumentsWithPreserve = new 
ArrayList<>(configureArguments);
-      configureArgumentsWithPreserve.add("preserve-repository=true");
-      if (executeDiagnosticCommand(JFR_CONFIGURE_COMMAND, 
configureArgumentsWithPreserve.toArray(new String[0]))) {
-        return true;
+    if (isStopped(sharedRecording._recording)) {
+      if (!restartRecording(sharedRecording, spec)) {
+        return false;
       }
-      LOGGER.warn("Failed to apply preserveRepository option for continuous 
JFR recording. Retrying without it");
+    } else if (!sharedRecording._spec.equals(spec) && 
!updateRecording(sharedRecording, spec)) {
+      return false;
     }
-    return executeDiagnosticCommand(JFR_CONFIGURE_COMMAND, 
configureArguments.toArray(new String[0]));
+    if (!spec._name.equals(_recordingName)) {
+      sharedRecording._referenceCount++;
+      _recordingName = spec._name;
+    }
+    return true;
   }
 
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private void updateRepositoryCleanupTask(
-      PinotConfiguration subset, String recordingName, RecordingState 
recordingState) {
-    if (!isRepositoryCleanupEnabled()) {
+  private void releaseRecordingReference() {
+    if (_recordingName == null) {
       return;
     }
-    String repositoryRoot = subset.getProperty(DIRECTORY, (String) null);
-    if (repositoryRoot == null || repositoryRoot.isEmpty()) {
-      cancelRepositoryCleanupTask(recordingState);
-      return;
+    SharedRecording sharedRecording = GLOBAL_RECORDINGS.get(_recordingName);
+    if (sharedRecording._referenceCount > 1) {
+      sharedRecording._referenceCount--;
+    } else {
+      stopRecording(_recordingName, sharedRecording._recording);
+      GLOBAL_RECORDINGS.remove(_recordingName);
     }
-    long maxTotalSizeBytes;
-    try {
-      maxTotalSizeBytes = 
DataSizeUtils.toBytes(subset.getProperty(REPOSITORY_MAX_TOTAL_SIZE,
-          DEFAULT_REPOSITORY_MAX_TOTAL_SIZE));
-    } catch (IllegalArgumentException | ClassCastException e) {
-      throw new RuntimeException("Failed to parse continuous JFR repository 
max total size from key '"
-          + REPOSITORY_MAX_TOTAL_SIZE + "'", e);
-    }
-    if (recordingState._cleanupTask != null && 
!recordingState._cleanupTask.isCancelled()
-        && Objects.equals(recordingState._repositoryRoot, repositoryRoot)
-        && recordingState._repositoryMaxTotalSizeBytes == maxTotalSizeBytes) {
-      return;
-    }
-    cancelRepositoryCleanupTask(recordingState);
-    recordingState._repositoryRoot = repositoryRoot;
-    recordingState._repositoryMaxTotalSizeBytes = maxTotalSizeBytes;
-    recordingState._cleanupTask = 
REPOSITORY_CLEANUP_EXECUTOR.scheduleWithFixedDelay(
-        () -> cleanupRepositoriesIfNeeded(recordingName, repositoryRoot, 
maxTotalSizeBytes), 0,
-        REPOSITORY_CLEANUP_INTERVAL.toMillis(), TimeUnit.MILLISECONDS);
+    _recordingName = null;
   }
 
+  /// Applies a new spec to a running recording. JFR can change all the 
options of a running recording, except the
+  /// `toDisk` option. For that option, this method starts a new recording and 
stops the old one.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private void cancelRepositoryCleanupTask(RecordingState recordingState) {
-    if (recordingState._cleanupTask != null) {
-      recordingState._cleanupTask.cancel(false);
-      recordingState._cleanupTask = null;
+  private boolean updateRecording(SharedRecording sharedRecording, 
RecordingSpec spec) {
+    if (sharedRecording._spec._toDisk != spec._toDisk) {
+      Recording recording = startRecording(spec);
+      if (recording == null) {
+        return false;
+      }
+      stopRecording(spec._name, sharedRecording._recording);
+      sharedRecording._recording = recording;
+    } else {
+      try {
+        applyOptions(sharedRecording._recording, spec);
+      } catch (RuntimeException e) {
+        LOGGER.warn("Failed to update continuous JFR recording '{}'", 
spec._name, e);
+        return false;
+      }
+      LOGGER.info("Updated continuous JFR recording '{}' to {}", spec._name, 
spec);
     }
+    sharedRecording._spec = spec;
+    return true;
   }
 
-  @VisibleForTesting
-  protected boolean isRepositoryCleanupEnabled() {
-    return true;
+  /// Returns whether the recording stopped without this class, for example 
with `jcmd <pid> JFR.stop`.
+  private static boolean isStopped(Recording recording) {
+    RecordingState state = recording.getState();
+    return state == RecordingState.STOPPED || state == RecordingState.CLOSED;
   }
 
-  private void cleanupRepositoriesIfNeeded(String recordingName, String 
repositoryRoot, long maxTotalSizeBytes) {
-    try {
-      Path repositoryRootPath = 
Path.of(repositoryRoot).toAbsolutePath().normalize();
-      Path activeRepositoryPath = resolveCurrentRepositoryPath();
-      cleanupOldRepositoriesInternal(repositoryRootPath, maxTotalSizeBytes, 
activeRepositoryPath);
-    } catch (Exception e) {
-      LOGGER.warn("Failed to clean up old JFR repositories for recording '{}' 
under '{}'", recordingName,
-          repositoryRoot, e);
+  /// Replaces a recording that stopped without this class with a new one. JFR 
cannot start a stopped or closed
+  /// recording again, and rejects any change to a closed one.
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean restartRecording(SharedRecording sharedRecording, 
RecordingSpec spec) {
+    LOGGER.warn("Continuous JFR recording '{}' is {}, but Pinot did not stop 
it. Starting a new recording", spec._name,
+        sharedRecording._recording.getState());
+    Recording recording = startRecording(spec);
+    if (recording == null) {
+      return false;
     }
+    stopRecording(spec._name, sharedRecording._recording);
+    sharedRecording._recording = recording;
+    sharedRecording._spec = spec;
+    return true;
   }
 
   @Nullable
-  private Path resolveCurrentRepositoryPath() {
-    if (!isDiagnosticCommandAvailable()) {
-      return null;
-    }
+  private static Recording startRecording(RecordingSpec spec) {
+    Recording recording = null;
     try {
-      Object result = _mBeanServer.invoke(
-          _diagnosticCommandObjectName,
-          JFR_CONFIGURE_COMMAND,
-          new Object[]{new String[0]},
-          new String[]{String[].class.getName()});
-      if (result == null) {
-        return null;
-      }
-      String output = result.toString();
-      for (String line : output.split("\\R")) {
-        String trimmedLine = line.trim();
-        if (trimmedLine.startsWith("Repository path:")) {
-          String repositoryPath = trimmedLine.substring("Repository 
path:".length()).trim();
-          if (!repositoryPath.isEmpty()) {
-            return Path.of(repositoryPath).toAbsolutePath().normalize();
-          }
-        }
+      recording = new Recording();
+      recording.setName(spec._name);
+      recording.setToDisk(spec._toDisk);
+      applyOptions(recording, spec);
+      recording.start();
+    } catch (RuntimeException e) {
+      LOGGER.warn("Failed to start continuous JFR recording '{}'", spec._name, 
e);
+      if (recording != null) {
+        recording.close();
       }
       return null;
-    } catch (Exception e) {
-      LOGGER.warn("Failed to resolve current JFR repository path from {}", 
JFR_CONFIGURE_COMMAND, e);
-      return null;
-    }
-  }
-
-  @VisibleForTesting
-  static void cleanupOldRepositoriesInternal(Path repositoryRootPath, long 
maxTotalSizeBytes,
-      @Nullable Path activeRepositoryPath) {
-    if (!Files.isDirectory(repositoryRootPath) || maxTotalSizeBytes < 0) {
-      return;
-    }
-    if (activeRepositoryPath != null && 
repositoryRootPath.equals(activeRepositoryPath)) {
-      return;
-    }
-    try {
-      List<PathWithMetadata> repositories = new ArrayList<>();
-      try (java.util.stream.Stream<Path> children = 
Files.list(repositoryRootPath)) {
-        children.filter(path -> activeRepositoryPath == null || 
!activeRepositoryPath.startsWith(path))
-            .forEach(path -> repositories.add(
-                new PathWithMetadata(path, computePathSize(path), 
readLastModifiedMillis(path))));
-      }
-      long totalSizeBytes = repositories.stream().mapToLong(pathWithMetadata 
-> pathWithMetadata._sizeBytes).sum();
-      if (activeRepositoryPath != null) {
-        totalSizeBytes += computePathSize(activeRepositoryPath);
-      }
-      if (totalSizeBytes <= maxTotalSizeBytes) {
-        return;
-      }
-      repositories.sort((left, right) -> 
Long.compare(left._lastModifiedMillis, right._lastModifiedMillis));
-      for (PathWithMetadata repository : repositories) {
-        if (totalSizeBytes <= maxTotalSizeBytes) {
-          break;
-        }
-        deletePathRecursively(repository._path);
-        totalSizeBytes -= repository._sizeBytes;
-      }
-    } catch (Exception e) {
-      LOGGER.warn("Failed to clean old JFR repositories under '{}'", 
repositoryRootPath, e);
     }
+    LOGGER.info("Started continuous JFR recording '{}' with {}", spec._name, 
spec);
+    return recording;
   }
 
-  private static long computePathSize(Path path) {
-    if (!Files.exists(path)) {
-      return 0;
-    }
-    SizeCollector sizeCollector = new SizeCollector();
-    try {
-      Files.walkFileTree(path, sizeCollector);
-      return sizeCollector._sizeBytes;
-    } catch (Exception e) {
-      throw new RuntimeException("Failed to compute size for path '" + path + 
"'", e);
+  private static void applyOptions(Recording recording, RecordingSpec spec) {
+    recording.setSettings(spec._settings);
+    recording.setDumpOnExit(spec._dumpOnExit);
+    if (spec._toDisk) {
+      recording.setMaxSize(spec._maxSizeBytes);
+      recording.setMaxAge(spec._maxAge);
     }
   }
 
-  private static long readLastModifiedMillis(Path path) {
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private static void stopRecording(String recordingName, Recording recording) 
{
     try {
-      return Files.getLastModifiedTime(path).toMillis();
-    } catch (Exception e) {
-      return Long.MAX_VALUE;
-    }
-  }
-
-  private static void deletePathRecursively(Path path) {
-    if (!Files.exists(path)) {
-      return;
+      recording.stop();
+    } catch (IllegalStateException e) {
+      // Another tool, for example jcmd, stopped or closed the recording 
before.
+      LOGGER.debug("Continuous JFR recording '{}' was not running", 
recordingName, e);
     }
-    try {
-      Files.walkFileTree(path, new RecursiveDeleteVisitor());
-    } catch (Exception e) {
-      throw new RuntimeException("Failed to delete path '" + path + "'", e);
+    Recording previous = STOPPED_RECORDINGS.put(recordingName, recording);

Review Comment:
   Minor: the kept stopped recording has the same name as the running one. 
After any disable/enable or `toDisk` change, `jcmd <pid> JFR.stop 
name=pinot-continuous` matches the stopped one first. It fails with "Can't stop 
an already stopped recording", and the running one keeps going (reproduced). 
Operators then need the numeric id. Consider renaming the kept recording, e.g. 
`<name>-stopped`, or documenting the id workaround.



##########
pinot-core/src/main/java/org/apache/pinot/core/util/trace/ContinuousJfrStarter.java:
##########
@@ -153,490 +131,417 @@ public class ContinuousJfrStarter implements 
PinotClusterConfigChangeListener {
   public static final String MAX_AGE = "maxAge";
   public static final String DEFAULT_MAX_AGE = "P7D";
 
-  /// A flag to track whether the JFR recording has been started.
-  /// This is specially useful for testing and quickstarts, where servers, 
brokers and other components are executed
-  /// in the same JVM.
-  public static final ContinuousJfrStarter INSTANCE = new 
ContinuousJfrStarter();
+  /// Keys that older versions read. The JVM owns these options now, so this 
class ignores them and logs the
+  /// replacement for each one.
+  private static final Map<String, String> REMOVED_KEYS = Map.of(
+      "directory", "-XX:FlightRecorderOptions:repository=<path>",
+      "dumpPath", "-XX:FlightRecorderOptions:dumppath=<path>",
+      "maxDumps", "a cleanup of the repository directory outside the JVM",
+      "preserveRepository", 
"-XX:FlightRecorderOptions:preserve-repository=true",
+      "repositoryMaxTotalSize", "a cleanup of the repository directory outside 
the JVM");
+
   private static final Object GLOBAL_RECORDING_LOCK = new Object();
+  /// The recordings that this class runs, by recording name. Several 
instances in the same JVM share an entry.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private static final Map<String, RecordingState> GLOBAL_RECORDINGS = new 
HashMap<>();
+  private static final Map<String, SharedRecording> GLOBAL_RECORDINGS = new 
HashMap<>();
+  /// The last stopped recording for each recording name.
+  ///
+  /// `Recording.stop()` keeps the recorded chunks in the repository, but 
`Recording.close()` deletes them. This class
+  /// keeps the last stopped recording open, so its data survives a disable or 
a restart of the recording. It closes
+  /// the recording before it, so the disk usage stays bounded.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private boolean _running = false;
+  private static final Map<String, Recording> STOPPED_RECORDINGS = new 
HashMap<>();
+
+  private final String _instanceId;
+  private final boolean _recordingStartedByJvmArgument;
   @GuardedBy("GLOBAL_RECORDING_LOCK")
+  @Nullable
   private Map<String, Object> _currentConfig;
+  /// The name of the recording this instance holds a reference to, or null if 
it does not record.
   @GuardedBy("GLOBAL_RECORDING_LOCK")
-  private String _recordingName;
-  private final MBeanServer _mBeanServer;
   @Nullable
-  private final ObjectName _diagnosticCommandObjectName;
+  private String _recordingName;
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean _jvmArgumentWarningLogged;
+  @GuardedBy("GLOBAL_RECORDING_LOCK")
+  private boolean _closed;
 
-  @VisibleForTesting
-  protected ContinuousJfrStarter() {
-    this(ManagementFactory.getPlatformMBeanServer(), 
createDiagnosticCommandObjectName());
+  public ContinuousJfrStarter(String instanceId) {
+    this(instanceId, ManagementFactory.getRuntimeMXBean().getInputArguments());
   }
 
   @VisibleForTesting
-  protected ContinuousJfrStarter(MBeanServer mBeanServer, @Nullable ObjectName 
diagnosticCommandObjectName) {
-    _mBeanServer = mBeanServer;
-    _diagnosticCommandObjectName = diagnosticCommandObjectName;
+  ContinuousJfrStarter(String instanceId, List<String> jvmArguments) {
+    _instanceId = instanceId;
+    _recordingStartedByJvmArgument = jvmArguments.stream()
+        .anyMatch(argument -> 
argument.startsWith(START_FLIGHT_RECORDING_ARGUMENT));
   }
 
   @Override
   public void onChange(Set<String> changedConfigs, Map<String, String> 
clusterConfigs) {
     boolean jfrChanged = changedConfigs.stream()
         .anyMatch(changedConfig -> 
changedConfig.startsWith(CommonConstants.JFR));
-    if (!jfrChanged && _currentConfig != null) {
-      LOGGER.debug("ChangedConfigs: {} does not contain any JFR config. 
Skipping updates", changedConfigs);
-      return;
-    }
-    PinotConfiguration config = new PinotConfiguration(clusterConfigs);
-    PinotConfiguration subset = config.subset(CommonConstants.JFR);
+    PinotConfiguration subset = new 
PinotConfiguration(clusterConfigs).subset(CommonConstants.JFR);
 
     synchronized (GLOBAL_RECORDING_LOCK) {
+      if (_closed) {
+        return;
+      }
+      if (!jfrChanged && _currentConfig != null) {
+        LOGGER.debug("ChangedConfigs: {} does not contain any JFR config. 
Skipping updates", changedConfigs);
+        return;
+      }
       Map<String, Object> newSubsetMap = subset.toMap();
-
-      if (_currentConfig != null && _currentConfig.equals(newSubsetMap)) {
-        // No change
+      if (newSubsetMap.equals(_currentConfig)) {
         LOGGER.debug("JFR config change detected, but no actual change in 
config");
         return;
       }
-
-      if (!applyConfig(subset, newSubsetMap)) {
-        LOGGER.warn("Failed to apply continuous JFR config update");
+      if (applyConfig(subset)) {
+        _currentConfig = newSubsetMap;
+      } else {
+        LOGGER.warn("Failed to apply continuous JFR config update on instance 
{}", _instanceId);
       }
     }
   }
 
+  /// Returns whether this instance holds a reference to a recording that is 
running.
   public boolean isRunning() {
     synchronized (GLOBAL_RECORDING_LOCK) {
-      return _running;
+      return _recordingName != null
+          && GLOBAL_RECORDINGS.get(_recordingName)._recording.getState() == 
RecordingState.RUNNING;
     }
   }
 
-  @VisibleForTesting
-  protected static void resetGlobalStateForTesting() {
+  /// Gives back the reference of this instance to its recording, and ignores 
later config changes. The component that
+  /// registered this instance calls this method when it stops. Without it, a 
component that restarts in the same JVM
+  /// (for example in tests) keeps a reference to the recording, and 
`enabled=false` no longer stops it.
+  public void close() {
     synchronized (GLOBAL_RECORDING_LOCK) {
-      GLOBAL_RECORDINGS.clear();
+      if (_closed) {
+        return;
+      }
+      _closed = true;
+      releaseRecordingReference();

Review Comment:
   **Major:** `close()` now stops the recording when it drops the last 
reference. That breaks `dumpOnExit=true` on a graceful shutdown. On SIGTERM, 
`AbstractBaseCommand` → `cleanup()` → `stopAll()` → `stop()` → `close()` runs 
in an application shutdown hook, at the same time as JFR's own hook. JFR's 
`ShutdownHook` only dumps recordings that are still `RUNNING`. With the JVM 
default `preserve-repository=false`, JFR then deletes the repository, so the 
data is gone.
   
   I reproduced this on JDK 25 with this class, `dumpOnExit=true`, then 
`System.exit`:
   - No `close()` (like base and `ec35ce69`): 5/5 exit dumps.
   - `close()` before exit: 0/5.
   - `close()` in a shutdown hook, as Pinot does: 17/20.
   
   One fix: when `close()` drops the last reference, give it back without 
stopping the recording (at least when `_spec._dumpOnExit` is true), and let a 
later acquire adopt it or JVM exit end it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to