On 06.04.21 21:00, Stefan Fuhrmann wrote:

On 06.04.21 08:12, Nathan Hartman wrote:
>
tasks-used-optimized-wc-status.patch applied with some offsets, so is
against a slightly out-of-date trunk, but applied without conflicts.
But my build had many test failures:

Summary of test results:
   1176 tests PASSED
   165 tests SKIPPED
   66 tests XFAILED (16 WORK-IN-PROGRESS)
   1223 tests FAILED
Python version: 2.7.16.
SUMMARY: Some tests failed

Sorry about that.  Might be side-effects of prototypical hacks to wc_*,
which I did months ago and had almost forgotten about until last night.

Turned out to be 2 segfaults which were easy to fix.

stat_tests.py 13: timestamp behaviour is still failing for 'cleanup'.
But I think this one can wait.

I noticed, however, that while the first two patches were rooted in
subversion, the third patch was rooted in subversion/libsvn_wc, so
perhaps there are other needed changes that didn't make it into the
patch? Otherwise, something is hosed... If you'd like, I'll provide
the fails.log. Be warned, it's 2.5MB!

No worries.  It's been a bit late last night.
I will try to create a streamlined patch tomorrow.

I attached the new patch. It is much cleaner, delivers good cold cache
performance (9x for large working copies) but with hot file caches, it
is still 3.5x slower than the "final product" (for very, very large w/c).

-- Stefan^2.
Index: build.conf
===================================================================
--- build.conf	(revision 1888439)
+++ build.conf	(working copy)
@@ -395,7 +395,7 @@
         private\svn_temp_serializer.h private\svn_io_private.h
         private\svn_sorts_private.h private\svn_auth_private.h
         private\svn_string_private.h private\svn_magic.h
-        private\svn_subr_private.h private\svn_mutex.h
+        private\svn_subr_private.h private\svn_mutex.h  private\svn_task.h
         private\svn_thread_cond.h private\svn_waitable_counter.h
         private\svn_packed_data.h private\svn_object_pool.h private\svn_cert.h
         private\svn_config_private.h private\svn_dirent_uri_private.h
Index: subversion/include/private/svn_task.h
===================================================================
--- subversion/include/private/svn_task.h	(revision 1888446)
+++ subversion/include/private/svn_task.h	(working copy)
@@ -181,8 +181,8 @@
  * into either @a svn_task__add() or @a svn_task__add_similar() - even if
  * you use a @c NULL process baton.
  *
- * @todo Consider replace these pools with a single pool at the parent
- * and turn this into a simple getter.  We will end up with a lot fewer
+ * @todo Consider replacing these pools with a single pool at the parent
+ * and turning this into a simple getter.  We will end up with a lot fewer
  * pools that OTOH may live a lot longer.
  */
 apr_pool_t *svn_task__create_process_pool(
@@ -189,7 +189,7 @@
   svn_task__t *parent);
 
 /**
- * Append a new sub-task to the current @a task with the given
+ * Append a new sub-task to the @a current task with the given
  * @a process_pool.  The latter must have been allocated with
  * @a svn_task__create_process_pool() for @a task.
  *
@@ -205,7 +205,7 @@
  * and / or processing will take place.
  */
 svn_error_t *svn_task__add(
-  svn_task__t *task,
+  svn_task__t *current,
   apr_pool_t *process_pool,
   void *partial_output,
   svn_task__process_func_t process_func,
@@ -220,7 +220,7 @@
  * as for the current task.  This is useful for recursive tasks.
  */
 svn_error_t *svn_task__add_similar(
-  svn_task__t *task,
+  svn_task__t *current,
   apr_pool_t *process_pool,
   void *partial_output,
   void *process_baton);
Index: subversion/libsvn_wc/status.c
===================================================================
--- subversion/libsvn_wc/status.c	(revision 1888427)
+++ subversion/libsvn_wc/status.c	(working copy)
@@ -51,6 +51,7 @@
 #include "private/svn_wc_private.h"
 #include "private/svn_fspath.h"
 #include "private/svn_editor.h"
+#include "private/svn_task.h"
 
 
 /* The file internal variant of svn_wc_status3_t, with slightly more
@@ -75,6 +76,28 @@
 } svn_wc__internal_status_t;
 
 
+/* Forward declare */
+struct walk_status_baton;
+
+typedef svn_error_t *
+(*dir_status_func_t)(const struct walk_status_baton *wb,
+                     const char *local_abspath,
+                     svn_boolean_t skip_this_dir,
+                     const char *parent_repos_root_url,
+                     const char *parent_repos_relpath,
+                     const char *parent_repos_uuid,
+                     const struct svn_wc__db_info_t *dir_info,
+                     const svn_io_dirent2_t *dirent,
+                     const apr_array_header_t *ignore_patterns,
+                     svn_depth_t depth,
+                     svn_boolean_t get_all,
+                     svn_boolean_t no_ignore,
+                     svn_wc_status_func4_t status_func,
+                     void *status_baton,
+                     svn_cancel_func_t cancel_func,
+                     void *cancel_baton,
+                     apr_pool_t *scratch_pool);
+
 /*** Baton used for walking the local status */
 struct walk_status_baton
 {
@@ -100,6 +123,9 @@
 
   /* Repository locks, if set. */
   apr_hash_t *repos_locks;
+
+  dir_status_func_t dir_status_func;
+  void *dir_status_baton;
 };
 
 /*** Editor batons ***/
@@ -1073,6 +1099,14 @@
  * allocations are made in SCRATCH_POOL.
  *
  * The remaining parameters correspond to get_dir_status(). */
+
+struct status_task_baton
+{
+  svn_task__t *task;
+  apr_pool_t *result_pool;
+  apr_array_header_t *partial_results;
+};
+
 static svn_error_t *
 one_child_status(const struct walk_status_baton *wb,
                  const char *local_abspath,
@@ -1123,15 +1157,15 @@
       if (depth == svn_depth_infinity
           && info->has_descendants /* is dir, or was dir and tc descendants */)
         {
-          SVN_ERR(get_dir_status(wb, local_abspath, TRUE,
-                                 dir_repos_root_url, dir_repos_relpath,
-                                 dir_repos_uuid, info,
-                                 dirent, ignore_patterns,
-                                 svn_depth_infinity, get_all,
-                                 no_ignore,
-                                 status_func, status_baton,
-                                 cancel_func, cancel_baton,
-                                 scratch_pool));
+          SVN_ERR(wb->dir_status_func(wb, local_abspath, TRUE,
+                                      dir_repos_root_url, dir_repos_relpath,
+                                      dir_repos_uuid, info,
+                                      dirent, ignore_patterns,
+                                      svn_depth_infinity, get_all,
+                                      no_ignore,
+                                      status_func, status_baton,
+                                      cancel_func, cancel_baton,
+                                      scratch_pool));
         }
 
       return SVN_NO_ERROR;
@@ -2457,6 +2491,329 @@
 
 
 
+
+/* Send svn_wc_status3_t * structures for the directory LOCAL_ABSPATH and
+  *   for all its child nodes (according to DEPTH) through STATUS_FUNC /
+  *   STATUS_BATON.
+  * 
+  *   If SKIP_THIS_DIR is TRUE, the directory's own status will not be reported.
+  *   All subdirs reached by recursion will be reported regardless of this
+  *   parameter's value.
+  * 
+  *   PARENT_REPOS_* parameters can be set to refer to LOCAL_ABSPATH's parent's
+  *   URL, i.e. the URL the WC reflects at the dirname of LOCAL_ABSPATH, to avoid
+  *   retrieving them again. Otherwise they must be NULL.
+  * 
+  *   DIR_INFO can be set to the information of LOCAL_ABSPATH, to avoid retrieving
+  *   it again. Otherwise it must be NULL.
+  * 
+  *   DIRENT is LOCAL_ABSPATH's own dirent and is only needed if it is reported,
+  *   so if SKIP_THIS_DIR is TRUE, DIRENT can be left NULL.
+  * 
+  *   Other arguments are the same as those passed to
+  *   svn_wc_get_status_editor5().  */
+ 
+typedef struct dir_status_process_baton_t
+{
+  const char *local_abspath;
+  svn_boolean_t skip_this_dir;
+  const char *parent_repos_root_url;
+  const char *parent_repos_relpath;
+  const char *parent_repos_uuid;
+  const struct svn_wc__db_info_t *dir_info;
+  const svn_io_dirent2_t *dirent;
+  const apr_array_header_t *ignore_patterns;
+  svn_depth_t depth;
+  svn_boolean_t get_all;
+  svn_boolean_t no_ignore;
+} dir_status_process_baton_t;
+
+typedef struct dir_status_result_t
+{
+  svn_wc_status3_t *status;
+  const char *local_abspath;
+} dir_status_result_t;
+
+static svn_error_t *
+add_status_result(void *baton,
+                  const char *local_abspath,
+                  const svn_wc_status3_t *status,
+                  apr_pool_t *scratch_pool)
+{
+  struct status_task_baton *b = baton;
+  apr_pool_t *result_pool = b->result_pool;
+  apr_array_header_t *results = b->partial_results;
+
+  dir_status_result_t *result = apr_pcalloc(result_pool, sizeof(*result));
+  result->status = svn_wc_dup_status3(status, result_pool);
+  result->local_abspath = apr_pstrdup(result_pool, local_abspath);
+
+  if (results == NULL)
+    {
+      results = apr_array_make(result_pool, 8, sizeof(dir_status_result_t*));
+      b->partial_results = results;
+    }
+
+  APR_ARRAY_PUSH(results, dir_status_result_t *) = result;
+
+  return SVN_NO_ERROR;
+}
+
+/* Structure returned by svn_wc__db_read_children_info.  Only has the
+ *   fields needed by status. */
+static struct svn_wc__db_info_t *
+db_info_dup(const struct svn_wc__db_info_t *info,
+            apr_pool_t *result_pool)
+{
+  struct svn_wc__db_moved_to_info_t **moves;
+  struct svn_wc__db_info_t *result = apr_pmemdup(result_pool, info,
+                                                 sizeof(*result));
+
+  result->repos_relpath  = apr_pstrdup(result_pool, info->repos_relpath);
+  result->repos_root_url = apr_pstrdup(result_pool, info->repos_root_url);
+  result->repos_uuid     = apr_pstrdup(result_pool, info->repos_uuid);
+
+  if (info->changelist)
+    result->changelist     = apr_pstrdup(result_pool, info->changelist);
+  if (info->changed_author)
+    result->changed_author = apr_pstrdup(result_pool, info->changed_author);
+
+  if (info->lock)
+    {
+      result->lock = apr_pmemdup(result_pool, info->lock,
+                                 sizeof(*result->lock));
+      result->lock->token = apr_pstrdup(result_pool, info->lock->token);
+      if (info->lock->owner)
+        result->lock->owner = apr_pstrdup(result_pool, info->lock->owner);
+      if (info->lock->comment)
+        result->lock->comment = apr_pstrdup(result_pool, info->lock->comment);
+    }
+  
+  for (moves = &result->moved_to; *moves; moves = &(*moves)->next)
+    {
+      *moves = apr_pmemdup(result_pool, *moves, sizeof(**moves));
+      (*moves)->moved_to_abspath =
+          apr_pstrdup(result_pool, (*moves)->moved_to_abspath);
+      (*moves)->shadow_op_root_abspath =
+          apr_pstrdup(result_pool, (*moves)->shadow_op_root_abspath);
+    }
+  
+  return result;
+}
+
+static svn_error_t *
+create_dir_status_task(const struct walk_status_baton *wb,
+                       const char *local_abspath,
+                       svn_boolean_t skip_this_dir,
+                       const char *parent_repos_root_url,
+                       const char *parent_repos_relpath,
+                       const char *parent_repos_uuid,
+                       const struct svn_wc__db_info_t *dir_info,
+                       const svn_io_dirent2_t *dirent,
+                       const apr_array_header_t *ignore_patterns,
+                       svn_depth_t depth,
+                       svn_boolean_t get_all,
+                       svn_boolean_t no_ignore,
+                       svn_wc_status_func4_t status_func,
+                       void *status_baton,
+                       svn_cancel_func_t cancel_func,
+                       void *cancel_baton,
+                       apr_pool_t *scratch_pool)
+{
+  struct status_task_baton *baton = wb->dir_status_baton;
+
+  apr_pool_t *process_pool = svn_task__create_process_pool(baton->task);
+  dir_status_process_baton_t *sub_baton =
+      apr_pcalloc(process_pool, sizeof(*sub_baton));
+
+  sub_baton->local_abspath =
+      apr_pstrdup(process_pool, local_abspath);
+  sub_baton->parent_repos_root_url =
+      apr_pstrdup(process_pool, parent_repos_root_url);
+  sub_baton->parent_repos_relpath =
+      apr_pstrdup(process_pool, parent_repos_relpath);
+  sub_baton->parent_repos_uuid =
+      apr_pstrdup(process_pool, parent_repos_uuid);
+  sub_baton->dirent =
+      svn_io_dirent2_dup(dirent, process_pool);
+  sub_baton->dir_info =
+      db_info_dup(dir_info, process_pool);
+  sub_baton->skip_this_dir = TRUE;
+
+  sub_baton->ignore_patterns = ignore_patterns;
+  sub_baton->depth = depth;
+  sub_baton->get_all = get_all;
+  sub_baton->no_ignore = no_ignore;
+
+  SVN_ERR(svn_task__add_similar(baton->task, process_pool,
+                                baton->partial_results &&
+                                baton->partial_results->nelts
+                                  ? baton->partial_results
+                                  : NULL,
+                                sub_baton));
+  baton->partial_results = NULL;
+
+  return SVN_NO_ERROR;
+}
+
+static svn_error_t *
+get_dir_status_process(void **result,
+                       svn_task__t *task,
+                       void *thread_context,
+                       void *process_baton,
+                       svn_cancel_func_t cancel_func,
+                       void *cancel_baton,
+                       apr_pool_t *result_pool,
+                       apr_pool_t *scratch_pool)
+{
+  dir_status_process_baton_t *baton = process_baton;
+  struct walk_status_baton *wb = thread_context;
+
+  struct status_task_baton status_baton;
+  status_baton.task = task;
+  status_baton.result_pool = result_pool;
+  status_baton.partial_results = apr_array_make(result_pool, 0,
+                                                sizeof(dir_status_result_t*));
+
+  wb->dir_status_func = create_dir_status_task;
+  wb->dir_status_baton = &status_baton;
+
+  SVN_ERR(get_dir_status(wb, baton->local_abspath,
+                         baton->skip_this_dir,
+                         baton->parent_repos_root_url,
+                         baton->parent_repos_relpath,
+                         baton->parent_repos_uuid,
+                         baton->dir_info,
+                         baton->dirent,
+                         baton->ignore_patterns,
+                         baton->depth,
+                         baton->get_all,
+                         baton->no_ignore,
+                         add_status_result,
+                         &status_baton,
+                         cancel_func,
+                         cancel_baton,
+                         scratch_pool));
+
+  *result = status_baton.partial_results && status_baton.partial_results->nelts
+          ? status_baton.partial_results
+          : NULL;
+  return SVN_NO_ERROR;
+}
+
+typedef struct dir_status_output_baton_t
+{
+  svn_wc_status_func4_t status_func;
+  void *status_baton;
+} dir_status_output_baton_t;
+
+static svn_error_t *
+get_dir_status_output(svn_task__t *task,
+                      void *result,
+                      void *output_baton,
+                      svn_cancel_func_t cancel_func,
+                      void *cancel_baton,
+                      apr_pool_t *result_pool,
+                      apr_pool_t *scratch_pool)
+{
+  apr_array_header_t *to_send = result;
+  dir_status_output_baton_t *baton = output_baton;
+  int i;
+
+  for (i = 0; i < to_send->nelts; ++i)
+    {
+      dir_status_result_t *status =
+          APR_ARRAY_IDX(to_send, i, dir_status_result_t *);
+      if (cancel_func)
+        SVN_ERR(cancel_func(cancel_baton));
+
+      if (baton->status_func)
+        SVN_ERR(baton->status_func(baton->status_baton,
+                                   status->local_abspath,
+                                   status->status,
+                                   scratch_pool));
+    }
+
+  return SVN_NO_ERROR;
+}
+
+static svn_error_t *
+get_dir_context_creator(void **thread_context,
+                        void *baton,
+                        apr_pool_t *result_pool,
+                        apr_pool_t *scratch_pool)
+{
+  const struct walk_status_baton *wb = baton;
+
+  struct walk_status_baton *result = apr_pcalloc(result_pool,
+                                                 sizeof(*result));
+  SVN_ERR(svn_wc__db_open(&result->db, NULL, FALSE, FALSE,
+                          result_pool, scratch_pool));
+  result->externals = wb->externals;
+  result->target_abspath = wb->target_abspath;
+  result->ignore_text_mods = wb->ignore_text_mods;
+  result->check_working_copy = wb->check_working_copy;
+  result->repos_root = NULL;
+  result->repos_locks = NULL;
+
+  *thread_context = result;
+  return SVN_NO_ERROR;
+}
+
+static svn_error_t *
+get_dir_status_parallel(struct walk_status_baton *wb,
+                        const char *local_abspath,
+                        svn_boolean_t skip_this_dir,
+                        const char *parent_repos_root_url,
+                        const char *parent_repos_relpath,
+                        const char *parent_repos_uuid,
+                        const struct svn_wc__db_info_t *dir_info,
+                        const svn_io_dirent2_t *dirent,
+                        const apr_array_header_t *ignore_patterns,
+                        svn_depth_t depth,
+                        svn_boolean_t get_all,
+                        svn_boolean_t no_ignore,
+                        svn_wc_status_func4_t status_func,
+                        void *status_baton,
+                        svn_cancel_func_t cancel_func,
+                        void *cancel_baton,
+                        apr_pool_t *scratch_pool)
+{
+  dir_status_process_baton_t process_baton;
+  dir_status_output_baton_t output_baton;
+
+  process_baton.local_abspath = local_abspath;
+  process_baton.skip_this_dir = skip_this_dir;
+  process_baton.parent_repos_root_url = parent_repos_root_url;
+  process_baton.parent_repos_relpath = parent_repos_relpath;
+  process_baton.parent_repos_uuid = parent_repos_uuid;
+  process_baton.dir_info = dir_info;
+  process_baton.dirent = dirent;
+  process_baton.ignore_patterns = ignore_patterns;
+  process_baton.depth = depth;
+  process_baton.get_all = get_all;
+  process_baton.no_ignore = no_ignore;
+
+  output_baton.status_func = status_func;
+  output_baton.status_baton = status_baton;
+
+  SVN_ERR(svn_task__run(24,
+                        get_dir_status_process, &process_baton,
+                        get_dir_status_output, &output_baton,
+                        get_dir_context_creator, wb,
+                        cancel_func, cancel_baton,
+                        scratch_pool, scratch_pool));
+/*
+  SVN_ERR(svn_task__run(1,
+                        get_dir_status_process, &process_baton,
+                        get_dir_status_output, &output_baton,
+                        get_dir_context_creator, wb,
+                        cancel_func, cancel_baton,
+                        scratch_pool, scratch_pool));
+*/
+  return SVN_NO_ERROR;
+}
+
 /*** Public API ***/
 
 svn_error_t *
@@ -2513,6 +2870,8 @@
   eb->wb.check_working_copy = check_working_copy;
   eb->wb.repos_locks      = NULL;
   eb->wb.repos_root       = NULL;
+  eb->wb.dir_status_func  = get_dir_status;
+  eb->wb.dir_status_baton = NULL;
 
   SVN_ERR(svn_wc__db_externals_defined_below(&eb->wb.externals,
                                              wc_ctx->db, eb->target_abspath,
@@ -2644,6 +3003,8 @@
   wb.check_working_copy = TRUE;
   wb.repos_root = NULL;
   wb.repos_locks = NULL;
+  wb.dir_status_func = get_dir_status;
+  wb.dir_status_baton = NULL;
 
   /* Use the caller-provided ignore patterns if provided; the build-time
      configured defaults otherwise. */
@@ -2690,19 +3051,19 @@
       && info->status != svn_wc__db_status_excluded
       && info->status != svn_wc__db_status_server_excluded)
     {
-      SVN_ERR(get_dir_status(&wb,
-                             local_abspath,
-                             FALSE /* skip_root */,
-                             NULL, NULL, NULL,
-                             info,
-                             dirent,
-                             ignore_patterns,
-                             depth,
-                             get_all,
-                             no_ignore,
-                             status_func, status_baton,
-                             cancel_func, cancel_baton,
-                             scratch_pool));
+      SVN_ERR(get_dir_status_parallel(&wb,
+                                      local_abspath,
+                                      FALSE /* skip_root */,
+                                      NULL, NULL, NULL,
+                                      info,
+                                      dirent,
+                                      ignore_patterns,
+                                      depth,
+                                      get_all,
+                                      no_ignore,
+                                      status_func, status_baton,
+                                      cancel_func, cancel_baton,
+                                      scratch_pool));
     }
   else
     {

Reply via email to