subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1545929 - in /subversion/trunk/subversion: include/private/svn_ra_svn_private.h libsvn_ra_svn/marshal.c svnserve/serve.c svnserve/server.h svnserve/svnserve.c
Date Wed, 27 Nov 2013 05:45:12 GMT
Author: stefan2
Date: Wed Nov 27 05:45:12 2013
New Revision: 1545929

URL: http://svn.apache.org/r1545929
Log:
Finally, enable svnserve to handle a very high number of connections.

The basic idea is that we introduce a variant of serve() that checks
for the system load between serving commands.  Under high load, the
connection will be put back into the queue of connections to process.
This effectively results in a round-robin scheme.

The new scheme is currently limited to systems that support thread pools
(i.e. APR 1.3+) and the handling of idle connections is sub-optimal.

* subversion/include/private/svn_ra_svn_private.h
  (svn_ra_svn__has_command): Declare new private API.
  (svn_ra_svn__handle_command): Publish former static function.

* subversion/libsvn_ra_svn/marshal.c
  (svn_ra_svn__has_item): New utility function.
  (svn_ra_svn__has_command): Implement the new API.

* subversion/svnserve/server.h
  (serve_interruptable): Declare the new, interruptable serve() variant.

* subversion/svnserve/serve.c
  (reopen_repos): New utility function to be called instead of to
                  construct_server_baton when serving a previously
                  suspended connection.
  (serve_interruptable): Implement taking initialization code from the
                         former serve_socket and using a command processing
                         loop similar to svn_ra_svn__handle_commands2.

* subversion/svnserve/svnserve.c
  (serve_socket): Simplify calling serve_interruptable instead of serve.
  (threads): Moved here from sub_main to able to add tasks from the
             serving functions.
  (is_busy): Load check callback used with serve_interruptable.
  (serve_thread): More sophisticated task implementation for thread
                  pooled mode where we will employ a round-robin scheme
                  under high load.                  
  (sub_main): Remove local thread pool variable.

Modified:
    subversion/trunk/subversion/include/private/svn_ra_svn_private.h
    subversion/trunk/subversion/libsvn_ra_svn/marshal.c
    subversion/trunk/subversion/svnserve/serve.c
    subversion/trunk/subversion/svnserve/server.h
    subversion/trunk/subversion/svnserve/svnserve.c

Modified: subversion/trunk/subversion/include/private/svn_ra_svn_private.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/include/private/svn_ra_svn_private.h?rev=1545929&r1=1545928&r2=1545929&view=diff
==============================================================================
--- subversion/trunk/subversion/include/private/svn_ra_svn_private.h (original)
+++ subversion/trunk/subversion/include/private/svn_ra_svn_private.h Wed Nov 27 05:45:12 2013
@@ -256,6 +256,33 @@ svn_ra_svn__read_cmd_response(svn_ra_svn
                               apr_pool_t *pool,
                               const char *fmt, ...);
 
+/** Check the receive buffer and socket of @a conn whether there is some
+ * unprocessed incomming data without waiting for new data to come in.
+ * If data is found, set @a *has_command to TRUE.  If the connection does
+ * not contain any more data and has been closed, set @a *terminated to
+ * TRUE.
+ */
+svn_error_t *
+svn_ra_svn__has_command(svn_boolean_t *has_command,
+                        svn_boolean_t *terminated,
+                        svn_ra_svn_conn_t *conn,
+                        apr_pool_t *pool);
+
+/** Accept a single command from @a conn and handle them according
+ * to @a cmd_hash.  Command handlers will be passed @a conn, @a pool,
+ * the parameters of the command, and @a baton.  @a *terminate will be
+ * set if either @a error_on_disconnect is FALSE and the connection got
+ * closed, or if the command being handled has the "terminate" flag set
+ * in the command table.
+ */
+svn_error_t *
+svn_ra_svn__handle_command(svn_boolean_t *terminate,
+                           apr_hash_t *cmd_hash,
+                           void *baton,
+                           svn_ra_svn_conn_t *conn,
+                           svn_boolean_t error_on_disconnect,
+                           apr_pool_t *pool);
+
 /** Accept commands over the network and handle them according to @a
  * commands.  Command handlers will be passed @a conn, a subpool of @a
  * pool (cleared after each command is handled), the parameters of the

Modified: subversion/trunk/subversion/libsvn_ra_svn/marshal.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_ra_svn/marshal.c?rev=1545929&r1=1545928&r2=1545929&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_ra_svn/marshal.c (original)
+++ subversion/trunk/subversion/libsvn_ra_svn/marshal.c Wed Nov 27 05:45:12 2013
@@ -1233,6 +1233,34 @@ svn_ra_svn__read_item(svn_ra_svn_conn_t 
   return read_item(conn, pool, *item, c, 0);
 }
 
+/* Drain existing whitespace from the receive buffer of CONN until either
+   there is no data in the underlying receive socket anymore or we found
+   a non-whitespace char.  Set *HAS_ITEM to TRUE in the latter case.
+ */
+static svn_error_t *
+svn_ra_svn__has_item(svn_boolean_t *has_item,
+                     svn_ra_svn_conn_t *conn,
+                     apr_pool_t *pool)
+{
+  do
+    {
+      if (conn->read_ptr == conn->read_end)
+        {
+          if (conn->write_pos)
+            SVN_ERR(writebuf_flush(conn, pool));
+
+          if (!svn_ra_svn__input_waiting(conn, pool))
+            break;
+
+          SVN_ERR(readbuf_fill(conn, pool));
+        }
+    }
+  while (svn_iswhitespace(*conn->read_ptr) && ++conn->read_ptr);
+
+  *has_item = conn->read_ptr != conn->read_end;
+  return SVN_NO_ERROR;
+}
+
 svn_error_t *
 svn_ra_svn__skip_leading_garbage(svn_ra_svn_conn_t *conn,
                                  apr_pool_t *pool)
@@ -1521,7 +1549,25 @@ svn_ra_svn__read_cmd_response(svn_ra_svn
                            status);
 }
 
-static svn_error_t *
+svn_error_t *
+svn_ra_svn__has_command(svn_boolean_t *has_command,
+                        svn_boolean_t *terminated,
+                        svn_ra_svn_conn_t *conn,
+                        apr_pool_t *pool)
+{
+  svn_error_t *err = svn_ra_svn__has_item(has_command, conn, pool);
+  if (err && err->apr_err == SVN_ERR_RA_SVN_CONNECTION_CLOSED)
+    {
+      *terminated = TRUE;
+      svn_error_clear(err);
+      return SVN_NO_ERROR;
+    }
+
+  *terminated = FALSE;
+  return svn_error_trace(err);
+}
+
+svn_error_t *
 svn_ra_svn__handle_command(svn_boolean_t *terminate,
                            apr_hash_t *cmd_hash,
                            void *baton,

Modified: subversion/trunk/subversion/svnserve/serve.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/svnserve/serve.c?rev=1545929&r1=1545928&r2=1545929&view=diff
==============================================================================
--- subversion/trunk/subversion/svnserve/serve.c (original)
+++ subversion/trunk/subversion/svnserve/serve.c Wed Nov 27 05:45:12 2013
@@ -3800,6 +3800,142 @@ construct_server_baton(server_baton_t **
   return SVN_NO_ERROR;
 }
 
+/* Open a svn_repos object in CONNECTION for the same repository and with
+   the same settings as last time.  The repository object remains valid
+   until POOL gets cleaned up at which point the respective pointer in
+   CONNECTION reset.  The repository in CONNECTION must have been opened
+   at some point in the past using construct_server_baton.
+ */
+static svn_error_t *
+reopen_repos(connection_t *connection,
+             apr_pool_t *pool)
+{
+  fs_warning_baton_t *warn_baton = apr_pcalloc(pool, sizeof(*warn_baton));
+  repository_t *repository = connection->baton->repository;
+
+  /* Open the repository and fill in b with the resulting information. */
+  SVN_ERR(svn_repos__repos_pool_get(&repository->repos,
+                                    connection->params->repos_pool,
+                                    repository->repos_root, repository->uuid,
+                                    pool));
+  repository->fs = svn_repos_fs(repository->repos);
+
+  /* Reset the REPOS pointer as soon as the REPOS will be returned to the
+     REPOS_POOL. */
+  apr_pool_cleanup_register(pool, repository, reset_repos,
+                            apr_pool_cleanup_null);
+
+  /* Configure svn_repos object */
+  SVN_ERR(svn_repos_remember_client_capabilities(repository->repos,
+                                                 repository->capabilities));
+  SVN_ERR(svn_repos_hooks_setenv(repository->repos, repository->hooks_env,
+                                 pool));
+  
+  warn_baton->server = connection->baton;
+  warn_baton->conn = connection->conn;
+  warn_baton->pool = svn_pool_create(pool);
+  svn_fs_set_warning_func(connection->baton->repository->fs,
+                          fs_warning_func, &warn_baton);
+
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+serve_interruptable(svn_boolean_t *terminate_p,
+                    connection_t *connection,
+                    svn_boolean_t (* is_busy)(connection_t *),
+                    apr_pool_t *pool)
+{
+  svn_boolean_t terminate = FALSE;
+  const svn_ra_svn_cmd_entry_t *command;
+  apr_pool_t *iterpool = svn_pool_create(pool);
+
+  /* Prepare command parser. */
+  apr_hash_t *cmd_hash = apr_hash_make(pool);
+  for (command = main_commands; command->cmdname; command++)
+    svn_hash_sets(cmd_hash, command->cmdname, command);
+
+  /* Auto-initialize connection & open repository */
+  if (connection->conn)
+    {
+      /* This is not the first call for CONNECTION. */
+      if (connection->baton->repository->repos == NULL)
+        SVN_ERR(reopen_repos(connection, pool));
+    }
+  else
+    {
+      apr_status_t status;
+
+      /* Enable TCP keep-alives on the socket so we time out when
+       * the connection breaks due to network-layer problems.
+       * If the peer has dropped the connection due to a network partition
+       * or a crash, or if the peer no longer considers the connection
+       * valid because we are behind a NAT and our public IP has changed,
+       * it will respond to the keep-alive probe with a RST instead of an
+       * acknowledgment segment, which will cause svn to abort the session
+       * even while it is currently blocked waiting for data from the peer. */
+      status = apr_socket_opt_set(connection->usock, APR_SO_KEEPALIVE, 1);
+      if (status)
+        {
+          /* It's not a fatal error if we cannot enable keep-alives. */
+        }
+
+      /* create the connection, configure ports etc. */
+      connection->conn
+        = svn_ra_svn_create_conn3(connection->usock, NULL, NULL,
+                                  connection->params->compression_level,
+                                  connection->params->zero_copy_limit,
+                                  connection->params->error_check_interval,
+                                  pool);
+
+      /* Construct server baton and open the repository for the first time. */
+      SVN_ERR(construct_server_baton(&connection->baton, connection->conn,
+                                     connection->params, pool));
+    }
+
+  /* Process incomming commands. */
+  while (!terminate)
+    {
+      svn_pool_clear(iterpool);
+      if (is_busy && is_busy(connection))
+        {
+          svn_boolean_t has_command;
+
+          /* If the server is busy, execute just one command and only if
+           * there is one currently waiting in our receive buffers.
+           */
+          SVN_ERR(svn_ra_svn__has_command(&has_command, &terminate,
+                                          connection->conn, iterpool));
+          if (has_command)
+            SVN_ERR(svn_ra_svn__handle_command(&terminate, cmd_hash,
+                                              connection->baton,
+                                              connection->conn,
+                                              FALSE, iterpool));
+
+          break;
+        }
+      else
+        {
+          /* The server is not busy, thus let's serve whichever command
+           * comes in next and whenever it comes in.  This requires the
+           * busy() callback test to return TRUE while there are still some
+           * resources left.
+           */
+          SVN_ERR(svn_ra_svn__handle_command(&terminate, cmd_hash,
+                                             connection->baton,
+                                             connection->conn,
+                                             FALSE, iterpool));
+        }
+    }
+
+  /* error or normal end of session. Close the connection */
+  svn_pool_destroy(iterpool);
+  if (terminate_p)
+    *terminate_p = terminate;
+
+  return SVN_NO_ERROR;
+}
+
 svn_error_t *serve(svn_ra_svn_conn_t *conn,
                    serve_params_t *params,
                    apr_pool_t *pool)

Modified: subversion/trunk/subversion/svnserve/server.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/svnserve/server.h?rev=1545929&r1=1545928&r2=1545929&view=diff
==============================================================================
--- subversion/trunk/subversion/svnserve/server.h (original)
+++ subversion/trunk/subversion/svnserve/server.h Wed Nov 27 05:45:12 2013
@@ -201,6 +201,23 @@ client_info_t * get_client_info(svn_ra_s
 svn_error_t *serve(svn_ra_svn_conn_t *conn, serve_params_t *params,
                    apr_pool_t *pool);
 
+/* Serve the connection CONNECTION for as long as IS_BUSY does not
+   return TRUE.  If IS_BUSY is NULL, serve the connection until it
+   either gets terminated or there is an error.  If TERMINATE_P is
+   not NULL, set *TERMINATE_P to TRUE if the connection got
+   terminated.
+
+   For the first call, CONNECTION->CONN may be NULL in which case we
+   will create an ra_svn connection object.  Subsequent calls will
+   check for an open repository and automatically re-open the repo
+   in pool if necessary.
+ */
+svn_error_t *
+serve_interruptable(svn_boolean_t *terminate_p,
+                    connection_t *connection,
+                    svn_boolean_t (* is_busy)(connection_t *),
+                    apr_pool_t *pool);
+
 /* Initialize the Cyrus SASL library. POOL is used for allocations. */
 svn_error_t *cyrus_init(apr_pool_t *pool);
 

Modified: subversion/trunk/subversion/svnserve/svnserve.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/svnserve/svnserve.c?rev=1545929&r1=1545928&r2=1545929&view=diff
==============================================================================
--- subversion/trunk/subversion/svnserve/svnserve.c (original)
+++ subversion/trunk/subversion/svnserve/svnserve.c Wed Nov 27 05:45:12 2013
@@ -526,33 +526,8 @@ static svn_error_t *
 serve_socket(connection_t *connection,
              apr_pool_t *pool)
 {
-  apr_status_t status;
-  svn_error_t *err;
-  
-  /* Enable TCP keep-alives on the socket so we time out when
-   * the connection breaks due to network-layer problems.
-   * If the peer has dropped the connection due to a network partition
-   * or a crash, or if the peer no longer considers the connection
-   * valid because we are behind a NAT and our public IP has changed,
-   * it will respond to the keep-alive probe with a RST instead of an
-   * acknowledgment segment, which will cause svn to abort the session
-   * even while it is currently blocked waiting for data from the peer. */
-  status = apr_socket_opt_set(connection->usock, APR_SO_KEEPALIVE, 1);
-  if (status)
-    {
-      /* It's not a fatal error if we cannot enable keep-alives. */
-    }
-
-  /* create the connection, configure ports etc. */
-  connection->conn
-    = svn_ra_svn_create_conn3(connection->usock, NULL, NULL,
-                              connection->params->compression_level,
-                              connection->params->zero_copy_limit,
-                              connection->params->error_check_interval,
-                              pool);
-
   /* process the actual request and log errors */
-  err = serve(connection->conn, connection->params, pool);
+  svn_error_t *err = serve_interruptable(NULL, connection, NULL, pool);
   if (err)
     logger__log_error(connection->params->logger, err, NULL,
                       get_client_info(connection->conn, connection->params,
@@ -567,6 +542,55 @@ serve_socket(connection_t *connection,
    There should be at most THREADPOOL_MAX_SIZE such pools. */
 svn_root_pools__t *connection_pools;
 
+#if HAVE_THREADPOOLS
+
+/* The global thread pool serving all connections. */
+apr_thread_pool_t *threads;
+
+/* Very simple load determination callback for serve_interruptable:
+   With less than have the threads in THREADS in use, we can afford to
+   wait in the socket read() function.  Otherwise, poll them round-robin. */
+static svn_boolean_t
+is_busy(connection_t *connection)
+{
+  return apr_thread_pool_threads_count(threads) * 2
+       > apr_thread_pool_thread_max_get(threads);
+}
+
+/* Serve the connection given by DATA.  Under high load, serve only
+   the current command (if any) and then put the connection back into
+   THREAD's task pool. */
+static void * APR_THREAD_FUNC serve_thread(apr_thread_t *tid, void *data)
+{
+  svn_boolean_t done;
+  connection_t *connection = data;
+  svn_error_t *err;
+
+  apr_pool_t *pool = svn_root_pools__acquire_pool(connection_pools);
+
+  /* process the actual request and log errors */
+  err = serve_interruptable(&done, connection, is_busy, pool);
+  if (err)
+    {
+      logger__log_error(connection->params->logger, err, NULL,
+                        get_client_info(connection->conn, connection->params,
+                                        pool));
+      svn_error_clear(err);
+    }
+  svn_root_pools__release_pool(pool, connection_pools);
+
+  /* Close or re-schedule connection. */
+  if (done)
+    close_connection(connection);
+  else
+    apr_thread_pool_push(threads, serve_thread, connection, 0, NULL);
+    
+  return NULL;
+}
+
+#else
+
+/* Fully serve the connection given by DATA. */
 static void * APR_THREAD_FUNC serve_thread(apr_thread_t *tid, void *data)
 {
   struct connection_t *connection = data;
@@ -584,6 +608,8 @@ static void * APR_THREAD_FUNC serve_thre
 }
 #endif
 
+#endif
+
 /* Write the PID of the current process as a decimal number, followed by a
    newline to the file FILENAME, using POOL for temporary allocations. */
 static svn_error_t *write_pid_file(const char *filename, apr_pool_t *pool)
@@ -643,14 +669,10 @@ sub_main(int *exit_code, int argc, const
   const char *arg;
   apr_status_t status;
   apr_proc_t proc;
-#if APR_HAS_THREADS
-#if HAVE_THREADPOOLS
-  apr_thread_pool_t *threads;
-#else
+#if APR_HAS_THREADS && !HAVE_THREADPOOLS
   apr_threadattr_t *tattr;
   apr_thread_t *tid;
 #endif
-#endif
   svn_boolean_t is_multi_threaded;
   enum connection_handling_mode handling_mode = CONNECTION_DEFAULT;
   apr_hash_t *fs_config = NULL;



Mime
View raw message