--- server/mpm/experimental/peruser/mpm_default.h 2009-05-27 15:09:19.000000000 +0300 +++ server/mpm/experimental/peruser/mpm_default.h 2009-05-28 10:10:42.000000000 +0300 @@ -77,6 +77,12 @@ #define DEFAULT_MIN_FREE_PROCESSORS 2 #endif +/* Maximum --- more than this, and idle processors will be killed (0 = disable) */ + +#ifndef DEFAULT_MAX_FREE_PROCESSORS +#define DEFAULT_MAX_FREE_PROCESSORS 0 +#endif + /* Maximum processors per ServerEnvironment */ #ifndef DEFAULT_MAX_PROCESSORS @@ -107,4 +113,50 @@ #define DEFAULT_MAX_REQUESTS_PER_CHILD 10000 #endif +/* Maximum multiplexers */ + +#ifndef DEFAULT_MAX_MULTIPLEXERS +#define DEFAULT_MAX_MULTIPLEXERS 20 +#endif + +/* Minimum multiplexers */ + +#ifndef DEFAULT_MIN_MULTIPLEXERS +#define DEFAULT_MIN_MULTIPLEXERS 3 +#endif + +/* Amount of time a child can run before it expires (0 = turn off) */ + +#ifndef DEFAULT_EXPIRE_TIMEOUT +#define DEFAULT_EXPIRE_TIMEOUT 1800 +#endif + +/* Amount of time a child can stay idle (0 = turn off) */ + +#ifndef DEFAULT_IDLE_TIMEOUT +#define DEFAULT_IDLE_TIMEOUT 900 +#endif + +/* Amount of time a multiplexer can stay idle (0 = turn off) */ + +#ifndef DEFAULT_MULTIPLEXER_IDLE_TIMEOUT +#define DEFAULT_MULTIPLEXER_IDLE_TIMEOUT 0 +#endif + +/* Amount of maximum time a multiplexer can wait for processor if it is busy (0 = never wait) + * This is decreased with every busy request + */ + +#ifndef DEFAULT_PROCESSOR_WAIT_TIMEOUT +#define DEFAULT_PROCESSOR_WAIT_TIMEOUT 5 +#endif + +/* The number of different levels there are when a multiplexer is waiting for processor + * (between maximum waiting time and no waiting) + */ + +#ifndef DEFAULT_PROCESSOR_WAIT_STEPS +#define DEFAULT_PROCESSOR_WAIT_STEPS 10 +#endif + #endif /* AP_MPM_DEFAULT_H */ --- server/mpm/experimental/peruser/mpm.h 2009-03-22 22:46:45.000000000 +0200 +++ server/mpm/experimental/peruser/mpm.h 2009-03-22 22:39:10.000000000 +0200 @@ -84,6 +84,7 @@ #define AP_MPM_USES_POD 1 #define MPM_CHILD_PID(i) (ap_scoreboard_image->parent[i].pid) #define MPM_NOTE_CHILD_KILLED(i) (MPM_CHILD_PID(i) = 0) +#define MPM_VALID_PID(p) (getpgid(p) == getpgrp()) #define MPM_ACCEPT_FUNC unixd_accept extern int ap_threads_per_child; --- server/mpm/experimental/peruser/peruser.c 2009-05-27 15:09:19.000000000 +0300 +++ server/mpm/experimental/peruser/peruser.c 2009-05-28 13:54:27.000000000 +0300 @@ -195,20 +195,30 @@ #define CHILD_STATUS_STANDBY 0 /* wait for a request before starting */ #define CHILD_STATUS_STARTING 1 /* wait for socket creation */ -#define CHILD_STATUS_READY 2 /* wait for mux to restart */ -#define CHILD_STATUS_ACTIVE 3 /* ready to take requests */ +#define CHILD_STATUS_READY 2 /* is ready to take requests */ +#define CHILD_STATUS_ACTIVE 3 /* is currently busy handling requests */ #define CHILD_STATUS_RESTART 4 /* child about to die and restart */ +/* cgroup settings */ +#define CGROUP_TASKS_FILE "/tasks" +#define CGROUP_TASKS_FILE_LEN 7 + /* config globals */ int ap_threads_per_child=0; /* Worker threads per child */ static apr_proc_mutex_t *accept_mutex; static int ap_min_processors=DEFAULT_MIN_PROCESSORS; static int ap_min_free_processors=DEFAULT_MIN_FREE_PROCESSORS; +static int ap_max_free_processors=DEFAULT_MAX_FREE_PROCESSORS; static int ap_max_processors=DEFAULT_MAX_PROCESSORS; +static int ap_min_multiplexers=DEFAULT_MIN_MULTIPLEXERS; +static int ap_max_multiplexers=DEFAULT_MAX_MULTIPLEXERS; static int ap_daemons_limit=0; /* MaxClients */ -static int expire_timeout=1800; -static int idle_timeout=900; +static int expire_timeout=DEFAULT_EXPIRE_TIMEOUT; +static int idle_timeout=DEFAULT_IDLE_TIMEOUT; +static int multiplexer_idle_timeout=DEFAULT_MULTIPLEXER_IDLE_TIMEOUT; +static int processor_wait_timeout=DEFAULT_PROCESSOR_WAIT_TIMEOUT; +static int processor_wait_steps=DEFAULT_PROCESSOR_WAIT_STEPS; static int server_limit = DEFAULT_SERVER_LIMIT; static int first_server_limit; static int changed_limit_at_restart; @@ -222,15 +232,21 @@ { int processor_id; + const char *name; /* Server environment's unique string identifier */ + /* security settings */ uid_t uid; /* user id */ gid_t gid; /* group id */ const char *chroot; /* directory to chroot() to, can be null */ + int nice_lvl; + const char *cgroup; /* cgroup directory, can be null */ /* resource settings */ int min_processors; int min_free_processors; + int max_free_processors; int max_processors; + int availability; /* sockets */ int input; /* The socket descriptor */ @@ -437,6 +453,25 @@ return "UNKNOWN"; } +char* scoreboard_status_string(int status) { + switch(status) + { + case SERVER_DEAD: return "DEAD"; + case SERVER_STARTING: return "STARTING"; + case SERVER_READY: return "READY"; + case SERVER_BUSY_READ: return "BUSY_READ"; + case SERVER_BUSY_WRITE: return "BUSY_WRITE"; + case SERVER_BUSY_KEEPALIVE: return "BUSY_KEEPALIVE"; + case SERVER_BUSY_LOG: return "BUSY_LOG"; + case SERVER_BUSY_DNS: return "BUSY_DNS"; + case SERVER_CLOSING: return "CLOSING"; + case SERVER_GRACEFUL: return "GRACEFUL"; + case SERVER_NUM_STATUS: return "NUM_STATUS"; + } + + return "UNKNOWN"; +} + void dump_child_table() { #ifdef MPM_PERUSER_DEBUG @@ -511,10 +546,6 @@ static void accept_mutex_on(void) { -/* for some reason this fails if we listen on the pipe_of_death. - fortunately I don't think we currently need it */ - -#if 0 apr_status_t rv = apr_proc_mutex_lock(accept_mutex); if (rv != APR_SUCCESS) { const char *msg = "couldn't grab the accept mutex"; @@ -529,12 +560,10 @@ exit(APEXIT_CHILDFATAL); } } -#endif } static void accept_mutex_off(void) { -#if 0 apr_status_t rv = apr_proc_mutex_unlock(accept_mutex); if (rv != APR_SUCCESS) { const char *msg = "couldn't release the accept mutex"; @@ -552,7 +581,6 @@ exit(APEXIT_CHILDFATAL); } } -#endif } /* On some architectures it's safe to do unserialized accept()s in the single @@ -715,8 +743,10 @@ ret = apr_socket_recv(lr->sd, &pipe_read_char, &n); if (APR_STATUS_IS_EAGAIN(ret)) { - /* It lost the lottery. It must continue to suffer - * through a life of servitude. */ + /* It lost the lottery. It must continue to suffer + * through a life of servitude. */ + _DBG("POD read EAGAIN"); + return ret; } else { @@ -1087,8 +1117,7 @@ for(i = 0, total = 0; i < NUM_CHILDS; ++i) { if(CHILD_INFO_TABLE[i].senv == CHILD_INFO_TABLE[child_num].senv && - (SCOREBOARD_STATUS(i) == SERVER_STARTING || - SCOREBOARD_STATUS(i) == SERVER_READY)) + (CHILD_INFO_TABLE[i].status == CHILD_STATUS_READY)) { total++; } @@ -1116,7 +1145,7 @@ apr_bucket *bucket; const apr_array_header_t *headers_in_array; const apr_table_entry_t *headers_in; - int counter; + int counter, wait_time, wait_step_size; apr_socket_t *thesock = ap_get_module_config(r->connection->conn_config, &core_module); @@ -1137,10 +1166,73 @@ apr_table_get(r->headers_in, "Host"), my_child_num, processor->senv->output); _DBG("r->the_request=\"%s\" len=%d", r->the_request, strlen(r->the_request)); - ap_get_brigade(r->connection->input_filters, bb, AP_MODE_EXHAUSTIVE, APR_NONBLOCK_READ, len); + wait_step_size = 100 / processor_wait_steps; - /* Scan the brigade looking for heap-buckets */ + /* Check if the processor is available */ + if (total_processors(processor->id) == processor->senv->max_processors && + idle_processors(processor->id) == 0) { + /* The processor is currently busy, try to wait (a little) */ + _DBG("processor seems to be busy, trying to wait for it"); + + if (processor->senv->availability == 0) { + processor->senv->availability = 0; + + _DBG("processor is very busy (availability = 0) - not passing request"); + + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + "Too many requests for processor %s, increase MaxProcessors", processor->senv->name); + + /* No point in waiting for the processor, it's very busy */ + return -1; + } + + /* We sleep a little (depending how available the processor usually is) */ + int i; + + wait_time = (processor_wait_timeout / processor_wait_steps) * 1000000; + + for(i = 0; i <= processor->senv->availability; i += wait_step_size) { + usleep(wait_time); + /* Check if the processor is ready */ + if (total_processors(processor->id) < processor->senv->max_processors || + idle_processors(processor->id) > 0) { + /* The processor has freed - lets use it */ + _DBG("processor freed before wait time expired"); + break; + } + } + + if (processor->senv->availability <= wait_step_size) { + processor->senv->availability = 0; + } + else processor->senv->availability -= wait_step_size; + + /* Check if we waited all the time */ + if (i > processor->senv->availability) { + _DBG("processor is busy - not passing request (availability = %d)", + processor->senv->availability); + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, ap_server_conf, + "Too many requests for processor %s, increase MaxProcessors", processor->senv->name); + return -1; + } + + /* We could increase the availability a little here, + * because the processor got freed eventually + */ + } + else { + /* Smoothly increment the availability back to 100 */ + if (processor->senv->availability >= 100-wait_step_size) { + processor->senv->availability = 100; + } + else processor->senv->availability += wait_step_size; + } + + ap_get_brigade(r->connection->input_filters, bb, AP_MODE_EXHAUSTIVE, APR_NONBLOCK_READ, len); + + /* Scan the brigade looking for heap-buckets */ + _DBG("Scanning the brigade",0); bucket = APR_BRIGADE_FIRST(bb); while (bucket != APR_BRIGADE_SENTINEL(bb) && @@ -1294,12 +1386,22 @@ /* -- receive data from socket -- */ apr_os_sock_get(&ctrl_sock_fd, lr->sd); _DBG("receiving from sock_fd=%d", ctrl_sock_fd); - ret = recvmsg(ctrl_sock_fd, &msg, 0); - if(ret == -1) - _DBG("recvmsg failed with error \"%s\"", strerror(errno)); - else - _DBG("recvmsg returned %d", ret); + // Don't block + ret = recvmsg(ctrl_sock_fd, &msg, MSG_DONTWAIT); + + if (ret == -1 && errno == EAGAIN) { + _DBG("receive_from_multiplexer recvmsg() EAGAIN, someone was faster"); + + return APR_EAGAIN; + } + else if (ret == -1) { + _DBG("recvmsg failed with error \"%s\"", strerror(errno)); + + // Error, better kill this child to be on the safe side + return APR_EGENERAL; + } + else _DBG("recvmsg returned %d", ret); /* -- extract socket from the cmsg -- */ memcpy(&trans_sock_fd, CMSG_DATA(cmsg), sizeof(trans_sock_fd)); @@ -1399,10 +1501,58 @@ return 0; } -static int peruser_setup_child(int childnum) +static int peruser_setup_cgroup(int childnum, server_env_t *senv, apr_pool_t *pool) +{ + apr_file_t *file; + int length; + apr_size_t content_len; + char *tasks_file, *content, *pos; + + _DBG("starting to add pid to cgroup %s", senv->cgroup); + + length = strlen(senv->cgroup) + CGROUP_TASKS_FILE_LEN; + tasks_file = malloc(length); + + if (!tasks_file) return -1; + + pos = apr_cpystrn(tasks_file, senv->cgroup, length); + apr_cpystrn(pos, CGROUP_TASKS_FILE, CGROUP_TASKS_FILE_LEN); + + /* Prepare the data to be written to tasks file */ + content = apr_itoa(pool, ap_my_pid); + content_len = strlen(content); + + _DBG("writing pid %s to tasks file %s", content, tasks_file); + + if (apr_file_open(&file, tasks_file, APR_WRITE, APR_OS_DEFAULT, pool)) { + ap_log_error(APLOG_MARK, APLOG_ALERT, errno, NULL, + "cgroup: unable to open file %s", + tasks_file); + free(tasks_file); + return OK; /* don't fail if cgroup not available */ + } + + if (apr_file_write(file, content, &content_len)) { + ap_log_error(APLOG_MARK, APLOG_ALERT, errno, NULL, + "cgroup: unable to write pid to file %s", + tasks_file); + } + + apr_file_close(file); + + free(tasks_file); + + return OK; +} + +static int peruser_setup_child(int childnum, apr_pool_t *pool) { server_env_t *senv = CHILD_INFO_TABLE[childnum].senv; + if (senv->nice_lvl != 0) { + nice(senv->nice_lvl); + } + if(senv->chroot) { _DBG("chdir to %s", senv->chroot); if(chdir(senv->chroot)) { @@ -1421,6 +1571,10 @@ } } + if(senv->cgroup) { + peruser_setup_cgroup(childnum, senv, pool); + } + if (senv->uid == -1 && senv->gid == -1) { return unixd_setup_child(); } @@ -1594,15 +1748,6 @@ { case CHILD_TYPE_MULTIPLEXER: _DBG("MULTIPLEXER %d", my_child_num); - - /* update status on processors that are ready to accept requests */ - _DBG("updating processor stati", 0); - for(i = 0; i < NUM_CHILDS; ++i) - { - if(CHILD_INFO_TABLE[i].status == CHILD_STATUS_READY) - CHILD_INFO_TABLE[i].status = CHILD_STATUS_ACTIVE; - } - break; case CHILD_TYPE_PROCESSOR: @@ -1626,7 +1771,7 @@ apr_os_sock_put(&pod_sock, &fd, pconf); listen_add(pconf, pod_sock, check_pipe_of_death); - if(peruser_setup_child(my_child_num) != 0) + if(peruser_setup_child(my_child_num, pchild) != 0) clean_child_exit(APEXIT_CHILDFATAL); ap_run_child_init(pchild, ap_server_conf); @@ -1670,14 +1815,19 @@ clean_child_exit(0); } - (void) ap_update_child_status(sbh, SERVER_READY, (request_rec *) NULL); + (void) ap_update_child_status(sbh, SERVER_READY, (request_rec *) NULL); + + CHILD_INFO_TABLE[my_child_num].status = CHILD_STATUS_READY; + _DBG("Child %d (%s) is now ready", my_child_num, child_type_string(CHILD_INFO_TABLE[my_child_num].type)); /* * Wait for an acceptable connection to arrive. */ - /* Lock around "accept", if necessary */ - SAFE_ACCEPT(accept_mutex_on()); + /* Lock around "accept", if necessary */ + if (CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_MULTIPLEXER) { + SAFE_ACCEPT(accept_mutex_on()); + } if (num_listensocks == 1) { offset = 0; @@ -1729,18 +1879,27 @@ * defer the exit */ status = listensocks[offset].accept_func((void *)&sock, &listensocks[offset], ptrans); - SAFE_ACCEPT(accept_mutex_off()); /* unlock after "accept" */ + + if (CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_MULTIPLEXER) { + SAFE_ACCEPT(accept_mutex_off()); /* unlock after "accept" */ + } if (status == APR_EGENERAL) { /* resource shortage or should-not-occur occurred */ clean_child_exit(1); } - else if (status != APR_SUCCESS || die_now) { + else if (status != APR_SUCCESS || die_now || sock == NULL) { continue; } + if (CHILD_INFO_TABLE[my_child_num].status == CHILD_STATUS_READY) { + CHILD_INFO_TABLE[my_child_num].status = CHILD_STATUS_ACTIVE; + _DBG("Child %d (%s) is now active", my_child_num, child_type_string(CHILD_INFO_TABLE[my_child_num].type)); + } + if (CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_PROCESSOR || - CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_WORKER) + CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_WORKER || + CHILD_INFO_TABLE[my_child_num].type == CHILD_TYPE_MULTIPLEXER) { _DBG("CHECKING IF WE SHOULD CLONE A CHILD..."); @@ -1754,8 +1913,11 @@ if(total_processors(my_child_num) < CHILD_INFO_TABLE[my_child_num].senv->max_processors && - idle_processors(my_child_num) <= - CHILD_INFO_TABLE[my_child_num].senv->min_free_processors) + (idle_processors(my_child_num) <= + CHILD_INFO_TABLE[my_child_num].senv->min_free_processors || + total_processors(my_child_num) < + CHILD_INFO_TABLE[my_child_num].senv->min_processors + )) { _DBG("CLONING CHILD"); child_clone(); @@ -1804,46 +1966,80 @@ clean_child_exit(0); } -static server_env_t* senv_add(int uid, int gid, const char* chroot) -{ +static server_env_t* find_senv_by_name(const char *name) { int i; - int socks[2]; - _DBG("Searching for matching senv..."); + if (name == NULL) return NULL; + + _DBG("name=%s", name); + + for(i = 0; i < NUM_SENV; i++) + { + if(SENV[i].name != NULL && !strcmp(SENV[i].name, name)) { + return &SENV[i]; + } + } + + return NULL; +} + +static server_env_t* find_matching_senv(server_env_t* senv) { + int i; + + _DBG("name=%s uid=%d gid=%d chroot=%s", senv->name, senv->uid, senv->gid, senv->chroot); for(i = 0; i < NUM_SENV; i++) - { - if(SENV[i].uid == uid && SENV[i].gid == gid && - (SENV[i].chroot == NULL || !strcmp(SENV[i].chroot, chroot))) { - _DBG("Found existing senv: %i", i); - return &SENV[i]; + if((senv->name != NULL && SENV[i].name != NULL && !strcmp(SENV[i].name, senv->name)) || + (senv->name == NULL && SENV[i].uid == senv->uid && SENV[i].gid == senv->gid && + ( + (SENV[i].chroot == NULL && senv->chroot == NULL) || + ((SENV[i].chroot != NULL || senv->chroot != NULL) && !strcmp(SENV[i].chroot, senv->chroot))) + ) + ) { + return &SENV[i]; + } } + + return NULL; +} + +static server_env_t* senv_add(server_env_t *senv) +{ + int socks[2]; + server_env_t *old_senv; + + _DBG("Searching for matching senv..."); + + old_senv = find_matching_senv(senv); + + if (old_senv) { + _DBG("Found existing senv"); + senv = old_senv; + return old_senv; } if(NUM_SENV >= server_limit) - { - _DBG("server_limit reached!"); - return NULL; - } + { + _DBG("server_limit reached!"); + return NULL; + } _DBG("Creating new senv"); - SENV[NUM_SENV].uid = uid; - SENV[NUM_SENV].gid = gid; - SENV[NUM_SENV].chroot = chroot; - - SENV[NUM_SENV].min_processors = ap_min_processors; - SENV[NUM_SENV].min_free_processors = ap_min_free_processors; - SENV[NUM_SENV].max_processors = ap_max_processors; + memcpy(&SENV[NUM_SENV], senv, sizeof(server_env_t)); + + SENV[NUM_SENV].availability = 100; socketpair(PF_UNIX, SOCK_STREAM, 0, socks); SENV[NUM_SENV].input = socks[0]; SENV[NUM_SENV].output = socks[1]; + senv = &SENV[NUM_SENV]; return &SENV[server_env_image->control->num++]; } + static const char* child_clone() { int i; @@ -1869,7 +2065,14 @@ new = &CHILD_INFO_TABLE[i]; new->senv = this->senv; - new->type = CHILD_TYPE_WORKER; + + if (this->type == CHILD_TYPE_MULTIPLEXER) { + new->type = CHILD_TYPE_MULTIPLEXER; + } + else { + new->type = CHILD_TYPE_WORKER; + } + new->sock_fd = this->sock_fd; new->status = CHILD_STATUS_STARTING; @@ -1878,7 +2081,7 @@ } static const char* child_add(int type, int status, - apr_pool_t *pool, uid_t uid, gid_t gid, const char* chroot) + apr_pool_t *pool, server_env_t *senv) { _DBG("adding child #%d", NUM_CHILDS); @@ -1888,10 +2091,10 @@ "Increase NumServers in your config file."; } - if (chroot && !ap_is_directory(pool, chroot)) - return apr_psprintf(pool, "Error: chroot directory [%s] does not exist", chroot); + if (senv->chroot && !ap_is_directory(pool, senv->chroot)) + return apr_psprintf(pool, "Error: chroot directory [%s] does not exist", senv->chroot); - CHILD_INFO_TABLE[NUM_CHILDS].senv = senv_add(uid, gid, chroot); + CHILD_INFO_TABLE[NUM_CHILDS].senv = senv_add(senv); if(CHILD_INFO_TABLE[NUM_CHILDS].senv == NULL) { @@ -1907,10 +2110,10 @@ CHILD_INFO_TABLE[NUM_CHILDS].status = status; _DBG("[%d] uid=%d gid=%d type=%d chroot=%s", - NUM_CHILDS, uid, gid, type, - chroot); + NUM_CHILDS, senv->uid, senv->gid, type, + senv->chroot); - if (uid == 0 || gid == 0) + if (senv->uid == 0 || senv->gid == 0) { _DBG("Assigning root user/group to a child.", 0); } @@ -1957,7 +2160,7 @@ (void) ap_update_child_status_from_indexes(slot, 0, SERVER_STARTING, (request_rec *) NULL); - CHILD_INFO_TABLE[slot].status = CHILD_STATUS_ACTIVE; + CHILD_INFO_TABLE[slot].status = CHILD_STATUS_READY; #ifdef _OSD_POSIX @@ -2062,19 +2265,31 @@ if(CHILD_INFO_TABLE[i].status == CHILD_STATUS_STARTING) make_child(ap_server_conf, i); } - else if(((CHILD_INFO_TABLE[i].type == CHILD_TYPE_PROCESSOR || - CHILD_INFO_TABLE[i].type == CHILD_TYPE_WORKER) && - ap_scoreboard_image->parent[i].pid > 1) && - (idle_processors (i) > 1 || total_processes (i) == 1) && ( - (expire_timeout > 0 && ap_scoreboard_image->servers[i][0].status != SERVER_DEAD && - apr_time_sec(now - ap_scoreboard_image->servers[i][0].last_used) > expire_timeout) || - (idle_timeout > 0 && ap_scoreboard_image->servers[i][0].status == SERVER_READY && - apr_time_sec(now - ap_scoreboard_image->servers[i][0].last_used) > idle_timeout))) + else if( + (((CHILD_INFO_TABLE[i].type == CHILD_TYPE_PROCESSOR || + CHILD_INFO_TABLE[i].type == CHILD_TYPE_WORKER) && + ap_scoreboard_image->parent[i].pid > 1) && + (idle_processors (i) > CHILD_INFO_TABLE[i].senv->min_free_processors || CHILD_INFO_TABLE[i].senv->min_free_processors == 0) && + total_processes (i) > CHILD_INFO_TABLE[i].senv->min_processors && + ( + (expire_timeout > 0 && ap_scoreboard_image->servers[i][0].status != SERVER_DEAD && + apr_time_sec(now - ap_scoreboard_image->servers[i][0].last_used) > expire_timeout) || + (idle_timeout > 0 && ap_scoreboard_image->servers[i][0].status == SERVER_READY && + apr_time_sec(now - ap_scoreboard_image->servers[i][0].last_used) > idle_timeout) || + (CHILD_INFO_TABLE[i].senv->max_free_processors > 0 && CHILD_INFO_TABLE[i].status == CHILD_STATUS_READY && + idle_processors(i) > CHILD_INFO_TABLE[i].senv->max_free_processors)) + ) + || (CHILD_INFO_TABLE[i].type == CHILD_TYPE_MULTIPLEXER && + (multiplexer_idle_timeout > 0 && ap_scoreboard_image->servers[i][0].status == SERVER_READY && + apr_time_sec(now - ap_scoreboard_image->servers[i][0].last_used) > multiplexer_idle_timeout) && + total_processors(i) > CHILD_INFO_TABLE[i].senv->min_processors + ) + ) { CHILD_INFO_TABLE[i].pid = 0; CHILD_INFO_TABLE[i].status = CHILD_STATUS_STANDBY; - if(CHILD_INFO_TABLE[i].type == CHILD_TYPE_WORKER) + if(CHILD_INFO_TABLE[i].type == CHILD_TYPE_WORKER || CHILD_INFO_TABLE[i].type == CHILD_TYPE_MULTIPLEXER) { /* completely free up this slot */ @@ -2173,7 +2388,6 @@ return 1; } -#if 0 #if APR_USE_SYSVSEM_SERIALIZE if (ap_accept_lock_mech == APR_LOCK_DEFAULT || ap_accept_lock_mech == APR_LOCK_SYSVSEM) { @@ -2189,7 +2403,6 @@ return 1; } } -#endif if (!is_graceful) { if (ap_run_pre_mpm(s->process->pool, SB_SHARED) != OK) { @@ -2598,7 +2811,10 @@ ap_listen_pre_config(); ap_min_processors = DEFAULT_MIN_PROCESSORS; ap_min_free_processors = DEFAULT_MIN_FREE_PROCESSORS; + ap_max_free_processors = DEFAULT_MAX_FREE_PROCESSORS; ap_max_processors = DEFAULT_MAX_PROCESSORS; + ap_min_multiplexers = DEFAULT_MIN_MULTIPLEXERS; + ap_max_multiplexers = DEFAULT_MAX_MULTIPLEXERS; ap_daemons_limit = server_limit; ap_pid_fname = DEFAULT_PIDLOG; ap_lock_fname = DEFAULT_LOCKFILE; @@ -2608,6 +2824,13 @@ ap_max_mem_free = APR_ALLOCATOR_MAX_FREE_UNLIMITED; #endif + expire_timeout = DEFAULT_EXPIRE_TIMEOUT; + idle_timeout = DEFAULT_IDLE_TIMEOUT; + multiplexer_idle_timeout = DEFAULT_MULTIPLEXER_IDLE_TIMEOUT; + processor_wait_timeout = DEFAULT_PROCESSOR_WAIT_TIMEOUT; + processor_wait_steps = DEFAULT_PROCESSOR_WAIT_STEPS; + + apr_cpystrn(ap_coredump_dir, ap_server_root, sizeof(ap_coredump_dir)); /* we need to know ServerLimit and ThreadLimit before we start processing @@ -2709,11 +2932,13 @@ server_env_image->control = (server_env_control*)shmem; shmem += sizeof(server_env_control*); server_env_image->table = (server_env_t*)shmem; + } + if(restart_num <= 2) { + _DBG("Cleaning server environments table"); + server_env_image->control->num = 0; - - for (i = 0; i < tmp_server_limit; i++) - { + for (i = 0; i < tmp_server_limit; i++) { SENV[i].processor_id = -1; SENV[i].uid = -1; SENV[i].gid = -1; @@ -2781,8 +3006,8 @@ if (pass_request(r, processor) == -1) { ap_log_error(APLOG_MARK, APLOG_ERR, 0, - ap_server_conf, "Could not pass request to proper " "child, request will not be honoured."); - return DECLINED; + ap_server_conf, "Could not pass request to processor %s (virtualhost %s), request will not be honoured.", + processor->senv->name, r->hostname); } _DBG("doing longjmp",0); longjmp(CHILD_INFO_TABLE[my_child_num].jmpbuffer, 1); @@ -2859,32 +3084,37 @@ ap_rputs("
\n", r); ap_rputs("

peruser status

\n", r); ap_rputs("\n", r); - ap_rputs("" - "" + ap_rputs("" + "" "" "" - "\n", r); + "" + "" + "\n", r); for (x = 0; x < NUM_CHILDS; x++) { senv = CHILD_INFO_TABLE[x].senv; - ap_rprintf(r, "" - "" + ap_rprintf(r, "" + "" "" - "\n", + "\n", CHILD_INFO_TABLE[x].id, CHILD_INFO_TABLE[x].pid, child_status_string(CHILD_INFO_TABLE[x].status), + scoreboard_status_string(SCOREBOARD_STATUS(x)), child_type_string(CHILD_INFO_TABLE[x].type), senv == NULL ? -1 : senv->uid, senv == NULL ? -1 : senv->gid, senv == NULL ? NULL : senv->chroot, + senv == NULL ? 0 : senv->nice_lvl, senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->input, senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->output, CHILD_INFO_TABLE[x].sock_fd, total_processors(x), senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->max_processors, idle_processors(x), - senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->min_free_processors + senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->min_free_processors, + senv == NULL ? -1 : CHILD_INFO_TABLE[x].senv->availability ); } ap_rputs("
IDPIDSTATUSTYPEUIDGIDCHROOTINPUT
IDPIDSTATUSSB STATUSTYPEUIDGIDCHROOTNICEINPUTOUTPUTSOCK_FDTOTAL PROCESSORSMAX PROCESSORSIDLE PROCESSORSMIN FREE PROCESSORS
IDLE PROCESSORSMIN FREE PROCESSORSAVAIL
%3d%5d%8s%12s%4d%4d%25s%5d
%3d%5d%8s%8s%12s%4d%4d%25s%3d%5d%6d%7d%d%d%d%d
%d%d%3d
\n", r); @@ -2938,50 +3168,183 @@ APR_OPTIONAL_HOOK(ap, status_hook, peruser_status_hook, NULL, NULL, APR_HOOK_MIDDLE); } -/* we define an Processor w/ specific uid/gid */ -static const char *cf_Processor(cmd_parms *cmd, void *dummy, - const char *user_name, const char *group_name, const char *chroot) +static const char *cf_Processor(cmd_parms *cmd, void *dummy, const char *arg) { - uid_t uid = ap_uname2id(user_name); - gid_t gid = ap_gname2id(group_name); + const char *user_name = NULL, *group_name = NULL, *directive; + server_env_t senv; + ap_directive_t *current; + + const char *endp = ap_strrchr_c(arg, '>'); + + if (endp == NULL) { + return apr_psprintf(cmd->temp_pool, + "Error: Directive %s> missing closing '>'", cmd->cmd->name); + } + + arg = apr_pstrndup(cmd->pool, arg, endp - arg); + + if (!arg) { + return apr_psprintf(cmd->temp_pool, + "Error: %s> must specify a processor name", cmd->cmd->name); + } + + senv.name = ap_getword_conf(cmd->pool, &arg); + _DBG("processor_name: %s", senv.name); + + if (strlen(senv.name) == 0) { + return apr_psprintf(cmd->temp_pool, + "Error: Directive %s> takes one argument", cmd->cmd->name); + } + + /* Check for existing processors on first launch and between gracefuls */ + if (restart_num == 1 || is_graceful) { + server_env_t *old_senv = find_senv_by_name(senv.name); + + if (old_senv) { + return apr_psprintf(cmd->temp_pool, + "Error: Processor %s already defined", senv.name); + } + } + + senv.nice_lvl = 0; + senv.chroot = NULL; + senv.cgroup = NULL; + senv.min_processors = ap_min_processors; + senv.min_free_processors = ap_min_free_processors; + senv.max_free_processors = ap_max_free_processors; + senv.max_processors = ap_max_processors; + + current = cmd->directive->first_child; + + int proc_temp = 0; + for(; current != NULL; current = current->next) { + directive = current->directive; + + if (!strcasecmp(directive, "user")) { + user_name = current->args; + } + else if (!strcasecmp(directive, "group")) { + group_name = current->args; + } + else if (!strcasecmp(directive, "chroot")) { + senv.chroot = ap_getword_conf(cmd->pool, ¤t->args); + } + else if (!strcasecmp(directive, "nicelevel")) { + senv.nice_lvl = atoi(current->args); + } + else if (!strcasecmp(directive, "maxprocessors")) { + proc_temp = atoi(current->args); + + if (proc_temp < 1) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MaxProcessors > 0, setting to 1"); + proc_temp = 1; + } + + senv.max_processors = proc_temp; + } + else if (!strcasecmp(directive, "minprocessors")) { + proc_temp = atoi(current->args); + + if (proc_temp < 0) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MinProcessors >= 0, setting to 0"); + proc_temp = 0; + } + + senv.min_processors = proc_temp; + } + else if (!strcasecmp(directive, "minspareprocessors")) { + proc_temp = atoi(current->args); - _DBG("user=%s:%d group=%s:%d chroot=%s", - user_name, uid, group_name, gid, chroot); + if (proc_temp < 0) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MinSpareProcessors >= 0, setting to 0"); + proc_temp = 0; + } + + senv.min_free_processors = proc_temp; + } + else if (!strcasecmp(directive, "maxspareprocessors")) { + proc_temp = atoi(current->args); + + if (proc_temp < 0) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MaxSpareProcessors >= 0, setting to 0"); + proc_temp = 0; + } + + senv.max_free_processors = proc_temp; + } + else if (!strcasecmp(directive, "cgroup")) { + senv.cgroup = ap_getword_conf(cmd->pool, ¤t->args); + } + else { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "Unknown directive %s in %s>", directive, cmd->cmd->name); + } + } + + if (user_name == NULL || group_name == NULL) { + return apr_psprintf(cmd->temp_pool, + "Error: User or Group must be set in %s>", cmd->cmd->name); + } + + senv.uid = ap_uname2id(user_name); + senv.gid = ap_gname2id(group_name); + + _DBG("name=%s user=%s:%d group=%s:%d chroot=%s nice_lvl=%d", + senv.name, user_name, senv.uid, group_name, senv.gid, senv.chroot, senv.nice_lvl); + + _DBG("min_processors=%d min_free_processors=%d max_spare_processors=%d max_processors=%d", + senv.min_processors, senv.min_free_processors, senv.max_free_processors, senv.max_processors); return child_add(CHILD_TYPE_PROCESSOR, CHILD_STATUS_STANDBY, - cmd->pool, uid, gid, chroot); + cmd->pool, &senv); } /* we define an Multiplexer child w/ specific uid/gid */ static const char *cf_Multiplexer(cmd_parms *cmd, void *dummy, const char *user_name, const char *group_name, const char *chroot) { - uid_t uid = ap_uname2id(user_name); - gid_t gid = ap_gname2id(group_name); + server_env_t senv; + + senv.name = NULL; + + senv.uid = ap_uname2id(user_name); + senv.gid = ap_gname2id(group_name); + senv.nice_lvl = 0; + senv.cgroup = NULL; + senv.chroot = chroot; + + senv.min_processors = ap_min_multiplexers; + senv.min_free_processors = ap_min_free_processors; + senv.max_free_processors = ap_max_free_processors; + senv.max_processors = ap_max_multiplexers; _DBG("user=%s:%d group=%s:%d chroot=%s [multiplexer id %d]", - user_name, uid, group_name, gid, chroot, NUM_CHILDS); + user_name, senv.uid, group_name, senv.gid, senv.chroot, NUM_CHILDS); return child_add(CHILD_TYPE_MULTIPLEXER, CHILD_STATUS_STARTING, - cmd->pool, uid, gid, chroot); + cmd->pool, &senv); } static const char* cf_ServerEnvironment(cmd_parms *cmd, void *dummy, - const char *user_name, const char *group_name, const char *chroot) + const char *name) { - int uid = ap_uname2id(user_name); - int gid = ap_gname2id(group_name); peruser_server_conf *sconf = PERUSER_SERVER_CONF(cmd->server->module_config); _DBG("function entered", 0); - if (chroot && !ap_is_directory(cmd->pool, chroot)) - return apr_psprintf(cmd->pool, "Error: chroot directory [%s] does not exist", chroot); + sconf->senv = find_senv_by_name(name); - sconf->senv = senv_add(uid, gid, chroot); + if (sconf->senv == NULL) { + return apr_psprintf(cmd->pool, + "Error: Processor %s not defined", name); + } - _DBG("user=%s:%d group=%s:%d chroot=%s numchilds=%d", - user_name, uid, group_name, gid, chroot, NUM_CHILDS); + _DBG("user=%d group=%d chroot=%s numchilds=%d", + sconf->senv->uid, sconf->senv->gid, sconf->senv->chroot, NUM_CHILDS); return NULL; } @@ -3046,10 +3409,10 @@ min_procs = atoi(arg); - if (min_procs < 1) { + if (min_procs < 0) { ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, - "WARNING: Require MaxProcessors > 0, setting to 1"); - min_procs = 1; + "WARNING: Require MinProcessors >= 0, setting to 0"); + min_procs = 0; } if (ap_check_cmd_context(cmd, NOT_IN_VIRTUALHOST) != NULL) { @@ -3075,10 +3438,10 @@ min_free_procs = atoi(arg); - if (min_free_procs < 1) { + if (min_free_procs < 0) { ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, - "WARNING: Require MinSpareProcessors > 0, setting to 1"); - min_free_procs = 1; + "WARNING: Require MinSpareProcessors >= 0, setting to 0"); + min_free_procs = 0; } if (ap_check_cmd_context(cmd, NOT_IN_VIRTUALHOST) != NULL) { @@ -3092,6 +3455,35 @@ return NULL; } +static const char *set_max_free_processors (cmd_parms *cmd, void *dummy, const char *arg) +{ + peruser_server_conf *sconf; + int max_free_procs; + const char *err = ap_check_cmd_context(cmd, NOT_IN_DIR_LOC_FILE|NOT_IN_LIMIT); + + if (err != NULL) { + return err; + } + + max_free_procs = atoi(arg); + + if (max_free_procs < 0) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MaxSpareProcessors >= 0, setting to 0"); + max_free_procs = 0; + } + + if (ap_check_cmd_context(cmd, NOT_IN_VIRTUALHOST) != NULL) { + sconf = PERUSER_SERVER_CONF(cmd->server->module_config); + sconf->senv->max_free_processors = max_free_procs; + } + else { + ap_max_free_processors = max_free_procs; + } + + return NULL; +} + static const char *set_max_processors (cmd_parms *cmd, void *dummy, const char *arg) { peruser_server_conf *sconf; @@ -3121,6 +3513,50 @@ return NULL; } +static const char *set_min_multiplexers (cmd_parms *cmd, void *dummy, const char *arg) +{ + int min_multiplexers; + const char *err = ap_check_cmd_context(cmd, NOT_IN_DIR_LOC_FILE|NOT_IN_LIMIT); + + if (err != NULL) { + return err; + } + + min_multiplexers = atoi(arg); + + if (min_multiplexers < 1) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MinMultiplexers > 0, setting to 1"); + min_multiplexers = 1; + } + + ap_min_multiplexers = min_multiplexers; + + return NULL; +} + +static const char *set_max_multiplexers (cmd_parms *cmd, void *dummy, const char *arg) +{ + int max_multiplexers; + const char *err = ap_check_cmd_context(cmd, NOT_IN_DIR_LOC_FILE|NOT_IN_LIMIT); + + if (err != NULL) { + return err; + } + + max_multiplexers = atoi(arg); + + if (max_multiplexers < 1) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require MaxMultiplexers > 0, setting to 1"); + max_multiplexers = 1; + } + + ap_max_multiplexers = max_multiplexers; + + return NULL; +} + static const char *set_server_limit (cmd_parms *cmd, void *dummy, const char *arg) { int tmp_server_limit; @@ -3183,6 +3619,42 @@ return NULL; } +static const char *set_multiplexer_idle_timeout (cmd_parms *cmd, void *dummy, const char *arg) { + const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY); + + if (err != NULL) { + return err; + } + + multiplexer_idle_timeout = atoi(arg); + + return NULL; +} + +static const char *set_processor_wait_timeout (cmd_parms *cmd, void *dummy, const char *timeout, const char *steps) { + const char *err = ap_check_cmd_context(cmd, GLOBAL_ONLY); + + if (err != NULL) { + return err; + } + + processor_wait_timeout = atoi(timeout); + + if (steps != NULL) { + int steps_tmp = atoi(steps); + + if (steps_tmp < 1) { + ap_log_error(APLOG_MARK, APLOG_STARTUP, 0, NULL, + "WARNING: Require ProcessorWaitTimeout steps > 0, setting to 1"); + steps_tmp = 1; + } + + processor_wait_steps = steps_tmp; + } + + return NULL; +} + static const command_rec peruser_cmds[] = { UNIX_DAEMON_COMMANDS, LISTEN_COMMANDS, @@ -3190,23 +3662,33 @@ "Minimum number of idle children, to handle request spikes"), AP_INIT_TAKE1("MinSpareServers", set_min_free_servers, NULL, RSRC_CONF, "Minimum number of idle children, to handle request spikes"), +AP_INIT_TAKE1("MaxSpareProcessors", set_max_free_processors, NULL, RSRC_CONF, + "Maximum number of idle children, 0 to disable"), AP_INIT_TAKE1("MaxClients", set_max_clients, NULL, RSRC_CONF, "Maximum number of children alive at the same time"), AP_INIT_TAKE1("MinProcessors", set_min_processors, NULL, RSRC_CONF, "Minimum number of processors per vhost"), AP_INIT_TAKE1("MaxProcessors", set_max_processors, NULL, RSRC_CONF, "Maximum number of processors per vhost"), +AP_INIT_TAKE1("MinMultiplexers", set_min_multiplexers, NULL, RSRC_CONF, + "Minimum number of multiplexers the server can have"), +AP_INIT_TAKE1("MaxMultiplexers", set_max_multiplexers, NULL, RSRC_CONF, + "Maximum number of multiplexers the server can have"), AP_INIT_TAKE1("ServerLimit", set_server_limit, NULL, RSRC_CONF, "Maximum value of MaxClients for this run of Apache"), AP_INIT_TAKE1("ExpireTimeout", set_expire_timeout, NULL, RSRC_CONF, - "Maximum idle time before a child is killed, 0 to disable"), + "Maximum time a child can live, 0 to disable"), AP_INIT_TAKE1("IdleTimeout", set_idle_timeout, NULL, RSRC_CONF, "Maximum time before a child is killed after being idle, 0 to disable"), +AP_INIT_TAKE1("MultiplexerIdleTimeout", set_multiplexer_idle_timeout, NULL, RSRC_CONF, + "Maximum time before a multiplexer is killed after being idle, 0 to disable"), +AP_INIT_TAKE12("ProcessorWaitTimeout", set_processor_wait_timeout, NULL, RSRC_CONF, + "Maximum time a multiplexer waits for the processor if it is busy"), AP_INIT_TAKE23("Multiplexer", cf_Multiplexer, NULL, RSRC_CONF, "Specify an Multiplexer Child configuration."), -AP_INIT_TAKE23("Processor", cf_Processor, NULL, RSRC_CONF, - "Specify a User and Group for a specific child process."), -AP_INIT_TAKE23("ServerEnvironment", cf_ServerEnvironment, NULL, RSRC_CONF, +AP_INIT_RAW_ARGS("