WWWWWW

Current Directory: /opt/cpanel/ea-ruby24/src/passenger-release-6.0.20/src/agent/Watchdog
Viewing File: /opt/cpanel/ea-ruby24/src/passenger-release-6.0.20/src/agent/Watchdog/WatchdogMain.cpp
/*
 *  Phusion Passenger - https://www.phusionpassenger.com/
 *  Copyright (c) 2010-2018 Phusion Holding B.V.
 *
 *  "Passenger", "Phusion Passenger" and "Union Station" are registered
 *  trademarks of Phusion Holding B.V.
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a copy
 *  of this software and associated documentation files (the "Software"), to deal
 *  in the Software without restriction, including without limitation the rights
 *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 *  copies of the Software, and to permit persons to whom the Software is
 *  furnished to do so, subject to the following conditions:
 *
 *  The above copyright notice and this permission notice shall be included in
 *  all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 *  THE SOFTWARE.
 */
// Include ev++.h early to avoid macro clash on EV_ERROR.
#include <ev++.h>

#include <oxt/thread.hpp>
#include <oxt/system_calls.hpp>
#include <boost/function.hpp>
#include <boost/bind/bind.hpp>
#include <boost/make_shared.hpp>
#include <boost/foreach.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <string>
#include <utility>
#include <vector>
#include <algorithm>

#if !defined(sun) && !defined(__sun)
	#define HAVE_FLOCK
#endif

#ifdef __linux__
	#include <sys/prctl.h>
#endif
#include <sys/select.h>
#include <sys/types.h>
#include <sys/time.h>
#ifdef HAVE_FLOCK
	#include <sys/file.h>
#endif
#include <sys/resource.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
#include <fcntl.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cerrno>

#include <jsoncpp/json.h>
#include <Shared/Fundamentals/Initialization.h>
#include <Shared/ApiServerUtils.h>
#include <Core/OptionParser.h>
#include <Watchdog/Config.h>
#include <Watchdog/ApiServer.h>
#include <JsonTools/Autocast.h>
#include <Constants.h>
#include <InstanceDirectory.h>
#include <FileDescriptor.h>
#include <FileTools/PathSecurityCheck.h>
#include <SystemTools/UserDatabase.h>
#include <SystemTools/ContainerHelpers.h>
#include <RandomGenerator.h>
#include <BackgroundEventLoop.h>
#include <LoggingKit/LoggingKit.h>
#include <Exceptions.h>
#include <StaticString.h>
#include <Hooks.h>
#include <IOTools/IOUtils.h>
#include <IOTools/MessageIO.h>
#include <Utils.h>
#include <Utils/Timer.h>
#include <Utils/ScopeGuard.h>
#include <StrIntTools/StrIntUtils.h>
#include <Utils/OptionParsing.h>
#include <Utils/VariantMap.h>

using namespace std;
using namespace boost;
using namespace oxt;
using namespace Passenger;
using namespace Passenger::Agent::Fundamentals;


enum OomFileType {
	OOM_ADJ,
	OOM_SCORE_ADJ
};

#define REQUEST_SOCKET_PASSWORD_SIZE     64

class InstanceDirToucher;
class AgentWatcher;


/***** Working objects *****/

namespace Passenger {
namespace Watchdog {
	struct WorkingObjects {
		RandomGenerator randomGenerator;
		EventFd errorEvent;
		EventFd exitEvent;
		uid_t defaultUid;
		gid_t defaultGid;
		InstanceDirectoryPtr instanceDir;
		int startupReportFile;
		int lockFile;
		vector<string> cleanupPidfiles;
		bool pidsCleanedUp;
		bool pidFileCleanedUp;
		string corePidFile;
		string fdPassingPassword;
		Json::Value extraConfigToPassToSubAgents;
		Json::Value controllerAddresses;
		Json::Value coreApiServerAddresses;
		Json::Value coreApiServerAuthorizations;
		Json::Value watchdogApiServerAddresses;
		Json::Value watchdogApiServerAuthorizations;

		int apiServerFds[SERVER_KIT_MAX_SERVER_ENDPOINTS];
		BackgroundEventLoop *bgloop;
		ServerKit::Context *serverKitContext;
		ServerKit::Schema serverKitSchema;
		ApiServer::ApiServer *apiServer;

		WorkingObjects()
			: errorEvent(__FILE__, __LINE__, "WorkingObjects: errorEvent"),
			  exitEvent(__FILE__, __LINE__, "WorkingObjects: exitEvent"),
			  startupReportFile(-1),
			  pidsCleanedUp(false),
			  pidFileCleanedUp(false),
			  extraConfigToPassToSubAgents(Json::objectValue),
			  controllerAddresses(Json::arrayValue),
			  coreApiServerAddresses(Json::arrayValue),
			  coreApiServerAuthorizations(Json::arrayValue),
			  watchdogApiServerAddresses(Json::arrayValue),
			  watchdogApiServerAuthorizations(Json::arrayValue),
			  bgloop(NULL),
			  serverKitContext(NULL),
			  apiServer(NULL)
		{
			for (unsigned int i = 0; i < SERVER_KIT_MAX_SERVER_ENDPOINTS; i++) {
				apiServerFds[i] = -1;
			}
		}
	};

	typedef boost::shared_ptr<WorkingObjects> WorkingObjectsPtr;
} // namespace Watchdog
} // namespace Passenger

using namespace Passenger::Watchdog;

static WrapperRegistry::Registry *watchdogWrapperRegistry;
static Schema *watchdogSchema;
static ConfigKit::Store *watchdogConfig;
static WorkingObjects *workingObjects;

static void cleanup(const WorkingObjectsPtr &wo);

#include "AgentWatcher.cpp"
#include "InstanceDirToucher.cpp"
#include "CoreWatcher.cpp"


/***** Functions *****/

#if !BOOST_OS_MACOS

struct WatchdogOomAdjustResult {
	struct Message {
		LoggingKit::Level level;
		string text;
	};

	string oldScore;
	// LoggingKit has not been initialized yet when setOomScoreNeverKill()
	// is called, so we store the messages here and print them after
	// LoggingKit initialization.
	vector<Message> messages;
};

static FILE *
openOomAdjFileGetType(const char *mode, OomFileType &type, string &path) {
	FILE *f = fopen("/proc/self/oom_score_adj", mode);
	if (f == NULL) {
		f = fopen("/proc/self/oom_adj", mode);
		if (f == NULL) {
			return NULL;
		} else {
			type = OOM_ADJ;
			path = "/proc/self/oom_adj";
			return f;
		}
	} else {
		type = OOM_SCORE_ADJ;
		path = "/proc/self/oom_score_adj";
		return f;
	}
}

/**
 * Set the current process's OOM score to "never kill".
 */
static WatchdogOomAdjustResult
setOomScoreNeverKill() {
	WatchdogOomAdjustResult result;
	FILE *f;
	string path;
	OomFileType type;
	int e;

	if (geteuid() != 0) {
		WatchdogOomAdjustResult::Message msg;
		msg.level = LoggingKit::DEBUG;
		msg.text = "Not adjusting Watchdog's OOM score because not running with root privileges";
		result.messages.push_back(msg);
		return result;
	}

	f = openOomAdjFileGetType("r", type, path);
	if (f == NULL) {
		e = errno;
		P_ERROR("Error adjusting Watchdog's OOM score: error opening both"
			" /proc/self/oom_score_adj and /proc/self/oom_adj for reading: " <<
			strerror(e) << " (errno=" << e << ")");
		return result;
	}
	// mark if this is a legacy score so we won't try to write it as OOM_SCORE_ADJ
	if (type == OOM_ADJ) {
		result.oldScore.append("l");
	}
	char buf[1024];
	size_t bytesRead;
	while (true) {
		bytesRead = fread(buf, 1, sizeof(buf), f);
		if (bytesRead == 0 && feof(f)) {
			break;
		} else if (bytesRead == 0 && ferror(f)) {
			P_ERROR("Error adjusting Watchdog's OOM score: error reading " << path);
			fclose(f);
			result.oldScore.clear();
			return result;
		} else {
			result.oldScore.append(buf, bytesRead);
		}
	}
	fclose(f);

	f = fopen(path.c_str(), "w");
	if (f == NULL) {
		e = errno;
		WatchdogOomAdjustResult::Message msg;
		msg.level = LoggingKit::ERROR;
		msg.text = "Error adjusting Watchdog's OOM score: error opening "
			+ path + " for writing: " + strerror(e) + " (errno="
			+ toString(e) + ")";
		result.messages.push_back(msg);
		result.oldScore.clear();
		return result;
	}
	if (type == OOM_SCORE_ADJ) {
		fprintf(f, "-1000\n");
	} else {
		assert(type == OOM_ADJ);
		fprintf(f, "-17\n");
	}

	e = fflush(f);
	if (e != 0) {
		e = errno;
		WatchdogOomAdjustResult::Message msg;
		if (autoDetectInContainer()) {
			msg.level = LoggingKit::INFO;
			msg.text = "Running in container, so couldn't adjust Watchdog's"
				" OOM score through " + path;
		} else {
			msg.level = LoggingKit::ERROR;
			msg.text = "Error adjusting Watchdog's OOM score: error writing to "
				+ path + ": " + strerror(e) + " (errno=" + toString(e) + ")";
		}
		result.messages.push_back(msg);
	}
	e = fclose(f);
	if (e == EOF) {
		e = errno;
		WatchdogOomAdjustResult::Message msg;
		msg.level = LoggingKit::ERROR;
		msg.text = "Error adjusting Watchdog's OOM score: error closing "
			+ path + ": " + strerror(e) + " (errno=" + toString(e) + ")";
		result.messages.push_back(msg);
	}

	return result;
}

static void
printOomAdjustResultMessages(const WatchdogOomAdjustResult &result) {
	vector<WatchdogOomAdjustResult::Message>::const_iterator it;

	for (it = result.messages.begin(); it != result.messages.end(); it++) {
		P_LOG(LoggingKit::context, it->level, __FILE__, __LINE__, it->text);
	}
}

#endif

static void
terminationHandler(int signo) {
	ssize_t ret = write(workingObjects->exitEvent.writerFd(), "x", 1);
	(void) ret; // Don't care about the result.
}

/**
 * Wait until the starter process has exited or sent us an exit command,
 * or until one of the watcher threads encounter an error. If a thread
 * encountered an error then the error message will be printed.
 *
 * Returns whether this watchdog should exit gracefully, which is only the
 * case if the web server sent us an exit command and no thread encountered
 * an error.
 */
static bool
waitForStarterProcessOrWatchers(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers) {
	TRACE_POINT();
	fd_set fds;
	int max = -1, ret;
	char x;

	wo->bgloop->start("Main event loop", 0);

	struct sigaction action;
	action.sa_handler = terminationHandler;
	action.sa_flags = SA_RESTART;
	sigemptyset(&action.sa_mask);
	sigaction(SIGINT, &action, NULL);
	sigaction(SIGTERM, &action, NULL);

	FD_ZERO(&fds);
	if (feedbackFdAvailable()) {
		FD_SET(FEEDBACK_FD, &fds);
		max = std::max(max, FEEDBACK_FD);
	}
	FD_SET(wo->errorEvent.fd(), &fds);
	max = std::max(max, wo->errorEvent.fd());
	FD_SET(wo->exitEvent.fd(), &fds);
	max = std::max(max, wo->exitEvent.fd());

	UPDATE_TRACE_POINT();
	ret = syscalls::select(max + 1, &fds, NULL, NULL, NULL);
	if (ret == -1) {
		int e = errno;
		P_ERROR("select() failed: " << strerror(e));
		return false;
	}

	action.sa_handler = SIG_DFL;
	sigaction(SIGINT, &action, NULL);
	sigaction(SIGTERM, &action, NULL);

	P_DEBUG("Stopping API server");
	wo->bgloop->stop();
	for (unsigned int i = 0; i < SERVER_KIT_MAX_SERVER_ENDPOINTS; i++) {
		if (wo->apiServerFds[i] != -1) {
			syscalls::close(wo->apiServerFds[i]);
		}
	}

	if (FD_ISSET(wo->errorEvent.fd(), &fds)) {
		UPDATE_TRACE_POINT();
		vector<AgentWatcherPtr>::const_iterator it;
		string message, backtrace, watcherName;

		for (it = watchers.begin(); it != watchers.end() && message.empty(); it++) {
			message   = (*it)->getErrorMessage();
			backtrace = (*it)->getErrorBacktrace();
			watcherName = (*it)->name();
		}

		if (!message.empty() && backtrace.empty()) {
			P_ERROR("Error in " << watcherName << " watcher:\n  " << message);
		} else if (!message.empty() && !backtrace.empty()) {
			P_ERROR("Error in " << watcherName << " watcher:\n  " <<
				message << "\n" << backtrace);
		}
		return false;
	} else if (FD_ISSET(wo->exitEvent.fd(), &fds)) {
		return true;
	} else {
		UPDATE_TRACE_POINT();
		assert(feedbackFdAvailable());
		ret = syscalls::read(FEEDBACK_FD, &x, 1);
		return ret == 1 && x == 'c';
	}
}

string
relative(string filename){
	string dir = filename.substr(filename.find_last_of('/')+1);
	return dir;
}

static vector<pid_t>
readCleanupPids(const WorkingObjectsPtr &wo) {
	vector<pid_t> result;

	foreach (string filename, wo->cleanupPidfiles) {
		FILE *f = fopen(relative(filename).c_str(), "r");
		if (f != NULL) {
			char buf[33];
			size_t ret;

			ret = fread(buf, 1, 32, f);
			fclose(f);
			if (ret > 0) {
				buf[ret] = '\0';
				result.push_back(atoi(buf));
			} else {
				P_WARN("Cannot read cleanup PID file " << relative(filename).c_str() << " (" << filename << ")");
			}
		} else {
			P_WARN("Cannot open cleanup PID file " << relative(filename).c_str() << " (" << filename << ")");
		}
	}

	return result;
}

static void
killCleanupPids(const vector<pid_t> &cleanupPids) {
	foreach (pid_t pid, cleanupPids) {
		P_DEBUG("Sending SIGTERM to cleanup PID " << pid);
		if(kill(pid, SIGTERM) == -1){
			int e = errno;
			P_WARN("Failed to send SIGTERM to " << pid << ", error: " << e << " " << strerror(e));
		}
	}
}

static void
killCleanupPids(const WorkingObjectsPtr &wo) {
	if (!wo->pidsCleanedUp) {
		killCleanupPids(readCleanupPids(wo));
		wo->pidsCleanedUp = true;
	}
}

static void
deletePidFile(const WorkingObjectsPtr &wo) {
	Json::Value pidFile = watchdogConfig->get("watchdog_pid_file");
	if (!pidFile.isNull() && !wo->pidFileCleanedUp && watchdogConfig->get("watchdog_pid_file_autodelete").asBool()) {
		syscalls::unlink(pidFile.asCString());
		wo->pidFileCleanedUp = true;
	}
}

static void
cleanupAgentsInBackground(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers, char *argv[]) {
	boost::this_thread::disable_interruption di;
	boost::this_thread::disable_syscall_interruption dsi;
	pid_t pid;
	int e;

	pid = fork();
	if (pid == 0) {
		// Child
		try {
			vector<AgentWatcherPtr>::const_iterator it;
			Timer<SystemTime::GRAN_10MSEC> timer(false);
			fd_set fds, fds2;
			int max, agentProcessesDone;
			unsigned long long deadline = 30000; // miliseconds

			#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(sun)
				// Change process title.
				strcpy(argv[0], "PassengerWatchdog (cleaning up...)");
			#endif

			P_DEBUG("Sending SIGTERM to all agent processes");
			for (it = watchers.begin(); it != watchers.end(); it++) {
				(*it)->signalShutdown();
			}

			max = 0;
			FD_ZERO(&fds);
			for (it = watchers.begin(); it != watchers.end(); it++) {
				FD_SET((*it)->getFeedbackFd(), &fds);
				if ((*it)->getFeedbackFd() > max) {
					max = (*it)->getFeedbackFd();
				}
			}

			P_DEBUG("Waiting until all agent processes have exited...");
			timer.start();
			agentProcessesDone = 0;
			while (agentProcessesDone != -1
			    && agentProcessesDone < (int) watchers.size()
			    && timer.elapsed() < deadline)
			{
				struct timeval timeout;

				#ifdef FD_COPY
					FD_COPY(&fds, &fds2);
				#else
					FD_ZERO(&fds2);
					for (it = watchers.begin(); it != watchers.end(); it++) {
						FD_SET((*it)->getFeedbackFd(), &fds2);
					}
				#endif

				timeout.tv_sec = 0;
				timeout.tv_usec = 10000;
				agentProcessesDone = syscalls::select(max + 1, &fds2, NULL, NULL, &timeout);
				if (agentProcessesDone > 0 && timer.elapsed() < deadline) {
					usleep(10000);
				}
			}

			if (agentProcessesDone == -1 || timer.elapsed() >= deadline) {
				// An error occurred or we've waited long enough. Kill all the
				// processes.
				P_WARN("Some " PROGRAM_NAME " agent processes did not exit " <<
					"in time, forcefully shutting down all.");
			} else {
				P_DEBUG("All " PROGRAM_NAME " agent processes have exited. Forcing all subprocesses to shut down.");
			}
			P_DEBUG("Sending SIGKILL to all agent processes");
			for (it = watchers.begin(); it != watchers.end(); it++) {
				(*it)->forceShutdown();
			}

			cleanup(wo);

			_exit(0);
		} catch (const std::exception &e) {
			P_CRITICAL("An exception occurred during cleaning up: " << e.what());
			_exit(1);
		} catch (...) {
			P_CRITICAL("An unknown exception occurred during cleaning up");
			_exit(1);
		}

	} else if (pid == -1) {
		// Error
		e = errno;
		throw SystemException("fork() failed", e);

	} else {
		// Parent

		// Let child process handle cleanup.
		wo->instanceDir->detach();
	}
}

static void
forceAllAgentsShutdown(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers) {
	vector<AgentWatcherPtr>::iterator it;

	P_DEBUG("Sending SIGTERM to all agent processes");
	for (it = watchers.begin(); it != watchers.end(); it++) {
		(*it)->signalShutdown();
	}
	usleep(1000000);
	P_DEBUG("Sending SIGKILL to all agent processes");
	for (it = watchers.begin(); it != watchers.end(); it++) {
		(*it)->forceShutdown();
	}
}

static void
runHookScriptAndThrowOnError(const char *name) {
	TRACE_POINT();
	HookScriptOptions options;

	options.name = name;
	options.spec = watchdogConfig->get(string("hook_") + name).asString();
	options.agentConfig = watchdogConfig->inspectEffectiveValues();

	if (!runHookScripts(options)) {
		throw RuntimeException(string("Hook script ") + name + " failed");
	}
}


static void
usage() {
	printf("Usage: " AGENT_EXE " watchdog <OPTIONS...>\n");
	printf("Runs the " PROGRAM_NAME " watchdog.\n\n");
	printf("The watchdog runs and supervises various " PROGRAM_NAME " agent processes,\n");
	printf("at this moment only the core (there was also a process called UstRouter but\n");
	printf("it no longer exists). Arguments marked with \"[A]\", e.g.\n");
	printf("--passenger-root and --log-level, are automatically passed to all supervised\n");
	printf("agents, unless you explicitly override them by passing extra arguments to a\n");
	printf("supervised agent specifically. You can pass arguments to a supervised agent by\n");
	printf("wrapping those arguments between --BC/--EC and --BU/--EU.\n");
	printf("\n");
	printf("  Example 1: pass some arguments to the core.\n\n");
	printf("  " SHORT_PROGRAM_NAME " watchdog --passenger-root /opt/passenger \\\n");
	printf("    --BC --listen tcp://127.0.0.1:4000 /webapps/foo\n");
	printf("\n");
	printf("  Example 2: pass some arguments to the core, and some others to the\n");
	printf("  UstRouter. The watchdog itself and the core will use logging\n");
	printf("  level 3, while the UstRouter will use logging level 1.\n\n");
	printf("  " SHORT_PROGRAM_NAME " watchdog --passenger-root /opt/passenger \\\n");
	printf("    --BC --listen tcp://127.0.0.1:4000 /webapps/foo --EC \\\n");
	printf("    --BU --log-level 1 --EU \\\n");
	printf("    --log-level 3\n");
	printf("\n");
	printf("Required options:\n");
	printf("       --passenger-root PATH  The location to the " PROGRAM_NAME " source\n");
	printf("                              directory [A]\n");
	printf("\n");
	printf("Argument passing options (optional):\n");
	printf("  --BC, --begin-core-args   Signals the beginning of arguments to pass to the\n");
	printf("                            Passenger core\n");
	printf("  --EC, --end-core-args     Signals the end of arguments to pass to the\n");
	printf("                            Passenger core\n");
	printf("  --BU, --begin-ust-router-args\n");
	printf("                            Signals the beginning of arguments to pass to the\n");
	printf("                            UstRouter\n");
	printf("  --EU, --end-ust-router-args\n");
	printf("                              Signals the end of arguments to pass to the\n");
	printf("                            UstRouter\n");
	printf("\n");
	printf("Other options (optional):\n");
	printf("      --api-listen ADDRESS  Listen on the given address for API commands.\n");
	printf("                            The address must be formatted as tcp://IP:PORT for\n");
	printf("                            TCP sockets, or unix:PATH for Unix domain sockets.\n");
	printf("                            You can specify this option multiple times (up to\n");
	printf("                            %u times) to listen on multiple addresses.\n",
		SERVER_KIT_MAX_SERVER_ENDPOINTS - 1);
	printf("      --authorize [LEVEL]:USERNAME:PASSWORDFILE\n");
	printf("                            Enables authentication on the API server, through\n");
	printf("                            the given API account. LEVEL indicates the\n");
	printf("                            privilege level (see below). PASSWORDFILE must\n");
	printf("                            point to a file containing the password\n");
	printf("\n");
	printf("      --instance-registry-dir  Directory to register instance into.\n");
	printf("                               Default: %s\n", getSystemTempDir());
	printf("\n");
	printf("      --spawn-dir              Directory for spawn handshakes.\n");
	printf("                               Default: %s\n", getSystemTempDir());
	printf("\n");
	printf("      --no-user-switching     Disables user switching support [A]\n");
	printf("      --default-user NAME     Default user to start apps as, when user\n");
	printf("                              switching is enabled. Default: " DEFAULT_WEB_APP_USER "\n");
	printf("      --default-group NAME    Default group to start apps as, when user\n");
	printf("                              switching is disabled. Default: the default\n");
	printf("                              user's primary group\n");
	printf("\n");
	printf("      --daemonize             Daemonize into the background\n");
	printf("      --user NAME             Lower privilege to the given user\n");
	printf("      --pid-file PATH         Store the watchdog's PID in the given file. The\n");
	printf("                              file is deleted on exit\n");
	printf("      --no-delete-pid-file    Do not delete PID file on exit\n");
	printf("      --log-file PATH         Log to the given file.\n");
	printf("      --log-level LEVEL       Logging level. [A] Default: %d\n", DEFAULT_LOG_LEVEL);
	printf("      --disable-log-prefix    Disables prefixing application logs with App PID stdout\n");
	printf("      --report-file PATH      Upon successful initialization, report instance\n");
	printf("                              information to the given file, in JSON format\n");
	printf("      --cleanup-pidfile PATH  Upon shutdown, kill the process specified by\n");
	printf("                              the given PID file\n");
	printf("\n");
	printf("      --ctl NAME=VALUE        Set custom internal option\n");
	printf("\n");
	printf("  -h, --help                  Show this help\n");
	printf("\n");
	printf("[A] = Automatically passed to supervised agents\n");
	printf("\n");
	printf("API account privilege levels (ordered from most to least privileges):\n");
	printf("  readonly    Read-only access\n");
	printf("  full        Full access (default)\n");
}

static void
parseOptions(int argc, const char *argv[], ConfigKit::Store &config) {
	OptionParser p(usage);
	Json::Value updates(Json::objectValue);
	int i = 2;

	while (i < argc) {
		if (p.isValueFlag(argc, i, argv[i], '\0', "--passenger-root")) {
			updates["passenger_root"] = argv[i + 1];
			i += 2;
		} else if (p.isFlag(argv[i], '\0', "--BC")
			|| p.isFlag(argv[i], '\0', "--begin-core-args"))
		{
			i++;
			while (i < argc) {
				if (p.isFlag(argv[i], '\0', "--EC")
				 || p.isFlag(argv[i], '\0', "--end-core-args"))
				{
					i++;
					break;
				} else if (p.isFlag(argv[i], '\0', "--BU")
				 || p.isFlag(argv[i], '\0', "--begin-ust-router-args"))
				{
					break;
				} else {
					Json::Value coreUpdates(Json::objectValue);
					if (!parseCoreOption(argc, argv, i, coreUpdates)) {
						fprintf(stderr, "ERROR: unrecognized core argument %s. Please "
							"type '%s core --help' for usage.\n", argv[i], argv[0]);
						exit(1);
					}

					Json::Value::iterator it, end = coreUpdates.end();
					for (it = coreUpdates.begin(); it != end; it++) {
						string translatedName = watchdogSchema->core.translator.
							reverseTranslateOne(it.name());
						updates[translatedName] = *it;
					}
				}
			}
		} else if (p.isFlag(argv[i], '\0', "--BU")
			|| p.isFlag(argv[i], '\0', "--begin-ust-router-args"))
		{
			i++;
			while (i < argc) {
				if (p.isFlag(argv[i], '\0', "--EU")
				 || p.isFlag(argv[i], '\0', "--end-ust-router-args"))
				{
					i++;
					break;
				} else if (p.isFlag(argv[i], '\0', "--BC")
				 || p.isFlag(argv[i], '\0', "--begin-core-args"))
				{
					break;
				} else {
					fprintf(stderr, "ERROR: unrecognized UstRouter argument %s. Please "
						"type '%s ust-router --help' for usage.\n", argv[i], argv[0]);
					exit(1);
				}
			}
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--api-listen")) {
			if (getSocketAddressType(argv[i + 1]) != SAT_UNKNOWN) {
				Json::Value &addresses = updates["watchdog_api_server_addresses"];
				if (addresses.size() == SERVER_KIT_MAX_SERVER_ENDPOINTS - 1) {
					fprintf(stderr, "ERROR: you may specify up to %u --api-listen addresses.\n",
						SERVER_KIT_MAX_SERVER_ENDPOINTS - 1);
					exit(1);
				}
				addresses.append(argv[i + 1]);
				i += 2;
			} else {
				fprintf(stderr, "ERROR: invalid address format for --api-listen. The address "
					"must be formatted as tcp://IP:PORT for TCP sockets, or unix:PATH "
					"for Unix domain sockets.\n");
				exit(1);
			}
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--authorize")) {
			vector<string> args;
			split(argv[i + 1], ':', args);
			if (args.size() < 2 || args.size() > 3) {
				fprintf(stderr, "ERROR: invalid format for --authorize. The syntax "
					"is \"[LEVEL:]USERNAME:PASSWORDFILE\".\n");
				exit(1);
			}

			updates["watchdog_api_server_authorizations"].append(argv[i + 1]);
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--instance-registry-dir")) {
			updates["instance_registry_dir"] = argv[i + 1];
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--spawn-dir")) {
			updates["spawn_dir"] = argv[i + 1];
			i += 2;
		} else if (p.isFlag(argv[i], '\0', "--no-user-switching")) {
			updates["user_switching"] = false;
			i++;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--default-user")) {
			updates["default_user"] = argv[i + 1];
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--default-group")) {
			updates["default_group"] = argv[i + 1];
			i += 2;
		} else if (p.isFlag(argv[i], '\0', "--daemonize")) {
			updates["daemonize"] = true;
			i++;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--user")) {
			updates["user"] = argv[i + 1];
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--pid-file")) {
			updates["watchdog_pid_file"] = argv[i + 1];
			i += 2;
		} else if (p.isFlag(argv[i], '\0', "--no-delete-pid-file")) {
			updates["watchdog_pid_file_autodelete"] = false;
			i++;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--log-level")) {
			updates["log_level"] = argv[i + 1];
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--report-file")) {
			updates["startup_report_file"] = argv[i + 1];
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--cleanup-pidfile")) {
			updates["pidfiles_to_delete_on_exit"].append(argv[i + 1]);
			i += 2;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--log-file")) {
			updates["log_target"] = argv[i + 1];
			i += 2;
		} else if (p.isFlag(argv[i], '\0', "--disable-log-prefix")) {
			updates["disable_log_prefix"] = true;
			i++;
		} else if (p.isValueFlag(argc, i, argv[i], '\0', "--ctl")) {
			const char *value = strchr(argv[i + 1], '=');
			if (value == NULL) {
				fprintf(stderr, "ERROR: '%s' is not a valid --ctl parameter. "
					"It must be in the form of NAME=VALUE.\n", argv[i + 1]);
				exit(1);
			}
			string name(argv[i + 1], value - argv[i + 1]);
			value++;
			if (*value == '\0') {
				fprintf(stderr, "ERROR: '%s' is not a valid --ctl parameter. "
					"The value must be non-empty.\n", argv[i + 1]);
				exit(1);
			}
			updates[name] = autocastValueToJson(value);
			i += 2;
		} else if (p.isFlag(argv[i], 'h', "--help")) {
			usage();
			exit(0);
		} else {
			fprintf(stderr, "ERROR: unrecognized argument %s. Please type "
				"'%s watchdog --help' for usage.\n", argv[i], argv[0]);
			exit(1);
		}
	}

	if (!updates.empty()) {
		vector<ConfigKit::Error> errors;
		if (!config.update(updates, errors)) {
			P_BUG("Unable to set initial configuration: " <<
				ConfigKit::toString(errors) << "\n"
				"Raw initial configuration: " << updates.toStyledString());
		}
	}
}

static void
initializeBareEssentials(int argc, char *argv[], WorkingObjectsPtr &wo) {
	/*
	 * Some Apache installations (like on OS X) redirect stdout to /dev/null,
	 * so that only stderr is redirected to the log file. We therefore
	 * forcefully redirect stdout to stderr so that everything ends up in the
	 * same place.
	 */
	dup2(2, 1);

	/*
	 * Most operating systems overcommit memory. We *know* that this watchdog process
	 * doesn't use much memory; on OS X it uses about 200 KB of private RSS. If the
	 * watchdog is killed by the system Out-Of-Memory Killer or then it's all over:
	 * the system administrator will have to restart the web server for Phusion
	 * Passenger to be usable again. So here we disable Linux's OOM killer
	 * for this watchdog. Note that the OOM score is inherited by child processes
	 * so we need to restore it after each fork().
	 */
#if !BOOST_OS_MACOS
	WatchdogOomAdjustResult oomAdjustResult = setOomScoreNeverKill();
#endif

	watchdogWrapperRegistry = new WrapperRegistry::Registry();
	watchdogWrapperRegistry->finalize();
	watchdogSchema = new Schema(watchdogWrapperRegistry);
	watchdogConfig = new ConfigKit::Store(*watchdogSchema);
	initializeAgent(argc, &argv, SHORT_PROGRAM_NAME " watchdog",
		*watchdogConfig, watchdogSchema->core.schema.loggingKit.translator,
		parseOptions, NULL, 2);

	// Start all sub-agents with this environment variable.
	setenv("PASSENGER_USE_FEEDBACK_FD", "true", 1);

	wo = boost::make_shared<WorkingObjects>();
	workingObjects = wo.get();
#if !BOOST_OS_MACOS
	printOomAdjustResultMessages(oomAdjustResult);
	wo->extraConfigToPassToSubAgents["oom_score"] = oomAdjustResult.oldScore;
#endif
}

static void
maybeSetsid() {
	/* Become the session leader so that Apache can't kill the
	 * watchdog with killpg() during shutdown, so that a
	 * Ctrl-C only affects the web server, and so that
	 * we can kill all of our subprocesses in a single killpg().
	 *
	 * WatchdogLauncher.h already calls setsid() before exec()ing
	 * the Watchdog, but Flying Passenger does not.
	 */
	if (watchdogConfig->get("setsid").asBool()) {
		setsid();
	}
}

static void
redirectStdinToNull() {
	int fd = open("/dev/null", O_RDONLY);
	if (fd != -1) {
		dup2(fd, 0);
		close(fd);
	}
}

static void
maybeDaemonize() {
	pid_t pid;
	int e;

	if (watchdogConfig->get("daemonize").asBool()) {
		pid = fork();
		if (pid == 0) {
			setsid();
			redirectStdinToNull();
		} else if (pid == -1) {
			e = errno;
			throw SystemException("Cannot fork", e);
		} else {
			_exit(0);
		}
	}
}

static void
createPidFile() {
	TRACE_POINT();
	Json::Value pidFile = watchdogConfig->get("watchdog_pid_file");
	if (!pidFile.isNull()) {
		char pidStr[32];

		snprintf(pidStr, sizeof(pidStr), "%lld", (long long) getpid());

		int fd = syscalls::open(pidFile.asCString(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
		if (fd == -1) {
			int e = errno;
			throw FileSystemException("Cannot create PID file " + pidFile.asString(),
				e, pidFile.asString());
		}

		UPDATE_TRACE_POINT();
		FdGuard guard(fd, __FILE__, __LINE__);
		writeExact(fd, pidStr, strlen(pidStr));
	}
}

static void
openStartupReportFile(const WorkingObjectsPtr &wo) {
	TRACE_POINT();
	Json::Value path = watchdogConfig->get("startup_report_file");
	if (!path.isNull()) {
		int fd = syscalls::open(path.asCString(), O_WRONLY | O_CREAT | O_TRUNC, 0600);
		if (fd == -1) {
			int e = errno;
			throw FileSystemException("Cannot open report file " + path.asString(),
				e, path.asString());
		}

		P_LOG_FILE_DESCRIPTOR_OPEN4(fd, __FILE__, __LINE__, "WorkingObjects: startupReportFile");
		wo->startupReportFile = fd;
	}
}

static void
chdirToTmpDir() {
	const Json::Value pidfiles = watchdogConfig->get("pidfiles_to_delete_on_exit");
	if (!pidfiles.empty()) {
		string str = pidfiles[0].asString();
		string dir = str.substr(0,str.find_last_of('/'));
		if (dir != "" && chdir(dir.c_str()) == -1) {
			throw RuntimeException("Cannot change working directory to " + dir);
		}
	}
}

static void
lowerPrivilege() {
	TRACE_POINT();
	string userName = watchdogConfig->get("user").asString();

	if (geteuid() == 0 && !userName.empty()) {
		OsUser osUser;
		if (!lookupSystemUserByName(userName, osUser)) {
			throw NonExistentUserException("Operating system user '" + userName
				+ "' does not exist");
		}

		gid_t gid = osUser.pwd.pw_gid;
		string groupName = lookupSystemGroupnameByGid(gid);

		if (initgroups(userName.c_str(), gid) != 0) {
			int e = errno;
			throw SystemException("Unable to lower " SHORT_PROGRAM_NAME " watchdog's privilege "
				"to that of user '" + userName + "' and group '" + groupName +
				"': cannot set supplementary groups", e);
		}
		if (setgid(gid) != 0) {
			int e = errno;
			throw SystemException("Unable to lower " SHORT_PROGRAM_NAME " watchdog's privilege "
				"to that of user '" + userName + "' and group '" + groupName +
				"': cannot set group ID to " + toString(gid), e);
		}
		if (setuid(osUser.pwd.pw_uid) != 0) {
			int e = errno;
			throw SystemException("Unable to lower " SHORT_PROGRAM_NAME " watchdog's privilege "
				"to that of user '" + userName + "' and group '" + groupName +
				"': cannot set user ID to " + toString(osUser.pwd.pw_uid), e);
		}
		#ifdef __linux__
			// When we change the uid, /proc/self/pid contents don't change owner,
			// causing us to lose access to our own /proc/self/pid files.
			// This prctl call changes those files' ownership.
			// References:
			// https://stackoverflow.com/questions/8337846/files-ownergroup-doesnt-change-at-location-proc-pid-after-setuid
			// http://man7.org/linux/man-pages/man5/proc.5.html (search for "dumpable")
			prctl(PR_SET_DUMPABLE, 1);
		#endif
		setenv("USER", osUser.pwd.pw_name, 1);
		setenv("HOME", osUser.pwd.pw_dir, 1);
		setenv("UID", toString(gid).c_str(), 1);
	}
}

static void
lookupDefaultUidGid(uid_t &uid, gid_t &gid) {
	const string defaultUser = watchdogConfig->get("default_user").asString();
	const string defaultGroup = watchdogConfig->get("default_group").asString();

	OsUser osUser;
	if (!lookupSystemUserByName(defaultUser, osUser)) {
		throw NonExistentUserException("Default user '" + defaultUser +
			"' does not exist");
	}
	uid = osUser.pwd.pw_uid;

	OsGroup osGroup;
	if (!lookupSystemGroupByName(defaultGroup, osGroup)) {
		throw NonExistentGroupException("Default group '" + defaultGroup +
			"' does not exist");
	}
	gid = osGroup.grp.gr_gid;
}

static void
warnIfInstanceDirVulnerable(const string &root) {
	TRACE_POINT();

	if (geteuid() != 0) {
		return; // Passenger is not root, so no escalation.
	}

	vector<string> errors, checkErrors;
	if (isPathProbablySecureForRootUse(root, errors, checkErrors)) {
		if (!checkErrors.empty()) {
			string message = "WARNING: unable to perform privilege escalation vulnerability detection:\n";
			foreach (string line, checkErrors) {
				message.append("\n - " + line);
			}
			P_WARN(message);
		}
	} else {
		string message = "WARNING: potential privilege escalation vulnerability detected. " \
			PROGRAM_NAME " is running as root, and part(s) of the " SHORT_PROGRAM_NAME
			" instance directory (" + root + ") can be changed by non-root user(s):\n";
		foreach (string line, errors) {
			message.append("\n - " + line);
		}
		foreach (string line, checkErrors) {
			message.append("\n - " + line);
		}
		message.append("\n\nPlease either fix up the permissions for the insecure paths, or use" \
					   " a different location for the instance dir that can only be modified by root.");
		P_WARN(message);
	}
}

static void
initializeWorkingObjects(const WorkingObjectsPtr &wo, InstanceDirToucherPtr &instanceDirToucher,
	uid_t uidBeforeLoweringPrivilege)
{
	TRACE_POINT();
	Json::Value doc;
	Json::Value::iterator it, end;

	UPDATE_TRACE_POINT();
	lookupDefaultUidGid(wo->defaultUid, wo->defaultGid);

	doc = watchdogConfig->get("pidfiles_to_delete_on_exit");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->cleanupPidfiles.push_back(it->asString());
	}

	UPDATE_TRACE_POINT();
	InstanceDirectory::CreationOptions instanceOptions;
	instanceOptions.userSwitching = watchdogConfig->get("user_switching").asBool();
	instanceOptions.originalUid = uidBeforeLoweringPrivilege;
	instanceOptions.defaultUid = wo->defaultUid;
	instanceOptions.defaultGid = wo->defaultGid;
	instanceOptions.properties["name"] = wo->randomGenerator.generateAsciiString(8);
	instanceOptions.properties["integration_mode"] = watchdogConfig->get("integration_mode").asString();
	instanceOptions.properties["server_software"] = watchdogConfig->get("server_software").asString();
	if (watchdogConfig->get("integration_mode").asString() == "standalone") {
		instanceOptions.properties["standalone_engine"] = watchdogConfig->get("standalone_engine").asString();
	}

	// check if path is safe
	warnIfInstanceDirVulnerable(watchdogConfig->get("instance_registry_dir").asString());
	warnIfInstanceDirVulnerable(watchdogConfig->get("spawn_dir").asString());

	wo->instanceDir = boost::make_shared<InstanceDirectory>(instanceOptions,
		watchdogConfig->get("instance_registry_dir").asString());
	wo->extraConfigToPassToSubAgents["instance_dir"] = wo->instanceDir->getPath();
	instanceDirToucher = boost::make_shared<InstanceDirToucher>(wo);

	UPDATE_TRACE_POINT();
	string lockFilePath = wo->instanceDir->getPath() + "/lock";
	wo->lockFile = syscalls::open(lockFilePath.c_str(), O_RDONLY);
	if (wo->lockFile == -1) {
		int e = errno;
		throw FileSystemException("Cannot open " + lockFilePath + " for reading",
			e, lockFilePath);
	}
	P_LOG_FILE_DESCRIPTOR_OPEN4(wo->lockFile, __FILE__, __LINE__, "WorkingObjects: lock file");

	createFile(wo->instanceDir->getPath() + "/watchdog.pid", toString(getpid()));

	UPDATE_TRACE_POINT();
	string readOnlyAdminPassword = wo->randomGenerator.generateAsciiString(24);
	string fullAdminPassword = wo->randomGenerator.generateAsciiString(24);
	if (geteuid() == 0 && !watchdogConfig->get("user_switching").asBool()) {
		createFile(wo->instanceDir->getPath() + "/read_only_admin_password.txt",
			readOnlyAdminPassword, S_IRUSR, wo->defaultUid, wo->defaultGid);
		createFile(wo->instanceDir->getPath() + "/full_admin_password.txt",
			fullAdminPassword, S_IRUSR, wo->defaultUid, wo->defaultGid);
	} else {
		createFile(wo->instanceDir->getPath() + "/read_only_admin_password.txt",
			readOnlyAdminPassword, S_IRUSR | S_IWUSR);
		createFile(wo->instanceDir->getPath() + "/full_admin_password.txt",
			fullAdminPassword, S_IRUSR | S_IWUSR);
	}

	if (watchdogConfig->get("core_pid_file").isNull()) {
		wo->corePidFile = wo->instanceDir->getPath() + "/core.pid";
	} else {
		wo->corePidFile = watchdogConfig->get("core_pid_file").asString();
	}
	wo->fdPassingPassword = wo->randomGenerator.generateAsciiString(24);

	UPDATE_TRACE_POINT();
	wo->controllerAddresses.append("unix:" + wo->instanceDir->getPath() + "/agents.s/core");
	doc = watchdogConfig->get("controller_addresses");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->controllerAddresses.append(*it);
	}

	wo->coreApiServerAddresses.append("unix:" + wo->instanceDir->getPath() + "/agents.s/core_api");
	doc = watchdogConfig->get("core_api_server_addresses");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->coreApiServerAddresses.append(*it);
	}

	UPDATE_TRACE_POINT();
	wo->coreApiServerAuthorizations.append(
		"readonly:ro_admin:" + wo->instanceDir->getPath() +
		"/read_only_admin_password.txt");
	wo->coreApiServerAuthorizations.append(
		"full:admin:" + wo->instanceDir->getPath() +
		"/full_admin_password.txt");
	doc = watchdogConfig->get("core_api_server_authorizations");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->coreApiServerAuthorizations.append(*it);
	}
}

static void
initializeAgentWatchers(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers) {
	TRACE_POINT();
	watchers.push_back(boost::make_shared<CoreWatcher>(wo));
}

static void
makeFileWorldReadableAndWritable(const string &path) {
	int ret;

	do {
		ret = chmod(path.c_str(), parseModeString("u=rw,g=rw,o=rw"));
	} while (ret == -1 && errno == EINTR);
}

static void
initializeApiServer(const WorkingObjectsPtr &wo) {
	TRACE_POINT();
	Json::Value doc;
	Json::Value::iterator it;
	string description;

	UPDATE_TRACE_POINT();
	wo->watchdogApiServerAuthorizations.append(
		"readonly:ro_admin:" + wo->instanceDir->getPath() +
		"/read_only_admin_password.txt");
	wo->watchdogApiServerAuthorizations.append(
		"full:admin:" + wo->instanceDir->getPath() +
		"/full_admin_password.txt");
	doc = watchdogConfig->get("watchdog_api_server_authorizations");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->watchdogApiServerAuthorizations.append(*it);
	}

	UPDATE_TRACE_POINT();
	wo->watchdogApiServerAddresses.append(
		"unix:" + wo->instanceDir->getPath() +
		"/agents.s/watchdog_api");
	doc = watchdogConfig->get("watchdog_api_server_addresses");
	for (it = doc.begin(); it != doc.end(); it++) {
		wo->watchdogApiServerAddresses.append(*it);
	}

	UPDATE_TRACE_POINT();
	for (unsigned int i = 0; i < wo->watchdogApiServerAddresses.size(); i++) {
		string address = wo->watchdogApiServerAddresses[i].asString();
		P_DEBUG("API server will listen on " << address);
		wo->apiServerFds[i] = createServer(address, 0, true,
			__FILE__, __LINE__);
		if (getSocketAddressType(address) == SAT_UNIX) {
			makeFileWorldReadableAndWritable(parseUnixSocketAddress(address));
		}
	}

	UPDATE_TRACE_POINT();
	Json::Value contextConfig = watchdogConfig->inspectEffectiveValues();
	wo->bgloop = new BackgroundEventLoop(true, true);
	wo->serverKitContext = new ServerKit::Context(
		watchdogSchema->apiServerKit.schema,
		contextConfig,
		watchdogSchema->apiServerKit.translator);
	wo->serverKitContext->libev = wo->bgloop->safe;
	wo->serverKitContext->libuv = wo->bgloop->libuv_loop;
	wo->serverKitContext->initialize();

	UPDATE_TRACE_POINT();
	Json::Value apiServerConfig = watchdogConfig->inspectEffectiveValues();
	apiServerConfig["fd_passing_password"] = wo->fdPassingPassword;
	apiServerConfig["authorizations"] = wo->watchdogApiServerAuthorizations;
	wo->apiServer = new ApiServer::ApiServer(
		wo->serverKitContext,
		watchdogSchema->apiServer.schema,
		apiServerConfig,
		watchdogSchema->apiServer.translator);
	wo->apiServer->exitEvent = &wo->exitEvent;
	wo->apiServer->initialize();
	for (unsigned int i = 0; i < wo->watchdogApiServerAddresses.size(); i++) {
		wo->apiServer->listen(wo->apiServerFds[i]);
	}
}

static void
startAgents(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers) {
	TRACE_POINT();
	foreach (AgentWatcherPtr watcher, watchers) {
		P_DEBUG("Starting agent: " << watcher->name());
		try {
			watcher->start();
		} catch (const std::exception &e) {
			if (feedbackFdAvailable()) {
				writeArrayMessage(FEEDBACK_FD,
					"Watchdog startup error",
					e.what(),
					NULL);
			} else {
				const oxt::tracable_exception *e2 =
					dynamic_cast<const oxt::tracable_exception *>(&e);
				if (e2 != NULL) {
					P_CRITICAL("ERROR: " << e2->what() << "\n" << e2->backtrace());
				} else {
					P_CRITICAL("ERROR: " << e.what());
				}
			}
			forceAllAgentsShutdown(wo, watchers);
			cleanup(wo);
			exit(1);
		}
		// Allow other exceptions to propagate and crash the watchdog.
	}
}

static void
beginWatchingAgents(const WorkingObjectsPtr &wo, vector<AgentWatcherPtr> &watchers) {
	foreach (AgentWatcherPtr watcher, watchers) {
		try {
			watcher->beginWatching();
		} catch (const std::exception &e) {
			writeArrayMessage(FEEDBACK_FD,
				"Watchdog startup error",
				e.what(),
				NULL);
			forceAllAgentsShutdown(wo, watchers);
			cleanup(wo);
			exit(1);
		}
		// Allow other exceptions to propagate and crash the watchdog.
	}
}

static void
reportStartupResult(const WorkingObjectsPtr &wo, const vector<AgentWatcherPtr> &watchers) {
	TRACE_POINT();
	Json::Value report;

	report["instance_dir"] = wo->instanceDir->getPath();

	foreach (AgentWatcherPtr watcher, watchers) {
		watcher->reportAgentStartupResult(report);
	}

	if (feedbackFdAvailable()) {
		writeArrayMessage(FEEDBACK_FD, "Agents information", NULL);
		writeScalarMessage(FEEDBACK_FD, report.toStyledString());
	}

	if (wo->startupReportFile != -1) {
		string str = report.toStyledString();
		writeExact(wo->startupReportFile, str.data(), str.size());
		close(wo->startupReportFile);
		P_LOG_FILE_DESCRIPTOR_CLOSE(wo->startupReportFile);
		wo->startupReportFile = -1;
	}
}

static void
finalizeInstanceDir(const WorkingObjectsPtr &wo) {
	TRACE_POINT();
	#ifdef HAVE_FLOCK
		if (flock(wo->lockFile, LOCK_EX) == -1) {
			int e = errno;
			throw SystemException("Cannot obtain exclusive lock on the "
				"instance directory lock file", e);
		}
	#endif
	wo->instanceDir->finalizeCreation();
}

static void
cleanup(const WorkingObjectsPtr &wo) {
	TRACE_POINT();

	// We need to call destroy() explicitly because of circular references.
	if (wo->instanceDir != NULL && wo->instanceDir->isOwner()) {
		wo->instanceDir->destroy();
		wo->instanceDir.reset();
	}

	killCleanupPids(wo);
	deletePidFile(wo);
}

int
watchdogMain(int argc, char *argv[]) {
	WorkingObjectsPtr wo;

	initializeBareEssentials(argc, argv, wo);
	P_NOTICE("Starting " SHORT_PROGRAM_NAME " watchdog...");
	InstanceDirToucherPtr instanceDirToucher;
	vector<AgentWatcherPtr> watchers;
	uid_t uidBeforeLoweringPrivilege = geteuid();

	try {
		TRACE_POINT();
		maybeSetsid();
		maybeDaemonize();
		createPidFile();
		openStartupReportFile(wo);
		chdirToTmpDir();
		lowerPrivilege();
		initializeWorkingObjects(wo, instanceDirToucher, uidBeforeLoweringPrivilege);
		initializeAgentWatchers(wo, watchers);
		initializeApiServer(wo);
		UPDATE_TRACE_POINT();
		runHookScriptAndThrowOnError("before_watchdog_initialization");
	} catch (const std::exception &e) {
		if (feedbackFdAvailable()) {
			writeArrayMessage(FEEDBACK_FD,
				"Watchdog startup error",
				e.what(),
				NULL);
		} else {
			const oxt::tracable_exception *e2 =
				dynamic_cast<const oxt::tracable_exception *>(&e);
			if (e2 != NULL) {
				P_CRITICAL("ERROR: " << e2->what() << "\n" << e2->backtrace());
			} else {
				P_CRITICAL("ERROR: " << e.what());
			}
		}
		if (wo != NULL) {
			cleanup(wo);
		}
		return 1;
	}
	// Allow other exceptions to propagate and crash the watchdog.

	try {
		TRACE_POINT();
		startAgents(wo, watchers);
		beginWatchingAgents(wo, watchers);
		reportStartupResult(wo, watchers);
		finalizeInstanceDir(wo);
		P_INFO("All " PROGRAM_NAME " agents started!");
		UPDATE_TRACE_POINT();
		runHookScriptAndThrowOnError("after_watchdog_initialization");

		UPDATE_TRACE_POINT();
		boost::this_thread::disable_interruption di;
		boost::this_thread::disable_syscall_interruption dsi;
		bool shouldExitGracefully = waitForStarterProcessOrWatchers(wo, watchers);
		if (shouldExitGracefully) {
			/* Fork a child process which cleans up all the agent processes in
			 * the background and exit this watchdog process so that we don't block
			 * the web server.
			 */
			P_DEBUG("Web server exited gracefully; gracefully shutting down all agents...");
		} else {
			P_DEBUG("Web server did not exit gracefully, forcing shutdown of all agents...");
		}
		UPDATE_TRACE_POINT();
		runHookScriptAndThrowOnError("before_watchdog_shutdown");
		UPDATE_TRACE_POINT();
		AgentWatcher::stopWatching(watchers);
		if (shouldExitGracefully) {
			UPDATE_TRACE_POINT();
			cleanupAgentsInBackground(wo, watchers, argv);
			// Child process will call cleanup()
		} else {
			UPDATE_TRACE_POINT();
			forceAllAgentsShutdown(wo, watchers);
			cleanup(wo);
		}
		UPDATE_TRACE_POINT();
		runHookScriptAndThrowOnError("after_watchdog_shutdown");

		return shouldExitGracefully ? 0 : 1;
	} catch (const tracable_exception &e) {
		P_CRITICAL("ERROR: " << e.what() << "\n" << e.backtrace());
		cleanup(wo);
		return 1;
	}
}
F1l3 Manager
