#!/usr/bin/perl

my $cfgbase = '/etc/vservers';
my $cgbase = '/dev/cgroup';
my $vkill = '/usr/sbin/vkill';
my $failcnt_rate_max = 500;
my $load_floor = 200;

my $host = `hostname`;
chomp $host;

sub guests {
	opendir(CFGBASE,$cfgbase);
	my @guests;
	while ( my $guest = readdir(CFGBASE) ) {
		open(MARK,"$cfgbase/$guest/apps/init/mark");
		my $mark = <MARK>;
		chomp $mark;
		push(@guests,$guest) if $mark eq 'default';
		close(MARK);
	}
	closedir(CFGBASE);
	return @guests;
}

sub restart_guest {
	my $guest = shift;
	my $ratio = shift;
	open(CTX,"$cfgbase/$guest/context") || return;
	my $ctx = <CTX>;
	chomp $ctx;
	return unless $ctx =~ /^[0-9]+/;
	my $kill_cmd = "$vkill --xid $ctx -s 9 -- -1";
	print "$kill_cmd\n";
	system"$kill_cmd";
	system"echo /usr/sbin/vserver $guest start | at now+5min";
	open(S,"|/usr/sbin/sendmail jeremy\@hq.newdream.net");
	print S "Subject: $guest on $host\n\n$kill_cmd\n$ratio\n";
	close(S);
	return "success";
}

sub monitor {
	while ( my $guest = shift ) {
#		print "$guest\n";
		open(FCNT,"$cgbase/$guest/memory.failcnt");
		my $fcnt = <FCNT>;
		chomp $fcnt;
		my $cleartime = (stat(FCNT))[9];
		my $now = time();
		close(FCNT);
		next if $fcnt == 0;
		my $timespan = $now - $cleartime;
		my $failspersec = $fcnt / $timespan;
		if ( $failspersec > $failcnt_rate_max ) {
			print "$guest OUT OF CONTROL\n";
			restart_guest($guest,$failspersec);
		}
		else {
			print "$failspersec fps on $guest within limits\n";
		}
	}
}

my $pid = fork();
if ( $pid == 0 ) {
	my $last_load = 0;
	while(1) {
		open(LOAD,'/proc/loadavg');
		my $load = <LOAD>;
		chomp $load;
		($load) = split(/\s+/,$load);
		if ( $load >= $last_load && $load > $load_floor ) {
			my @guests = guests();
			monitor(@guests);
		}
		$last_load = $load;
		sleep 30;
	}
}
else {
	open(PID,">/var/run/vservermonitor.pid");
	print PID $pid;
	close(PID);
	exit;
}
