2021-04-08 12:54:49 +02:00
|
|
|
<?php
|
|
|
|
|
|
|
|
|
2022-04-27 14:24:48 +02:00
|
|
|
namespace App\Repository;
|
2021-04-08 12:54:49 +02:00
|
|
|
|
|
|
|
|
2022-05-13 15:24:05 +02:00
|
|
|
use App\Entity\Job;
|
2022-04-27 14:24:48 +02:00
|
|
|
use App\Entity\Run;
|
|
|
|
use App\Service\Secret;
|
2021-04-13 14:07:11 +02:00
|
|
|
use DateTime;
|
2022-04-27 14:24:48 +02:00
|
|
|
use Doctrine\ORM\EntityRepository;
|
2021-05-24 18:36:16 +02:00
|
|
|
use GuzzleHttp\Client;
|
2021-10-18 15:17:36 +02:00
|
|
|
use GuzzleHttp\Exception\GuzzleException;
|
2021-05-24 18:36:16 +02:00
|
|
|
use phpseclib3\Crypt\PublicKeyLoader;
|
|
|
|
use phpseclib3\Net\SSH2;
|
2021-04-08 12:54:49 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
*
|
|
|
|
*/
|
2022-04-27 14:24:48 +02:00
|
|
|
class JobRepository extends EntityRepository
|
2021-04-08 12:54:49 +02:00
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-11-19 15:13:20 +01:00
|
|
|
public function getFailingJobs()
|
|
|
|
{
|
2022-04-27 14:24:48 +02:00
|
|
|
$runRepo = $this->getEntityManager()->getRepository(Run::class);
|
2022-05-16 18:00:44 +02:00
|
|
|
/** @var Job[] $jobs */
|
|
|
|
$jobs = $this->getAllJobs();
|
2021-11-19 15:13:20 +01:00
|
|
|
|
2022-05-16 18:00:44 +02:00
|
|
|
$return = [];
|
|
|
|
foreach($jobs as $job) {
|
2022-05-17 16:06:46 +02:00
|
|
|
if($job->getData('needschecking')) {
|
2022-05-16 18:00:44 +02:00
|
|
|
$return[] = $job;
|
2021-11-19 15:13:20 +01:00
|
|
|
}
|
|
|
|
}
|
2022-05-16 18:00:44 +02:00
|
|
|
return $return;
|
2021-11-19 15:13:20 +01:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @return array
|
|
|
|
*/
|
|
|
|
public function getRunningJobs(): array
|
2022-02-04 14:21:42 +01:00
|
|
|
{
|
2022-05-16 18:00:44 +02:00
|
|
|
$qb = $this->createQueryBuilder('job');
|
2022-05-17 16:06:46 +02:00
|
|
|
return $qb
|
|
|
|
->where('job.running != 0')
|
|
|
|
->getQuery()->getResult();
|
2022-05-16 18:00:44 +02:00
|
|
|
}
|
2022-02-04 14:21:42 +01:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param bool $idiskey
|
|
|
|
* @return array
|
|
|
|
*/
|
|
|
|
public function getAllJobs(bool $idiskey = false): array
|
2022-05-16 18:00:44 +02:00
|
|
|
{
|
|
|
|
$qb = $this->createQueryBuilder('job');
|
2022-05-18 11:23:41 +02:00
|
|
|
|
|
|
|
$jobs = $qb->where('job.id = job.id');
|
|
|
|
|
|
|
|
if($idiskey) {
|
|
|
|
$jobs = $jobs->orderBy('job.id');
|
|
|
|
} else {
|
|
|
|
$jobs = $jobs
|
|
|
|
->orderBy('job.name')
|
|
|
|
->addOrderBy("JSON_VALUE(job.data, '$.host')")
|
|
|
|
->addOrderBy("JSON_VALUE(job.data, '$.service')");
|
|
|
|
}
|
|
|
|
|
|
|
|
/** @var Job $jobs */
|
|
|
|
$jobs = $jobs->getQuery()->getResult();
|
2022-05-16 18:00:44 +02:00
|
|
|
|
|
|
|
return $this->parseJobs($jobs);
|
2022-02-04 14:21:42 +01:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param array $jobs
|
|
|
|
* @return array
|
|
|
|
*/
|
2022-05-16 18:00:44 +02:00
|
|
|
public function parseJobs(array $jobs): array
|
2021-04-08 12:54:49 +02:00
|
|
|
{
|
2022-04-27 14:24:48 +02:00
|
|
|
$runRepo = $this->getEntityManager()->getRepository(Run::class);
|
2022-05-13 15:24:05 +02:00
|
|
|
|
2021-04-08 12:54:49 +02:00
|
|
|
foreach ($jobs as $key=>&$job) {
|
2022-05-13 15:24:05 +02:00
|
|
|
$jobData = $job->getData();
|
2022-05-18 13:21:09 +02:00
|
|
|
$job->setData('host-displayname', $jobData['host']);
|
|
|
|
$job->setData('host', $jobData['host']);
|
|
|
|
$job->setData('service', $jobData['service'] ?? '');
|
|
|
|
$job->setData('norun', $job->getLastrun() !== null && $job->getNextrun() > $job->getLastrun());
|
|
|
|
$job->setData('running', $job->getRunning() != 0);
|
2022-05-19 20:25:05 +02:00
|
|
|
$failedruns = $runRepo->getRunsForJob($job, true, $jobData['fail-days']);
|
2022-05-16 18:00:44 +02:00
|
|
|
$failed = count($failedruns);
|
2022-05-19 20:25:05 +02:00
|
|
|
$all = count($runRepo->getRunsForJob($job, false, $jobData['fail-days']));
|
2022-05-20 11:06:22 +02:00
|
|
|
$job->setData('lastfail', isset($failedruns[0]) ? $failedruns[0]->toArray() : NULL);
|
2022-05-18 13:21:09 +02:00
|
|
|
$job->setData('needschecking', $all > 0 && (($failed / $all) * 100) > $jobData['fail-pct']);
|
2022-05-13 15:24:05 +02:00
|
|
|
if(!empty($jobData['containertype']) && $jobData['containertype'] != 'none') {
|
2022-05-18 13:21:09 +02:00
|
|
|
$job->setData('host-displayname', $jobData['service'] . ' on ' . $jobData['host']);
|
2021-05-28 12:25:22 +02:00
|
|
|
}
|
2021-04-08 12:54:49 +02:00
|
|
|
}
|
2021-07-20 16:29:03 +02:00
|
|
|
|
2021-04-08 12:54:49 +02:00
|
|
|
return $jobs;
|
|
|
|
}
|
2021-04-13 14:07:11 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @return array
|
|
|
|
*/
|
|
|
|
public function getJobsDue(): array
|
2021-05-24 18:36:16 +02:00
|
|
|
{
|
2022-05-17 16:06:46 +02:00
|
|
|
$qb = $this->createQueryBuilder('job');
|
|
|
|
return $qb
|
|
|
|
->where(
|
|
|
|
$qb->expr()->andX(
|
|
|
|
$qb->expr()->lte('job.nextrun', ':timestamp'),
|
|
|
|
$qb->expr()->orX(
|
|
|
|
$qb->expr()->isNull('job.lastrun'),
|
|
|
|
$qb->expr()->gt('job.lastrun', ':timestamp')
|
|
|
|
),
|
|
|
|
$qb->expr()->in('job.running', [0,2])
|
|
|
|
)
|
|
|
|
)
|
|
|
|
->orWhere(
|
|
|
|
$qb->expr()->andX(
|
|
|
|
$qb->expr()->notIn('job.running', [0,1,2]),
|
|
|
|
$qb->expr()->lt('job.running', ':timestamp')
|
|
|
|
)
|
|
|
|
)
|
|
|
|
->orWhere('job.running = 2')
|
|
|
|
->orderBy('job.running', 'DESC')
|
|
|
|
->addOrderBy('job.nextrun', 'ASC')
|
|
|
|
->setParameter(':timestamp', time())
|
|
|
|
->getQuery()->getResult();
|
2021-05-24 18:36:16 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @param bool $status
|
|
|
|
* @return void
|
|
|
|
*/
|
|
|
|
public function setJobRunning(Job $job, bool $status): void
|
2021-10-22 12:16:21 +02:00
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
2021-10-22 12:16:21 +02:00
|
|
|
|
2022-05-21 10:50:37 +02:00
|
|
|
if(in_array($job->getRunning(), [0,1,2])) $job->setRunning($status ? 1 : 0);
|
2021-10-22 12:16:21 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-10-22 12:16:21 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @param string $name
|
|
|
|
* @param mixed $value
|
|
|
|
* @return void
|
|
|
|
*/
|
|
|
|
public function setTempVar(Job &$job, string $name, mixed $value): void
|
2021-05-24 18:36:16 +02:00
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$job->setData('temp_vars.' . $name, $value);
|
2021-05-24 18:36:16 +02:00
|
|
|
}
|
|
|
|
|
2021-05-28 18:19:12 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @param string|null $name
|
|
|
|
* @return void
|
|
|
|
*/
|
|
|
|
public function deleteTempVar(Job &$job, ?string $name = NULL ): void
|
2021-05-28 18:19:12 +02:00
|
|
|
{
|
2022-05-21 10:50:37 +02:00
|
|
|
$job->removeData('temp_vars.' . ($name !== NULL ? '.' . $name : ''));
|
2021-05-28 18:19:12 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @param string $name
|
|
|
|
* @param mixed|NULL $default
|
|
|
|
* @return mixed
|
|
|
|
*/
|
|
|
|
public function getTempVar(Job $job, string $name, mixed $default = NULL): mixed
|
2021-05-28 18:19:12 +02:00
|
|
|
{
|
2022-05-21 10:50:37 +02:00
|
|
|
return $job->getData('temp_vars.' . $name) ?? $default;
|
2021-05-28 18:19:12 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @return array
|
|
|
|
*/
|
|
|
|
private function runHttpJob(Job &$job): array
|
2021-05-24 18:36:16 +02:00
|
|
|
{
|
2021-05-28 09:59:55 +02:00
|
|
|
$client = new Client();
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2022-05-20 14:48:22 +02:00
|
|
|
$url = $job->getData('url');
|
|
|
|
$user = $job->getData('basicauth-username');
|
2022-05-17 16:06:46 +02:00
|
|
|
if(!empty($job->getData('vars'))) {
|
|
|
|
foreach($job->getData('vars') as $key => $var) {
|
2022-05-20 14:48:22 +02:00
|
|
|
if (!empty($user)) $user = str_replace('{' . $key . '}', ($var['issecret'] ? Secret::decrypt(base64_decode($var['value'])) : $var['value']), $job->getData('basicauth-username'));
|
|
|
|
$url = str_replace('{' . $key . '}', ($var['issecret'] ? Secret::decrypt(base64_decode($var['value'])) : $var['value']), $job->getData('url'));
|
2021-05-24 18:36:16 +02:00
|
|
|
}
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2021-05-28 09:59:55 +02:00
|
|
|
$options['http_errors'] = false;
|
2022-05-20 14:48:22 +02:00
|
|
|
$options['auth'] = !empty($user) ? [$user, Secret::decrypt(base64_decode($job->getData('basicauth-password')))] : NULL;
|
2021-09-21 09:37:22 +02:00
|
|
|
try {
|
|
|
|
$res = $client->request('GET', $url, $options);
|
|
|
|
$return['exitcode'] = $res->getStatusCode();
|
|
|
|
$return['output'] = $res->getBody();
|
2022-05-17 16:06:46 +02:00
|
|
|
$return['failed'] = !in_array($return['exitcode'], $job->getData('http-status'));
|
2021-10-18 15:17:36 +02:00
|
|
|
} catch(GuzzleException $exception) {
|
2021-09-21 09:37:22 +02:00
|
|
|
$return['exitcode'] = $exception->getCode();
|
|
|
|
$return['output'] = $exception->getMessage();
|
|
|
|
$return['failed'] = true;
|
|
|
|
}
|
|
|
|
|
2021-05-28 09:59:55 +02:00
|
|
|
return $return;
|
|
|
|
}
|
2021-05-27 20:24:03 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @return array
|
|
|
|
*/
|
|
|
|
private function runCommandJob(Job &$job): array
|
2021-05-28 09:59:55 +02:00
|
|
|
{
|
2022-05-20 14:48:22 +02:00
|
|
|
$command = $job->getData('command');
|
2022-05-17 16:06:46 +02:00
|
|
|
if(!empty($job->getData('vars'))) {
|
|
|
|
foreach ($job->getData('vars') as $key => $var) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$command = str_replace('{' . $key . '}', $var['value'], $job->getData('command'));
|
2021-05-27 20:24:03 +02:00
|
|
|
}
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2022-05-17 16:06:46 +02:00
|
|
|
if ($job->getData('containertype') == 'docker') {
|
|
|
|
$command = $this->prepareDockerCommand($command, $job->getData('service'), $job->getData('container-user'));
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-09-22 10:26:52 +02:00
|
|
|
try {
|
2022-05-17 16:06:46 +02:00
|
|
|
if($job->getData('hosttype') == 'local') {
|
2021-09-22 10:26:52 +02:00
|
|
|
$return = $this->runLocalCommand($command);
|
2022-05-17 16:06:46 +02:00
|
|
|
} elseif($job->getData('hosttype') == 'ssh') {
|
2022-05-20 14:48:22 +02:00
|
|
|
$return = $this->runSshCommand($command, $job->getData('host'), $job->getData('user'), Secret::decrypt(base64_decode($job->getData('ssh-privkey'))), Secret::decrypt(base64_decode($job->getData('privkey-password'))));
|
2021-09-22 10:26:52 +02:00
|
|
|
}
|
2022-05-17 16:06:46 +02:00
|
|
|
$return['failed'] = !in_array($return['exitcode'], $job->getData('response'));
|
2021-09-22 10:26:52 +02:00
|
|
|
} catch (\RuntimeException $exception) {
|
|
|
|
$return['exitcode'] = $exception->getCode();
|
|
|
|
$return['output'] = $exception->getMessage();
|
|
|
|
$return['failed'] = true;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-09-22 10:26:52 +02:00
|
|
|
|
2021-07-14 13:09:23 +02:00
|
|
|
return $return;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param string $command
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-05-28 09:59:55 +02:00
|
|
|
private function runLocalCommand(string $command): array
|
|
|
|
{
|
2021-07-14 13:46:56 +02:00
|
|
|
if(function_exists('pcntl_signal')) pcntl_signal(SIGCHLD, SIG_DFL);
|
2021-05-28 09:59:55 +02:00
|
|
|
$return['exitcode'] = NULL;
|
|
|
|
$return['output'] = NULL;
|
|
|
|
exec($command . ' 2>&1', $return['output'], $return['exitcode']);
|
2022-05-19 11:56:06 +02:00
|
|
|
if(function_exists('pcntl_signal')) pcntl_signal(SIGCHLD, SIG_IGN);
|
2021-05-28 09:59:55 +02:00
|
|
|
$return['output'] = implode("\n", $return['output']);
|
|
|
|
return $return;
|
|
|
|
}
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param string $command
|
|
|
|
* @param string $host
|
|
|
|
* @param string $user
|
|
|
|
* @param string|null $privkey
|
|
|
|
* @param string|null $password
|
|
|
|
* @return array
|
|
|
|
*/
|
2022-05-17 16:06:46 +02:00
|
|
|
private function runSshCommand(string $command, string $host, string $user, ?string $privkey, ?string $password): array
|
2021-05-28 09:59:55 +02:00
|
|
|
{
|
|
|
|
$ssh = new SSH2($host);
|
|
|
|
$key = null;
|
|
|
|
if(!empty($privkey)) {
|
|
|
|
if(!empty($password)) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$key = PublicKeyLoader::load($privkey, $password);
|
2021-05-28 09:59:55 +02:00
|
|
|
} else {
|
2022-05-20 14:48:22 +02:00
|
|
|
$key = PublicKeyLoader::load($privkey);
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
|
|
|
} elseif (!empty($password)) {
|
2021-05-28 17:28:02 +02:00
|
|
|
$key = $password;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
|
|
|
if (!$ssh->login($user, $key)) {
|
2021-05-28 11:45:30 +02:00
|
|
|
$return['output'] = "Login failed";
|
|
|
|
$return['exitcode'] = 255;
|
|
|
|
return $return;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-07-02 22:06:18 +02:00
|
|
|
$ssh->setTimeout(0);
|
2021-05-29 15:50:19 +02:00
|
|
|
$return['output'] = $ssh->exec($command);
|
|
|
|
$return['exitcode'] = $ssh->getExitStatus();
|
|
|
|
$return['exitcode'] = (empty($return['exitcode'])) ? 0 : $return['exitcode'];
|
2021-05-28 09:59:55 +02:00
|
|
|
return $return;
|
|
|
|
}
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @param float $starttime
|
|
|
|
* @param bool $manual
|
|
|
|
* @return array|string[]
|
|
|
|
* @throws \Doctrine\DBAL\Exception
|
|
|
|
*/
|
|
|
|
private function runRebootJob(Job &$job, float &$starttime, bool &$manual): array
|
2021-05-28 09:59:55 +02:00
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
2022-05-18 13:05:55 +02:00
|
|
|
if($job->getRunning() == 1) {
|
2022-05-19 11:56:06 +02:00
|
|
|
$this->setTempVar($job, 'starttime', $starttime);
|
|
|
|
$this->setTempVar($job, 'manual', $manual);
|
2022-05-20 11:32:42 +02:00
|
|
|
$rebootcommand = $job->getData('reboot-command');
|
|
|
|
$rebootcommand = str_replace('{reboot-delay}', $job->getData('reboot-delay'), $rebootcommand);
|
|
|
|
$rebootcommand = str_replace('{reboot-delay-secs}', $job->getData('reboot-delay-secs'), $rebootcommand);
|
2022-05-18 13:05:55 +02:00
|
|
|
|
|
|
|
if (!empty($job->getData('vars'))) {
|
|
|
|
foreach ($job->getData('vars') as $key => $var) {
|
2022-05-20 11:32:42 +02:00
|
|
|
$rebootcommand = str_replace('{' . $key . '}', $var['value'], $rebootcommand);
|
2021-05-27 21:17:10 +02:00
|
|
|
}
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
$job->setRunning(time() + $job->getData('reboot-delay-secs') + ($job->getData('reboot-duration') * 60));
|
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-04-27 14:24:48 +02:00
|
|
|
try {
|
2022-05-18 13:05:55 +02:00
|
|
|
if($job->getData('hosttype') == 'local') {
|
2022-05-20 11:32:42 +02:00
|
|
|
$this->runLocalCommand($rebootcommand);
|
2022-05-18 13:05:55 +02:00
|
|
|
} elseif($job->getData('hosttype') == 'ssh') {
|
2022-05-20 14:48:22 +02:00
|
|
|
$this->runSshCommand($rebootcommand, $job->getData('host'), $job->getData('user'), Secret::decrypt(base64_decode($job->getData('ssh-privkey'))) ?? '', Secret::decrypt(base64_decode($job->getData('privkey-password'))) ?? '');
|
2022-04-27 14:24:48 +02:00
|
|
|
}
|
|
|
|
} catch (\RuntimeException $exception) {
|
|
|
|
$return['exitcode'] = $exception->getCode();
|
|
|
|
$return['output'] = $exception->getMessage();
|
|
|
|
$return['failed'] = true;
|
|
|
|
return $return;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-07-15 13:14:36 +02:00
|
|
|
return ['status' => 'deferred'];
|
2021-05-28 09:59:55 +02:00
|
|
|
|
2022-05-18 13:05:55 +02:00
|
|
|
} elseif($job->getRunning() != 0) {
|
|
|
|
if($job->getRunning() > time()) {
|
2021-07-15 13:14:36 +02:00
|
|
|
return ['status' => 'deferred'];
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2022-05-19 11:56:06 +02:00
|
|
|
$starttime = (float)$this->getTempVar($job, 'starttime');
|
|
|
|
$this->deleteTempVar($job, 'starttime');
|
|
|
|
$manual = $this->getTempVar($job, 'manual');
|
|
|
|
$this->deleteTempVar($job, 'manual');
|
2021-05-29 11:56:29 +02:00
|
|
|
|
2022-05-21 10:50:37 +02:00
|
|
|
$job->setRunning(1);
|
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-05-27 21:17:10 +02:00
|
|
|
|
2022-05-20 14:48:22 +02:00
|
|
|
$getservicescommand = $job->getData('getservices-command');
|
2022-05-18 13:05:55 +02:00
|
|
|
if (!empty($job->getData('vars'))) {
|
|
|
|
foreach ($job->getData('vars') as $key => $var) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$getservicescommand = str_replace('{' . $key . '}', $var['value'], $job->getData('getservices-command'));
|
2021-05-27 21:17:10 +02:00
|
|
|
}
|
|
|
|
}
|
2022-04-27 14:24:48 +02:00
|
|
|
try {
|
2022-05-18 13:05:55 +02:00
|
|
|
if($job->getData('hosttype') == 'local') {
|
2022-05-20 14:48:22 +02:00
|
|
|
$return = $this->runLocalCommand($getservicescommand);
|
2022-05-18 13:05:55 +02:00
|
|
|
} elseif($job->getData('hosttype') == 'ssh') {
|
2022-05-20 14:48:22 +02:00
|
|
|
$return = $this->runSshCommand($getservicescommand, $job->getData('host'), $job->getData('user'), Secret::decrypt(base64_decode($job->getData('ssh-privkey'))) ?? '', Secret::decrypt(base64_decode($job->getData('privkey-password'))) ?? '');
|
2022-04-27 14:24:48 +02:00
|
|
|
}
|
|
|
|
} catch (\RuntimeException $exception) {
|
|
|
|
$return['exitcode'] = $exception->getCode();
|
|
|
|
$return['output'] = $exception->getMessage();
|
|
|
|
$return['failed'] = true;
|
|
|
|
return $return;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2022-05-18 13:05:55 +02:00
|
|
|
$return['failed'] = !in_array($return['exitcode'], $job->getData('getservices-response'));
|
2021-05-28 18:19:12 +02:00
|
|
|
return $return;
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2022-05-18 13:05:55 +02:00
|
|
|
return ['success' => false, 'message' => 'You probably did something clearly wrong'];
|
2021-05-28 09:59:55 +02:00
|
|
|
}
|
2021-05-28 18:19:12 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param $job
|
|
|
|
* @param $console
|
|
|
|
* @return array
|
|
|
|
* @throws \Doctrine\DBAL\Exception
|
|
|
|
*/
|
|
|
|
public function runNow(Job &$job, $console = false) {
|
2022-05-19 15:48:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
2022-04-27 14:24:48 +02:00
|
|
|
$runRepo = $this->getEntityManager()->getRepository(Run::class);
|
2021-08-02 10:54:50 +02:00
|
|
|
|
2022-05-19 15:48:06 +02:00
|
|
|
if($console == false && ($runRepo->isSlowJob($job)) || count($job->getRuns()) == 0 || $job->getData('crontype') === 'reboot') {
|
|
|
|
if(in_array($job->getRunning(), [0,1,2])) {
|
|
|
|
$job->setRunning(2);
|
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
|
|
|
}
|
2021-06-01 17:41:10 +02:00
|
|
|
} else {
|
2022-05-19 11:56:06 +02:00
|
|
|
$output = $this->runJob($job, true);
|
2021-07-15 13:14:36 +02:00
|
|
|
if(!(isset($output['status']) && $output['status'] == 'deferred'))
|
2021-06-01 17:41:10 +02:00
|
|
|
return [
|
2021-06-01 20:21:47 +02:00
|
|
|
'status' => 'ran',
|
2021-07-15 12:41:34 +02:00
|
|
|
'output' => ($console) ? $output['output'] : htmlentities($output['output']),
|
2021-06-01 17:41:10 +02:00
|
|
|
'exitcode' => $output['exitcode'],
|
2021-07-02 21:03:21 +02:00
|
|
|
'runtime' => (float)$output['runtime'],
|
2022-04-27 14:24:48 +02:00
|
|
|
'success' => !str_contains($output['flags'], RunRepository::FAILED)
|
2021-06-01 17:41:10 +02:00
|
|
|
];
|
|
|
|
}
|
2022-05-24 18:09:14 +02:00
|
|
|
return [
|
|
|
|
'success' => NULL,
|
|
|
|
'status' => 'deferred'
|
|
|
|
];
|
2021-06-01 17:41:10 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param string $command
|
|
|
|
* @param string $service
|
|
|
|
* @param string|null $user
|
|
|
|
* @return string
|
|
|
|
*/
|
|
|
|
private function prepareDockerCommand(string $command, string $service, ?string $user): string
|
2021-05-28 09:59:55 +02:00
|
|
|
{
|
|
|
|
$prepend = 'docker exec ';
|
|
|
|
$prepend .= (!empty($user)) ? ' --user=' . $user . ' ' : '';
|
2021-05-28 10:16:08 +02:00
|
|
|
$prepend .= $service . ' ';
|
2021-05-28 09:59:55 +02:00
|
|
|
return $prepend . $command;
|
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param int $job
|
|
|
|
* @param bool $manual
|
|
|
|
* @return array|string[]
|
|
|
|
* @throws \Doctrine\DBAL\Exception
|
|
|
|
*/
|
|
|
|
public function runJob(Job &$job, bool $manual): array
|
2021-05-28 09:59:55 +02:00
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
2021-05-28 14:24:33 +02:00
|
|
|
$starttime = microtime(true);
|
2022-05-17 16:06:46 +02:00
|
|
|
if ($job->getData('crontype') == 'http') {
|
2021-05-28 09:59:55 +02:00
|
|
|
$result = $this->runHttpJob($job);
|
2022-05-17 16:06:46 +02:00
|
|
|
} elseif ($job->getData('crontype') == 'command') {
|
2021-05-28 09:59:55 +02:00
|
|
|
$result = $this->runCommandJob($job);
|
2022-05-17 16:06:46 +02:00
|
|
|
} elseif ($job->getData('crontype') == 'reboot') {
|
2021-06-01 13:45:57 +02:00
|
|
|
$result = $this->runRebootJob($job, $starttime, $manual);
|
2021-07-15 13:14:36 +02:00
|
|
|
if(isset($result['status']) && $result['status'] == 'deferred') return $result;
|
2021-05-24 18:36:16 +02:00
|
|
|
}
|
2021-05-28 14:24:33 +02:00
|
|
|
$endtime = microtime(true);
|
2021-05-29 11:51:34 +02:00
|
|
|
$runtime = $endtime - $starttime;
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2021-06-01 13:45:57 +02:00
|
|
|
// setting flags
|
|
|
|
$flags = [];
|
2021-06-01 17:41:10 +02:00
|
|
|
if ($result['failed'] === true) {
|
2022-04-27 14:24:48 +02:00
|
|
|
$flags[] = RunRepository::FAILED;
|
2021-06-01 13:45:57 +02:00
|
|
|
} else {
|
2022-04-27 14:24:48 +02:00
|
|
|
$flags[] = RunRepository::SUCCESS;
|
2021-06-01 13:45:57 +02:00
|
|
|
}
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2021-06-01 17:41:10 +02:00
|
|
|
if ($manual === true) {
|
2022-04-27 14:24:48 +02:00
|
|
|
$flags[] = RunRepository::MANUAL;
|
2021-06-01 13:45:57 +02:00
|
|
|
}
|
2021-11-30 13:54:17 +01:00
|
|
|
|
|
|
|
// Remove secrets from output
|
2022-05-17 16:06:46 +02:00
|
|
|
if(!empty($job->getData('vars'))) {
|
|
|
|
foreach($job->getData('vars') as $key => $var) {
|
2021-12-04 10:25:13 +01:00
|
|
|
if ($var['issecret']) {
|
|
|
|
$result['output'] = str_replace($var['value'], '{'.$key.'}', $result['output']);
|
|
|
|
}
|
2021-11-30 13:54:17 +01:00
|
|
|
}
|
|
|
|
}
|
2022-05-19 11:56:06 +02:00
|
|
|
|
2021-06-01 13:45:57 +02:00
|
|
|
// saving to database
|
2022-05-19 11:56:06 +02:00
|
|
|
$em->getConnection()->close();
|
|
|
|
$runRepo = $em->getRepository(Run::class);
|
2022-05-19 20:25:05 +02:00
|
|
|
$runRepo->addRun($job, $result['exitcode'], floor($starttime), $runtime, $result['output'], $flags);
|
2021-06-01 17:41:10 +02:00
|
|
|
if (!$manual){
|
|
|
|
// setting nextrun to next run
|
2022-05-17 16:06:46 +02:00
|
|
|
$nextrun = $job->getNextrun();
|
2021-06-01 17:41:10 +02:00
|
|
|
do {
|
2022-05-17 16:06:46 +02:00
|
|
|
$nextrun = $nextrun + $job->getInterval();
|
2021-06-01 17:41:10 +02:00
|
|
|
} while ($nextrun < time());
|
2021-05-24 18:36:16 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
$job->setNextrun($nextrun);
|
|
|
|
$this->deleteTempVar($job);
|
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-06-01 17:41:10 +02:00
|
|
|
}
|
2022-05-17 16:06:46 +02:00
|
|
|
return ['job_id' => $job->getId(), 'exitcode' => $result['exitcode'], 'timestamp' =>floor($starttime), 'runtime' => $runtime, 'output' => (string)$result['output'], 'flags' => implode("", $flags)];
|
2021-05-24 18:36:16 +02:00
|
|
|
}
|
2021-05-29 14:20:05 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param int $id
|
|
|
|
* @return void
|
|
|
|
* @throws \Doctrine\DBAL\Exception
|
|
|
|
*/
|
2022-05-19 15:48:06 +02:00
|
|
|
public function unlockJob(?Job $job = NULL): void
|
2021-05-29 14:20:05 +02:00
|
|
|
{
|
2022-05-19 15:48:06 +02:00
|
|
|
$qb = $this->createQueryBuilder('job');
|
|
|
|
$qry = $qb
|
|
|
|
->update()
|
|
|
|
->set('job.running', 0)
|
|
|
|
->where('job.running = 1');
|
|
|
|
|
|
|
|
if($job !== NULL) {
|
|
|
|
$qry = $qry
|
|
|
|
->andWhere('job = :job')
|
|
|
|
->setParameter(':job', $job);
|
2021-05-29 14:20:05 +02:00
|
|
|
}
|
2022-05-19 15:48:06 +02:00
|
|
|
$qry->getQuery()->execute();
|
2021-05-29 14:20:05 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param Job $job
|
|
|
|
* @return bool
|
|
|
|
*/
|
|
|
|
public function isLockedJob(Job $job): bool
|
2021-07-15 13:22:52 +02:00
|
|
|
{
|
2022-05-18 17:16:39 +02:00
|
|
|
return $job->getRunning() != 0;
|
2021-07-15 13:22:52 +02:00
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param array $values
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-04-13 14:07:11 +02:00
|
|
|
public function addJob(array $values)
|
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
2021-05-19 13:24:38 +02:00
|
|
|
if(empty($values['crontype']) ||
|
2021-04-13 14:07:11 +02:00
|
|
|
empty($values['name']) ||
|
2021-05-06 15:53:21 +02:00
|
|
|
empty($values['interval']) ||
|
2021-04-13 14:07:11 +02:00
|
|
|
empty($values['nextrun'])
|
|
|
|
) {
|
2021-05-21 13:09:48 +02:00
|
|
|
throw new \InvalidArgumentException('Some fields are empty');
|
2021-04-13 14:07:11 +02:00
|
|
|
}
|
|
|
|
|
2022-05-18 17:14:25 +02:00
|
|
|
$job = $this->prepareJob($values);
|
2021-05-21 13:09:48 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-05-21 13:09:48 +02:00
|
|
|
return ['success' => true, 'message' => 'Cronjob succesfully added'];
|
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param int $id
|
|
|
|
* @param array $values
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-05-21 13:09:48 +02:00
|
|
|
public function editJob(int $id, array $values)
|
|
|
|
{
|
2022-05-19 11:56:06 +02:00
|
|
|
$em = $this->getEntityManager();
|
|
|
|
|
2021-05-21 13:09:48 +02:00
|
|
|
if(empty($values['crontype']) ||
|
|
|
|
empty($values['name']) ||
|
|
|
|
empty($values['interval']) ||
|
|
|
|
empty($values['nextrun'])
|
|
|
|
) {
|
|
|
|
throw new \InvalidArgumentException('Some fields are empty');
|
|
|
|
}
|
2022-05-18 17:14:25 +02:00
|
|
|
$job = $this->find($id);
|
|
|
|
$job = $this->prepareJob($values, $job);
|
2021-05-21 13:09:48 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
$em->persist($job);
|
|
|
|
$em->flush();
|
2021-05-21 13:09:48 +02:00
|
|
|
return ['success' => true, 'message' => 'Cronjob succesfully edited'];
|
|
|
|
}
|
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param array $values
|
|
|
|
* @param Job|null $job
|
|
|
|
* @return Job
|
|
|
|
*/
|
2022-05-18 17:14:25 +02:00
|
|
|
public function prepareJob(array $values, ?Job $job = NULL): Job
|
2021-05-21 13:09:48 +02:00
|
|
|
{
|
2022-05-18 17:14:25 +02:00
|
|
|
if ($job === NULL) {
|
|
|
|
$job = new Job();
|
|
|
|
$job->setRunning(0);
|
|
|
|
}
|
|
|
|
$job->setName($values['name']);
|
|
|
|
$job->setInterval($values['interval']);
|
|
|
|
|
2021-05-26 13:34:19 +02:00
|
|
|
if(empty($values['lastrun']) || (isset($values['lastrun-eternal']) && $values['lastrun-eternal'] == 'true')) {
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setLastrun(NULL);
|
2021-04-13 14:07:11 +02:00
|
|
|
} else {
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setLastrun(DateTime::createFromFormat('d/m/Y H:i:s',$values['lastrun'])->getTimestamp());
|
2021-04-13 14:07:11 +02:00
|
|
|
}
|
|
|
|
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setNextrun(DateTime::createFromFormat('d/m/Y H:i:s', $values['nextrun'])->getTimestamp());
|
|
|
|
$job->setData('retention', !empty($values['retention']) ? (int)$values['retention'] : NULL);
|
2021-07-20 16:29:03 +02:00
|
|
|
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('crontype', $values['crontype'] ?? NULL);
|
|
|
|
$job->setData('hosttype', $values['hosttype']);
|
|
|
|
$job->setData('containertype', $values['containertype']);
|
|
|
|
$job->setData('fail-pct', !empty($values['fail-pct']) ? (int)$values['fail-pct'] : 50);
|
|
|
|
$job->setData('fail-days', !empty($values['fail-days']) ? (int)$values['fail-days'] : 7);
|
2021-04-13 14:07:11 +02:00
|
|
|
|
2022-05-18 17:14:25 +02:00
|
|
|
if(!$job->hasData('crontype')) {
|
2021-05-24 12:28:47 +02:00
|
|
|
throw new \InvalidArgumentException("Crontype cannot be empty");
|
|
|
|
}
|
2022-05-18 17:14:25 +02:00
|
|
|
switch($job->getData('crontype'))
|
2021-04-13 14:07:11 +02:00
|
|
|
{
|
2021-05-19 13:24:38 +02:00
|
|
|
case 'command':
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('command', $values['command']);
|
|
|
|
$job->setData('response', explode(',', $values['response']));
|
2021-04-15 13:52:27 +02:00
|
|
|
break;
|
2021-05-06 14:30:35 +02:00
|
|
|
case 'reboot':
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('reboot-command', $values['reboot-command']);
|
|
|
|
$job->setData('getservices-command', $values['getservices-command']);
|
|
|
|
$job->setData('getservices-response', explode(',',$values['getservices-response']));
|
|
|
|
$job->setData('reboot-duration', $values['reboot-duration']);
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->setData('reboot-delay', (int)$values['reboot-delay']);
|
|
|
|
$job->setData('reboot-delay-secs', (int)$values['reboot-delay'] * 60);
|
2021-05-06 14:30:35 +02:00
|
|
|
break;
|
2021-04-13 14:07:11 +02:00
|
|
|
case 'http':
|
|
|
|
$parsedUrl = parse_url($values['url']);
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('url', $values['url']);
|
|
|
|
$job->setData('http-status', explode(',', $values['http-status']));
|
|
|
|
$job->setData('basicauth-username', $values['basicauth-username']);
|
2021-04-13 14:07:11 +02:00
|
|
|
if(empty($parsedUrl['host'])) {
|
2021-05-21 13:09:48 +02:00
|
|
|
throw new \InvalidArgumentException('Some data was invalid');
|
2021-04-13 14:07:11 +02:00
|
|
|
}
|
2021-05-06 13:30:12 +02:00
|
|
|
if(!empty($values['basicauth-password'])) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->setData('basicauth-password', base64_encode(Secret::encrypt($values['basicauth-password'])));
|
2021-05-06 13:30:12 +02:00
|
|
|
}
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('host', $parsedUrl['host']);
|
2021-04-13 14:07:11 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2022-05-18 17:14:25 +02:00
|
|
|
switch($job->getData('hosttype')) {
|
2021-05-24 12:28:47 +02:00
|
|
|
default:
|
2022-05-20 19:34:45 +02:00
|
|
|
if($job->getData('crontype') == 'http') break;
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('hosttype', 'local');
|
2021-05-19 13:24:38 +02:00
|
|
|
case 'local':
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('host', 'localhost');
|
2021-05-19 13:24:38 +02:00
|
|
|
break;
|
|
|
|
case 'ssh':
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('host', $values['host']);
|
|
|
|
$job->setData('user', $values['user']);
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->removeData('privkey-password');
|
2021-05-19 13:24:38 +02:00
|
|
|
if(!empty($values['privkey-password'])) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->setData('privkey-password', base64_encode(Secret::encrypt($values['privkey-password'])));
|
2021-05-19 13:24:38 +02:00
|
|
|
}
|
|
|
|
if(!empty($_FILES['privkey']['tmp_name'])) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->setData('ssh-privkey', base64_encode(Secret::encrypt(file_get_contents($_FILES['privkey']['tmp_name']))));
|
2021-05-19 13:24:38 +02:00
|
|
|
}
|
2021-05-28 17:28:02 +02:00
|
|
|
if(isset($values['privkey-keep']) && $values['privkey-keep'] == true) {
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->setData('ssh-privkey', base64_encode(Secret::encrypt($values['privkey-orig'])));
|
2021-05-24 12:28:47 +02:00
|
|
|
}
|
2021-05-19 13:24:38 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-05-20 13:06:53 +02:00
|
|
|
|
2022-05-20 14:48:22 +02:00
|
|
|
switch($job->getData('containertype')) {
|
2021-05-24 12:28:47 +02:00
|
|
|
default:
|
2022-05-18 17:14:25 +02:00
|
|
|
if($job->getData('crontype') == 'http' || $job->getData('crontype') == 'reboot' ) break;
|
|
|
|
$job->setData('containertype', 'none');
|
2021-05-24 12:28:47 +02:00
|
|
|
case 'none':
|
|
|
|
// No options for no container
|
|
|
|
break;
|
2021-05-20 13:06:53 +02:00
|
|
|
case 'docker':
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('service', $values['service']);
|
|
|
|
$job->setData('container-user', $values['container-user']);
|
2021-05-20 13:06:53 +02:00
|
|
|
break;
|
|
|
|
}
|
2022-05-20 14:48:22 +02:00
|
|
|
$job->removeData('vars');
|
2021-05-06 13:30:12 +02:00
|
|
|
if(!empty($values['var-value'])) {
|
2022-03-21 11:30:30 +01:00
|
|
|
foreach($values['var-value'] as $key => $value) {
|
|
|
|
if(!empty($value) || $value == 0) {
|
2021-05-06 14:30:35 +02:00
|
|
|
if(isset($values['var-issecret'][$key]) && $values['var-issecret'][$key] != false) {
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('vars.' . $values['var-id'][$key] . '.issecret', true);
|
|
|
|
$job->setData('vars.' . $values['var-id'][$key] . '.value', base64_encode(Secret::encrypt($values['var-value'][$key])));
|
2021-05-06 13:30:12 +02:00
|
|
|
} else {
|
2022-05-18 17:14:25 +02:00
|
|
|
$job->setData('vars.' . $values['var-id'][$key] . '.issecret', false);
|
|
|
|
$job->setData('vars.' . $values['var-id'][$key] . '.value', $values['var-value'][$key]);
|
2021-05-06 13:30:12 +02:00
|
|
|
}
|
|
|
|
}
|
2021-04-13 14:44:58 +02:00
|
|
|
}
|
|
|
|
}
|
2022-05-18 17:14:25 +02:00
|
|
|
return $job;
|
2021-04-13 14:07:11 +02:00
|
|
|
}
|
2021-04-13 14:44:58 +02:00
|
|
|
|
2022-05-19 11:56:06 +02:00
|
|
|
/**
|
|
|
|
* @param int $id
|
|
|
|
* @return array
|
|
|
|
*/
|
2021-05-24 14:08:30 +02:00
|
|
|
public function deleteJob(int $id)
|
|
|
|
{
|
2022-05-18 11:12:35 +02:00
|
|
|
$em = $this->getEntityManager();
|
|
|
|
|
|
|
|
$job = $this->find($id);
|
|
|
|
$em->remove($job);
|
|
|
|
$em->flush();
|
2021-05-24 14:08:30 +02:00
|
|
|
|
|
|
|
return ['success' => true, 'message' => 'Cronjob succesfully deleted'];
|
|
|
|
}
|
2021-04-08 12:54:49 +02:00
|
|
|
}
|