From 3edcf564bc8f41a13a1ed9be4b1cc40976cf6527 Mon Sep 17 00:00:00 2001 From: Eudyptula Date: Mon, 10 Aug 2009 16:48:30 -0400 Subject: Major backend reorganization - moved much of backend.php into build class, reformated bundlers to be the same format as modules, fixed build status tracking to be properly resumable throughout the process (except stages that create files that get in their own way); etc. --- backend/backend.php | 93 ++++---------- backend/bundlers/cd-common.php | 6 + backend/bundlers/cd-head.php | 8 ++ backend/bundlers/cd.inc | 12 -- backend/bundlers/ext2.php | 12 -- backend/bundlers/ext2/bundle.php | 6 + backend/bundlers/ext2/cp.php | 6 + backend/bundlers/ext2/mkfs.php | 4 + backend/bundlers/ext2/tar.php | 3 + backend/bundlers/installcd.php | 8 -- backend/bundlers/installcd/assemble.php | 4 + backend/bundlers/installcd/bundle.php | 6 + backend/bundlers/installcd/cd-common.php | 1 + backend/bundlers/installcd/cd-head.php | 1 + backend/bundlers/jffs2.php | 6 - backend/bundlers/jffs2/bundle.php | 4 + backend/bundlers/jffs2/mkfs.php | 3 + backend/bundlers/livecd.php | 9 -- backend/bundlers/livecd/assemble.php | 4 + backend/bundlers/livecd/bundle.php | 7 ++ backend/bundlers/livecd/cd-common.php | 1 + backend/bundlers/livecd/cd-head.php | 1 + backend/bundlers/livecd/livecd-tools.php | 3 + backend/bundlers/tbz2.php | 6 - backend/bundlers/tbz2/bundle.php | 4 + backend/bundlers/tbz2/tar.php | 3 + backend/bundlers/tgz.php | 6 - backend/bundlers/tgz/bundle.php | 4 + backend/bundlers/tgz/tar.php | 3 + backend/functions/api.php | 1 - backend/functions/signals.php | 4 +- backend/include/includes.php | 4 - backend/modules/gentoo_portage/base-system.php | 2 +- backend/modules/gentoo_portage/build.php | 8 +- backend/modules/gentoo_portage/hostname.php | 2 +- backend/modules/gentoo_portage/init.d.php | 2 +- backend/modules/gentoo_portage/portage.php | 10 +- backend/modules/gentoo_portage/setup.php | 10 +- backend/modules/gentoo_portage/timezone.php | 6 +- frontend/pages/builds/delete.php | 5 +- frontend/pages/logout.php | 1 + frontend/routing.csv | 1 + gentoo-steps | 2 +- shared/classes/build.php | 168 +++++++++++++++++-------- shared/classes/task.php | 37 +++--- shared/config.php | 3 +- shared/functions/load_config.php | 2 +- shared/include/defaults.php | 7 +- shared/include/includes.php | 5 +- todo | 9 +- 50 files changed, 276 insertions(+), 247 deletions(-) create mode 100644 backend/bundlers/cd-common.php create mode 100644 backend/bundlers/cd-head.php delete mode 100644 backend/bundlers/cd.inc delete mode 100644 backend/bundlers/ext2.php create mode 100644 backend/bundlers/ext2/bundle.php create mode 100644 backend/bundlers/ext2/cp.php create mode 100644 backend/bundlers/ext2/mkfs.php create mode 100644 backend/bundlers/ext2/tar.php delete mode 100644 backend/bundlers/installcd.php create mode 100644 backend/bundlers/installcd/assemble.php create mode 100644 backend/bundlers/installcd/bundle.php create mode 120000 backend/bundlers/installcd/cd-common.php create mode 120000 backend/bundlers/installcd/cd-head.php delete mode 100644 backend/bundlers/jffs2.php create mode 100644 backend/bundlers/jffs2/bundle.php create mode 100644 backend/bundlers/jffs2/mkfs.php delete mode 100644 backend/bundlers/livecd.php create mode 100644 backend/bundlers/livecd/assemble.php create mode 100644 backend/bundlers/livecd/bundle.php create mode 120000 backend/bundlers/livecd/cd-common.php create mode 120000 backend/bundlers/livecd/cd-head.php create mode 100644 backend/bundlers/livecd/livecd-tools.php delete mode 100644 backend/bundlers/tbz2.php create mode 100644 backend/bundlers/tbz2/bundle.php create mode 100644 backend/bundlers/tbz2/tar.php delete mode 100644 backend/bundlers/tgz.php create mode 100644 backend/bundlers/tgz/bundle.php create mode 100644 backend/bundlers/tgz/tar.php delete mode 100644 backend/include/includes.php diff --git a/backend/backend.php b/backend/backend.php index 2874b1a..161b7b2 100755 --- a/backend/backend.php +++ b/backend/backend.php @@ -50,7 +50,7 @@ require_once(SHARED.'/include/dbinit.php'); while (true) { // TODO check first for builds that need to be resumed (and figure out how to resume things) while (true) { - $r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","upload_failed","building","got_signal") ORDER BY `ctime` ASC LIMIT 1'); + $r=query('SELECT * FROM `builds` WHERE `backend`="'.$S['conf']['backend_id'].'" AND `status` IN ("queued","cancel","uploading","building","bundling") AND `failed`!="true" ORDER BY `ctime` ASC LIMIT 1'); if ($r->rowCount()) break; else { @@ -60,76 +60,37 @@ while (true) { } } $build=new sql_build($r->fetch(PDO::FETCH_ASSOC)); - if (!isset($build->start)) { - $build->start=time(); - $build->write(); - } debug('Starting build id='.$build->id); $file=null; $owner=$build->get_owner(); - $workdir=WORK."/build-$build->id"; - if (($image=$build->build($workdir)) !== false) { - try { - $bundler=$build->get_opt('bundler'); - $bundle_proc="bundle_$bundler"; - if (!function_exists($bundle_proc)) - throw_exception("No bundler function defined for bundler $bundler"); - $opts=$build->get_opts(); - $file=$bundle_proc($image, $workdir, $opts); - end_internal_task(0); // Just in case - } catch (Exception $e) { - log_msg('Caught exception: '.$e->getMessage()); - end_internal_task(1); - $build->status='failed'; - $build->write(); - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed in bundling stage. You can find more information at id").'">'.url("build/$build->id").''); - } - $build->finish=time(); - debug('Finished with build id='.$build->id); - if (isset($file)) { - debug("Completed build successfully"); - if ($S['conf']['split_setup']) { - $build->status='uploading'; - $build->write(); - $key=randstring(30); - $build->set_opt('uploadkey', $key); - $c=curl_init(url('backend/upload_image')); - curl_setopt($c, CURLOPT_POST, 1); - curl_setopt($c, CURLOPT_POSTFIELDS, array( - 'build' => $build->id, - 'key' => $key, - 'file' => "@$file" - )); - curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); - $result=curl_exec($c); - if ($S['conf']['debug'] && is_string($result)) { - debug($result); - } - if ($result === false || strpos($result, 'Upload successful') === false) { - $build->status='upload_failed'; - $build->write(); - } else { - debug("Transferred $file... unlinking it"); - unlink($file); - $build->status='complete'; - $build->write(); - shell_exec('rm -rf '.escapeshellarg($workdir)); - } - } else { - $build->status='complete'; - $base=basename($file); - $ext=substr($base, strpos($base, '.')); - rename($file, COMPLETED.'/build-'.$build->id.$ext); - $build->write(); - shell_exec('rm -rf '.escapeshellarg($workdir)); - } - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has completed successfully. You can find more information and download the completed image at id").'">'.url("build/$build->id").''); - } - } else { - $build->status='failed'; + $workdir=WORK."/$build->id"; + if ($build->status == 'cancel') { + shell_exec('rm -rf '.escapeshellarg($workdir)); + $build->status='canceled'; $build->write(); - xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build failed', 'Your build has failed. You can find more information at id").'">'.url("build/$build->id").''); + continue; + } + if (isset($build->finish)) + $build->finish=null; + elseif (!isset($build->start)) + $build->start=time(); + $build->write(); + $success=$imagedir=$build->build($workdir); + if ($success && !$build->is_canceled()) { + $success=$file=$build->bundle($imagedir, $workdir); + if ($success && !$build->is_canceled()) { + $success=$build->upload($file); + } + } + if ($success) { + debug("Completed build successfully"); + execute_non_vital_command('Delete work directory', 'rm -rf '.escapeshellarg($workdir)); + $build->status='complete'; } + $build->finish=time(); + $build->write(); + debug('Finished with build id='.$build->id); + xhtmlemail('"'.$owner->name.'" <'.$owner->email.'>', null, $S['conf']['title'].' build finished', 'Your build has '.($success?'completed successfully':'failed').'. You can find more information and download the completed image at id").'">'.url("build/$build->id").''); unset($build); } ?> diff --git a/backend/bundlers/cd-common.php b/backend/bundlers/cd-common.php new file mode 100644 index 0000000..0a7f3fb --- /dev/null +++ b/backend/bundlers/cd-common.php @@ -0,0 +1,6 @@ + diff --git a/backend/bundlers/cd-head.php b/backend/bundlers/cd-head.php new file mode 100644 index 0000000..2e96bba --- /dev/null +++ b/backend/bundlers/cd-head.php @@ -0,0 +1,8 @@ +get_headers(); +if (strpos($headers['chost'], 'x86_64') === false) + $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; +else + $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; +?> diff --git a/backend/bundlers/cd.inc b/backend/bundlers/cd.inc deleted file mode 100644 index 6fb5aac..0000000 --- a/backend/bundlers/cd.inc +++ /dev/null @@ -1,12 +0,0 @@ -get_headers(); -if (strpos($headers['chost'], 'x86_64') === false) - $minimaliso=CACHE.'/cd/install-x86-minimal-20090623.iso'; -else - $minimaliso=CACHE.'/cd/install-amd64-minimal-20090625.iso'; -makedir("$I/boot"); -execute_command('Extract kernel, initrd, and squashfs from CD image ', LIB."/bkisofs-cli '$minimaliso' extract /isolinux/gentoo '$I/boot/kernel' extract /isolinux/gentoo.igz '$I/boot/initrd' extract /image.squashfs '$W/'"); -//file_put_contents("$W/unsquashfs-files", "/lib64/modules\n/lib/modules\n"); -execute_command('Extract kernel modules from SquashFS to image', "unsquashfs -i -d '$W/modules' '$W/image.squashfs' /lib/modules /lib64/modules; cp -avT '$W/modules' '$I'; rm -rf '$W/modules'"); -?> diff --git a/backend/bundlers/ext2.php b/backend/bundlers/ext2.php deleted file mode 100644 index 239b5bb..0000000 --- a/backend/bundlers/ext2.php +++ /dev/null @@ -1,12 +0,0 @@ - diff --git a/backend/bundlers/ext2/bundle.php b/backend/bundlers/ext2/bundle.php new file mode 100644 index 0000000..2418453 --- /dev/null +++ b/backend/bundlers/ext2/bundle.php @@ -0,0 +1,6 @@ + diff --git a/backend/bundlers/ext2/cp.php b/backend/bundlers/ext2/cp.php new file mode 100644 index 0000000..befbd91 --- /dev/null +++ b/backend/bundlers/ext2/cp.php @@ -0,0 +1,6 @@ + diff --git a/backend/bundlers/ext2/mkfs.php b/backend/bundlers/ext2/mkfs.php new file mode 100644 index 0000000..925c93f --- /dev/null +++ b/backend/bundlers/ext2/mkfs.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/ext2/tar.php b/backend/bundlers/ext2/tar.php new file mode 100644 index 0000000..5e51e1d --- /dev/null +++ b/backend/bundlers/ext2/tar.php @@ -0,0 +1,3 @@ + diff --git a/backend/bundlers/installcd.php b/backend/bundlers/installcd.php deleted file mode 100644 index 63ffb5a..0000000 --- a/backend/bundlers/installcd.php +++ /dev/null @@ -1,8 +0,0 @@ - diff --git a/backend/bundlers/installcd/assemble.php b/backend/bundlers/installcd/assemble.php new file mode 100644 index 0000000..c4cedc0 --- /dev/null +++ b/backend/bundlers/installcd/assemble.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/installcd/bundle.php b/backend/bundlers/installcd/bundle.php new file mode 100644 index 0000000..384490e --- /dev/null +++ b/backend/bundlers/installcd/bundle.php @@ -0,0 +1,6 @@ + diff --git a/backend/bundlers/installcd/cd-common.php b/backend/bundlers/installcd/cd-common.php new file mode 120000 index 0000000..70f9287 --- /dev/null +++ b/backend/bundlers/installcd/cd-common.php @@ -0,0 +1 @@ +../cd-common.php \ No newline at end of file diff --git a/backend/bundlers/installcd/cd-head.php b/backend/bundlers/installcd/cd-head.php new file mode 120000 index 0000000..8a38a03 --- /dev/null +++ b/backend/bundlers/installcd/cd-head.php @@ -0,0 +1 @@ +../cd-head.php \ No newline at end of file diff --git a/backend/bundlers/jffs2.php b/backend/bundlers/jffs2.php deleted file mode 100644 index c8e5de2..0000000 --- a/backend/bundlers/jffs2.php +++ /dev/null @@ -1,6 +0,0 @@ - diff --git a/backend/bundlers/jffs2/bundle.php b/backend/bundlers/jffs2/bundle.php new file mode 100644 index 0000000..de198a6 --- /dev/null +++ b/backend/bundlers/jffs2/bundle.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/jffs2/mkfs.php b/backend/bundlers/jffs2/mkfs.php new file mode 100644 index 0000000..bf9e702 --- /dev/null +++ b/backend/bundlers/jffs2/mkfs.php @@ -0,0 +1,3 @@ + diff --git a/backend/bundlers/livecd.php b/backend/bundlers/livecd.php deleted file mode 100644 index 740c72b..0000000 --- a/backend/bundlers/livecd.php +++ /dev/null @@ -1,9 +0,0 @@ - diff --git a/backend/bundlers/livecd/assemble.php b/backend/bundlers/livecd/assemble.php new file mode 100644 index 0000000..0371103 --- /dev/null +++ b/backend/bundlers/livecd/assemble.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/livecd/bundle.php b/backend/bundlers/livecd/bundle.php new file mode 100644 index 0000000..465404b --- /dev/null +++ b/backend/bundlers/livecd/bundle.php @@ -0,0 +1,7 @@ + diff --git a/backend/bundlers/livecd/cd-common.php b/backend/bundlers/livecd/cd-common.php new file mode 120000 index 0000000..70f9287 --- /dev/null +++ b/backend/bundlers/livecd/cd-common.php @@ -0,0 +1 @@ +../cd-common.php \ No newline at end of file diff --git a/backend/bundlers/livecd/cd-head.php b/backend/bundlers/livecd/cd-head.php new file mode 120000 index 0000000..8a38a03 --- /dev/null +++ b/backend/bundlers/livecd/cd-head.php @@ -0,0 +1 @@ +../cd-head.php \ No newline at end of file diff --git a/backend/bundlers/livecd/livecd-tools.php b/backend/bundlers/livecd/livecd-tools.php new file mode 100644 index 0000000..13132a3 --- /dev/null +++ b/backend/bundlers/livecd/livecd-tools.php @@ -0,0 +1,3 @@ + diff --git a/backend/bundlers/tbz2.php b/backend/bundlers/tbz2.php deleted file mode 100644 index ee69825..0000000 --- a/backend/bundlers/tbz2.php +++ /dev/null @@ -1,6 +0,0 @@ - diff --git a/backend/bundlers/tbz2/bundle.php b/backend/bundlers/tbz2/bundle.php new file mode 100644 index 0000000..6024c3e --- /dev/null +++ b/backend/bundlers/tbz2/bundle.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/tbz2/tar.php b/backend/bundlers/tbz2/tar.php new file mode 100644 index 0000000..2aead89 --- /dev/null +++ b/backend/bundlers/tbz2/tar.php @@ -0,0 +1,3 @@ + diff --git a/backend/bundlers/tgz.php b/backend/bundlers/tgz.php deleted file mode 100644 index 3b604d3..0000000 --- a/backend/bundlers/tgz.php +++ /dev/null @@ -1,6 +0,0 @@ - diff --git a/backend/bundlers/tgz/bundle.php b/backend/bundlers/tgz/bundle.php new file mode 100644 index 0000000..b283ee7 --- /dev/null +++ b/backend/bundlers/tgz/bundle.php @@ -0,0 +1,4 @@ + diff --git a/backend/bundlers/tgz/tar.php b/backend/bundlers/tgz/tar.php new file mode 100644 index 0000000..09fbf0c --- /dev/null +++ b/backend/bundlers/tgz/tar.php @@ -0,0 +1,3 @@ + diff --git a/backend/functions/api.php b/backend/functions/api.php index d8d648c..cffbf65 100644 --- a/backend/functions/api.php +++ b/backend/functions/api.php @@ -57,7 +57,6 @@ function log_msg($msg, $nl=true) { if (!isset($task)) { start_internal_task($msg); debug('log_msg creating task... this is bad'); - return; } $msg.=$nl?"\n":''; debug($msg); diff --git a/backend/functions/signals.php b/backend/functions/signals.php index 70231c7..62f5fe3 100644 --- a/backend/functions/signals.php +++ b/backend/functions/signals.php @@ -5,6 +5,7 @@ function handle_signal($sig=null) { unlink($pidfile); if (isset($build)) { if (isset($task)) { + log_msg("\nGot signal $sig\n"); $task->finish=time(); $task->exit=-$sig; $task->write(); @@ -13,10 +14,7 @@ function handle_signal($sig=null) { debug('$task not set'); } $build->finish=time(); - $build->status='got_signal'; - $build->details=$sig; $build->write(); - debug("build $build->id given status $build->status"); } debug("\nGot signal $sig - exiting"); exit; diff --git a/backend/include/includes.php b/backend/include/includes.php deleted file mode 100644 index d20c3ac..0000000 --- a/backend/include/includes.php +++ /dev/null @@ -1,4 +0,0 @@ - diff --git a/backend/modules/gentoo_portage/base-system.php b/backend/modules/gentoo_portage/base-system.php index c99447e..0c730e9 100644 --- a/backend/modules/gentoo_portage/base-system.php +++ b/backend/modules/gentoo_portage/base-system.php @@ -1,6 +1,6 @@ stage3; -execute_command('Unpack base system', "tar -xvjpf '$file' -C '$I'"); +execute_command('Unpack base system', "tar -xvjpf '$file' -C '$imagedir'"); if ($opts['basesystem'] == 'user_prune' && $opts['prunepkgs']) { emerge($opts['prunepkgs'], 'Prune base system packages', '-C'); } elseif ($opts['basesystem'] == 'auto_prune') { diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php index 781aaef..2d2f32f 100644 --- a/backend/modules/gentoo_portage/build.php +++ b/backend/modules/gentoo_portage/build.php @@ -2,9 +2,9 @@ require_once(dirname(__FILE__).'/packages.php'); // __DIR__ 5.3.0 $profile=new sql_gentoo_profile($opts['profile']); $headers=$profile->get_headers(); -$I="$workdir/image"; -$C="$workdir/config_root"; -$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $C); +$imagedir="$workdir/image"; +$confdir="$workdir/config_root"; +$S['prtg_cfgrt']=array('PORTAGE_CONFIGROOT' => $confdir); add_step('setup'); add_step('base-system'); $extra=explode(' ', $opts['options']); @@ -22,5 +22,5 @@ if (strlen($opts['pkgsets'])) add_step('pkgsets'); if (strlen($opts['install_packages'])) add_step('misc-pkgs'); -return $I; +return $imagedir; ?> diff --git a/backend/modules/gentoo_portage/hostname.php b/backend/modules/gentoo_portage/hostname.php index 11c7d20..e710560 100644 --- a/backend/modules/gentoo_portage/hostname.php +++ b/backend/modules/gentoo_portage/hostname.php @@ -1,4 +1,4 @@ diff --git a/backend/modules/gentoo_portage/init.d.php b/backend/modules/gentoo_portage/init.d.php index deef101..9ab61e6 100644 --- a/backend/modules/gentoo_portage/init.d.php +++ b/backend/modules/gentoo_portage/init.d.php @@ -2,7 +2,7 @@ start_internal_task('Remove unwanted init scripts'); foreach (explode(' ', $opts['pruneinit']) as $init) { list($name, $runlevel)=explode(':', $init, 2); - log_status("$name ($runlevel)", unlink("$I/etc/runlevels/$runlevel/$name")); + log_status("$name ($runlevel)", unlink("$imagedir/etc/runlevels/$runlevel/$name")); } end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/portage.php b/backend/modules/gentoo_portage/portage.php index a1eb3ed..bcfa7d7 100644 --- a/backend/modules/gentoo_portage/portage.php +++ b/backend/modules/gentoo_portage/portage.php @@ -3,14 +3,14 @@ $file=glob(CACHE.'/portage/*.tar.bz2'); if ($file) { $file=array_reverse($file); $file=$file[0]; - execute_command('Unpack portage snapshot', "tar -xvjpf '$file' -C '$I/usr'"); + execute_command('Unpack portage snapshot', "tar -xvjpf '$file' -C '$imagedir/usr'"); } else { start_internal_task('Copy local portage tree to image'); foreach (glob("{$S['conf']['portdir']}/*") as $from) { $file=substr($from, strlen($S['conf']['portdir'])+1); // Skip distfiles, binary packages, and potential overlay directories if ($file == 'distfiles' || $file == 'packages' || $file == 'local' || $file == 'overlay') continue; - $cmd="cp -av -t '$I/usr/portage/' '$from'"; + $cmd="cp -av -t '$imagedir/usr/portage/' '$from'"; error_get_last(); @shell_exec($cmd); end_internal_task((int)(bool)error_get_last()); @@ -33,8 +33,8 @@ foreach ($makeconf as $name => $val) $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); log_msg("/etc/make.conf:\n$contents"); -log_status('Writing /etc/make.conf', file_put_contents("$I/etc/make.conf", $contents)); -log_status('Remove previous make.profile', unlink("$I/etc/make.profile"), false); -log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$I/etc/make.profile")); +log_status('Writing /etc/make.conf', file_put_contents("$imagedir/etc/make.conf", $contents)); +log_status('Remove previous make.profile', unlink("$imagedir/etc/make.profile"), false); +log_status("Symlink make.profile -> /usr/portage/profiles/{$headers['profile']}", symlink("/usr/portage/profiles/{$headers['profile']}", "$imagedir/etc/make.profile")); end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/setup.php b/backend/modules/gentoo_portage/setup.php index d96eb51..93dbad8 100644 --- a/backend/modules/gentoo_portage/setup.php +++ b/backend/modules/gentoo_portage/setup.php @@ -1,12 +1,12 @@ .", symlink('.', "$C/etc")); +makedirs($imagedir, $confdir, "$workdir/log", "$workdir/tmp"); +log_status("Make symlink $confdir/etc -> .", symlink('.', "$confdir/etc")); $makeconf=array( 'pkgdir' => $S['conf']['pkgdir_root'].'/'.$profile->pkgdir, 'chost' => $headers['chost'], 'accept_keywords' => $headers['accept_keywords'], - 'root' => $I, + 'root' => $imagedir, 'port_logdir' => "$workdir/log", 'emerge_log_dir' => "$workdir/log", 'portage_tmpdir' => "$workdir/tmp" @@ -15,8 +15,8 @@ $contents=''; foreach ($makeconf as $name => $val) $contents.=strtoupper($name).'="'.str_replace('"', '\"', $val)."\"\n"; unset($makeconf); -log_status("Write $C/etc/make.conf", file_put_contents("$C/etc/make.conf", $contents)); +log_status("Write $confdir/etc/make.conf", file_put_contents("$confdir/etc/make.conf", $contents)); unset($contents); -log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')); +log_status('Make make.profile symlink to '.$S['conf']['portdir'].'/profiles/'.$headers['profile'], symlink($S['conf']['portdir'].'/profiles/'.$headers['profile'], $confdir.'/etc/make.profile')); end_internal_task(0); ?> diff --git a/backend/modules/gentoo_portage/timezone.php b/backend/modules/gentoo_portage/timezone.php index e5cc196..b5d4e6e 100644 --- a/backend/modules/gentoo_portage/timezone.php +++ b/backend/modules/gentoo_portage/timezone.php @@ -1,9 +1,9 @@ diff --git a/frontend/pages/builds/delete.php b/frontend/pages/builds/delete.php index 6fc6ae3..b92d28b 100644 --- a/frontend/pages/builds/delete.php +++ b/frontend/pages/builds/delete.php @@ -13,11 +13,9 @@ function body_builds_delete(&$S) { case 'queued': if (isset($S['build']->backend)) die(print_warning('Oops', 'You tried to delete this build just as it was about to start being built. Please try to cancel it in a moment.')); - case 'upload_failed': case 'canceled': - case 'failed': + case 'queued': case 'complete': - case 'got_signal': $S['build']->delete(); echo print_success('Build deleted.'); break; @@ -27,6 +25,7 @@ function body_builds_delete(&$S) { case 'uploading': case 'building': default: + $S['build']->failed='false'; // Otherwise doesn't get noticed by backend $S['build']->status='cancel'; $S['build']->write(); echo print_success('Build queued for cancellation.'); diff --git a/frontend/pages/logout.php b/frontend/pages/logout.php index 71f8c11..a60ee2d 100644 --- a/frontend/pages/logout.php +++ b/frontend/pages/logout.php @@ -4,6 +4,7 @@ function init_logout(&$S) { $S['session']->delete(); } setcookie($S['conf']['cookiename'], '', 1, $S['cookie_dir'], '', false, true); + unset($S['user']); if (isset($_REQUEST['go'])) { header('Location: '.url($_REQUEST['go'])); } diff --git a/frontend/routing.csv b/frontend/routing.csv index 3151d27..7183a43 100644 --- a/frontend/routing.csv +++ b/frontend/routing.csv @@ -19,6 +19,7 @@ build/([a-z0-9]{6})/([0-9]+)/([0-9]+) builds/task build task page build/([a-zA-Z0-9]{6})/download builds/download build build/([a-zA-Z0-9]{6})/history builds/history build build/([a-zA-Z0-9]{6})/delete builds/delete build +build/([a-zA-Z0-9]{6})/cancel builds/delete build #build/([a-z0-9]{6})/live builds/live build # Configurations create configurations/wizard diff --git a/gentoo-steps b/gentoo-steps index d17d61b..e0e4fe6 100644 --- a/gentoo-steps +++ b/gentoo-steps @@ -2,7 +2,7 @@ X*Set the date Install a stage tarball Install a portage snapshot (from host sytem - support tarball snapshots?) -*Set up make.conf +Set up make.conf ?*Mirror selection Profile selection *Locale selection (selected from /usr/share/i18n/SUPPORTED) diff --git a/shared/classes/build.php b/shared/classes/build.php index 64730fe..10b2043 100644 --- a/shared/classes/build.php +++ b/shared/classes/build.php @@ -36,18 +36,25 @@ class sql_build extends conf_build_common { ), 'status' => array ( 'type' => 'ENUM', - 'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'upload_failed\',\'canceled\',\'failed\',\'got_signal\',\'building\',\'bundling\'', + 'length' => '\'queued\',\'uploading\',\'cancel\',\'complete\',\'canceled\',\'building\',\'bundling\'', 'not_null' => true ), - 'details' => array ( + 'build_step' => array ( 'type' => 'TINYINT', - 'length' => 4 + 'length' => 4, + 'unsigned' => true ), - 'build_step' => array ( + 'num_steps' => array ( 'type' => 'TINYINT', - 'length' => 3, + 'length' => 4, 'unsigned' => true ), + 'failed' => array ( + 'type' => 'ENUM', + 'length' => '\'false\',\'true\'', + 'not_null' => true, + 'default' => 'false' + ), 'ctime' => array ( 'type' => 'INT', 'length' => 10, @@ -71,49 +78,52 @@ class sql_build extends conf_build_common { $perms=$this->visibility == 'public' || owner_or_admin($this->id); $html='
'.(isset($this->name) && strlen($this->name)?htmlentities($this->name):'Unnamed Build').' '; $links=array(); - if ($this->status == 'queued') { + switch ($this->status) { + case 'queued': $total=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued"')->fetch(PDO::FETCH_COLUMN); $num=query('SELECT COUNT(*) FROM `builds` WHERE `status`="queued" AND `ctime` <= '.$this->ctime)->fetch(PDO::FETCH_COLUMN); $html.="[queued ($num/$total)]"; - } elseif ($this->status == 'uploading') { + break; + case 'uploading': $html.='[uploading]'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'cancel') { + break; + case 'cancel': $html.='[pending cancellation]'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'building') { - // TODO stage x/y - $html.='[building ('.$this->build_step.'/'.$this->details.')]'; + break; + case 'building': + case 'bundling': + $html.='['.$this->status.' ('.$this->build_step.'/'.$this->num_steps.')]'; if ($perms) { //$links['Watch']="build/$this->id/live"; $links['Build Log']="build/$this->id"; } - } elseif ($this->status == 'complete') { - $r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC); - $d=($perms && $r['count']?'id/history").'">':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'':'').'
(last at '.date($format, $r['time']).')':''); + break; + case 'complete': + $url="build/$this->id/history"; + if ($perms && $S['request'] != $url) { + $r=query('SELECT COUNT(*) as `count`, MAX(`time`) as `time` FROM `downloads` WHERE `build`="'.$this->id.'"')->fetch(PDO::FETCH_ASSOC); + $d=($r['count']?'':'').$r['count'].' download'.($r['count'] != 1?'s':'').($r['count']?($perms?'':'').'
(last at '.date($format, $r['time']).')':''); + } else + $d=''; $html.=''.$d.'[successful]'; $links['Download image']="build/$this->id/download"; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'upload_failed') { - $html.='[upload failed]'; - if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'failed') { - $html.='[failed after step '.$this->build_step.']'; - if ($perms) { - //$links['View output of failed command']="build/$this->id/failure"; - $links['Build log']="build/$this->id"; - } - } elseif ($this->status == 'canceled') { + break; + case 'canceled': $html.='[canceled]'; if ($perms) $links['Build log']="build/$this->id"; - } elseif ($this->status == 'got_signal') { - $html.='[failed: got signal '.$this->details.' after step '.$this->build_step.']'; - if ($perms) $links['Build log']="build/$this->id"; - } else { + break; + default: $html.='[UNKNOWN STATUS: '.$this->status.']'; } - if ($perms && ($this->status == 'upload_failed' || $this->status == 'failed' || $this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->status == 'got_signal')) - $links['Delete']="build/$this->id/delete"; + if ($perms) { + if ($this->status == 'canceled' || $this->status == 'queued' || $this->status == 'complete' || $this->failed == 'true') + $links['Delete']="build/$this->id/delete"; + elseif ($this->status != 'cancel') + $links['Cancel']="build/$this->id/cancel"; + } if ($links) { foreach ($links as $label => $url) { if ($S['request'] == $url) @@ -141,13 +151,6 @@ class sql_build extends conf_build_common { $html.='
'; return $html; } - public function queued_tasks() { - global $S; - static $cache; - if (!isset($cache)) - $cache=query('SELECT COUNT(`order`) FROM `tasks` WHERE `start` IS NULL AND `build`="'.$this->id.'"')->fetch(PDO::FETCH_COLUMN); - return $cache; - } public function delete() { global $S; query('DELETE FROM `buildlogs` WHERE `build`="'.$this->id.'"'); @@ -161,43 +164,108 @@ class sql_build extends conf_build_common { public function build($workdir) { global $S; try { - if (!is_dir($workdir)) - log_status('Create work directory '.$workdir, mkdir($workdir, 0700)); $opts=$this->get_opts(); $S['build_steps']=array(); if (!is_readable(BACKEND."/modules/$this->module/build.php")) throw_exception("No build script for module $this->module"); - $dir=require(BACKEND."/modules/$this->module/build.php"); + $imagedir=require(BACKEND."/modules/$this->module/build.php"); switch ($this->status) { case 'queued': - $this->build_step=0; - case 'got_signal': - case 'failed': $this->status='building'; - $this->details=count($S['build_steps']); + $this->build_step=0; + $this->num_steps=count($S['build_steps']); $this->write(); case 'building': $step=$this->build_step; break; - case 'uploading': - case 'upload_failed': - case 'cancel': - case 'bundling': default: - $step=count($S['build_steps']); + return $imagedir; } + if (!is_dir($workdir)) + log_status('Create work directory '.$workdir, mkdir($workdir, 0700)); while ($step < count($S['build_steps'])) { require(BACKEND."/modules/$this->module/{$S['build_steps'][$step]}.php"); $step++; $this->build_step=$step; $this->write(); + if ($this->is_canceled()) return false; } - return $dir; + return $imagedir; } catch(Exception $e) { log_msg('Caught exception: '.$e->getMessage()); end_internal_task(1); + $this->failed(); return false; } } + public function bundle($imagedir, $workdir) { + global $S; + try { + $opts=$this->get_opts(); + $bundler=$opts['bundler']; + if (!is_readable(BACKEND."/bundlers/$bundler/bundle.php")) + throw_exception("No bundle script for bundler $bundler"); + $S['build_steps']=array(); + $file=require(BACKEND."/bundlers/$bundler/bundle.php"); + switch($this->status) { + case 'building': + $this->status='bundling'; + $this->build_step=0; + $this->num_steps=count($S['build_steps']); + $this->write(); + case 'bundling': + $step=$this->build_step; + break; + default: + return $file; + } + print_r($S['build_steps']); + while ($step < count($S['build_steps'])) { + require(BACKEND."/bundlers/$bundler/{$S['build_steps'][$step]}.php"); + $step++; + $this->build_step=$step; + $this->write(); + if ($this->is_canceled()) return false; + } + return $file; + } catch (Exception $e) { + log_msg('Caught exception: '.$e->getMessage()); + end_internal_task(1); + $this->failed(); + return false; + } + } + public function upload($file) { + $this->status='uploading'; + $this->write(); + $key=randstring(30); + $this->set_opt('uploadkey', $key); + $c=curl_init(url('backend/upload_image')); + curl_setopt($c, CURLOPT_POST, 1); + curl_setopt($c, CURLOPT_POSTFIELDS, array( + 'build' => $this->id, + 'key' => $key, + 'file' => "@$file" + )); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + $result=curl_exec($c); + $result($result !== false && strpos($result, 'Upload successful') !== false); + if ($result) { + $this->status='complete'; + $this->finish=time(); + $this->write(); + } else + $this->failed(); + return $result; + } + private function failed() { + $this->failed='true'; + $this->finish=time(); + $this->write(); + } + public function is_canceled() { + $this->load(); + return ($this->status == 'cancel'); + } } ?> diff --git a/shared/classes/task.php b/shared/classes/task.php index d763107..e249e0f 100644 --- a/shared/classes/task.php +++ b/shared/classes/task.php @@ -51,30 +51,25 @@ class sql_task extends sql_row_obj { global $S; $link="build/$this->build/$this->order"; $html='
'.htmlentities($this->description).'
'.($S['request'] == $link || strpos($S['request'], "$link/") === 0?'':'[log] ').($this->command?'env?' title="'.htmlentities(str_replace("\n", '; ', $this->env)).'"':'').'>'.htmlentities($this->command).' ':''); - if (isset($this->start)) { - if (isset($this->finish)) { - $html.='[successful'; - } else { - $html.='failed">['; - if (isset($this->exit)) { - if ($this->exit > 0) - $html.='exit status '.$this->exit; - elseif ($this->exit == -128) - $html.='got unknown signal'; - else - $html.='got signal '.-$this->exit; - } else - $html.='failed to execute'; - } - $html.='] Finished in '.display_time($this->finish-$this->start).''; + if (isset($this->finish)) { + $html.='[successful'; } else { - $html.='[running] Running for '.display_time(time()-$this->start).''; + $html.='failed">['; + if (isset($this->exit)) { + if ($this->exit > 0) + $html.='exit status '.$this->exit; + elseif ($this->exit == -128) + $html.='got unknown signal'; + else + $html.='got signal '.-$this->exit; + } else + $html.='failed to execute'; } + $html.='] Finished in '.display_time($this->finish-$this->start).''; } else { - $num=query('SELECT COUNT(*) FROM `tasks` WHERE `builds`="'.$this->build.'" AND `start` IS NULL AND `order` <= '.$this->order)->fetch(PDO::FETCH_ASSOC); - $html.="[queued $num/".$build->queued_tasks()."]"; + $html.='[running] Running for '.display_time(time()-$this->start).''; } $html.='
'; return $html; diff --git a/shared/config.php b/shared/config.php index 9270d2d..8a93454 100644 --- a/shared/config.php +++ b/shared/config.php @@ -8,14 +8,13 @@ $sqlpass='socpassword'; // MySQL password $sqldb='soc'; // MySQL database $debug=true; // Whether to print debugging information // $modules='All non-hidden dirs in frontend/modules'; // Space-separated list of modules to offer the user -// $bundlers='All non-hidden .php files in backend/bundlers'; // Space-separated list of bundlers to offer the user +// $bundlers='All non-hidden dirs in backend/bundlers' that contain a bundle.php; // Space-separated list of bundlers to offer the user // $cookiename='ingenueid'; // Name of the cookie to send for keeping sessions // $sessionlength=1814400; // Time in seconds before sessions are purged // $mod_rewrite=true; // Use mod_rewrite for pretty URLs // $timezone_root='/usr/share/zoneinfo'; // Directory to search for timezone data (sys-libs/timezone-data) $emailfrom='noreply@gentoo.org'; // Used as the From: field in emails $check_email_dns=true; // Use DNS to check the domain of submitted emails for validity -// $split_setup=true; // Whether the frontend and backend are running on different hosts // Frontend options: // $registration=false; // Whether users can create new accounts without an invite // $invite='admin'; // Who can use the invite function: true or 'user'=users; admin=admins; false=nobody diff --git a/shared/functions/load_config.php b/shared/functions/load_config.php index a212b16..d719fb9 100644 --- a/shared/functions/load_config.php +++ b/shared/functions/load_config.php @@ -3,7 +3,7 @@ function load_config() { require(SHARED.'/include/defaults.php'); require(SHARED.'/config.php'); $modules=explode(' ', $modules); - foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns split_setup registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id') as $var) { + foreach (explode(' ', 'title url sqlhost sqluser sqlpass sqldb debug modules bundlers cookiename sessionlength mod_rewrite timezone_root emailfrom check_email_dns registration invite logview_max progressbar_width pkgdir_root emerge_default_opts portdir backend_id') as $var) { if (isset($$var)) { $GLOBALS['S']['conf'][$var]=$$var; } diff --git a/shared/include/defaults.php b/shared/include/defaults.php index ced0e01..95cbfc2 100644 --- a/shared/include/defaults.php +++ b/shared/include/defaults.php @@ -14,10 +14,8 @@ foreach (glob(FRONTEND.'/modules/*.info') as $module) { } $modules=implode(' ', $modules); $bundlers=array(); -foreach (glob(BACKEND.'/bundlers/*.php') as $bundler) { - $bundler=basename($bundler); - $bundlers[]=substr($bundler, 0, strlen($bundler)-4); -} +foreach (glob(BACKEND.'/bundlers/*/bundle.php') as $bundler) + $bundlers[]=basename(dirname($bundler)); $bundlers=implode(' ', $bundlers); $cookiename='ingenueid'; $sessionlength=1814400; @@ -25,7 +23,6 @@ $mod_rewrite=true; $timezone_root='/usr/share/zoneinfo'; $emailfrom='noreply@noreply.net'; $check_email_dns=false; -$split_setup=true; $registration=false; $invite='admin'; $logview_max=1000; diff --git a/shared/include/includes.php b/shared/include/includes.php index 410177f..8a75b58 100644 --- a/shared/include/includes.php +++ b/shared/include/includes.php @@ -9,9 +9,6 @@ foreach (array('functions', 'classes') as $type) { } } } -$includes=($_SERVER['DOCUMENT_ROOT']?FRONTEND:BACKEND).'/include/includes.php'; -if (is_readable($includes)) - require_once($includes); -unset($dir, $file, $type, $includes); +unset($dir, $file, $type); load_config(); ?> diff --git a/todo b/todo index 9540fde..cca8939 100644 --- a/todo +++ b/todo @@ -6,7 +6,6 @@ Add cleanup functions to the frontend and backend (tasks dir in backend containi Allow config viewing for builds, not just configurations Add `flags` column to configurations, builds, use it to implement public and private things Add safe shutdown to backend so it will stop once it gets to the end of a step that can be resumed -Add 'cancel', option to builds, allow deletion of currently running builds (have backend check if canceled before and after each task) Add build->configuration and configuration duplication Consider adding `configuration` col to builds Add map file for liveCD, load it into DB, etc. (currently hardcoded = evil) @@ -18,14 +17,10 @@ Add option to upload a kernel Add option to upload an arbitrary tar.gz/bz2 to be unzipped over the finished image *** Implement selected items from gentoo-steps *** *** Documentation *** -Ponder whether to remove the split-setup option and always upload to the frontend -Make backend not retry indefinitely on failure (`failures` column?) Offer FTP upload Offer SCP upload? Ask someone to add the necessary USE flags to php on tinderbox Add rollback to backend so it can resume after a partial task -Use 'bundling' status on builds, consider adding 'bundling_failed' -Handle 'cancel' status in backend, offer it in frontend -Change builds->display() to use a switch($this->status) +Offer option in frontend to submit a failed build for resume +Change builds->display() to handle `failed` column Confirm that backend stops after success -*** Tidy up backend, move the proper parts into build->build() -- cgit v1.2.3-65-gdbad