diff options
author | 2008-10-10 00:03:50 +0530 | |
---|---|---|
committer | 2008-10-10 00:03:50 +0530 | |
commit | ef2a000cead37d503f7d209138bb7935d04192d8 (patch) | |
tree | 5e368b013974c290ac1628850a180a1e82936615 | |
parent | Changes to the release script (do-release.sh) (diff) | |
download | autotua-ef2a000cead37d503f7d209138bb7935d04192d8.tar.gz autotua-ef2a000cead37d503f7d209138bb7935d04192d8.tar.bz2 autotua-ef2a000cead37d503f7d209138bb7935d04192d8.zip |
Basic slave-master stateful encrypted interaction
* Slave can "take" jobs from the master now
- Asymmetrical encryption is used via GPG
- models.Slave stores the gpg fingerprint in models.GPGFingerprintField
- Slave imports the master's GPG key (/slave_api/autotua_master.asc)
* Currently, Slave registration is manual (./manage.py shell)
- Slave does fancy encrypted pickle talking (autotua.talk()) :)
* "Take" jobs via autotua.Jobs().takejob(maintainer, job_name)
- slave/autotua/crypt/__init__.py:
* Implements the glue with `gpg` (maybe pygnupg later?)
* Crypto() object. Has encrypt() and decrypt()
- Also see autotua.decrypt_if_required()
- GNUPGHOME for the slave is /var/tmp/autotua
* => Job().fetch() requires root access (userpriv/sandbox later)
* Phases store state to allow pausing/stopping and resuming of jobs
- Future feature, not really used ATM
- Job().everything() has prelim support for "resume"
* Various small bug fixes and tweaks
- Yes, I know I need to make this stuff more atomic :p
-rw-r--r-- | master/TODO | 3 | ||||
-rw-r--r-- | master/master/.gitignore | 1 | ||||
-rw-r--r-- | master/master/const.py | 3 | ||||
-rw-r--r-- | master/master/models.py | 26 | ||||
-rw-r--r-- | master/master/process/__init__.py | 4 | ||||
-rw-r--r-- | master/master/slave_api.py | 60 | ||||
-rw-r--r-- | master/master/urls.py | 6 | ||||
-rwxr-xr-x | master/setup-master.py | 10 | ||||
-rw-r--r-- | slave/autotua/__init__.py | 105 | ||||
-rw-r--r-- | slave/autotua/config.py | 2 | ||||
-rw-r--r-- | slave/autotua/crypt/__init__.py | 111 | ||||
-rw-r--r-- | slave/config/slave.cfg | 1 |
12 files changed, 294 insertions, 38 deletions
diff --git a/master/TODO b/master/TODO index 0944b7f..8c21c35 100644 --- a/master/TODO +++ b/master/TODO @@ -1,8 +1,7 @@ TODO: * Implement input via the webinterface (manual via command line atm) * Better jobuild dependency resolution for atom list (basic right now) - * Job status tracking and management (none right now) - * Asymmetrical key authentication for accepting jobs + * Job status tracking and management (skeleton right now) setup-master.py: * Should prompt for super-user diff --git a/master/master/.gitignore b/master/master/.gitignore new file mode 100644 index 0000000..c6bc574 --- /dev/null +++ b/master/master/.gitignore @@ -0,0 +1 @@ +gnupg/ diff --git a/master/master/const.py b/master/master/const.py index abdd4e4..a01eef2 100644 --- a/master/master/const.py +++ b/master/master/const.py @@ -7,10 +7,13 @@ # from autotua import config +import os # FIXME: insecure tmpdir. TMPDIR = '/tmp/master' JOBTAGE = config.JOBTAGE_DIR +MASTER_DIR = os.path.abspath(os.path.dirname(__file__)) +GPGHOME = MASTER_DIR+'/gnupg' STAGE_TYPES = ( ('stage1', 'stage1'), ('stage2', 'stage2'), diff --git a/master/master/models.py b/master/master/models.py index 6cf9330..6e2cd22 100644 --- a/master/master/models.py +++ b/master/master/models.py @@ -16,18 +16,30 @@ import const, random, urllib2 ### Models begin ### #################### +class GPGFingerprintField(models.CharField): + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 41 + kwargs['unique'] = True + kwargs['editable'] = False + super(GPGFingerprintField, self).__init__(*args, **kwargs) + class Slave(models.Model): - name = models.CharField(max_length=30, unique=True) + name = models.CharField(max_length=30) + # GPG Key Fingerprint + gpg_fp = GPGFingerprintField() # User which owns this slave owner = models.ForeignKey(User) - # Groups which can use this slave + # Groups which can offer jobs to this slave users = models.ManyToManyField(Group) # Status of slave state = models.CharField(max_length=30, default='U', choices=const.SLAVE_STATES) + class Meta: + unique_together = ['owner', 'name'] + def __unicode__(self): - return "%s (%s)" % (self.name, self.state) + return "%s/%s (%s)" % (self.owner, self.name, self.state) class Provider(models.Model): # Identifier for the mirror provider. @@ -88,9 +100,9 @@ class Mirror(models.Model): return '%s/%s (%s)' % (self.server, self.prefix, self.owner.name) def save(self): - if not self.server.endswith('/'): + if self.server[-1] != '/': self.server += '/' - if not self.prefix.endswith('/'): + if self.prefix[-1] != '/': self.prefix += '/' super(Mirror, self).save() @@ -108,7 +120,7 @@ class Job(models.Model): # Release (gives implicit info about provider) release = models.ForeignKey(Release) # i686, amd64, g4, etc. - arch = models.ForeignKey(Arch, limit_choices_to=release.archs.all()) + arch = models.ForeignKey(Arch) # Type of stage; hardened, uclibc, etc #type = CharField(maxlength=25) # const.stage_types # Revision of jobtage tree to use (not to be entered by user) @@ -124,7 +136,7 @@ class Job(models.Model): unique_together = ['name', 'maintainer'] def __unicode__(self): - return '%s/%s' % (self.maintainer, self.name) + return '%s/%s (%s)' % (self.maintainer, self.name, self.state) def save(self): atoms = self._get_deplist(self.atoms.split(), self.jobtagerev) diff --git a/master/master/process/__init__.py b/master/master/process/__init__.py index b0610e2..f66021b 100644 --- a/master/master/process/__init__.py +++ b/master/master/process/__init__.py @@ -10,11 +10,11 @@ import random from ..models import Mirror def generate_stage_url(job): - mirror = random.choice(Mirror.objects.filter(owner=job.provider)) + mirror = random.choice(Mirror.objects.filter(owner=job.release.provider)) url = mirror.server+mirror.prefix+mirror.structure data = {} data['owner'] = mirror.owner.name - data['stage'] = job.stage.name + data['stage'] = job.stage data['arch'] = job.arch.specific data['gen_arch'] = job.arch.generic data['release'] = job.release.name diff --git a/master/master/slave_api.py b/master/master/slave_api.py index 7c65bed..e08f4d7 100644 --- a/master/master/slave_api.py +++ b/master/master/slave_api.py @@ -7,24 +7,22 @@ # import cPickle as pickle +import os + +from autotua import crypt from django.conf import settings -from django.http import HttpResponse +from django.http import HttpResponse, Http404 from django.shortcuts import * -from master.models import Job, User -import process +from master.models import Job, User, Slave +import process, const -def job_list(request, **kwargs): - filters = {} - if kwargs.has_key('username'): - filters['maintainer'] = get_object_or_404(User, username=kwargs['username']) - if kwargs.has_key('job_name'): - filters['name'] = kwargs['job_name'] - jobs = [] - for job in get_list_or_404(Job, **filters): - jobs.append(job_data(job)) - return HttpResponse(pickle.dumps(jobs, 2), mimetype='application/octet-stream') +def _pickled_http_response(response, crypto=None, recipient=None): + response = pickle.dumps(response, 0) + if crypto: + response = crypto.encrypt(response, recipient) + return HttpResponse(response, mimetype='application/octet-stream') def job_data(job): data = {'maintainer': {'username': job.maintainer.username, @@ -35,3 +33,39 @@ def job_data(job): 'stage': process.generate_stage_url(job), 'atoms': job.atoms,} return data + +def job_list(request, **kwargs): + filters = {} + if kwargs.has_key('username'): + filters['maintainer'] = get_object_or_404(User, username=kwargs['username']) + if kwargs.has_key('job_name'): + filters['name'] = kwargs['job_name'] + jobs = [] + for job in get_list_or_404(Job, **filters): + jobs.append(job_data(job)) + if kwargs.has_key('job_name') and len(jobs) == 1: + jobs = jobs[0] + return _pickled_http_response(jobs) + +def accept_job(request): + if request.method == 'GET' or not request.POST.has_key('data'): + raise Http404 + crypto = crypt.Crypto(gpghome=const.GPGHOME) + (data, sender) = crypto.decrypt(request.POST['data']) + data = pickle.loads(data) + data['maintainer'] = get_object_or_404(User, username=data['maintainer']) + job = get_object_or_404(Job, **data) + slave = get_object_or_404(Slave, gpg_fp=sender) + job.slaves.add(slave) + response = 'Job Registered' + # Encrypting predictable responses is a security risk + # The private key becomes vulnerable to detection + # Hence, do not encrypt this response. + return _pickled_http_response(response) + +def get_pubkey(request): + pubkey_file = '%s/autotua_master.asc' % const.GPGHOME + if not os.path.exists(pubkey_file): + crypto = crypt.Crypto(gpghome=const.GPGHOME) + crypto.export_pubkey(pubkey_file, 'AutotuA Master') + return HttpResponse(open(pubkey_file).read()) diff --git a/master/master/urls.py b/master/master/urls.py index c00ad82..2888411 100644 --- a/master/master/urls.py +++ b/master/master/urls.py @@ -23,8 +23,10 @@ urlpatterns += patterns('master.views', urlpatterns += patterns('master.slave_api', (r'^slave_api/jobs/$', 'job_list'), - (r'^slave_api/jobs/(?P<username>[a-zA-Z0-9_]+)/$', 'job_list'), - (r'^slave_api/jobs/(?P<username>[a-zA-Z0-9_]+)/(?P<job_name>[^/]+)/$', 'job_list'), + (r'^slave_api/jobs/~(?P<username>[a-zA-Z0-9_]+)/$', 'job_list'), + (r'^slave_api/jobs/~(?P<username>[a-zA-Z0-9_]+)/(?P<job_name>[^/]+)/$', 'job_list'), + ('^slave_api/autotua_master.asc', 'get_pubkey'), + (r'^slave_api/slaves/accept/$', 'accept_job'), ) # Static media serving for development purposes diff --git a/master/setup-master.py b/master/setup-master.py index 5bfcfef..5be5803 100755 --- a/master/setup-master.py +++ b/master/setup-master.py @@ -109,6 +109,15 @@ def syncdb_master(): serverobj.prefix = server[1] serverobj.save() +def setup_gpg(): + from autotua import crypt + from master import const + data = {'name': 'AutotuA Master', + 'email': 'autotua@localhost', + 'expire': '1m'} + print 'Creating a "sample" gpg key (expires in 1 month)' + crypt.Crypto(gpghome=const.GPGHOME).init_gpghome(**data) + def setup_sample_job(): from sample_data import sample_job job = Job() @@ -156,6 +165,7 @@ elif sys.argv[1] == 'syncdb': # Start stuff syncdb_master() setup_sample_job() + setup_gpg() print "All done! Now you can start the master with `python manage.py runserver`" else: print_help() diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py index 3e23fc3..5f02947 100644 --- a/slave/autotua/__init__.py +++ b/slave/autotua/__init__.py @@ -9,25 +9,94 @@ import os, shutil, urllib2, atexit import os.path as osp import cPickle as pickle -from autotua import fetch, config, sync, chroot, jobuild +from urllib import urlencode +from autotua import fetch, config, sync, chroot, jobuild, crypt + +def decrypt_if_required(data, crypto): + gpg_header = '-----BEGIN PGP MESSAGE-----' + if data.split('\n')[0] != gpg_header: + return data + if not crypto: + raise Exception('Encryption selected, but no "crypto"') + return crypto.decrypt(data)[0] + +def talk(url, data=None, crypto=None, encrypt=False): + """ + Talk to the master server + @param url: relative URL to talk to on the master server + @type url: string + + @param data: Data to POST to the server + @type data: Anything! + + @param encrypt: Whether to encrypt the data to the POSTed + @type encrypt: bool + """ + # We not wantz leading '/' + if url[0] == '/': + url = url[1:] + url = urllib2.quote(url) + url = '/'.join([config.AUTOTUA_MASTER, url]) + if data: + # ASCII till I figure out str->bytes + data = pickle.dumps(data, 0) + if encrypt: + if not crypto: + raise Exception('Encryption selected, but no "crypto"') + data = crypto.encrypt(data) + data = urlencode({'data': data}) + data = urllib2.urlopen(url, data).read() + data = decrypt_if_required(data, crypto) + data = pickle.loads(data) + return data class Jobs: - """Interface to jobs on the master server that we can do""" + """Interface to jobs on the master server""" def __init__(self): - self.pubkey = '' + self.crypto = crypt.Crypto() + + def import_master_pubkey(self): + pubkey = 'autotua_master.asc' + url = '/'.join([config.AUTOTUA_MASTER, 'slave_api', pubkey]) + pubkey = urllib2.urlopen(url).read() + self.crypto.import_pubkey(pubkey) - def getjobs(self): + def getlist(self, maintainer=None): """ Get a list of jobs - (skeleton code atm) """ jobs = [] - job_list = pickle.load(urllib2.urlopen(config.AUTOTUA_MASTER+'/slave_api/jobs')) + url = 'slave_api/' + if maintainer: + url += '~%s/' % maintainer + url += 'jobs/' + job_list = talk(url, crypto=self.crypto) for job_data in job_list: jobs.append(Job(job_data)) return jobs + def getjob(self, maintainer, job_name): + """ + Get a job's data + """ + url = 'slave_api/jobs/~%s/%s' % (maintainer, job_name) + job_data = talk(url, crypto=self.crypto) + return Job(job_data) + + def takejob(self, maintainer, job_name): + """ + Take a specific job for running + """ + job = self.getjob(maintainer, job_name) + url = 'slave_api/slaves/accept/' + data = {'maintainer': job.maint, 'name': job.name} + ret = talk(url, data, crypto=self.crypto, encrypt=True) + if not ret: + raise Exception('Unable to register job') + print ret + return job + class Job: """A Job.""" @@ -42,6 +111,7 @@ class Job: self.atoms = job_data['atoms'] self.jobuilds = [] self.chroot = chroot.WorkChroot(self.jobdir, self.stage.filename) + self.next_phase = 'fetch' atexit.register(self.tidy) def __repr__(self): @@ -92,6 +162,7 @@ class Job: rev=self.jobtagerev, scheme="git-export").sync() ## Read config, get portage snapshot if required #self._fetch_portage_snapshot() + self.next_phase = 'prepare' def prepare(self): # Chroot setup needs to be done before parsing jobuilds @@ -103,6 +174,7 @@ class Job: self.jobuilds.append(jobuild.Jobuild(self.jobtagedir, atom)) print 'Fetch jobuild SRC_URI and hardlink/copy into chroot' self._setup_jobfiles() + self.next_phase = 'run' def run(self): processor = jobuild.Processor(None, self.chroot) @@ -110,10 +182,12 @@ class Job: processor.jobuild = jbld print 'Running jobuild "%s"' % jbld.atom processor.run_phase('all') + self.next_phase = 'tidy' def tidy(self): print 'Tidying up..' self.chroot.tidy() + self.next_phase = '' def clean(self): # Tidy up before cleaning @@ -121,12 +195,19 @@ class Job: shutil.rmtree(self.jobdir) os.removedirs(osp.join(config.WORKDIR, self.maint)) + def everything(self): + while self.next_phase: + exec('self.%s()' % self.next_phase) + print 'Everything done.' + if __name__ == "__main__": - job = Jobs().getjobs()[0] - job.fetch() + jobs = Jobs() + print 'Importing server public key' + jobs.import_master_pubkey() + print 'Registering sample job "Sample AutotuA job" for running' + job = jobs.takejob('test_user', 'Sample AutotuA job') + job.next_phase = 'prepare' if os.getuid() == 0: - job.prepare() - job.run() - job.tidy() + job.everything() else: - print 'You need to be root to run job.prepare(), job.run() and job.tidy()' + print 'You need to be root to run job.everything()' diff --git a/slave/autotua/config.py b/slave/autotua/config.py index 11fe625..7ab4267 100644 --- a/slave/autotua/config.py +++ b/slave/autotua/config.py @@ -18,8 +18,10 @@ IGNORE_PROXY = False LOGFILE = '/var/log/autotua/slave.log' TMPDIR = '/var/tmp/autotua' +GPGHOME = '/var/tmp/autotua/.gnupg' AUTOTUA_MASTER = '' +MASTER_PUBKEY = 'autotua_master.asc' JOBTAGE_URI = 'git://git.overlays.gentoo.org/proj/jobtage.git' # Bind mounted inside the chroots for use if defined diff --git a/slave/autotua/crypt/__init__.py b/slave/autotua/crypt/__init__.py new file mode 100644 index 0000000..bb97d17 --- /dev/null +++ b/slave/autotua/crypt/__init__.py @@ -0,0 +1,111 @@ +# vim: set sw=4 sts=4 et : +# Copyright: 2008 Gentoo Foundation +# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com> +# License: GPL-3 +# + +import subprocess, os +from .. import config + +class Crypto(object): + """ + Data Encrypter/Decrypter + """ + + def __init__(self, gpghome=config.GPGHOME): + """ + @param gpghome: Home directory for GPG + @type gpghome: string + """ + self.gpghome = gpghome + self.gpgcmd = 'gpg -a --keyid-format long --trust-model always ' + self.gpgcmd += '--homedir="%s" ' % self.gpghome + if not os.path.exists(self.gpghome+'/secring.gpg'): + raise Exception('"%s": Invalid GPG homedir' % self.gpghome) + + def _get_fp_from_keyid(self, keyid): + gpg_args = '--with-colons --fingerprint --list-keys "%s"' % keyid + process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True, + stdout=subprocess.PIPE) + output = process.stdout.readlines() + process.wait() + for line in output: + # Fingerprint line + if line.startswith('fpr'): + # Fingerprint + return line.split(':')[-2] + + def export_pubkey(self, file, which): + gpg_args = '--export "%s" > "%s"' % (which, file) + print self.gpgcmd+gpg_args + subprocess.check_call(self.gpgcmd+gpg_args, shell=True) + + def import_pubkey(self, pubkey): + gpg_args = '--import <<<"%s"' % pubkey + subprocess.check_call(self.gpgcmd+gpg_args, shell=True) + + def init_gpghome(self, name='Test AutotuA Slave', email='test_slave@test.org', + expire='5y', length='4096'): + """ + Initialize a GnuPG home by generating keys + """ + params = (('Key-Type', 'DSA'), + ('Key-Length', '1024'), + ('Subkey-Type', 'ELG-E'), + ('Subkey-Length', length), + ('Name-Real', name), + ('Name-Email', email), + ('Expire-Date', expire),) + # Batch mode + gpg_args = '--batch --gen-key <<<"' + gpg_args += '%echo Generating keys.. [Worship the gods of randomness :p]' + for param in params: + gpg_args += '\n%s: %s' % param + gpg_args += '"' + subprocess.check_call(self.gpgcmd+gpg_args, shell=True) + print 'Done.' + + def encrypt(self, data, recipient='Autotua Master'): + """ + @param data: Data to be encrypted + @type data: string + + @param recipient: Recipient for the data + @type recipient: string + + returns: encrypted_data + """ + gpg_args = '--encrypt --sign --recipient "%s" <<<"%s"' % (recipient, data) + process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True, + stdout=subprocess.PIPE) + data = process.stdout.read() + process.wait() + if process.returncode < 0: + raise Exception('Unable to encrypt, something went wrong :(') + return data + + def decrypt(self, data): + """ + @param data: Data to be encrypted + @type data: string + + returns: (decrypted_data, sender) + """ + gpg_args = '--decrypt <<<"%s"' % data + process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ddata = process.stdout.read()[:-1] # Extra \n at the end :-/ + # Get the output to stderr + gpg_out = process.stderr.readlines() + process.wait() + if process.returncode < 0 or not ddata: + raise Exception('Unable to decrypt, something went wrong') + # Get the line with the DSA long key ID + for line in gpg_out: + if line.find('using DSA key') != -1: + # Get the long key ID + gpg_out = line.split()[-1] + break + # Get the fingerprint + sender = self._get_fp_from_keyid(gpg_out) + return (ddata, sender) diff --git a/slave/config/slave.cfg b/slave/config/slave.cfg index dda3226..47216ca 100644 --- a/slave/config/slave.cfg +++ b/slave/config/slave.cfg @@ -8,6 +8,7 @@ verbose = True ;ignore_proxy = False ;logfile = '/var/log/autotua/slave.log' ;tmpdir = '/var/tmp/autotua' +;gpg_home = '/var/tmp/autotua/.gnupg' # You need to set this if you're running a local AutotuA master ;autotua_master = '' ;jobtage_uri = 'git://git.overlays.gentoo.org/proj/jobtage.git' |