aboutsummaryrefslogtreecommitdiff
blob: 5751ca0081d8fef93768f55121c7bf878758595f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
# -*- coding: utf-8 -*-

"""
    fetch.py
    ~~~~~~~~
    
    This module implements a Python class responsible to fetch and update
    the package database and the auxiliary files.
    
    Used only by the live version of g-octave.
    
    :copyright: (c) 2009-2010 by Rafael Goncalves Martins
    :license: GPL-2, see LICENSE for more details.
"""

from __future__ import absolute_import, print_function

__all__ = ['fetch']

from .config import Config
conf = Config(True) # fetch phase

from .exception import FetchException
from .compat import py3k, open as open_

if py3k:
    import urllib.request as urllib
else:
    import urllib2 as urllib

import glob
import json
import os
import re
import shutil
import subprocess
import sys
import tarfile

from contextlib import closing

def clean_db():
    for f in ['info.json', 'patches', 'octave-forge']:
        current = os.path.join(conf.db, f)
        if os.path.isdir(current):
            shutil.rmtree(current)
        elif os.path.isfile(current):
            os.unlink(current)

class GitHub:
    
    re_db_mirror = re.compile(r'github://(?P<user>[^/]+)/(?P<repo>[^/]+)/?')
    
    def __init__(self, user, repo):
        self.user = user
        self.repo = repo
        self.api_url = 'http://github.com/api/v2/json'
        self.url = 'http://github.com'
    
    def need_update(self):
        return not os.path.exists(os.path.join(
            conf.db, 'cache', 'commit_id'
        ))
    
    def get_commits(self, branch='master'):
        url = '%s/commits/list/%s/%s/%s/' % (
            self.api_url,
            self.user,
            self.repo,
            branch
        )
        commits = {}
        with closing(urllib.urlopen(url)) as fp:
            commits = json.load(fp)
        return commits['commits']
    
    def fetch_db(self, branch='master'):
        cache = os.path.join(conf.db, 'cache')
        commit_id = os.path.join(cache, 'commit_id')
        if not os.path.exists(cache):
            os.makedirs(cache)
        last_commit = self.get_commits()[0]['id']
        if os.path.exists(commit_id):
            with open_(commit_id) as fp:
                if fp.read().strip() == last_commit:
                    return False
        dest = os.path.join(cache, 'octave-forge-%s.tar.gz' % last_commit)
        return_value = subprocess.call([
            'wget',
            '--continue',
            '--output-document', dest,
            '%s/%s/%s/tarball/%s/' % (
                self.url,
                self.user,
                self.repo,
                branch
            )
        ])
        if return_value == os.EX_OK:
            with open_(os.path.join(cache, 'commit_id'), 'w') as fp:
                fp.write(last_commit)
        return True

    def extract(self):
        clean_db()
        cache = os.path.join(conf.db, 'cache')
        commit_id = os.path.join(cache, 'commit_id')
        tarball = None
        if os.path.exists(commit_id):
            with open_(commit_id) as fp:
                tarball = os.path.join(
                    cache,
                    'octave-forge-%s.tar.gz' % fp.read().strip()
                )
        if tarball is not None:
            if tarfile.is_tarfile(tarball):
                with closing(tarfile.open(tarball, 'r')) as fp:
                    fp.extractall(conf.db)
                dirs = glob.glob('%s/%s-%s*' % (conf.db, self.user, self.repo))
                if len(dirs) != 1:
                    raise FetchException('Failed to extract the tarball.')
                    return
                for f in os.listdir(dirs[0]):
                    shutil.move(os.path.join(dirs[0], f), conf.db)
                os.rmdir(dirs[0])

# TODO: Implement gitweb support

__modules__ = [
    GitHub
]

def fetch():
    for module in __modules__:
        match = module.re_db_mirror.match(conf.db_mirror)
        if match is not None:
            return module(**match.groupdict())