source: rtems-source-builder/source-builder/sb/download.py @ 5f44fcd

4.104.114.95
Last change on this file since 5f44fcd was 5f44fcd, checked in by Chris Johns <chrisj@…>, on 04/21/13 at 11:43:17

Fix CVS downloads.
  • Property mode set to 100644
File size: 9.8 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import os
26import stat
27import sys
28import urllib2
29import urlparse
30
31import cvs
32import error
33import git
34import log
35import path
36
37def _http_parser(source, config, opts):
38    #
39    # Is the file compressed ?
40    #
41    esl = source['ext'].split('.')
42    if esl[-1:][0] == 'gz':
43        source['compressed'] = '%{__gzip} -dc'
44    elif esl[-1:][0] == 'bz2':
45        source['compressed'] = '%{__bzip2} -dc'
46    elif esl[-1:][0] == 'zip':
47        source['compressed'] = '%{__zip} -u'
48    elif esl[-1:][0] == 'xz':
49        source['compressed'] = '%{__xz} -dc'
50
51def _git_parser(source, config, opts):
52    #
53    # Symlink.
54    #
55    us = source['url'].split('?')
56    source['path'] = path.dirname(us[0])
57    source['file'] = path.basename(us[0])
58    source['name'], source['ext'] = path.splitext(source['file'])
59    if len(us) > 1:
60        source['args'] = us[1:]
61    source['local'] = \
62        path.join(source['local_prefix'], 'git', source['file'])
63    source['symlink'] = source['local']
64
65def _cvs_parser(source, config, opts):
66    #
67    # Symlink.
68    #
69    if not source['url'].startswith('cvs://'):
70        raise error.general('invalid cvs path: %s' % (source['url']))
71    us = source['url'].split('?')
72    try:
73        url = us[0]
74        source['file'] = \
75            url[url[6:].index(':') + 7:].replace('/', '_').replace('@', '_').replace('.', '_')
76        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
77    except:
78        raise error.general('invalid cvs path: %s' % (source['url']))
79    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
80    if 'src_prefix' in source:
81        source['symlink'] = path.join(source['local'])
82    else:
83        source['symlink'] = source['local']
84
85def _file_parser(source, config, opts):
86    #
87    # Symlink.
88    #
89    source['symlink'] = source['local']
90
91parsers = { 'http': _http_parser,
92            'ftp':  _http_parser,
93            'git':  _git_parser,
94            'cvs':  _cvs_parser,
95            'file': _file_parser }
96
97def parse_url(url, pathkey, config, opts):
98    #
99    # Split the source up into the parts we need.
100    #
101    source = {}
102    source['url'] = url
103    source['path'] = path.dirname(url)
104    source['file'] = path.basename(url)
105    source['name'], source['ext'] = path.splitext(source['file'])
106    #
107    # Get the file. Checks the local source directory first.
108    #
109    source['local'] = None
110    for p in config.define(pathkey).split(':'):
111        local = path.join(path.abspath(p), source['file'])
112        if source['local'] is None:
113            source['local_prefix'] = path.abspath(p)
114            source['local'] = local
115        if path.exists(local):
116            source['local_prefix'] = path.abspath(p)
117            source['local'] = local
118            break
119    source['script'] = ''
120    for p in parsers:
121        if url.startswith(p):
122            source['type'] = p
123            if parsers[p](source, config, opts):
124                break
125    return source
126
127def _http_downloader(url, local, config, opts):
128    if path.exists(local):
129        return True
130    #
131    # Hack for GitHub.
132    #
133    if url.startswith('https://api.github.com'):
134        url = urlparse.urljoin(url, config.expand('tarball/%{version}'))
135    log.notice('download: %s -> %s' % (url, os.path.relpath(path.host(local))))
136    failed = False
137    if not opts.dry_run():
138        _in = None
139        _out = None
140        try:
141            _in = urllib2.urlopen(url)
142            _out = open(path.host(local), 'wb')
143            _out.write(_in.read())
144        except IOError, err:
145            log.notice('download: %s: error: %s' % (url, str(err)))
146            if path.exists(local):
147                os.remove(path.host(local))
148            failed = True
149        except ValueError, err:
150            log.notice('download: %s: error: %s' % (url, str(err)))
151            if path.exists(local):
152                os.remove(path.host(local))
153            failed = True
154        except:
155            msg = 'download: %s: error' % (url)
156            log.stderr(msd)
157            log.notice(msg)
158            if _out is not None:
159                _out.close()
160            raise
161        if _out is not None:
162            _out.close()
163        if _in is not None:
164            del _in
165        if not failed:
166            if not path.isfile(local):
167                raise error.general('source is not a file: %s' % (path.host(local)))
168    return not failed
169
170def _git_downloader(url, local, config, opts):
171    rlp = os.path.relpath(path.host(local))
172    us = url.split('?')
173    repo = git.repo(local, opts, config.macros)
174    if not repo.valid():
175        log.notice('git: clone: %s -> %s' % (us[0], rlp))
176        if not opts.dry_run():
177            repo.clone(us[0], local)
178    for a in us[1:]:
179        _as = a.split('=')
180        if _as[0] == 'branch':
181            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
182            if not opts.dry_run():
183                repo.checkout(_as[1])
184        elif _as[0] == 'pull':
185            log.notice('git: pull: %s' % (us[0]))
186            if not opts.dry_run():
187                repo.pull()
188        elif _as[0] == 'fetch':
189            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
190            if not opts.dry_run():
191                repo.fetch()
192        elif _as[0] == 'reset':
193            arg = []
194            if len(_as) > 1:
195                arg = ['--%s' % (_as[1])]
196            log.notice('git: reset: %s' % (us[0]))
197            if not opts.dry_run():
198                repo.reset(arg)
199    return True
200
201def _cvs_downloader(url, local, config, opts):
202    rlp = os.path.relpath(path.host(local))
203    us = url.split('?')
204    module = None
205    tag = None
206    date = None
207    src_prefix = None
208    for a in us[1:]:
209        _as = a.split('=')
210        if _as[0] == 'module':
211            if len(_as) != 2:
212                raise error.general('invalid cvs module: %s' % (a))
213            module = _as[1]
214        elif _as[0] == 'src-prefix':
215            if len(_as) != 2:
216                raise error.general('invalid cvs src-prefix: %s' % (a))
217            src_prefix = _as[1]
218        elif _as[0] == 'tag':
219            if len(_as) != 2:
220                raise error.general('invalid cvs tag: %s' % (a))
221            tag = _as[1]
222        elif _as[0] == 'date':
223            if len(_as) != 2:
224                raise error.general('invalid cvs date: %s' % (a))
225            date = _as[1]
226    repo = cvs.repo(local, opts, config.macros, src_prefix)
227    if not repo.valid():
228        log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
229        if not path.isdir(local):
230            log.notice('Creating source directory: %s' % \
231                           (os.path.relpath(path.host(local))))
232            log.output('making dir: %s' % (path.host(path.dirname(local))))
233            if not opts.dry_run():
234                path.mkdir(local)
235            repo.checkout(':%s' % (us[0][6:]), module, tag, date)
236    for a in us[1:]:
237        _as = a.split('=')
238        if _as[0] == 'update':
239            log.notice('cvs: update: %s' % (us[0]))
240            if not opts.dry_run():
241                repo.update()
242        elif _as[0] == 'reset':
243            log.notice('cvs: reset: %s' % (us[0]))
244            if not opts.dry_run():
245                repo.reset()
246    return True
247
248def _file_downloader(url, local, config, opts):
249    if path.exists(local):
250        return True
251    return path.isdir(url)
252
253downloaders = { 'http': _http_downloader,
254                'ftp':  _http_downloader,
255                'git':  _git_downloader,
256                'cvs':  _cvs_downloader,
257                'file': _file_downloader }
258
259def get_file(url, local, opts, config):
260    if local is None:
261        raise error.general('source/patch path invalid')
262    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
263        log.notice('Creating source directory: %s' % \
264                       (os.path.relpath(path.host(path.dirname(local)))))
265    log.output('making dir: %s' % (path.host(path.dirname(local))))
266    if not opts.dry_run():
267        path.mkdir(path.dirname(local))
268    if not path.exists(local) and opts.download_disabled():
269        raise error.general('source not found: %s' % (path.host(local)))
270    #
271    # Check if a URL hasbeen provided on the command line.
272    #
273    url_bases = opts.urls()
274    urls = []
275    if url_bases is not None:
276        for base in url_bases:
277            if base[-1:] != '/':
278                base += '/'
279            url_path = urlparse.urlsplit(url)[2]
280            slash = url_path.rfind('/')
281            if slash < 0:
282                url_file = url_path
283            else:
284                url_file = url_path[slash + 1:]
285            urls.append(urlparse.urljoin(base, url_file))
286    urls.append(url)
287    log.trace('_url: %s -> %s' % (','.join(urls), local))
288    for url in urls:
289        for dl in downloaders:
290            if url.startswith(dl):
291                if downloaders[dl](url, local, config, opts):
292                    return
293    if not opts.dry_run():
294        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.