source: rtems-source-builder/source-builder/sb/download.py @ 12f253c

4.11
Last change on this file since 12f253c was 12f253c, checked in by Chris Johns <chrisj@…>, on 12/14/15 at 05:09:27

Download source from RTEMS if a release.

Download source from ftp.rtems.org before the package's URL if
a release.

If a user adds a URL via the command line that is used then the
RTEMS release path then the package's URL.

A user can add --url=file://path/../morepath to have the RSB use
a local cache of source on their hard disk. This is useful if you
need to wipe the RSB and start again. Save away the 'sources' and
'patches' directories to a common directory and provide it with via
the --url option using the 'file://' prefix.

Closes #2482.

  • Property mode set to 100644
File size: 20.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import hashlib
26import os
27import stat
28import sys
29import urllib2
30import urlparse
31
32import cvs
33import error
34import git
35import log
36import path
37import sources
38import version
39
40def _do_download(opts):
41    download = True
42    if opts.dry_run():
43        download = False
44        wa = opts.with_arg('download')
45        if wa is not None:
46            if wa[0] == 'with_download' and wa[1] == 'yes':
47                download = True
48    return download
49
50def _humanize_bytes(bytes, precision = 1):
51    abbrevs = (
52        (1 << 50L, 'PB'),
53        (1 << 40L, 'TB'),
54        (1 << 30L, 'GB'),
55        (1 << 20L, 'MB'),
56        (1 << 10L, 'kB'),
57        (1, ' bytes')
58    )
59    if bytes == 1:
60        return '1 byte'
61    for factor, suffix in abbrevs:
62        if bytes >= factor:
63            break
64    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
65
66def _hash_check(file_, absfile, macros, remove = True):
67    failed = False
68    hash = sources.get_hash(file_.lower(), macros)
69    if hash is not None:
70        hash = hash.split()
71        if len(hash) != 2:
72            raise error.internal('invalid hash format: %s' % (file_))
73        try:
74            hashlib_algorithms = hashlib.algorithms
75        except:
76            hashlib_algorithms = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
77        if hash[0] not in hashlib_algorithms:
78            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
79        hasher = None
80        _in = None
81        try:
82            hasher = hashlib.new(hash[0])
83            _in = open(path.host(absfile), 'rb')
84            hasher.update(_in.read())
85        except IOError, err:
86            log.notice('hash: %s: read error: %s' % (file_, str(err)))
87            failed = True
88        except:
89            msg = 'hash: %s: error' % (file_)
90            log.stderr(msg)
91            log.notice(msg)
92            if _in is not None:
93                _in.close()
94            raise
95        if _in is not None:
96            _in.close()
97        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
98        if hasher.hexdigest() != hash[1]:
99            log.warning('checksum error: %s' % (file_))
100            failed = True
101        if failed and remove:
102            log.warning('removing: %s' % (file_))
103            if path.exists(absfile):
104                try:
105                    os.remove(path.host(absfile))
106                except IOError, err:
107                    raise error.general('hash: %s: remove: %s' % (absfile, str(err)))
108                except:
109                    raise error.general('hash: %s: remove error' % (file_))
110        if hasher is not None:
111            del hasher
112    else:
113        if version.released():
114            raise error.general('%s: no hash found in released RSB' % (file_))
115        log.warning('%s: no hash found' % (file_))
116    return not failed
117
118def _local_path(source, pathkey, config):
119    for p in config.define(pathkey).split(':'):
120        local = path.join(path.abspath(p), source['file'])
121        if source['local'] is None:
122            source['local_prefix'] = path.abspath(p)
123            source['local'] = local
124        if path.exists(local):
125            source['local_prefix'] = path.abspath(p)
126            source['local'] = local
127            _hash_check(source['file'], local, config.macros)
128            break
129
130def _http_parser(source, pathkey, config, opts):
131    #
132    # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
133    #
134    if 'gitweb.cgi' in source['url']:
135        url = source['url']
136        if '?' not in url:
137            raise error.general('invalid gitweb.cgi request: %s' % (url))
138        req = url.split('?')[1]
139        if len(req) == 0:
140            raise error.general('invalid gitweb.cgi request: %s' % (url))
141        #
142        # The gitweb.cgi request should have:
143        #    p=<what>
144        #    a=patch
145        #    h=<hash>
146        # so extract the p and h parts to make the local name.
147        #
148        p = None
149        a = None
150        h = None
151        for r in req.split(';'):
152            if '=' not in r:
153                raise error.general('invalid gitweb.cgi path: %s' % (url))
154            rs = r.split('=')
155            if rs[0] == 'p':
156                p = rs[1].replace('.', '-')
157            elif rs[0] == 'a':
158                a = rs[1]
159            elif rs[0] == 'h':
160                h = rs[1]
161        if p is None or h is None:
162            raise error.general('gitweb.cgi path missing p or h: %s' % (url))
163        source['file'] = '%s-%s.patch' % (p, h)
164    #
165    # Check local path
166    #
167    _local_path(source, pathkey, config)
168    #
169    # Is the file compressed ?
170    #
171    esl = source['ext'].split('.')
172    if esl[-1:][0] == 'gz':
173        source['compressed-type'] = 'gzip'
174        source['compressed'] = '%{__gzip} -dc'
175    elif esl[-1:][0] == 'bz2':
176        source['compressed-type'] = 'bzip2'
177        source['compressed'] = '%{__bzip2} -dc'
178    elif esl[-1:][0] == 'zip':
179        source['compressed-type'] = 'zip'
180        source['compressed'] = '%{__unzip} -u'
181    elif esl[-1:][0] == 'xz':
182        source['compressed-type'] = 'xz'
183        source['compressed'] = '%{__xz} -dc'
184
185def _patchworks_parser(source, pathkey, config, opts):
186    #
187    # Check local path
188    #
189    _local_path(source, pathkey, config)
190    source['url'] = 'http%s' % (source['path'][2:])
191
192def _git_parser(source, pathkey, config, opts):
193    #
194    # Check local path
195    #
196    _local_path(source, pathkey, config)
197    #
198    # Symlink.
199    #
200    us = source['url'].split('?')
201    source['path'] = path.dirname(us[0])
202    source['file'] = path.basename(us[0])
203    source['name'], source['ext'] = path.splitext(source['file'])
204    if len(us) > 1:
205        source['args'] = us[1:]
206    source['local'] = \
207        path.join(source['local_prefix'], 'git', source['file'])
208    source['symlink'] = source['local']
209
210def _cvs_parser(source, pathkey, config, opts):
211    #
212    # Check local path
213    #
214    _local_path(source, pathkey, config)
215    #
216    # Symlink.
217    #
218    if not source['url'].startswith('cvs://'):
219        raise error.general('invalid cvs path: %s' % (source['url']))
220    us = source['url'].split('?')
221    try:
222        url = us[0]
223        source['file'] = url[url[6:].index(':') + 7:]
224        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
225    except:
226        raise error.general('invalid cvs path: %s' % (source['url']))
227    for a in us[1:]:
228        _as = a.split('=')
229        if _as[0] == 'module':
230            if len(_as) != 2:
231                raise error.general('invalid cvs module: %s' % (a))
232            source['module'] = _as[1]
233        elif _as[0] == 'src-prefix':
234            if len(_as) != 2:
235                raise error.general('invalid cvs src-prefix: %s' % (a))
236            source['src_prefix'] = _as[1]
237        elif _as[0] == 'tag':
238            if len(_as) != 2:
239                raise error.general('invalid cvs tag: %s' % (a))
240            source['tag'] = _as[1]
241        elif _as[0] == 'date':
242            if len(_as) != 2:
243                raise error.general('invalid cvs date: %s' % (a))
244            source['date'] = _as[1]
245    if 'date' in source and 'tag' in source:
246        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
247    # Do here to ensure an ordered path, the URL can include options in any order
248    if 'module' in source:
249        source['file'] += '_%s' % (source['module'])
250    if 'tag' in source:
251        source['file'] += '_%s' % (source['tag'])
252    if 'date' in source:
253        source['file'] += '_%s' % (source['date'])
254    for c in '/@#%.-':
255        source['file'] = source['file'].replace(c, '_')
256    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
257    if 'src_prefix' in source:
258        source['symlink'] = path.join(source['local'], source['src_prefix'])
259    else:
260        source['symlink'] = source['local']
261
262def _file_parser(source, pathkey, config, opts):
263    #
264    # Check local path
265    #
266    _local_path(source, pathkey, config)
267    #
268    # Get the paths sorted.
269    #
270    source['file'] = source['url'][6:]
271
272parsers = { 'http': _http_parser,
273            'ftp':  _http_parser,
274            'pw':   _patchworks_parser,
275            'git':  _git_parser,
276            'cvs':  _cvs_parser,
277            'file': _file_parser }
278
279def parse_url(url, pathkey, config, opts):
280    #
281    # Split the source up into the parts we need.
282    #
283    source = {}
284    source['url'] = url
285    colon = url.find(':')
286    if url[colon + 1:colon + 3] != '//':
287        raise error.general('malforned URL (no protocol prefix): %s' % (url))
288    source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
289    source['file'] = path.basename(url)
290    source['name'], source['ext'] = path.splitext(source['file'])
291    if source['name'].endswith('.tar'):
292        source['name'] = source['name'][:-4]
293        source['ext'] = '.tar' + source['ext']
294    #
295    # Get the file. Checks the local source directory first.
296    #
297    source['local'] = None
298    for p in parsers:
299        if url.startswith(p):
300            source['type'] = p
301            if parsers[p](source, pathkey, config, opts):
302                break
303    source['script'] = ''
304    return source
305
306def _http_downloader(url, local, config, opts):
307    if path.exists(local):
308        return True
309    #
310    # Hack for GitHub.
311    #
312    if url.startswith('https://api.github.com'):
313        url = urlparse.urljoin(url, config.expand('tarball/%{version}'))
314    dst = os.path.relpath(path.host(local))
315    log.notice('download: %s -> %s' % (url, dst))
316    failed = False
317    if _do_download(opts):
318        _in = None
319        _out = None
320        _length = None
321        _have = 0
322        _chunk_size = 256 * 1024
323        _chunk = None
324        _last_percent = 200.0
325        _last_msg = ''
326        _wipe_output = False
327        try:
328            try:
329                _in = None
330                _ssl_context = None
331                try:
332                    import ssl
333                    _ssl_context = ssl._create_unverified_context()
334                    _in = urllib2.urlopen(url, context = _ssl_context)
335                except:
336                    _ssl_context = None
337                if _ssl_context is None:
338                    _in = urllib2.urlopen(url)
339                if url != _in.geturl():
340                    log.notice(' redirect: %s' % (_in.geturl()))
341                _out = open(path.host(local), 'wb')
342                try:
343                    _length = int(_in.info().getheader('Content-Length').strip())
344                except:
345                    pass
346                while True:
347                    _msg = '\rdownloading: %s - %s ' % (dst, _humanize_bytes(_have))
348                    if _length:
349                        _percent = round((float(_have) / _length) * 100, 2)
350                        if _percent != _last_percent:
351                            _msg += 'of %s (%0.0f%%) ' % (_humanize_bytes(_length), _percent)
352                    if _msg != _last_msg:
353                        extras = (len(_last_msg) - len(_msg))
354                        log.stdout_raw('%s%s' % (_msg, ' ' * extras + '\b' * extras))
355                        _last_msg = _msg
356                    _chunk = _in.read(_chunk_size)
357                    if not _chunk:
358                        break
359                    _out.write(_chunk)
360                    _have += len(_chunk)
361                if _wipe_output:
362                    log.stdout_raw('\r%s\r' % (' ' * len(_last_msg)))
363                else:
364                    log.stdout_raw('\n')
365            except:
366                log.stdout_raw('\n')
367                raise
368        except IOError, err:
369            log.notice('download: %s: error: %s' % (url, str(err)))
370            if path.exists(local):
371                os.remove(path.host(local))
372            failed = True
373        except ValueError, err:
374            log.notice('download: %s: error: %s' % (url, str(err)))
375            if path.exists(local):
376                os.remove(path.host(local))
377            failed = True
378        except:
379            msg = 'download: %s: error' % (url)
380            log.stderr(msg)
381            log.notice(msg)
382            if _in is not None:
383                _in.close()
384            if _out is not None:
385                _out.close()
386            raise
387        if _out is not None:
388            _out.close()
389        if _in is not None:
390            _in.close()
391            del _in
392        if not failed:
393            if not path.isfile(local):
394                raise error.general('source is not a file: %s' % (path.host(local)))
395            if not _hash_check(path.basename(local), local, config.macros, False):
396                raise error.general('checksum failure file: %s' % (dst))
397    return not failed
398
399def _git_downloader(url, local, config, opts):
400    repo = git.repo(local, opts, config.macros)
401    rlp = os.path.relpath(path.host(local))
402    us = url.split('?')
403    #
404    # Handle the various git protocols.
405    #
406    # remove 'git' from 'git://xxxx/xxxx?protocol=...'
407    #
408    url_base = us[0][len('git'):]
409    for a in us[1:]:
410        _as = a.split('=')
411        if _as[0] == 'protocol':
412            if len(_as) != 2:
413                raise error.general('invalid git protocol option: %s' % (_as))
414            if _as[1] == 'none':
415                # remove the rest of the protocol header leaving nothing.
416                us[0] = url_base[len('://'):]
417            else:
418                if _as[1] not in ['ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync']:
419                    raise error.general('unknown git protocol: %s' % (_as[1]))
420                us[0] = _as[1] + url_base
421    if not repo.valid():
422        log.notice('git: clone: %s -> %s' % (us[0], rlp))
423        if _do_download(opts):
424            repo.clone(us[0], local)
425    else:
426        repo.clean(['-f', '-d'])
427        repo.reset('--hard')
428        repo.checkout('master')
429    for a in us[1:]:
430        _as = a.split('=')
431        if _as[0] == 'branch' or _as[0] == 'checkout':
432            if len(_as) != 2:
433                raise error.general('invalid git branch/checkout: %s' % (_as))
434            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
435            if _do_download(opts):
436                repo.checkout(_as[1])
437        elif _as[0] == 'submodule':
438            if len(_as) != 2:
439                raise error.general('invalid git submodule: %s' % (_as))
440            log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
441            if _do_download(opts):
442                repo.submodule(_as[1])
443        elif _as[0] == 'fetch':
444            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
445            if _do_download(opts):
446                repo.fetch()
447        elif _as[0] == 'merge':
448            log.notice('git: merge: %s' % (us[0]))
449            if _do_download(opts):
450                repo.merge()
451        elif _as[0] == 'pull':
452            log.notice('git: pull: %s' % (us[0]))
453            if _do_download(opts):
454                repo.pull()
455        elif _as[0] == 'reset':
456            arg = []
457            if len(_as) > 1:
458                arg = ['--%s' % (_as[1])]
459            log.notice('git: reset: %s' % (us[0]))
460            if _do_download(opts):
461                repo.reset(arg)
462        elif _as[0] == 'protocol':
463            pass
464        else:
465            raise error.general('invalid git option: %s' % (_as))
466    return True
467
468def _cvs_downloader(url, local, config, opts):
469    rlp = os.path.relpath(path.host(local))
470    us = url.split('?')
471    module = None
472    tag = None
473    date = None
474    src_prefix = None
475    for a in us[1:]:
476        _as = a.split('=')
477        if _as[0] == 'module':
478            if len(_as) != 2:
479                raise error.general('invalid cvs module: %s' % (a))
480            module = _as[1]
481        elif _as[0] == 'src-prefix':
482            if len(_as) != 2:
483                raise error.general('invalid cvs src-prefix: %s' % (a))
484            src_prefix = _as[1]
485        elif _as[0] == 'tag':
486            if len(_as) != 2:
487                raise error.general('invalid cvs tag: %s' % (a))
488            tag = _as[1]
489        elif _as[0] == 'date':
490            if len(_as) != 2:
491                raise error.general('invalid cvs date: %s' % (a))
492            date = _as[1]
493    repo = cvs.repo(local, opts, config.macros, src_prefix)
494    if not repo.valid():
495        if not path.isdir(local):
496            log.notice('Creating source directory: %s' % \
497                           (os.path.relpath(path.host(local))))
498            if _do_download(opts):
499                path.mkdir(local)
500            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
501            if _do_download(opts):
502                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
503    for a in us[1:]:
504        _as = a.split('=')
505        if _as[0] == 'update':
506            log.notice('cvs: update: %s' % (us[0]))
507            if _do_download(opts):
508                repo.update()
509        elif _as[0] == 'reset':
510            log.notice('cvs: reset: %s' % (us[0]))
511            if _do_download(opts):
512                repo.reset()
513    return True
514
515def _file_downloader(url, local, config, opts):
516    if not path.exists(local):
517        try:
518            src = url[7:]
519            dst = local
520            log.notice('download: copy %s -> %s' % (src, dst))
521            path.copy(src, dst)
522        except:
523            return False
524    return True
525
526downloaders = { 'http': _http_downloader,
527                'ftp':  _http_downloader,
528                'pw':   _http_downloader,
529                'git':  _git_downloader,
530                'cvs':  _cvs_downloader,
531                'file': _file_downloader }
532
533def get_file(url, local, opts, config):
534    if local is None:
535        raise error.general('source/patch path invalid')
536    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
537        log.notice('Creating source directory: %s' % \
538                       (os.path.relpath(path.host(path.dirname(local)))))
539    log.output('making dir: %s' % (path.host(path.dirname(local))))
540    if _do_download(opts):
541        path.mkdir(path.dirname(local))
542    if not path.exists(local) and opts.download_disabled():
543        raise error.general('source not found: %s' % (path.host(local)))
544    #
545    # Check if a URL has been provided on the command line. If the package is
546    # release push to the start the RTEMS URL.
547    #
548    url_bases = opts.urls()
549    if version.released():
550        rtems_release_url = config.macros.expand('%{rtems_release_url}/%{rsb_version}/sources')
551        log.trace('release url: %s' % (rtems_release_url))
552        #
553        # If the URL being fetched is under the release path do not add the
554        # sources release path because it is already there.
555        #
556        if not url.startswith(rtems_release_url):
557            if url_bases is None:
558                url_bases = [rtems_release_url]
559            else:
560                url_bases.append(rtems_release_url)
561    urls = []
562    if url_bases is not None:
563        #
564        # Split up the URL we are being asked to download.
565        #
566        url_path = urlparse.urlsplit(url)[2]
567        slash = url_path.rfind('/')
568        if slash < 0:
569            url_file = url_path
570        else:
571            url_file = url_path[slash + 1:]
572        log.trace('url_file: %s' %(url_file))
573        for base in url_bases:
574            if base[-1:] != '/':
575                base += '/'
576            next_url = urlparse.urljoin(base, url_file)
577            log.trace('url: %s' %(next_url))
578            urls.append(next_url)
579    urls += url.split()
580    log.trace('_url: %s -> %s' % (','.join(urls), local))
581    for url in urls:
582        for dl in downloaders:
583            if url.startswith(dl):
584                if downloaders[dl](url, local, config, opts):
585                    return
586    if _do_download(opts):
587        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.