source: rtems-source-builder/source-builder/sb/download.py @ d3fa158

4.104.95
Last change on this file since d3fa158 was d3fa158, checked in by Chris Johns <chrisj@…>, on 03/10/16 at 05:19:58

sb: Add a download option --with-release-url/--without-release-url.

The default without the option is to prepend the release URL to the URL
list to download from the RTEMS FTP site first if the RSB is released. This
option can force the RTEMS FTP to be tried first when not a release,
or you can supply a different URL to download from or you can request
no RTEMS URL is tried first. Eg:

--with-release-url
--with-release-url=​file://opt/local/cache
--without-release-url

Move the RTEMS release URL to the default.mc file. Change the URL
to the RTEMS FTP server and do not use the https method of access.

The option's with_arg call was cleaned up to make sense.

Remove the log's raw output of an extra space.

Some download error message formating was cleaned up.

Closes #2636.
  • Property mode set to 100644
File size: 21.6 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25from __future__ import print_function
26
27import hashlib
28import os
29import stat
30import sys
31try:
32    import urllib.request as urllib_request
33    import urllib.parse as urllib_parse
34except ImportError:
35    import urllib as urllib_request
36    import urlparse as urllib_parse
37
38import cvs
39import error
40import git
41import log
42import path
43import sources
44import version
45
46def _do_download(opts):
47    download = True
48    if opts.dry_run():
49        download = False
50        wa = opts.with_arg('download')
51        if wa is not None:
52            if wa[0] == 'with_download' and wa[1] == 'yes':
53                download = True
54    return download
55
56def _humanize_bytes(bytes, precision = 1):
57    abbrevs = (
58        (1 << 50, 'PB'),
59        (1 << 40, 'TB'),
60        (1 << 30, 'GB'),
61        (1 << 20, 'MB'),
62        (1 << 10, 'kB'),
63        (1, ' bytes')
64    )
65    if bytes == 1:
66        return '1 byte'
67    for factor, suffix in abbrevs:
68        if bytes >= factor:
69            break
70    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
71
72def _hash_check(file_, absfile, macros, remove = True):
73    failed = False
74    hash = sources.get_hash(file_.lower(), macros)
75    if hash is not None:
76        hash = hash.split()
77        if len(hash) != 2:
78            raise error.internal('invalid hash format: %s' % (file_))
79        try:
80            hashlib_algorithms = hashlib.algorithms
81        except:
82            hashlib_algorithms = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
83        if hash[0] not in hashlib_algorithms:
84            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
85        hasher = None
86        _in = None
87        try:
88            hasher = hashlib.new(hash[0])
89            _in = open(path.host(absfile), 'rb')
90            hasher.update(_in.read())
91        except IOError as err:
92            log.notice('hash: %s: read error: %s' % (file_, str(err)))
93            failed = True
94        except:
95            msg = 'hash: %s: error' % (file_)
96            log.stderr(msg)
97            log.notice(msg)
98            if _in is not None:
99                _in.close()
100            raise
101        if _in is not None:
102            _in.close()
103        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
104        if hasher.hexdigest() != hash[1]:
105            log.warning('checksum error: %s' % (file_))
106            failed = True
107        if failed and remove:
108            log.warning('removing: %s' % (file_))
109            if path.exists(absfile):
110                try:
111                    os.remove(path.host(absfile))
112                except IOError as err:
113                    raise error.general('hash: %s: remove: %s' % (absfile, str(err)))
114                except:
115                    raise error.general('hash: %s: remove error' % (file_))
116        if hasher is not None:
117            del hasher
118    else:
119        if version.released():
120            raise error.general('%s: no hash found in released RSB' % (file_))
121        log.warning('%s: no hash found' % (file_))
122    return not failed
123
124def _local_path(source, pathkey, config):
125    for p in config.define(pathkey).split(':'):
126        local = path.join(path.abspath(p), source['file'])
127        if source['local'] is None:
128            source['local_prefix'] = path.abspath(p)
129            source['local'] = local
130        if path.exists(local):
131            source['local_prefix'] = path.abspath(p)
132            source['local'] = local
133            _hash_check(source['file'], local, config.macros)
134            break
135
136def _http_parser(source, pathkey, config, opts):
137    #
138    # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
139    #
140    if 'gitweb.cgi' in source['url']:
141        url = source['url']
142        if '?' not in url:
143            raise error.general('invalid gitweb.cgi request: %s' % (url))
144        req = url.split('?')[1]
145        if len(req) == 0:
146            raise error.general('invalid gitweb.cgi request: %s' % (url))
147        #
148        # The gitweb.cgi request should have:
149        #    p=<what>
150        #    a=patch
151        #    h=<hash>
152        # so extract the p and h parts to make the local name.
153        #
154        p = None
155        a = None
156        h = None
157        for r in req.split(';'):
158            if '=' not in r:
159                raise error.general('invalid gitweb.cgi path: %s' % (url))
160            rs = r.split('=')
161            if rs[0] == 'p':
162                p = rs[1].replace('.', '-')
163            elif rs[0] == 'a':
164                a = rs[1]
165            elif rs[0] == 'h':
166                h = rs[1]
167        if p is None or h is None:
168            raise error.general('gitweb.cgi path missing p or h: %s' % (url))
169        source['file'] = '%s-%s.patch' % (p, h)
170    #
171    # Check the source file name for any extra request query data and remove if
172    # found. Some hosts do not like file names containing them.
173    #
174    if '?' in source['file']:
175        qmark = source['file'].find('?')
176        source['file'] = source['file'][:qmark]
177    #
178    # Check local path
179    #
180    _local_path(source, pathkey, config)
181    #
182    # Is the file compressed ?
183    #
184    esl = source['ext'].split('.')
185    if esl[-1:][0] == 'gz':
186        source['compressed-type'] = 'gzip'
187        source['compressed'] = '%{__gzip} -dc'
188    elif esl[-1:][0] == 'bz2':
189        source['compressed-type'] = 'bzip2'
190        source['compressed'] = '%{__bzip2} -dc'
191    elif esl[-1:][0] == 'zip':
192        source['compressed-type'] = 'zip'
193        source['compressed'] = '%{__unzip} -u'
194    elif esl[-1:][0] == 'xz':
195        source['compressed-type'] = 'xz'
196        source['compressed'] = '%{__xz} -dc'
197
198def _patchworks_parser(source, pathkey, config, opts):
199    #
200    # Check local path
201    #
202    _local_path(source, pathkey, config)
203    source['url'] = 'http%s' % (source['path'][2:])
204
205def _git_parser(source, pathkey, config, opts):
206    #
207    # Check local path
208    #
209    _local_path(source, pathkey, config)
210    #
211    # Symlink.
212    #
213    us = source['url'].split('?')
214    source['path'] = path.dirname(us[0])
215    source['file'] = path.basename(us[0])
216    source['name'], source['ext'] = path.splitext(source['file'])
217    if len(us) > 1:
218        source['args'] = us[1:]
219    source['local'] = \
220        path.join(source['local_prefix'], 'git', source['file'])
221    source['symlink'] = source['local']
222
223def _cvs_parser(source, pathkey, config, opts):
224    #
225    # Check local path
226    #
227    _local_path(source, pathkey, config)
228    #
229    # Symlink.
230    #
231    if not source['url'].startswith('cvs://'):
232        raise error.general('invalid cvs path: %s' % (source['url']))
233    us = source['url'].split('?')
234    try:
235        url = us[0]
236        source['file'] = url[url[6:].index(':') + 7:]
237        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
238    except:
239        raise error.general('invalid cvs path: %s' % (source['url']))
240    for a in us[1:]:
241        _as = a.split('=')
242        if _as[0] == 'module':
243            if len(_as) != 2:
244                raise error.general('invalid cvs module: %s' % (a))
245            source['module'] = _as[1]
246        elif _as[0] == 'src-prefix':
247            if len(_as) != 2:
248                raise error.general('invalid cvs src-prefix: %s' % (a))
249            source['src_prefix'] = _as[1]
250        elif _as[0] == 'tag':
251            if len(_as) != 2:
252                raise error.general('invalid cvs tag: %s' % (a))
253            source['tag'] = _as[1]
254        elif _as[0] == 'date':
255            if len(_as) != 2:
256                raise error.general('invalid cvs date: %s' % (a))
257            source['date'] = _as[1]
258    if 'date' in source and 'tag' in source:
259        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
260    # Do here to ensure an ordered path, the URL can include options in any order
261    if 'module' in source:
262        source['file'] += '_%s' % (source['module'])
263    if 'tag' in source:
264        source['file'] += '_%s' % (source['tag'])
265    if 'date' in source:
266        source['file'] += '_%s' % (source['date'])
267    for c in '/@#%.-':
268        source['file'] = source['file'].replace(c, '_')
269    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
270    if 'src_prefix' in source:
271        source['symlink'] = path.join(source['local'], source['src_prefix'])
272    else:
273        source['symlink'] = source['local']
274
275def _file_parser(source, pathkey, config, opts):
276    #
277    # Check local path
278    #
279    _local_path(source, pathkey, config)
280    #
281    # Get the paths sorted.
282    #
283    source['file'] = source['url'][6:]
284
285parsers = { 'http': _http_parser,
286            'ftp':  _http_parser,
287            'pw':   _patchworks_parser,
288            'git':  _git_parser,
289            'cvs':  _cvs_parser,
290            'file': _file_parser }
291
292def parse_url(url, pathkey, config, opts):
293    #
294    # Split the source up into the parts we need.
295    #
296    source = {}
297    source['url'] = url
298    colon = url.find(':')
299    if url[colon + 1:colon + 3] != '//':
300        raise error.general('malforned URL (no protocol prefix): %s' % (url))
301    source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
302    source['file'] = path.basename(url)
303    source['name'], source['ext'] = path.splitext(source['file'])
304    if source['name'].endswith('.tar'):
305        source['name'] = source['name'][:-4]
306        source['ext'] = '.tar' + source['ext']
307    #
308    # Get the file. Checks the local source directory first.
309    #
310    source['local'] = None
311    for p in parsers:
312        if url.startswith(p):
313            source['type'] = p
314            if parsers[p](source, pathkey, config, opts):
315                break
316    source['script'] = ''
317    return source
318
319def _http_downloader(url, local, config, opts):
320    if path.exists(local):
321        return True
322    #
323    # Hack for GitHub.
324    #
325    if url.startswith('https://api.github.com'):
326        url = urllib_parse.urljoin(url, config.expand('tarball/%{version}'))
327    dst = os.path.relpath(path.host(local))
328    log.notice('download: %s -> %s' % (url, dst))
329    failed = False
330    if _do_download(opts):
331        _in = None
332        _out = None
333        _length = None
334        _have = 0
335        _chunk_size = 256 * 1024
336        _chunk = None
337        _last_percent = 200.0
338        _last_msg = ''
339        _have_status_output = False
340        try:
341            try:
342                _in = None
343                _ssl_context = None
344                _urllib_url = url
345                try:
346                    import ssl
347                    _ssl_context = ssl._create_unverified_context()
348                    _in = urllib_request.urlopen(_urllib_url, context = _ssl_context)
349                except:
350                    _ssl_context = None
351                if _ssl_context is None:
352                    _in = urllib_request.urlopen(_urllib_url)
353                if url != _in.geturl():
354                    log.notice(' redirect: %s' % (_in.geturl()))
355                _out = open(path.host(local), 'wb')
356                try:
357                    _length = int(_in.info().getheader('Content-Length').strip())
358                except:
359                    pass
360                while True:
361                    _msg = '\rdownloading: %s - %s ' % (dst, _humanize_bytes(_have))
362                    if _length:
363                        _percent = round((float(_have) / _length) * 100, 2)
364                        if _percent != _last_percent:
365                            _msg += 'of %s (%0.0f%%) ' % (_humanize_bytes(_length), _percent)
366                    if _msg != _last_msg:
367                        extras = (len(_last_msg) - len(_msg))
368                        log.stdout_raw('%s%s' % (_msg, ' ' * extras + '\b' * extras))
369                        _last_msg = _msg
370                        _have_status_output = True
371                    _chunk = _in.read(_chunk_size)
372                    if not _chunk:
373                        break
374                    _out.write(_chunk)
375                    _have += len(_chunk)
376                log.stdout_raw('\n\r')
377            except:
378                if _have_status_output:
379                    log.stdout_raw('\n\r')
380                raise
381        except IOError as err:
382            log.notice('download: %s: error: %s' % (url, str(err)))
383            if path.exists(local):
384                os.remove(path.host(local))
385            failed = True
386        except ValueError as err:
387            log.notice('download: %s: error: %s' % (url, str(err)))
388            if path.exists(local):
389                os.remove(path.host(local))
390            failed = True
391        except:
392            msg = 'download: %s: error' % (url)
393            log.stderr(msg)
394            log.notice(msg)
395            if _in is not None:
396                _in.close()
397            if _out is not None:
398                _out.close()
399            raise
400        if _out is not None:
401            _out.close()
402        if _in is not None:
403            _in.close()
404            del _in
405        if not failed:
406            if not path.isfile(local):
407                raise error.general('source is not a file: %s' % (path.host(local)))
408            if not _hash_check(path.basename(local), local, config.macros, False):
409                raise error.general('checksum failure file: %s' % (dst))
410    return not failed
411
412def _git_downloader(url, local, config, opts):
413    repo = git.repo(local, opts, config.macros)
414    rlp = os.path.relpath(path.host(local))
415    us = url.split('?')
416    #
417    # Handle the various git protocols.
418    #
419    # remove 'git' from 'git://xxxx/xxxx?protocol=...'
420    #
421    url_base = us[0][len('git'):]
422    for a in us[1:]:
423        _as = a.split('=')
424        if _as[0] == 'protocol':
425            if len(_as) != 2:
426                raise error.general('invalid git protocol option: %s' % (_as))
427            if _as[1] == 'none':
428                # remove the rest of the protocol header leaving nothing.
429                us[0] = url_base[len('://'):]
430            else:
431                if _as[1] not in ['ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync']:
432                    raise error.general('unknown git protocol: %s' % (_as[1]))
433                us[0] = _as[1] + url_base
434    if not repo.valid():
435        log.notice('git: clone: %s -> %s' % (us[0], rlp))
436        if _do_download(opts):
437            repo.clone(us[0], local)
438    else:
439        repo.clean(['-f', '-d'])
440        repo.reset('--hard')
441        repo.checkout('master')
442    for a in us[1:]:
443        _as = a.split('=')
444        if _as[0] == 'branch' or _as[0] == 'checkout':
445            if len(_as) != 2:
446                raise error.general('invalid git branch/checkout: %s' % (_as))
447            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
448            if _do_download(opts):
449                repo.checkout(_as[1])
450        elif _as[0] == 'submodule':
451            if len(_as) != 2:
452                raise error.general('invalid git submodule: %s' % (_as))
453            log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
454            if _do_download(opts):
455                repo.submodule(_as[1])
456        elif _as[0] == 'fetch':
457            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
458            if _do_download(opts):
459                repo.fetch()
460        elif _as[0] == 'merge':
461            log.notice('git: merge: %s' % (us[0]))
462            if _do_download(opts):
463                repo.merge()
464        elif _as[0] == 'pull':
465            log.notice('git: pull: %s' % (us[0]))
466            if _do_download(opts):
467                repo.pull()
468        elif _as[0] == 'reset':
469            arg = []
470            if len(_as) > 1:
471                arg = ['--%s' % (_as[1])]
472            log.notice('git: reset: %s' % (us[0]))
473            if _do_download(opts):
474                repo.reset(arg)
475        elif _as[0] == 'protocol':
476            pass
477        else:
478            raise error.general('invalid git option: %s' % (_as))
479    return True
480
481def _cvs_downloader(url, local, config, opts):
482    rlp = os.path.relpath(path.host(local))
483    us = url.split('?')
484    module = None
485    tag = None
486    date = None
487    src_prefix = None
488    for a in us[1:]:
489        _as = a.split('=')
490        if _as[0] == 'module':
491            if len(_as) != 2:
492                raise error.general('invalid cvs module: %s' % (a))
493            module = _as[1]
494        elif _as[0] == 'src-prefix':
495            if len(_as) != 2:
496                raise error.general('invalid cvs src-prefix: %s' % (a))
497            src_prefix = _as[1]
498        elif _as[0] == 'tag':
499            if len(_as) != 2:
500                raise error.general('invalid cvs tag: %s' % (a))
501            tag = _as[1]
502        elif _as[0] == 'date':
503            if len(_as) != 2:
504                raise error.general('invalid cvs date: %s' % (a))
505            date = _as[1]
506    repo = cvs.repo(local, opts, config.macros, src_prefix)
507    if not repo.valid():
508        if not path.isdir(local):
509            log.notice('Creating source directory: %s' % \
510                           (os.path.relpath(path.host(local))))
511            if _do_download(opts):
512                path.mkdir(local)
513            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
514            if _do_download(opts):
515                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
516    for a in us[1:]:
517        _as = a.split('=')
518        if _as[0] == 'update':
519            log.notice('cvs: update: %s' % (us[0]))
520            if _do_download(opts):
521                repo.update()
522        elif _as[0] == 'reset':
523            log.notice('cvs: reset: %s' % (us[0]))
524            if _do_download(opts):
525                repo.reset()
526    return True
527
528def _file_downloader(url, local, config, opts):
529    if not path.exists(local):
530        try:
531            src = url[7:]
532            dst = local
533            log.notice('download: copy %s -> %s' % (src, dst))
534            path.copy(src, dst)
535        except:
536            return False
537    return True
538
539downloaders = { 'http': _http_downloader,
540                'ftp':  _http_downloader,
541                'pw':   _http_downloader,
542                'git':  _git_downloader,
543                'cvs':  _cvs_downloader,
544                'file': _file_downloader }
545
546def get_file(url, local, opts, config):
547    if local is None:
548        raise error.general('source/patch path invalid')
549    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
550        log.notice('Creating source directory: %s' % \
551                       (os.path.relpath(path.host(path.dirname(local)))))
552    log.output('making dir: %s' % (path.host(path.dirname(local))))
553    if _do_download(opts):
554        path.mkdir(path.dirname(local))
555    if not path.exists(local) and opts.download_disabled():
556        raise error.general('source not found: %s' % (path.host(local)))
557    #
558    # Check if a URL has been provided on the command line. If the package is
559    # released push to the start the RTEMS URL unless overrided by the command
560    # line option --with-release-url. The variant --without-release-url can
561    # override the released check.
562    #
563    url_bases = opts.urls()
564    rtems_release_url_value = config.macros.expand('%{rtems_release_url}/%{rsb_version}/sources')
565    rtems_release_url = None
566    if version.released():
567        rtems_release_url = rtems_release_url_value
568    with_rel_url = opts.with_arg('release-url')
569    if with_rel_url[0] == 'with_release-url':
570        if with_rel_url[1] == 'yes':
571            rtems_release_url = rtems_release_url_value
572        elif with_rel_url[1] == 'no':
573            pass
574        else:
575            rtems_release_url = with_rel_url[1]
576    elif with_rel_url[0] == 'without_release-url' and with_rel_url[1] == 'no':
577        rtems_release_url = rtems_release_url_value
578    if rtems_release_url is not None:
579        log.trace('release url: %s' % (rtems_release_url))
580        #
581        # If the URL being fetched is under the release path do not add the
582        # sources release path because it is already there.
583        #
584        if not url.startswith(rtems_release_url):
585            if url_bases is None:
586                url_bases = [rtems_release_url]
587            else:
588                url_bases.append(rtems_release_url)
589    urls = []
590    if url_bases is not None:
591        #
592        # Split up the URL we are being asked to download.
593        #
594        url_path = urllib_parse.urlsplit(url)[2]
595        slash = url_path.rfind('/')
596        if slash < 0:
597            url_file = url_path
598        else:
599            url_file = url_path[slash + 1:]
600        log.trace('url_file: %s' %(url_file))
601        for base in url_bases:
602            if base[-1:] != '/':
603                base += '/'
604            next_url = urllib_parse.urljoin(base, url_file)
605            log.trace('url: %s' %(next_url))
606            urls.append(next_url)
607    urls += url.split()
608    log.trace('_url: %s -> %s' % (','.join(urls), local))
609    for url in urls:
610        for dl in downloaders:
611            if url.startswith(dl):
612                if downloaders[dl](url, local, config, opts):
613                    return
614    if _do_download(opts):
615        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.