source: rtems-source-builder/source-builder/sb/download.py @ 2e548833

4.11
Last change on this file since 2e548833 was 2e548833, checked in by Chris Johns <chrisj@…>, on 03/10/16 at 05:48:38

sb: Fix --without-release-url logic.

Updates #2636.
  • Property mode set to 100644
File size: 21.6 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25from __future__ import print_function
26
27import hashlib
28import os
29import stat
30import sys
31try:
32    import urllib.request as urllib_request
33    import urllib.parse as urllib_parse
34except ImportError:
35    import urllib as urllib_request
36    import urlparse as urllib_parse
37
38import cvs
39import error
40import git
41import log
42import path
43import sources
44import version
45
46def _do_download(opts):
47    download = True
48    if opts.dry_run():
49        download = False
50        wa = opts.with_arg('download')
51        if wa is not None:
52            if wa[0] == 'with_download' and wa[1] == 'yes':
53                download = True
54    return download
55
56def _humanize_bytes(bytes, precision = 1):
57    abbrevs = (
58        (1 << 50, 'PB'),
59        (1 << 40, 'TB'),
60        (1 << 30, 'GB'),
61        (1 << 20, 'MB'),
62        (1 << 10, 'kB'),
63        (1, ' bytes')
64    )
65    if bytes == 1:
66        return '1 byte'
67    for factor, suffix in abbrevs:
68        if bytes >= factor:
69            break
70    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
71
72def _hash_check(file_, absfile, macros, remove = True):
73    failed = False
74    hash = sources.get_hash(file_.lower(), macros)
75    if hash is not None:
76        hash = hash.split()
77        if len(hash) != 2:
78            raise error.internal('invalid hash format: %s' % (file_))
79        try:
80            hashlib_algorithms = hashlib.algorithms
81        except:
82            hashlib_algorithms = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
83        if hash[0] not in hashlib_algorithms:
84            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
85        hasher = None
86        _in = None
87        try:
88            hasher = hashlib.new(hash[0])
89            _in = open(path.host(absfile), 'rb')
90            hasher.update(_in.read())
91        except IOError as err:
92            log.notice('hash: %s: read error: %s' % (file_, str(err)))
93            failed = True
94        except:
95            msg = 'hash: %s: error' % (file_)
96            log.stderr(msg)
97            log.notice(msg)
98            if _in is not None:
99                _in.close()
100            raise
101        if _in is not None:
102            _in.close()
103        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
104        if hasher.hexdigest() != hash[1]:
105            log.warning('checksum error: %s' % (file_))
106            failed = True
107        if failed and remove:
108            log.warning('removing: %s' % (file_))
109            if path.exists(absfile):
110                try:
111                    os.remove(path.host(absfile))
112                except IOError as err:
113                    raise error.general('hash: %s: remove: %s' % (absfile, str(err)))
114                except:
115                    raise error.general('hash: %s: remove error' % (file_))
116        if hasher is not None:
117            del hasher
118    else:
119        if version.released():
120            raise error.general('%s: no hash found in released RSB' % (file_))
121        log.warning('%s: no hash found' % (file_))
122    return not failed
123
124def _local_path(source, pathkey, config):
125    for p in config.define(pathkey).split(':'):
126        local = path.join(path.abspath(p), source['file'])
127        if source['local'] is None:
128            source['local_prefix'] = path.abspath(p)
129            source['local'] = local
130        if path.exists(local):
131            source['local_prefix'] = path.abspath(p)
132            source['local'] = local
133            _hash_check(source['file'], local, config.macros)
134            break
135
136def _http_parser(source, pathkey, config, opts):
137    #
138    # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
139    #
140    if 'gitweb.cgi' in source['url']:
141        url = source['url']
142        if '?' not in url:
143            raise error.general('invalid gitweb.cgi request: %s' % (url))
144        req = url.split('?')[1]
145        if len(req) == 0:
146            raise error.general('invalid gitweb.cgi request: %s' % (url))
147        #
148        # The gitweb.cgi request should have:
149        #    p=<what>
150        #    a=patch
151        #    h=<hash>
152        # so extract the p and h parts to make the local name.
153        #
154        p = None
155        a = None
156        h = None
157        for r in req.split(';'):
158            if '=' not in r:
159                raise error.general('invalid gitweb.cgi path: %s' % (url))
160            rs = r.split('=')
161            if rs[0] == 'p':
162                p = rs[1].replace('.', '-')
163            elif rs[0] == 'a':
164                a = rs[1]
165            elif rs[0] == 'h':
166                h = rs[1]
167        if p is None or h is None:
168            raise error.general('gitweb.cgi path missing p or h: %s' % (url))
169        source['file'] = '%s-%s.patch' % (p, h)
170    #
171    # Check the source file name for any extra request query data and remove if
172    # found. Some hosts do not like file names containing them.
173    #
174    if '?' in source['file']:
175        qmark = source['file'].find('?')
176        source['file'] = source['file'][:qmark]
177    #
178    # Check local path
179    #
180    _local_path(source, pathkey, config)
181    #
182    # Is the file compressed ?
183    #
184    esl = source['ext'].split('.')
185    if esl[-1:][0] == 'gz':
186        source['compressed-type'] = 'gzip'
187        source['compressed'] = '%{__gzip} -dc'
188    elif esl[-1:][0] == 'bz2':
189        source['compressed-type'] = 'bzip2'
190        source['compressed'] = '%{__bzip2} -dc'
191    elif esl[-1:][0] == 'zip':
192        source['compressed-type'] = 'zip'
193        source['compressed'] = '%{__unzip} -u'
194    elif esl[-1:][0] == 'xz':
195        source['compressed-type'] = 'xz'
196        source['compressed'] = '%{__xz} -dc'
197
198def _patchworks_parser(source, pathkey, config, opts):
199    #
200    # Check local path
201    #
202    _local_path(source, pathkey, config)
203    source['url'] = 'http%s' % (source['path'][2:])
204
205def _git_parser(source, pathkey, config, opts):
206    #
207    # Check local path
208    #
209    _local_path(source, pathkey, config)
210    #
211    # Symlink.
212    #
213    us = source['url'].split('?')
214    source['path'] = path.dirname(us[0])
215    source['file'] = path.basename(us[0])
216    source['name'], source['ext'] = path.splitext(source['file'])
217    if len(us) > 1:
218        source['args'] = us[1:]
219    source['local'] = \
220        path.join(source['local_prefix'], 'git', source['file'])
221    source['symlink'] = source['local']
222
223def _cvs_parser(source, pathkey, config, opts):
224    #
225    # Check local path
226    #
227    _local_path(source, pathkey, config)
228    #
229    # Symlink.
230    #
231    if not source['url'].startswith('cvs://'):
232        raise error.general('invalid cvs path: %s' % (source['url']))
233    us = source['url'].split('?')
234    try:
235        url = us[0]
236        source['file'] = url[url[6:].index(':') + 7:]
237        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
238    except:
239        raise error.general('invalid cvs path: %s' % (source['url']))
240    for a in us[1:]:
241        _as = a.split('=')
242        if _as[0] == 'module':
243            if len(_as) != 2:
244                raise error.general('invalid cvs module: %s' % (a))
245            source['module'] = _as[1]
246        elif _as[0] == 'src-prefix':
247            if len(_as) != 2:
248                raise error.general('invalid cvs src-prefix: %s' % (a))
249            source['src_prefix'] = _as[1]
250        elif _as[0] == 'tag':
251            if len(_as) != 2:
252                raise error.general('invalid cvs tag: %s' % (a))
253            source['tag'] = _as[1]
254        elif _as[0] == 'date':
255            if len(_as) != 2:
256                raise error.general('invalid cvs date: %s' % (a))
257            source['date'] = _as[1]
258    if 'date' in source and 'tag' in source:
259        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
260    # Do here to ensure an ordered path, the URL can include options in any order
261    if 'module' in source:
262        source['file'] += '_%s' % (source['module'])
263    if 'tag' in source:
264        source['file'] += '_%s' % (source['tag'])
265    if 'date' in source:
266        source['file'] += '_%s' % (source['date'])
267    for c in '/@#%.-':
268        source['file'] = source['file'].replace(c, '_')
269    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
270    if 'src_prefix' in source:
271        source['symlink'] = path.join(source['local'], source['src_prefix'])
272    else:
273        source['symlink'] = source['local']
274
275def _file_parser(source, pathkey, config, opts):
276    #
277    # Check local path
278    #
279    _local_path(source, pathkey, config)
280    #
281    # Get the paths sorted.
282    #
283    source['file'] = source['url'][6:]
284
285parsers = { 'http': _http_parser,
286            'ftp':  _http_parser,
287            'pw':   _patchworks_parser,
288            'git':  _git_parser,
289            'cvs':  _cvs_parser,
290            'file': _file_parser }
291
292def parse_url(url, pathkey, config, opts):
293    #
294    # Split the source up into the parts we need.
295    #
296    source = {}
297    source['url'] = url
298    colon = url.find(':')
299    if url[colon + 1:colon + 3] != '//':
300        raise error.general('malforned URL (no protocol prefix): %s' % (url))
301    source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
302    source['file'] = path.basename(url)
303    source['name'], source['ext'] = path.splitext(source['file'])
304    if source['name'].endswith('.tar'):
305        source['name'] = source['name'][:-4]
306        source['ext'] = '.tar' + source['ext']
307    #
308    # Get the file. Checks the local source directory first.
309    #
310    source['local'] = None
311    for p in parsers:
312        if url.startswith(p):
313            source['type'] = p
314            if parsers[p](source, pathkey, config, opts):
315                break
316    source['script'] = ''
317    return source
318
319def _http_downloader(url, local, config, opts):
320    if path.exists(local):
321        return True
322    #
323    # Hack for GitHub.
324    #
325    if url.startswith('https://api.github.com'):
326        url = urllib_parse.urljoin(url, config.expand('tarball/%{version}'))
327    dst = os.path.relpath(path.host(local))
328    log.notice('download: %s -> %s' % (url, dst))
329    failed = False
330    if _do_download(opts):
331        _in = None
332        _out = None
333        _length = None
334        _have = 0
335        _chunk_size = 256 * 1024
336        _chunk = None
337        _last_percent = 200.0
338        _last_msg = ''
339        _have_status_output = False
340        try:
341            try:
342                _in = None
343                _ssl_context = None
344                _urllib_url = url
345                try:
346                    import ssl
347                    _ssl_context = ssl._create_unverified_context()
348                    _in = urllib_request.urlopen(_urllib_url, context = _ssl_context)
349                except:
350                    _ssl_context = None
351                if _ssl_context is None:
352                    _in = urllib_request.urlopen(_urllib_url)
353                if url != _in.geturl():
354                    log.notice(' redirect: %s' % (_in.geturl()))
355                _out = open(path.host(local), 'wb')
356                try:
357                    _length = int(_in.info().getheader('Content-Length').strip())
358                except:
359                    pass
360                while True:
361                    _msg = '\rdownloading: %s - %s ' % (dst, _humanize_bytes(_have))
362                    if _length:
363                        _percent = round((float(_have) / _length) * 100, 2)
364                        if _percent != _last_percent:
365                            _msg += 'of %s (%0.0f%%) ' % (_humanize_bytes(_length), _percent)
366                    if _msg != _last_msg:
367                        extras = (len(_last_msg) - len(_msg))
368                        log.stdout_raw('%s%s' % (_msg, ' ' * extras + '\b' * extras))
369                        _last_msg = _msg
370                        _have_status_output = True
371                    _chunk = _in.read(_chunk_size)
372                    if not _chunk:
373                        break
374                    _out.write(_chunk)
375                    _have += len(_chunk)
376                log.stdout_raw('\n\r')
377            except:
378                if _have_status_output:
379                    log.stdout_raw('\n\r')
380                raise
381        except IOError as err:
382            log.notice('download: %s: error: %s' % (url, str(err)))
383            if path.exists(local):
384                os.remove(path.host(local))
385            failed = True
386        except ValueError as err:
387            log.notice('download: %s: error: %s' % (url, str(err)))
388            if path.exists(local):
389                os.remove(path.host(local))
390            failed = True
391        except:
392            msg = 'download: %s: error' % (url)
393            log.stderr(msg)
394            log.notice(msg)
395            if _in is not None:
396                _in.close()
397            if _out is not None:
398                _out.close()
399            raise
400        if _out is not None:
401            _out.close()
402        if _in is not None:
403            _in.close()
404            del _in
405        if not failed:
406            if not path.isfile(local):
407                raise error.general('source is not a file: %s' % (path.host(local)))
408            if not _hash_check(path.basename(local), local, config.macros, False):
409                raise error.general('checksum failure file: %s' % (dst))
410    return not failed
411
412def _git_downloader(url, local, config, opts):
413    repo = git.repo(local, opts, config.macros)
414    rlp = os.path.relpath(path.host(local))
415    us = url.split('?')
416    #
417    # Handle the various git protocols.
418    #
419    # remove 'git' from 'git://xxxx/xxxx?protocol=...'
420    #
421    url_base = us[0][len('git'):]
422    for a in us[1:]:
423        _as = a.split('=')
424        if _as[0] == 'protocol':
425            if len(_as) != 2:
426                raise error.general('invalid git protocol option: %s' % (_as))
427            if _as[1] == 'none':
428                # remove the rest of the protocol header leaving nothing.
429                us[0] = url_base[len('://'):]
430            else:
431                if _as[1] not in ['ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync']:
432                    raise error.general('unknown git protocol: %s' % (_as[1]))
433                us[0] = _as[1] + url_base
434    if not repo.valid():
435        log.notice('git: clone: %s -> %s' % (us[0], rlp))
436        if _do_download(opts):
437            repo.clone(us[0], local)
438    else:
439        repo.clean(['-f', '-d'])
440        repo.reset('--hard')
441        repo.checkout('master')
442    for a in us[1:]:
443        _as = a.split('=')
444        if _as[0] == 'branch' or _as[0] == 'checkout':
445            if len(_as) != 2:
446                raise error.general('invalid git branch/checkout: %s' % (_as))
447            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
448            if _do_download(opts):
449                repo.checkout(_as[1])
450        elif _as[0] == 'submodule':
451            if len(_as) != 2:
452                raise error.general('invalid git submodule: %s' % (_as))
453            log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
454            if _do_download(opts):
455                repo.submodule(_as[1])
456        elif _as[0] == 'fetch':
457            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
458            if _do_download(opts):
459                repo.fetch()
460        elif _as[0] == 'merge':
461            log.notice('git: merge: %s' % (us[0]))
462            if _do_download(opts):
463                repo.merge()
464        elif _as[0] == 'pull':
465            log.notice('git: pull: %s' % (us[0]))
466            if _do_download(opts):
467                repo.pull()
468        elif _as[0] == 'reset':
469            arg = []
470            if len(_as) > 1:
471                arg = ['--%s' % (_as[1])]
472            log.notice('git: reset: %s' % (us[0]))
473            if _do_download(opts):
474                repo.reset(arg)
475        elif _as[0] == 'protocol':
476            pass
477        else:
478            raise error.general('invalid git option: %s' % (_as))
479    return True
480
481def _cvs_downloader(url, local, config, opts):
482    rlp = os.path.relpath(path.host(local))
483    us = url.split('?')
484    module = None
485    tag = None
486    date = None
487    src_prefix = None
488    for a in us[1:]:
489        _as = a.split('=')
490        if _as[0] == 'module':
491            if len(_as) != 2:
492                raise error.general('invalid cvs module: %s' % (a))
493            module = _as[1]
494        elif _as[0] == 'src-prefix':
495            if len(_as) != 2:
496                raise error.general('invalid cvs src-prefix: %s' % (a))
497            src_prefix = _as[1]
498        elif _as[0] == 'tag':
499            if len(_as) != 2:
500                raise error.general('invalid cvs tag: %s' % (a))
501            tag = _as[1]
502        elif _as[0] == 'date':
503            if len(_as) != 2:
504                raise error.general('invalid cvs date: %s' % (a))
505            date = _as[1]
506    repo = cvs.repo(local, opts, config.macros, src_prefix)
507    if not repo.valid():
508        if not path.isdir(local):
509            log.notice('Creating source directory: %s' % \
510                           (os.path.relpath(path.host(local))))
511            if _do_download(opts):
512                path.mkdir(local)
513            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
514            if _do_download(opts):
515                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
516    for a in us[1:]:
517        _as = a.split('=')
518        if _as[0] == 'update':
519            log.notice('cvs: update: %s' % (us[0]))
520            if _do_download(opts):
521                repo.update()
522        elif _as[0] == 'reset':
523            log.notice('cvs: reset: %s' % (us[0]))
524            if _do_download(opts):
525                repo.reset()
526    return True
527
528def _file_downloader(url, local, config, opts):
529    if not path.exists(local):
530        try:
531            src = url[7:]
532            dst = local
533            log.notice('download: copy %s -> %s' % (src, dst))
534            path.copy(src, dst)
535        except:
536            return False
537    return True
538
539downloaders = { 'http': _http_downloader,
540                'ftp':  _http_downloader,
541                'pw':   _http_downloader,
542                'git':  _git_downloader,
543                'cvs':  _cvs_downloader,
544                'file': _file_downloader }
545
546def get_file(url, local, opts, config):
547    if local is None:
548        raise error.general('source/patch path invalid')
549    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
550        log.notice('Creating source directory: %s' % \
551                       (os.path.relpath(path.host(path.dirname(local)))))
552    log.output('making dir: %s' % (path.host(path.dirname(local))))
553    if _do_download(opts):
554        path.mkdir(path.dirname(local))
555    if not path.exists(local) and opts.download_disabled():
556        raise error.general('source not found: %s' % (path.host(local)))
557    #
558    # Check if a URL has been provided on the command line. If the package is
559    # released push to the start the RTEMS URL unless overrided by the command
560    # line option --with-release-url. The variant --without-release-url can
561    # override the released check.
562    #
563    url_bases = opts.urls()
564    rtems_release_url_value = config.macros.expand('%{rtems_release_url}/%{rsb_version}/sources')
565    rtems_release_url = None
566    if version.released():
567        rtems_release_url = rtems_release_url_value
568    with_rel_url = opts.with_arg('release-url')
569    if with_rel_url[0] == 'with_release-url':
570        if with_rel_url[1] == 'yes':
571            rtems_release_url = rtems_release_url_value
572        elif with_rel_url[1] == 'no':
573            pass
574        else:
575            rtems_release_url = with_rel_url[1]
576    elif with_rel_url[0] == 'without_release-url' and with_rel_url[1] == 'yes':
577        rtems_release_url = None
578    if rtems_release_url is not None:
579        log.trace('release url: %s' % (rtems_release_url))
580        #
581        # If the URL being fetched is under the release path do not add the
582        # sources release path because it is already there.
583        #
584        if not url.startswith(rtems_release_url):
585            if url_bases is None:
586                url_bases = [rtems_release_url]
587            else:
588                url_bases.append(rtems_release_url)
589    urls = []
590    if url_bases is not None:
591        #
592        # Split up the URL we are being asked to download.
593        #
594        url_path = urllib_parse.urlsplit(url)[2]
595        slash = url_path.rfind('/')
596        if slash < 0:
597            url_file = url_path
598        else:
599            url_file = url_path[slash + 1:]
600        log.trace('url_file: %s' %(url_file))
601        for base in url_bases:
602            if base[-1:] != '/':
603                base += '/'
604            next_url = urllib_parse.urljoin(base, url_file)
605            log.trace('url: %s' %(next_url))
606            urls.append(next_url)
607    urls += url.split()
608    log.trace('_url: %s -> %s' % (','.join(urls), local))
609    for url in urls:
610        for dl in downloaders:
611            if url.startswith(dl):
612                if downloaders[dl](url, local, config, opts):
613                    return
614    if _do_download(opts):
615        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.