source: rtems-source-builder/source-builder/sb/download.py @ 383f7e6

4.104.95
Last change on this file since 383f7e6 was 383f7e6, checked in by Chris Johns <chrisj@…>, on 04/15/16 at 01:44:27

sb: Add --rsb-file options to %source and %patch to set a file name.

Override the automatic file name of a downloaded file and use the file
name provided by the option. This is useful if the URL has no meanful
file that can be automatically extracted from the URL.
  • Property mode set to 100644
File size: 23.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25from __future__ import print_function
26
27import hashlib
28import os
29import stat
30import sys
31try:
32    import urllib.request as urllib_request
33    import urllib.parse as urllib_parse
34except ImportError:
35    import urllib2 as urllib_request
36    import urlparse as urllib_parse
37
38import cvs
39import error
40import git
41import log
42import path
43import sources
44import version
45
46def _do_download(opts):
47    download = True
48    if opts.dry_run():
49        download = False
50        wa = opts.with_arg('download')
51        if wa is not None:
52            if wa[0] == 'with_download' and wa[1] == 'yes':
53                download = True
54    return download
55
56def _humanize_bytes(bytes, precision = 1):
57    abbrevs = (
58        (1 << 50, 'PB'),
59        (1 << 40, 'TB'),
60        (1 << 30, 'GB'),
61        (1 << 20, 'MB'),
62        (1 << 10, 'kB'),
63        (1, ' bytes')
64    )
65    if bytes == 1:
66        return '1 byte'
67    for factor, suffix in abbrevs:
68        if bytes >= factor:
69            break
70    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
71
72def _sensible_url(url, used = 0):
73    space = 200
74    if len(url) > space:
75        size = int(space - 14)
76        url = url[:size] + '...<see log>'
77    return url
78
79def _hash_check(file_, absfile, macros, remove = True):
80    failed = False
81    hash = sources.get_hash(file_.lower(), macros)
82    if hash is not None:
83        hash = hash.split()
84        if len(hash) != 2:
85            raise error.internal('invalid hash format: %s' % (file_))
86        try:
87            hashlib_algorithms = hashlib.algorithms
88        except:
89            hashlib_algorithms = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
90        if hash[0] not in hashlib_algorithms:
91            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
92        hasher = None
93        _in = None
94        try:
95            hasher = hashlib.new(hash[0])
96            _in = open(path.host(absfile), 'rb')
97            hasher.update(_in.read())
98        except IOError as err:
99            log.notice('hash: %s: read error: %s' % (file_, str(err)))
100            failed = True
101        except:
102            msg = 'hash: %s: error' % (file_)
103            log.stderr(msg)
104            log.notice(msg)
105            if _in is not None:
106                _in.close()
107            raise
108        if _in is not None:
109            _in.close()
110        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
111        if hasher.hexdigest() != hash[1]:
112            log.warning('checksum error: %s' % (file_))
113            failed = True
114        if failed and remove:
115            log.warning('removing: %s' % (file_))
116            if path.exists(absfile):
117                try:
118                    os.remove(path.host(absfile))
119                except IOError as err:
120                    raise error.general('hash: %s: remove: %s' % (absfile, str(err)))
121                except:
122                    raise error.general('hash: %s: remove error' % (file_))
123        if hasher is not None:
124            del hasher
125    else:
126        if version.released():
127            raise error.general('%s: no hash found in released RSB' % (file_))
128        log.warning('%s: no hash found' % (file_))
129    return not failed
130
131def _local_path(source, pathkey, config):
132    for p in config.define(pathkey).split(':'):
133        local_prefix = path.abspath(p)
134        local = path.join(local_prefix, source['file'])
135        if source['local'] is None:
136            source['local_prefix'] = local_prefix
137            source['local'] = local
138        if path.exists(local):
139            source['local_prefix'] = local_prefix
140            source['local'] = local
141            _hash_check(source['file'], local, config.macros)
142            break
143
144def _http_parser(source, pathkey, config, opts):
145    #
146    # If the file has not been overrided attempt to recover a possible file name.
147    #
148    if 'file-override' not in source['options']:
149        #
150        # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
151        #
152        if 'gitweb.cgi' in source['url']:
153            url = source['url']
154            if '?' not in url:
155                raise error.general('invalid gitweb.cgi request: %s' % (url))
156            req = url.split('?')[1]
157            if len(req) == 0:
158                raise error.general('invalid gitweb.cgi request: %s' % (url))
159            #
160            # The gitweb.cgi request should have:
161            #    p=<what>
162            #    a=patch
163            #    h=<hash>
164            # so extract the p and h parts to make the local name.
165            #
166            p = None
167            a = None
168            h = None
169            for r in req.split(';'):
170                if '=' not in r:
171                    raise error.general('invalid gitweb.cgi path: %s' % (url))
172                rs = r.split('=')
173                if rs[0] == 'p':
174                    p = rs[1].replace('.', '-')
175                elif rs[0] == 'a':
176                    a = rs[1]
177                elif rs[0] == 'h':
178                    h = rs[1]
179            if p is None or h is None:
180                raise error.general('gitweb.cgi path missing p or h: %s' % (url))
181            source['file'] = '%s-%s.patch' % (p, h)
182        #
183        # Check the source file name for any extra request query data and remove if
184        # found. Some hosts do not like file names containing them.
185        #
186        if '?' in source['file']:
187            qmark = source['file'].find('?')
188            source['file'] = source['file'][:qmark]
189    #
190    # Check local path
191    #
192    _local_path(source, pathkey, config)
193    #
194    # Is the file compressed ?
195    #
196    esl = source['ext'].split('.')
197    if esl[-1:][0] == 'gz':
198        source['compressed-type'] = 'gzip'
199        source['compressed'] = '%{__gzip} -dc'
200    elif esl[-1:][0] == 'bz2':
201        source['compressed-type'] = 'bzip2'
202        source['compressed'] = '%{__bzip2} -dc'
203    elif esl[-1:][0] == 'zip':
204        source['compressed-type'] = 'zip'
205        source['compressed'] = '%{__unzip} -u'
206    elif esl[-1:][0] == 'xz':
207        source['compressed-type'] = 'xz'
208        source['compressed'] = '%{__xz} -dc'
209
210def _patchworks_parser(source, pathkey, config, opts):
211    #
212    # Check local path
213    #
214    _local_path(source, pathkey, config)
215    source['url'] = 'http%s' % (source['path'][2:])
216
217def _git_parser(source, pathkey, config, opts):
218    #
219    # Check local path
220    #
221    _local_path(source, pathkey, config)
222    #
223    # Symlink.
224    #
225    us = source['url'].split('?')
226    source['path'] = path.dirname(us[0])
227    source['file'] = path.basename(us[0])
228    source['name'], source['ext'] = path.splitext(source['file'])
229    if len(us) > 1:
230        source['args'] = us[1:]
231    source['local'] = \
232        path.join(source['local_prefix'], 'git', source['file'])
233    source['symlink'] = source['local']
234
235def _cvs_parser(source, pathkey, config, opts):
236    #
237    # Check local path
238    #
239    _local_path(source, pathkey, config)
240    #
241    # Symlink.
242    #
243    if not source['url'].startswith('cvs://'):
244        raise error.general('invalid cvs path: %s' % (source['url']))
245    us = source['url'].split('?')
246    try:
247        url = us[0]
248        source['file'] = url[url[6:].index(':') + 7:]
249        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
250    except:
251        raise error.general('invalid cvs path: %s' % (source['url']))
252    for a in us[1:]:
253        _as = a.split('=')
254        if _as[0] == 'module':
255            if len(_as) != 2:
256                raise error.general('invalid cvs module: %s' % (a))
257            source['module'] = _as[1]
258        elif _as[0] == 'src-prefix':
259            if len(_as) != 2:
260                raise error.general('invalid cvs src-prefix: %s' % (a))
261            source['src_prefix'] = _as[1]
262        elif _as[0] == 'tag':
263            if len(_as) != 2:
264                raise error.general('invalid cvs tag: %s' % (a))
265            source['tag'] = _as[1]
266        elif _as[0] == 'date':
267            if len(_as) != 2:
268                raise error.general('invalid cvs date: %s' % (a))
269            source['date'] = _as[1]
270    if 'date' in source and 'tag' in source:
271        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
272    # Do here to ensure an ordered path, the URL can include options in any order
273    if 'module' in source:
274        source['file'] += '_%s' % (source['module'])
275    if 'tag' in source:
276        source['file'] += '_%s' % (source['tag'])
277    if 'date' in source:
278        source['file'] += '_%s' % (source['date'])
279    for c in '/@#%.-':
280        source['file'] = source['file'].replace(c, '_')
281    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
282    if 'src_prefix' in source:
283        source['symlink'] = path.join(source['local'], source['src_prefix'])
284    else:
285        source['symlink'] = source['local']
286
287def _file_parser(source, pathkey, config, opts):
288    #
289    # Check local path
290    #
291    _local_path(source, pathkey, config)
292    #
293    # Get the paths sorted.
294    #
295    source['file'] = source['url'][6:]
296
297parsers = { 'http': _http_parser,
298            'ftp':  _http_parser,
299            'pw':   _patchworks_parser,
300            'git':  _git_parser,
301            'cvs':  _cvs_parser,
302            'file': _file_parser }
303
304def parse_url(url, pathkey, config, opts, file_override = None):
305    #
306    # Split the source up into the parts we need.
307    #
308    source = {}
309    source['url'] = url
310    source['options'] = []
311    colon = url.find(':')
312    if url[colon + 1:colon + 3] != '//':
313        raise error.general('malforned URL (no protocol prefix): %s' % (url))
314    source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
315    if file_override is None:
316        source['file'] = path.basename(url)
317    else:
318        bad_chars = [c for c in ['/', '\\', '?', '*'] if c in file_override]
319        if len(bad_chars) > 0:
320            raise error.general('bad characters in file name: %s' % (file_override))
321
322        log.output('download: file-override: %s' % (file_override))
323        source['file'] = file_override
324        source['options'] += ['file-override']
325    source['name'], source['ext'] = path.splitext(source['file'])
326    if source['name'].endswith('.tar'):
327        source['name'] = source['name'][:-4]
328        source['ext'] = '.tar' + source['ext']
329    #
330    # Get the file. Checks the local source directory first.
331    #
332    source['local'] = None
333    for p in parsers:
334        if url.startswith(p):
335            source['type'] = p
336            if parsers[p](source, pathkey, config, opts):
337                break
338    source['script'] = ''
339    return source
340
341def _http_downloader(url, local, config, opts):
342    if path.exists(local):
343        return True
344    #
345    # Hack for GitHub.
346    #
347    if url.startswith('https://api.github.com'):
348        url = urllib_parse.urljoin(url, config.expand('tarball/%{version}'))
349    dst = os.path.relpath(path.host(local))
350    log.output('download: (full) %s -> %s' % (url, dst))
351    log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst))
352    failed = False
353    if _do_download(opts):
354        _in = None
355        _out = None
356        _length = None
357        _have = 0
358        _chunk_size = 256 * 1024
359        _chunk = None
360        _last_percent = 200.0
361        _last_msg = ''
362        _have_status_output = False
363        _url = url
364        try:
365            try:
366                _in = None
367                _ssl_context = None
368                # See #2656
369                _req = urllib_request.Request(_url)
370                _req.add_header('User-Agent', 'Wget/1.16.3 (freebsd10.1)')
371                try:
372                    import ssl
373                    _ssl_context = ssl._create_unverified_context()
374                    _in = urllib_request.urlopen(_req, context = _ssl_context)
375                except:
376                    log.output('download: no ssl context')
377                    _ssl_context = None
378                if _ssl_context is None:
379                    _in = urllib_request.urlopen(_req)
380                if _url != _in.geturl():
381                    _url = _in.geturl()
382                    log.output(' redirect: %s' % (_url))
383                    log.notice(' redirect: %s' % (_sensible_url(_url)))
384                _out = open(path.host(local), 'wb')
385                try:
386                    _length = int(_in.info()['Content-Length'].strip())
387                except:
388                    pass
389                while True:
390                    _msg = '\rdownloading: %s - %s ' % (dst, _humanize_bytes(_have))
391                    if _length:
392                        _percent = round((float(_have) / _length) * 100, 2)
393                        if _percent != _last_percent:
394                            _msg += 'of %s (%0.0f%%) ' % (_humanize_bytes(_length), _percent)
395                    if _msg != _last_msg:
396                        extras = (len(_last_msg) - len(_msg))
397                        log.stdout_raw('%s%s' % (_msg, ' ' * extras + '\b' * extras))
398                        _last_msg = _msg
399                        _have_status_output = True
400                    _chunk = _in.read(_chunk_size)
401                    if not _chunk:
402                        break
403                    _out.write(_chunk)
404                    _have += len(_chunk)
405                log.stdout_raw('\n\r')
406            except:
407                if _have_status_output:
408                    log.stdout_raw('\n\r')
409                raise
410        except IOError as err:
411            log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err)))
412            if path.exists(local):
413                os.remove(path.host(local))
414            failed = True
415        except ValueError as err:
416            log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err)))
417            if path.exists(local):
418                os.remove(path.host(local))
419            failed = True
420        except:
421            msg = 'download: %s: error' % (_sensible_url(_url))
422            log.stderr(msg)
423            log.notice(msg)
424            if _in is not None:
425                _in.close()
426            if _out is not None:
427                _out.close()
428            raise
429        if _out is not None:
430            _out.close()
431        if _in is not None:
432            _in.close()
433            del _in
434        if not failed:
435            if not path.isfile(local):
436                raise error.general('source is not a file: %s' % (path.host(local)))
437            if not _hash_check(path.basename(local), local, config.macros, False):
438                raise error.general('checksum failure file: %s' % (dst))
439    return not failed
440
441def _git_downloader(url, local, config, opts):
442    repo = git.repo(local, opts, config.macros)
443    rlp = os.path.relpath(path.host(local))
444    us = url.split('?')
445    #
446    # Handle the various git protocols.
447    #
448    # remove 'git' from 'git://xxxx/xxxx?protocol=...'
449    #
450    url_base = us[0][len('git'):]
451    for a in us[1:]:
452        _as = a.split('=')
453        if _as[0] == 'protocol':
454            if len(_as) != 2:
455                raise error.general('invalid git protocol option: %s' % (_as))
456            if _as[1] == 'none':
457                # remove the rest of the protocol header leaving nothing.
458                us[0] = url_base[len('://'):]
459            else:
460                if _as[1] not in ['ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync']:
461                    raise error.general('unknown git protocol: %s' % (_as[1]))
462                us[0] = _as[1] + url_base
463    if not repo.valid():
464        log.notice('git: clone: %s -> %s' % (us[0], rlp))
465        if _do_download(opts):
466            repo.clone(us[0], local)
467    else:
468        repo.clean(['-f', '-d'])
469        repo.reset('--hard')
470        repo.checkout('master')
471    for a in us[1:]:
472        _as = a.split('=')
473        if _as[0] == 'branch' or _as[0] == 'checkout':
474            if len(_as) != 2:
475                raise error.general('invalid git branch/checkout: %s' % (_as))
476            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
477            if _do_download(opts):
478                repo.checkout(_as[1])
479        elif _as[0] == 'submodule':
480            if len(_as) != 2:
481                raise error.general('invalid git submodule: %s' % (_as))
482            log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
483            if _do_download(opts):
484                repo.submodule(_as[1])
485        elif _as[0] == 'fetch':
486            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
487            if _do_download(opts):
488                repo.fetch()
489        elif _as[0] == 'merge':
490            log.notice('git: merge: %s' % (us[0]))
491            if _do_download(opts):
492                repo.merge()
493        elif _as[0] == 'pull':
494            log.notice('git: pull: %s' % (us[0]))
495            if _do_download(opts):
496                repo.pull()
497        elif _as[0] == 'reset':
498            arg = []
499            if len(_as) > 1:
500                arg = ['--%s' % (_as[1])]
501            log.notice('git: reset: %s' % (us[0]))
502            if _do_download(opts):
503                repo.reset(arg)
504        elif _as[0] == 'protocol':
505            pass
506        else:
507            raise error.general('invalid git option: %s' % (_as))
508    return True
509
510def _cvs_downloader(url, local, config, opts):
511    rlp = os.path.relpath(path.host(local))
512    us = url.split('?')
513    module = None
514    tag = None
515    date = None
516    src_prefix = None
517    for a in us[1:]:
518        _as = a.split('=')
519        if _as[0] == 'module':
520            if len(_as) != 2:
521                raise error.general('invalid cvs module: %s' % (a))
522            module = _as[1]
523        elif _as[0] == 'src-prefix':
524            if len(_as) != 2:
525                raise error.general('invalid cvs src-prefix: %s' % (a))
526            src_prefix = _as[1]
527        elif _as[0] == 'tag':
528            if len(_as) != 2:
529                raise error.general('invalid cvs tag: %s' % (a))
530            tag = _as[1]
531        elif _as[0] == 'date':
532            if len(_as) != 2:
533                raise error.general('invalid cvs date: %s' % (a))
534            date = _as[1]
535    repo = cvs.repo(local, opts, config.macros, src_prefix)
536    if not repo.valid():
537        if not path.isdir(local):
538            log.notice('Creating source directory: %s' % \
539                           (os.path.relpath(path.host(local))))
540            if _do_download(opts):
541                path.mkdir(local)
542            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
543            if _do_download(opts):
544                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
545    for a in us[1:]:
546        _as = a.split('=')
547        if _as[0] == 'update':
548            log.notice('cvs: update: %s' % (us[0]))
549            if _do_download(opts):
550                repo.update()
551        elif _as[0] == 'reset':
552            log.notice('cvs: reset: %s' % (us[0]))
553            if _do_download(opts):
554                repo.reset()
555    return True
556
557def _file_downloader(url, local, config, opts):
558    if not path.exists(local):
559        try:
560            src = url[7:]
561            dst = local
562            log.notice('download: copy %s -> %s' % (src, dst))
563            path.copy(src, dst)
564        except:
565            return False
566    return True
567
568downloaders = { 'http': _http_downloader,
569                'ftp':  _http_downloader,
570                'pw':   _http_downloader,
571                'git':  _git_downloader,
572                'cvs':  _cvs_downloader,
573                'file': _file_downloader }
574
575def get_file(url, local, opts, config):
576    if local is None:
577        raise error.general('source/patch path invalid')
578    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
579        log.notice('Creating source directory: %s' % \
580                       (os.path.relpath(path.host(path.dirname(local)))))
581    log.output('making dir: %s' % (path.host(path.dirname(local))))
582    if _do_download(opts):
583        path.mkdir(path.dirname(local))
584    if not path.exists(local) and opts.download_disabled():
585        raise error.general('source not found: %s' % (path.host(local)))
586    #
587    # Check if a URL has been provided on the command line. If the package is
588    # released push to the start the RTEMS URL unless overrided by the command
589    # line option --with-release-url. The variant --without-release-url can
590    # override the released check.
591    #
592    url_bases = opts.urls()
593    try:
594        rtems_release_url_value = config.macros.expand('%{rtems_release_url}/%{rsb_version}/sources')
595    except:
596        rtems_release_url_value = None
597        log.output('RTEMS release URL could not be expanded')
598    rtems_release_url = None
599    if version.released() and rtems_release_url_value:
600        rtems_release_url = rtems_release_url_value
601    with_rel_url = opts.with_arg('release-url')
602    if with_rel_url[1] == 'not-found':
603        if config.defined('without_release_url'):
604            with_rel_url = ('without_release-url', 'yes')
605    if with_rel_url[0] == 'with_release-url':
606        if with_rel_url[1] == 'yes':
607            if rtems_release_url_value is None:
608                raise error.general('no valid release URL')
609            rtems_release_url = rtems_release_url_value
610        elif with_rel_url[1] == 'no':
611            pass
612        else:
613            rtems_release_url = with_rel_url[1]
614    elif with_rel_url[0] == 'without_release-url' and with_rel_url[1] == 'yes':
615        rtems_release_url = None
616    if rtems_release_url is not None:
617        log.trace('release url: %s' % (rtems_release_url))
618        #
619        # If the URL being fetched is under the release path do not add the
620        # sources release path because it is already there.
621        #
622        if not url.startswith(rtems_release_url):
623            if url_bases is None:
624                url_bases = [rtems_release_url]
625            else:
626                url_bases.append(rtems_release_url)
627    urls = []
628    if url_bases is not None:
629        #
630        # Split up the URL we are being asked to download.
631        #
632        url_path = urllib_parse.urlsplit(url)[2]
633        slash = url_path.rfind('/')
634        if slash < 0:
635            url_file = url_path
636        else:
637            url_file = url_path[slash + 1:]
638        log.trace('url_file: %s' %(url_file))
639        for base in url_bases:
640            if base[-1:] != '/':
641                base += '/'
642            next_url = urllib_parse.urljoin(base, url_file)
643            log.trace('url: %s' %(next_url))
644            urls.append(next_url)
645    urls += url.split()
646    log.trace('_url: %s -> %s' % (','.join(urls), local))
647    for url in urls:
648        for dl in downloaders:
649            if url.startswith(dl):
650                if downloaders[dl](url, local, config, opts):
651                    return
652    if _do_download(opts):
653        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.