source: rtems-source-builder/source-builder/sb/download.py @ 3237c8e

4.104.114.95
Last change on this file since 3237c8e was 3237c8e, checked in by Chris Johns <chrisj@…>, on 03/29/15 at 04:35:00

sb: Provide an unverified SSL context to the URL.

The RTEMS servers are causing an exception when downloading patches. The solution is
provided in PEP-0476 (​https://www.python.org/dev/peps/pep-0476/#opting-out).
  • Property mode set to 100644
File size: 19.2 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import hashlib
26import os
27import stat
28import sys
29import urllib2
30import urlparse
31
32import cvs
33import error
34import git
35import log
36import path
37import sources
38
39def _do_download(opts):
40    download = True
41    if opts.dry_run():
42        download = False
43        wa = opts.with_arg('download')
44        if wa is not None:
45            if wa[0] == 'with_download' and wa[1] == 'yes':
46                download = True
47    return download
48
49def _humanize_bytes(bytes, precision = 1):
50    abbrevs = (
51        (1 << 50L, 'PB'),
52        (1 << 40L, 'TB'),
53        (1 << 30L, 'GB'),
54        (1 << 20L, 'MB'),
55        (1 << 10L, 'kB'),
56        (1, ' bytes')
57    )
58    if bytes == 1:
59        return '1 byte'
60    for factor, suffix in abbrevs:
61        if bytes >= factor:
62            break
63    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
64
65def _hash_check(file_, absfile, macros, remove = True):
66    failed = False
67    hash = sources.get_hash(file_.lower(), macros)
68    if hash is not None:
69        hash = hash.split()
70        if len(hash) != 2:
71            raise error.internal('invalid hash format: %s' % (file_))
72        try:
73            hashlib_algorithms = hashlib.algorithms
74        except:
75            hashlib_algorithms = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
76        if hash[0] not in hashlib_algorithms:
77            raise error.general('invalid hash algorithm for %s: %s' % (file_, hash[0]))
78        hasher = None
79        _in = None
80        try:
81            hasher = hashlib.new(hash[0])
82            _in = open(path.host(absfile), 'rb')
83            hasher.update(_in.read())
84        except IOError, err:
85            log.notice('hash: %s: read error: %s' % (file_, str(err)))
86            failed = True
87        except:
88            msg = 'hash: %s: error' % (file_)
89            log.stderr(msg)
90            log.notice(msg)
91            if _in is not None:
92                _in.close()
93            raise
94        if _in is not None:
95            _in.close()
96        log.output('checksums: %s: %s => %s' % (file_, hasher.hexdigest(), hash[1]))
97        if hasher.hexdigest() != hash[1]:
98            log.warning('checksum error: %s' % (file_))
99            failed = True
100        if failed and remove:
101            log.warning('removing: %s' % (file_))
102            if path.exists(absfile):
103                try:
104                    os.remove(path.host(absfile))
105                except IOError, err:
106                    raise error.general('hash: %s: remove: %s' % (absfile, str(err)))
107                except:
108                    raise error.general('hash: %s: remove error' % (file_))
109        if hasher is not None:
110            del hasher
111    else:
112        log.warning('%s: no hash found' % (file_))
113    return not failed
114
115def _local_path(source, pathkey, config):
116    for p in config.define(pathkey).split(':'):
117        local = path.join(path.abspath(p), source['file'])
118        if source['local'] is None:
119            source['local_prefix'] = path.abspath(p)
120            source['local'] = local
121        if path.exists(local):
122            source['local_prefix'] = path.abspath(p)
123            source['local'] = local
124            _hash_check(source['file'], local, config.macros)
125            break
126
127def _http_parser(source, pathkey, config, opts):
128    #
129    # Hack for gitweb.cgi patch downloads. We rewrite the various fields.
130    #
131    if 'gitweb.cgi' in source['url']:
132        url = source['url']
133        if '?' not in url:
134            raise error.general('invalid gitweb.cgi request: %s' % (url))
135        req = url.split('?')[1]
136        if len(req) == 0:
137            raise error.general('invalid gitweb.cgi request: %s' % (url))
138        #
139        # The gitweb.cgi request should have:
140        #    p=<what>
141        #    a=patch
142        #    h=<hash>
143        # so extract the p and h parts to make the local name.
144        #
145        p = None
146        a = None
147        h = None
148        for r in req.split(';'):
149            if '=' not in r:
150                raise error.general('invalid gitweb.cgi path: %s' % (url))
151            rs = r.split('=')
152            if rs[0] == 'p':
153                p = rs[1].replace('.', '-')
154            elif rs[0] == 'a':
155                a = rs[1]
156            elif rs[0] == 'h':
157                h = rs[1]
158        if p is None or h is None:
159            raise error.general('gitweb.cgi path missing p or h: %s' % (url))
160        source['file'] = '%s-%s.patch' % (p, h)
161    #
162    # Check local path
163    #
164    _local_path(source, pathkey, config)
165    #
166    # Is the file compressed ?
167    #
168    esl = source['ext'].split('.')
169    if esl[-1:][0] == 'gz':
170        source['compressed-type'] = 'gzip'
171        source['compressed'] = '%{__gzip} -dc'
172    elif esl[-1:][0] == 'bz2':
173        source['compressed-type'] = 'bzip2'
174        source['compressed'] = '%{__bzip2} -dc'
175    elif esl[-1:][0] == 'zip':
176        source['compressed-type'] = 'zip'
177        source['compressed'] = '%{__unzip} -u'
178    elif esl[-1:][0] == 'xz':
179        source['compressed-type'] = 'xz'
180        source['compressed'] = '%{__xz} -dc'
181
182def _patchworks_parser(source, pathkey, config, opts):
183    #
184    # Check local path
185    #
186    _local_path(source, pathkey, config)
187    source['url'] = 'http%s' % (source['path'][2:])
188
189def _git_parser(source, pathkey, config, opts):
190    #
191    # Check local path
192    #
193    _local_path(source, pathkey, config)
194    #
195    # Symlink.
196    #
197    us = source['url'].split('?')
198    source['path'] = path.dirname(us[0])
199    source['file'] = path.basename(us[0])
200    source['name'], source['ext'] = path.splitext(source['file'])
201    if len(us) > 1:
202        source['args'] = us[1:]
203    source['local'] = \
204        path.join(source['local_prefix'], 'git', source['file'])
205    source['symlink'] = source['local']
206
207def _cvs_parser(source, pathkey, config, opts):
208    #
209    # Check local path
210    #
211    _local_path(source, pathkey, config)
212    #
213    # Symlink.
214    #
215    if not source['url'].startswith('cvs://'):
216        raise error.general('invalid cvs path: %s' % (source['url']))
217    us = source['url'].split('?')
218    try:
219        url = us[0]
220        source['file'] = url[url[6:].index(':') + 7:]
221        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
222    except:
223        raise error.general('invalid cvs path: %s' % (source['url']))
224    for a in us[1:]:
225        _as = a.split('=')
226        if _as[0] == 'module':
227            if len(_as) != 2:
228                raise error.general('invalid cvs module: %s' % (a))
229            source['module'] = _as[1]
230        elif _as[0] == 'src-prefix':
231            if len(_as) != 2:
232                raise error.general('invalid cvs src-prefix: %s' % (a))
233            source['src_prefix'] = _as[1]
234        elif _as[0] == 'tag':
235            if len(_as) != 2:
236                raise error.general('invalid cvs tag: %s' % (a))
237            source['tag'] = _as[1]
238        elif _as[0] == 'date':
239            if len(_as) != 2:
240                raise error.general('invalid cvs date: %s' % (a))
241            source['date'] = _as[1]
242    if 'date' in source and 'tag' in source:
243        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
244    # Do here to ensure an ordered path, the URL can include options in any order
245    if 'module' in source:
246        source['file'] += '_%s' % (source['module'])
247    if 'tag' in source:
248        source['file'] += '_%s' % (source['tag'])
249    if 'date' in source:
250        source['file'] += '_%s' % (source['date'])
251    for c in '/@#%.-':
252        source['file'] = source['file'].replace(c, '_')
253    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
254    if 'src_prefix' in source:
255        source['symlink'] = path.join(source['local'], source['src_prefix'])
256    else:
257        source['symlink'] = source['local']
258
259def _file_parser(source, pathkey, config, opts):
260    #
261    # Check local path
262    #
263    _local_path(source, pathkey, config)
264    #
265    # Symlink.
266    #
267    source['symlink'] = source['local']
268
269parsers = { 'http': _http_parser,
270            'ftp':  _http_parser,
271            'pw':   _patchworks_parser,
272            'git':  _git_parser,
273            'cvs':  _cvs_parser,
274            'file': _file_parser }
275
276def parse_url(url, pathkey, config, opts):
277    #
278    # Split the source up into the parts we need.
279    #
280    source = {}
281    source['url'] = url
282    colon = url.find(':')
283    if url[colon + 1:colon + 3] != '//':
284        raise error.general('malforned URL (no protocol prefix): %s' % (url))
285    source['path'] = url[:colon + 3] + path.dirname(url[colon + 3:])
286    source['file'] = path.basename(url)
287    source['name'], source['ext'] = path.splitext(source['file'])
288    if source['name'].endswith('.tar'):
289        source['name'] = source['name'][:-4]
290        source['ext'] = '.tar' + source['ext']
291    #
292    # Get the file. Checks the local source directory first.
293    #
294    source['local'] = None
295    for p in parsers:
296        if url.startswith(p):
297            source['type'] = p
298            if parsers[p](source, pathkey, config, opts):
299                break
300    source['script'] = ''
301    return source
302
303def _http_downloader(url, local, config, opts):
304    if path.exists(local):
305        return True
306    #
307    # Hack for GitHub.
308    #
309    if url.startswith('https://api.github.com'):
310        url = urlparse.urljoin(url, config.expand('tarball/%{version}'))
311    dst = os.path.relpath(path.host(local))
312    log.notice('download: %s -> %s' % (url, dst))
313    failed = False
314    if _do_download(opts):
315        _in = None
316        _out = None
317        _length = None
318        _have = 0
319        _chunk_size = 256 * 1024
320        _chunk = None
321        _last_percent = 200.0
322        _last_msg = ''
323        _wipe_output = False
324        try:
325            try:
326                _in = None
327                _ssl_context = None
328                try:
329                    import ssl
330                    _ssl_context = ssl._create_unverified_context()
331                except:
332                    pass
333                _in = urllib2.urlopen(url, context = _ssl_context)
334                if url != _in.geturl():
335                    log.notice(' redirect: %s' % (_in.geturl()))
336                _out = open(path.host(local), 'wb')
337                try:
338                    _length = int(_in.info().getheader('Content-Length').strip())
339                except:
340                    pass
341                while True:
342                    _msg = '\rdownloading: %s - %s ' % (dst, _humanize_bytes(_have))
343                    if _length:
344                        _percent = round((float(_have) / _length) * 100, 2)
345                        if _percent != _last_percent:
346                            _msg += 'of %s (%0.0f%%) ' % (_humanize_bytes(_length), _percent)
347                    if _msg != _last_msg:
348                        extras = (len(_last_msg) - len(_msg))
349                        log.stdout_raw('%s%s' % (_msg, ' ' * extras + '\b' * extras))
350                        _last_msg = _msg
351                    _chunk = _in.read(_chunk_size)
352                    if not _chunk:
353                        break
354                    _out.write(_chunk)
355                    _have += len(_chunk)
356                if _wipe_output:
357                    log.stdout_raw('\r%s\r' % (' ' * len(_last_msg)))
358                else:
359                    log.stdout_raw('\n')
360            except:
361                log.stdout_raw('\n')
362                raise
363        except IOError, err:
364            log.notice('download: %s: error: %s' % (url, str(err)))
365            if path.exists(local):
366                os.remove(path.host(local))
367            failed = True
368        except ValueError, err:
369            log.notice('download: %s: error: %s' % (url, str(err)))
370            if path.exists(local):
371                os.remove(path.host(local))
372            failed = True
373        except:
374            msg = 'download: %s: error' % (url)
375            log.stderr(msg)
376            log.notice(msg)
377            if _in is not None:
378                _in.close()
379            if _out is not None:
380                _out.close()
381            raise
382        if _out is not None:
383            _out.close()
384        if _in is not None:
385            _in.close()
386            del _in
387        if not failed:
388            if not path.isfile(local):
389                raise error.general('source is not a file: %s' % (path.host(local)))
390            if not _hash_check(path.basename(local), local, config.macros, False):
391                raise error.general('checksum failure file: %s' % (dst))
392    return not failed
393
394def _git_downloader(url, local, config, opts):
395    repo = git.repo(local, opts, config.macros)
396    rlp = os.path.relpath(path.host(local))
397    us = url.split('?')
398    #
399    # Handle the various git protocols.
400    #
401    # remove 'git' from 'git://xxxx/xxxx?protocol=...'
402    #
403    url_base = us[0][len('git'):]
404    for a in us[1:]:
405        _as = a.split('=')
406        if _as[0] == 'protocol':
407            if len(_as) != 2:
408                raise error.general('invalid git protocol option: %s' % (_as))
409            if _as[1] == 'none':
410                # remove the rest of the protocol header leaving nothing.
411                us[0] = url_base[len('://'):]
412            else:
413                if _as[1] not in ['ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync']:
414                    raise error.general('unknown git protocol: %s' % (_as[1]))
415                us[0] = _as[1] + url_base
416    if not repo.valid():
417        log.notice('git: clone: %s -> %s' % (us[0], rlp))
418        if _do_download(opts):
419            repo.clone(us[0], local)
420    else:
421        repo.clean(['-f', '-d'])
422        repo.reset('--hard')
423        repo.checkout('master')
424    for a in us[1:]:
425        _as = a.split('=')
426        if _as[0] == 'branch' or _as[0] == 'checkout':
427            if len(_as) != 2:
428                raise error.general('invalid git branch/checkout: %s' % (_as))
429            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
430            if _do_download(opts):
431                repo.checkout(_as[1])
432        elif _as[0] == 'submodule':
433            if len(_as) != 2:
434                raise error.general('invalid git submodule: %s' % (_as))
435            log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
436            if _do_download(opts):
437                repo.submodule(_as[1])
438        elif _as[0] == 'fetch':
439            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
440            if _do_download(opts):
441                repo.fetch()
442        elif _as[0] == 'merge':
443            log.notice('git: merge: %s' % (us[0]))
444            if _do_download(opts):
445                repo.merge()
446        elif _as[0] == 'pull':
447            log.notice('git: pull: %s' % (us[0]))
448            if _do_download(opts):
449                repo.pull()
450        elif _as[0] == 'reset':
451            arg = []
452            if len(_as) > 1:
453                arg = ['--%s' % (_as[1])]
454            log.notice('git: reset: %s' % (us[0]))
455            if _do_download(opts):
456                repo.reset(arg)
457        elif _as[0] == 'protocol':
458            pass
459        else:
460            raise error.general('invalid git option: %s' % (_as))
461    return True
462
463def _cvs_downloader(url, local, config, opts):
464    rlp = os.path.relpath(path.host(local))
465    us = url.split('?')
466    module = None
467    tag = None
468    date = None
469    src_prefix = None
470    for a in us[1:]:
471        _as = a.split('=')
472        if _as[0] == 'module':
473            if len(_as) != 2:
474                raise error.general('invalid cvs module: %s' % (a))
475            module = _as[1]
476        elif _as[0] == 'src-prefix':
477            if len(_as) != 2:
478                raise error.general('invalid cvs src-prefix: %s' % (a))
479            src_prefix = _as[1]
480        elif _as[0] == 'tag':
481            if len(_as) != 2:
482                raise error.general('invalid cvs tag: %s' % (a))
483            tag = _as[1]
484        elif _as[0] == 'date':
485            if len(_as) != 2:
486                raise error.general('invalid cvs date: %s' % (a))
487            date = _as[1]
488    repo = cvs.repo(local, opts, config.macros, src_prefix)
489    if not repo.valid():
490        if not path.isdir(local):
491            log.notice('Creating source directory: %s' % \
492                           (os.path.relpath(path.host(local))))
493            if _do_download(opts):
494                path.mkdir(local)
495            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
496            if _do_download(opts):
497                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
498    for a in us[1:]:
499        _as = a.split('=')
500        if _as[0] == 'update':
501            log.notice('cvs: update: %s' % (us[0]))
502            if _do_download(opts):
503                repo.update()
504        elif _as[0] == 'reset':
505            log.notice('cvs: reset: %s' % (us[0]))
506            if _do_download(opts):
507                repo.reset()
508    return True
509
510def _file_downloader(url, local, config, opts):
511    if path.exists(local):
512        return True
513    return path.isdir(url)
514
515downloaders = { 'http': _http_downloader,
516                'ftp':  _http_downloader,
517                'pw':   _http_downloader,
518                'git':  _git_downloader,
519                'cvs':  _cvs_downloader,
520                'file': _file_downloader }
521
522def get_file(url, local, opts, config):
523    if local is None:
524        raise error.general('source/patch path invalid')
525    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
526        log.notice('Creating source directory: %s' % \
527                       (os.path.relpath(path.host(path.dirname(local)))))
528    log.output('making dir: %s' % (path.host(path.dirname(local))))
529    if _do_download(opts):
530        path.mkdir(path.dirname(local))
531    if not path.exists(local) and opts.download_disabled():
532        raise error.general('source not found: %s' % (path.host(local)))
533    #
534    # Check if a URL has been provided on the command line.
535    #
536    url_bases = opts.urls()
537    urls = []
538    if url_bases is not None:
539        for base in url_bases:
540            if base[-1:] != '/':
541                base += '/'
542            url_path = urlparse.urlsplit(url)[2]
543            slash = url_path.rfind('/')
544            if slash < 0:
545                url_file = url_path
546            else:
547                url_file = url_path[slash + 1:]
548            urls.append(urlparse.urljoin(base, url_file))
549    urls += url.split()
550    log.trace('_url: %s -> %s' % (','.join(urls), local))
551    for url in urls:
552        for dl in downloaders:
553            if url.startswith(dl):
554                if downloaders[dl](url, local, config, opts):
555                    return
556    if _do_download(opts):
557        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.