Changeset 1bbb766 in rtems-source-builder


Ignore:
Timestamp:
Mar 17, 2016, 5:47:31 AM (4 years ago)
Author:
Chris Johns <chrisj@…>
Branches:
4.10, 4.9, master
Children:
38ed59a
Parents:
b480086
git-author:
Chris Johns <chrisj@…> (03/17/16 05:47:31)
git-committer:
Chris Johns <chrisj@…> (03/17/16 05:49:56)
Message:

sb: Change urlib to urllib2 on Python2 and add user agent support.

Some sites block the standard python user agent. Change to Wget
as it seems to make the tested sites behave.

Limit the size of long URLs when logging to the user.

Closes #2656.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • source-builder/sb/download.py

    rb480086 r1bbb766  
    3333    import urllib.parse as urllib_parse
    3434except ImportError:
    35     import urllib as urllib_request
     35    import urllib2 as urllib_request
    3636    import urlparse as urllib_parse
    3737
     
    6969            break
    7070    return '%.*f%s' % (precision, float(bytes) / factor, suffix)
     71
     72def _sensible_url(url, used = 0):
     73    space = 150 - used - 15
     74    if len(url) > space:
     75        size = (space - 5) / 2
     76        url = url[:size] + ' ... ' + url[-size:]
     77    return url
    7178
    7279def _hash_check(file_, absfile, macros, remove = True):
     
    326333        url = urllib_parse.urljoin(url, config.expand('tarball/%{version}'))
    327334    dst = os.path.relpath(path.host(local))
    328     log.notice('download: %s -> %s' % (url, dst))
     335    log.output('download: %s -> %s' % (url, dst))
     336    log.notice('download: %s -> %s' % (_sensible_url(url, len(dst)), dst))
    329337    failed = False
    330338    if _do_download(opts):
     
    338346        _last_msg = ''
    339347        _have_status_output = False
     348        _url = url
    340349        try:
    341350            try:
    342351                _in = None
    343352                _ssl_context = None
    344                 _urllib_url = url
     353                # See #2656
     354                _req = urllib_request.Request(_url)
     355                _req.add_header('User-Agent', 'Wget/1.16.3 (freebsd10.1)')
    345356                try:
    346357                    import ssl
    347358                    _ssl_context = ssl._create_unverified_context()
    348                     _in = urllib_request.urlopen(_urllib_url, context = _ssl_context)
     359                    _in = urllib_request.urlopen(_req, context = _ssl_context)
    349360                except:
    350361                    _ssl_context = None
    351362                if _ssl_context is None:
    352                     _in = urllib_request.urlopen(_urllib_url)
    353                 if url != _in.geturl():
    354                     log.notice(' redirect: %s' % (_in.geturl()))
     363                    _in = urllib_request.urlopen(_req)
     364                if _url != _in.geturl():
     365                    _url = _in.geturl()
     366                    log.output(' redirect: %s' % (_url))
     367                    log.notice(' redirect: %s' % (_sensible_url(_url)))
    355368                _out = open(path.host(local), 'wb')
    356369                try:
    357                     _length = int(_in.info().getheader('Content-Length').strip())
     370                    _length = int(_in.info()['Content-Length'].strip())
    358371                except:
    359372                    pass
     
    380393                raise
    381394        except IOError as err:
    382             log.notice('download: %s: error: %s' % (url, str(err)))
     395            log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err)))
    383396            if path.exists(local):
    384397                os.remove(path.host(local))
    385398            failed = True
    386399        except ValueError as err:
    387             log.notice('download: %s: error: %s' % (url, str(err)))
     400            log.notice('download: %s: error: %s' % (_sensible_url(_url), str(err)))
    388401            if path.exists(local):
    389402                os.remove(path.host(local))
    390403            failed = True
    391404        except:
    392             msg = 'download: %s: error' % (url)
     405            msg = 'download: %s: error' % (_sensible_url(_url))
    393406            log.stderr(msg)
    394407            log.notice(msg)
Note: See TracChangeset for help on using the changeset viewer.