source: rtems-source-builder/source-builder/sb/download.py @ c65b9eb

4.104.114.95
Last change on this file since c65b9eb was c65b9eb, checked in by Chris Johns <chrisj@…>, on 04/25/13 at 05:06:20

Change the CVS download path to include the module and tag/date.
  • Property mode set to 100644
File size: 11.0 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import os
26import stat
27import sys
28import urllib2
29import urlparse
30
31import cvs
32import error
33import git
34import log
35import path
36
37def _http_parser(source, config, opts):
38    #
39    # Is the file compressed ?
40    #
41    esl = source['ext'].split('.')
42    if esl[-1:][0] == 'gz':
43        source['compressed'] = '%{__gzip} -dc'
44    elif esl[-1:][0] == 'bz2':
45        source['compressed'] = '%{__bzip2} -dc'
46    elif esl[-1:][0] == 'zip':
47        source['compressed'] = '%{__zip} -u'
48    elif esl[-1:][0] == 'xz':
49        source['compressed'] = '%{__xz} -dc'
50
51def _git_parser(source, config, opts):
52    #
53    # Symlink.
54    #
55    us = source['url'].split('?')
56    source['path'] = path.dirname(us[0])
57    source['file'] = path.basename(us[0])
58    source['name'], source['ext'] = path.splitext(source['file'])
59    if len(us) > 1:
60        source['args'] = us[1:]
61    source['local'] = \
62        path.join(source['local_prefix'], 'git', source['file'])
63    source['symlink'] = source['local']
64
65def _cvs_parser(source, config, opts):
66    #
67    # Symlink.
68    #
69    if not source['url'].startswith('cvs://'):
70        raise error.general('invalid cvs path: %s' % (source['url']))
71    us = source['url'].split('?')
72    try:
73        url = us[0]
74        source['file'] = url[url[6:].index(':') + 7:]
75        source['cvsroot'] = ':%s:' % (url[6:url[6:].index('/') + 6:])
76    except:
77        raise error.general('invalid cvs path: %s' % (source['url']))
78    for a in us[1:]:
79        _as = a.split('=')
80        if _as[0] == 'module':
81            if len(_as) != 2:
82                raise error.general('invalid cvs module: %s' % (a))
83            source['module'] = _as[1]
84        elif _as[0] == 'src-prefix':
85            if len(_as) != 2:
86                raise error.general('invalid cvs src-prefix: %s' % (a))
87            source['src_prefix'] = _as[1]
88        elif _as[0] == 'tag':
89            if len(_as) != 2:
90                raise error.general('invalid cvs tag: %s' % (a))
91            source['tag'] = _as[1]
92        elif _as[0] == 'date':
93            if len(_as) != 2:
94                raise error.general('invalid cvs date: %s' % (a))
95            source['date'] = _as[1]
96    if 'date' in source and 'tag' in source:
97        raise error.general('cvs URL cannot have a date and tag: %s' % (source['url']))
98    # Do here to ensure an ordered path, the URL can include options in any order
99    if 'module' in source:
100        source['file'] += '_%s' % (source['module'])
101    if 'tag' in source:
102        source['file'] += '_%s' % (source['tag'])
103    if 'date' in source:
104        source['file'] += '_%s' % (source['date'])
105    for c in '/@#%.-':
106        source['file'] = source['file'].replace(c, '_')
107    source['local'] = path.join(source['local_prefix'], 'cvs', source['file'])
108    if 'src_prefix' in source:
109        source['symlink'] = path.join(source['local'], source['src_prefix'])
110    else:
111        source['symlink'] = source['local']
112
113def _file_parser(source, config, opts):
114    #
115    # Symlink.
116    #
117    source['symlink'] = source['local']
118
119parsers = { 'http': _http_parser,
120            'ftp':  _http_parser,
121            'git':  _git_parser,
122            'cvs':  _cvs_parser,
123            'file': _file_parser }
124
125def parse_url(url, pathkey, config, opts):
126    #
127    # Split the source up into the parts we need.
128    #
129    source = {}
130    source['url'] = url
131    source['path'] = path.dirname(url)
132    source['file'] = path.basename(url)
133    source['name'], source['ext'] = path.splitext(source['file'])
134    #
135    # Get the file. Checks the local source directory first.
136    #
137    source['local'] = None
138    for p in config.define(pathkey).split(':'):
139        local = path.join(path.abspath(p), source['file'])
140        if source['local'] is None:
141            source['local_prefix'] = path.abspath(p)
142            source['local'] = local
143        if path.exists(local):
144            source['local_prefix'] = path.abspath(p)
145            source['local'] = local
146            break
147    source['script'] = ''
148    for p in parsers:
149        if url.startswith(p):
150            source['type'] = p
151            if parsers[p](source, config, opts):
152                break
153    return source
154
155def _http_downloader(url, local, config, opts):
156    if path.exists(local):
157        return True
158    #
159    # Hack for GitHub.
160    #
161    if url.startswith('https://api.github.com'):
162        url = urlparse.urljoin(url, config.expand('tarball/%{version}'))
163    log.notice('download: %s -> %s' % (url, os.path.relpath(path.host(local))))
164    failed = False
165    if not opts.dry_run():
166        _in = None
167        _out = None
168        try:
169            _in = urllib2.urlopen(url)
170            _out = open(path.host(local), 'wb')
171            _out.write(_in.read())
172        except IOError, err:
173            log.notice('download: %s: error: %s' % (url, str(err)))
174            if path.exists(local):
175                os.remove(path.host(local))
176            failed = True
177        except ValueError, err:
178            log.notice('download: %s: error: %s' % (url, str(err)))
179            if path.exists(local):
180                os.remove(path.host(local))
181            failed = True
182        except:
183            msg = 'download: %s: error' % (url)
184            log.stderr(msd)
185            log.notice(msg)
186            if _out is not None:
187                _out.close()
188            raise
189        if _out is not None:
190            _out.close()
191        if _in is not None:
192            del _in
193        if not failed:
194            if not path.isfile(local):
195                raise error.general('source is not a file: %s' % (path.host(local)))
196    return not failed
197
198def _git_downloader(url, local, config, opts):
199    rlp = os.path.relpath(path.host(local))
200    us = url.split('?')
201    repo = git.repo(local, opts, config.macros)
202    if not repo.valid():
203        log.notice('git: clone: %s -> %s' % (us[0], rlp))
204        if not opts.dry_run():
205            repo.clone(us[0], local)
206    for a in us[1:]:
207        _as = a.split('=')
208        if _as[0] == 'branch':
209            log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
210            if not opts.dry_run():
211                repo.checkout(_as[1])
212        elif _as[0] == 'pull':
213            log.notice('git: pull: %s' % (us[0]))
214            if not opts.dry_run():
215                repo.pull()
216        elif _as[0] == 'fetch':
217            log.notice('git: fetch: %s -> %s' % (us[0], rlp))
218            if not opts.dry_run():
219                repo.fetch()
220        elif _as[0] == 'reset':
221            arg = []
222            if len(_as) > 1:
223                arg = ['--%s' % (_as[1])]
224            log.notice('git: reset: %s' % (us[0]))
225            if not opts.dry_run():
226                repo.reset(arg)
227    return True
228
229def _cvs_downloader(url, local, config, opts):
230    rlp = os.path.relpath(path.host(local))
231    us = url.split('?')
232    module = None
233    tag = None
234    date = None
235    src_prefix = None
236    for a in us[1:]:
237        _as = a.split('=')
238        if _as[0] == 'module':
239            if len(_as) != 2:
240                raise error.general('invalid cvs module: %s' % (a))
241            module = _as[1]
242        elif _as[0] == 'src-prefix':
243            if len(_as) != 2:
244                raise error.general('invalid cvs src-prefix: %s' % (a))
245            src_prefix = _as[1]
246        elif _as[0] == 'tag':
247            if len(_as) != 2:
248                raise error.general('invalid cvs tag: %s' % (a))
249            tag = _as[1]
250        elif _as[0] == 'date':
251            if len(_as) != 2:
252                raise error.general('invalid cvs date: %s' % (a))
253            date = _as[1]
254    repo = cvs.repo(local, opts, config.macros, src_prefix)
255    if not repo.valid():
256        if not path.isdir(local):
257            log.notice('Creating source directory: %s' % \
258                           (os.path.relpath(path.host(local))))
259            if not opts.dry_run():
260                path.mkdir(local)
261            log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
262            if not opts.dry_run():
263                repo.checkout(':%s' % (us[0][6:]), module, tag, date)
264    for a in us[1:]:
265        _as = a.split('=')
266        if _as[0] == 'update':
267            log.notice('cvs: update: %s' % (us[0]))
268            if not opts.dry_run():
269                repo.update()
270        elif _as[0] == 'reset':
271            log.notice('cvs: reset: %s' % (us[0]))
272            if not opts.dry_run():
273                repo.reset()
274    return True
275
276def _file_downloader(url, local, config, opts):
277    if path.exists(local):
278        return True
279    return path.isdir(url)
280
281downloaders = { 'http': _http_downloader,
282                'ftp':  _http_downloader,
283                'git':  _git_downloader,
284                'cvs':  _cvs_downloader,
285                'file': _file_downloader }
286
287def get_file(url, local, opts, config):
288    if local is None:
289        raise error.general('source/patch path invalid')
290    if not path.isdir(path.dirname(local)) and not opts.download_disabled():
291        log.notice('Creating source directory: %s' % \
292                       (os.path.relpath(path.host(path.dirname(local)))))
293    log.output('making dir: %s' % (path.host(path.dirname(local))))
294    if not opts.dry_run():
295        path.mkdir(path.dirname(local))
296    if not path.exists(local) and opts.download_disabled():
297        raise error.general('source not found: %s' % (path.host(local)))
298    #
299    # Check if a URL hasbeen provided on the command line.
300    #
301    url_bases = opts.urls()
302    urls = []
303    if url_bases is not None:
304        for base in url_bases:
305            if base[-1:] != '/':
306                base += '/'
307            url_path = urlparse.urlsplit(url)[2]
308            slash = url_path.rfind('/')
309            if slash < 0:
310                url_file = url_path
311            else:
312                url_file = url_path[slash + 1:]
313            urls.append(urlparse.urljoin(base, url_file))
314    urls.append(url)
315    log.trace('_url: %s -> %s' % (','.join(urls), local))
316    for url in urls:
317        for dl in downloaders:
318            if url.startswith(dl):
319                if downloaders[dl](url, local, config, opts):
320                    return
321    if not opts.dry_run():
322        raise error.general('downloading %s: all paths have failed, giving up' % (url))
Note: See TracBrowser for help on using the repository browser.