source: rtems-source-builder/source-builder/sb/path.py

Last change on this file was 574839d, checked in by Chris Johns <chrisj@…>, on 04/22/23 at 05:05:14

sb/path: Handle unicode filenames in a source path

The change leaves the encoding in that currently exists because
I cannot remember why it is there. If an encoding error happens
return the same path to see if it will work.
  • Property mode set to 100644
File size: 12.0 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# Manage paths locally. The internally the path is in Unix or shell format and
22# we convert to the native format when performing operations at the Python
23# level. This allows macro expansion to work.
24#
25
26from __future__ import print_function
27
28import os
29import shutil
30import stat
31import string
32import sys
33
34from . import error
35from . import log
36
37windows_posix = sys.platform == 'msys'
38windows = os.name == 'nt'
39
40win_maxpath = 254
41
42def host(path):
43    if path is not None:
44        while '//' in path:
45            path = path.replace('//', '/')
46        if windows:
47            if len(path) > 2 and \
48               path[0] == '/' and path[2] == '/' and \
49               (path[1] in string.ascii_lowercase or \
50                path[1] in string.ascii_uppercase):
51                path = '%s:%s' % (path[1], path[2:])
52            path = path.replace('/', '\\')
53            if len(path) > win_maxpath:
54                if path.startswith('\\\\?\\'):
55                    path = path[4:]
56                path = u'\\'.join([u'\\\\?', path])
57    return path
58
59def shell(path):
60    if isinstance(path, bytes):
61        path = path.decode('utf8')
62    if path is not None:
63        if windows or windows_posix:
64            path = path.encode('ascii', 'ignore').decode('ascii')
65            if path.startswith('\\\\?\\'):
66                path = path[4:]
67            if len(path) > 1 and path[1] == ':':
68                path = '/%s%s' % (path[0].lower(), path[2:])
69            path = path.replace('\\', '/')
70        while '//' in path:
71            path = path.replace('//', '/')
72    return path
73
74def host_encode(dpath):
75    '''Encoding the path was present in the RSB however on a ZFS pool I am
76       seeing a failure with a go test in gcc:
77          gcc/testsuite/go.test/test/fixedbugs/issue27836.dir
78       Not encoding works however I am not sure why the encoding was added
79       so the following keeps the encoding and falls back to not encoded
80       if there is an error.0
81    '''
82    try:
83        return host(dpath).encode('utf8')
84    except:
85        pass
86    return dpath
87
88def basename(path):
89    path = shell(path)
90    return shell(os.path.basename(host(path)))
91
92def dirname(path):
93    path = shell(path)
94    return shell(os.path.dirname(path))
95
96def is_abspath(path):
97    if path is not None and len(path) > 0:
98        return '/' == path[0]
99    return False
100
101def join(path, *args):
102    path = shell(path)
103    for arg in args:
104        if len(path):
105            path += '/' + shell(arg)
106        else:
107            path = shell(arg)
108    return shell(path)
109
110def abspath(path):
111    path = shell(path)
112    return shell(os.path.abspath(host(path)))
113
114def relpath(path, start = None):
115    path = shell(path)
116    if start is None:
117        path = os.path.relpath(host(path))
118    else:
119        path = os.path.relpath(host(path), start)
120    return shell(path)
121
122def splitext(path):
123    path = shell(path)
124    root, ext = os.path.splitext(host(path))
125    return shell(root), ext
126
127def listdir(path):
128    path = shell(path)
129    hp = host(path)
130    if not os.path.exists(hp):
131        return []
132    return os.listdir(hp)
133
134def exists(paths):
135    def _exists(p):
136        if not is_abspath(p):
137            p = shell(join(os.getcwd(), host(p)))
138        return basename(p) in ['.'] + listdir(dirname(p))
139
140    if type(paths) == list:
141        results = []
142        for p in paths:
143            results += [_exists(shell(p))]
144        return results
145    return _exists(shell(paths))
146
147def isdir(path):
148    path = shell(path)
149    return os.path.isdir(host(path))
150
151def isfile(path):
152    path = shell(path)
153    return os.path.isfile(host(path))
154
155def isabspath(path):
156    path = shell(path)
157    return path[0] == '/'
158
159def iswritable(path):
160    path = shell(path)
161    return os.access(host(path), os.W_OK)
162
163def ispathwritable(path):
164    path = shell(path)
165    while len(path) > 1:
166        if exists(path):
167            return iswritable(path)
168        path = dirname(path)
169    return False
170
171def mkdir(path):
172    path = shell(path)
173    if exists(path):
174        if not isdir(path):
175            raise error.general('path exists and is not a directory: %s' % (path))
176    else:
177        if windows:
178            try:
179                os.makedirs(host(path))
180            except IOError as err:
181                raise error.general('cannot make directory: %s' % (path))
182            except OSError as err:
183                raise error.general('cannot make directory: %s' % (path))
184            except WindowsError as err:
185                raise error.general('cannot make directory: %s' % (path))
186        else:
187            try:
188                os.makedirs(host(path))
189            except IOError as err:
190                raise error.general('cannot make directory: %s' % (path))
191            except OSError as err:
192                raise error.general('cannot make directory: %s' % (path))
193
194def chdir(path):
195    path = shell(path)
196    os.chdir(host(path))
197
198def removeall(path):
199    #
200    # Perform the removal of the directory tree manually so we can
201    # make sure on Windows the files are correctly encoded to avoid
202    # the file name size limit. On Windows the os.walk fails once we
203    # get to the max path length on Windows.
204    #
205    def _isdir(path):
206        hpath = host_encode(path)
207        return os.path.isdir(hpath) and not os.path.islink(hpath)
208
209    def _remove_node(path):
210        hpath = host_encode(path)
211        if not os.path.islink(hpath) and not os.access(hpath, os.W_OK):
212            os.chmod(hpath, stat.S_IWUSR)
213        if _isdir(path):
214            os.rmdir(hpath)
215        else:
216            os.unlink(hpath)
217
218    def _remove(path):
219        dirs = []
220        for name in listdir(path):
221            path_ = join(path, name)
222            hname = host(path_)
223            if _isdir(path_):
224                dirs += [name]
225            else:
226                _remove_node(path_)
227        for name in dirs:
228            dir = join(path, name)
229            _remove(dir)
230            _remove_node(dir)
231
232    path = shell(path)
233    hpath = host_encode(path)
234
235    if os.path.exists(hpath):
236        _remove(path)
237        _remove_node(path)
238
239def expand(name, paths):
240    l = []
241    for p in paths:
242        l += [join(shell(p), name)]
243    return l
244
245def copy(src, dst):
246    src = shell(src)
247    dst = shell(dst)
248    hsrc = host(src)
249    hdst = host(dst)
250    try:
251        shutil.copy(hsrc, hdst)
252    except OSError as why:
253        if windows:
254            if WindowsError is not None and isinstance(why, WindowsError):
255                pass
256        else:
257            raise error.general('copying tree (1): %s -> %s: %s' % (hsrc, hdst, str(why)))
258
259def copy_tree(src, dst):
260    trace = False
261
262    hsrc = host(src)
263    hdst = host(dst)
264
265    if exists(src):
266        names = listdir(src)
267    else:
268        names = []
269
270    if trace:
271        print('path.copy_tree:')
272        print('   src: "%s"' % (src))
273        print('  hsrc: "%s"' % (hsrc))
274        print('   dst: "%s"' % (dst))
275        print('  hdst: "%s"' % (hdst))
276        print(' names: %r' % (names))
277
278    if not os.path.isdir(hdst):
279        if trace:
280            print(' mkdir: %s' % (hdst))
281        try:
282            os.makedirs(hdst)
283        except OSError as why:
284            raise error.general('copying tree: cannot create target directory %s: %s' % \
285                                (hdst, str(why)))
286
287    for name in names:
288        srcname = host(os.path.join(hsrc, name))
289        dstname = host(os.path.join(hdst, name))
290        try:
291            if os.path.islink(srcname):
292                linkto = os.readlink(srcname)
293                if exists(shell(dstname)):
294                    if os.path.islink(dstname):
295                        dstlinkto = os.readlink(dstname)
296                        if linkto != dstlinkto:
297                            log.warning('copying tree: link does not match: %s -> %s' % \
298                                            (dstname, dstlinkto))
299                            os.remove(dstname)
300                    else:
301                        log.warning('copying tree: destination is not a link: %s' % \
302                                        (dstname))
303                        os.remove(dstname)
304                else:
305                    os.symlink(linkto, dstname)
306            elif os.path.isdir(srcname):
307                copy_tree(srcname, dstname)
308            else:
309                shutil.copyfile(host(srcname), host(dstname))
310                shutil.copystat(host(srcname), host(dstname))
311        except shutil.Error as err:
312            raise error.general('copying tree (2): %s -> %s: %s' % \
313                                (hsrc, hdst, str(err)))
314        except EnvironmentError as why:
315            raise error.general('copying tree (3): %s -> %s: %s' % \
316                                (srcname, dstname, str(why)))
317    try:
318        shutil.copystat(hsrc, hdst)
319    except OSError as why:
320        if windows:
321            if WindowsError is not None and isinstance(why, WindowsError):
322                pass
323        else:
324            raise error.general('copying tree (4): %s -> %s: %s' % (hsrc, hdst, str(why)))
325
326def get_size(path, depth = -1):
327    #
328    # Get the size the directory tree manually to the required depth.
329    # This makes sure on Windows the files are correctly encoded to avoid
330    # the file name size limit. On Windows the os.walk fails once we
331    # get to the max path length on Windows.
332    #
333    def _isdir(path):
334        hpath = host_encode(path)
335        return os.path.isdir(hpath) and not os.path.islink(hpath)
336
337    def _node_size(path):
338        hpath = host_encode(path)
339        size = 0
340        if not os.path.islink(hpath):
341            size = os.path.getsize(hpath)
342        return size
343
344    def _get_size(path, depth, level = 0):
345        level += 1
346        dirs = []
347        size = 0
348        for name in listdir(path):
349            path_ = join(path, shell(name))
350            hname = host(path_)
351            if _isdir(path_):
352                dirs += [shell(name)]
353            else:
354                size += _node_size(path_)
355        if depth < 0 or level < depth:
356            for name in dirs:
357                dir = join(path, name)
358                size += _get_size(dir, depth, level)
359        return size
360
361    path = shell(path)
362    hpath = host_encode(path)
363    size = 0
364
365    if os.path.exists(hpath):
366        size = _get_size(path, depth)
367
368    return size
369
370def get_humanize_size(path, depth = -1):
371    size = get_size(path, depth)
372    for unit in ['','K','M','G','T','P','E','Z']:
373        if abs(size) < 1024.0:
374            return "%5.3f%sB" % (size, unit)
375        size /= 1024.0
376    return "%.3f%sB" % (size, 'Y')
377
378if __name__ == '__main__':
379    print(host('/a/b/c/d-e-f'))
380    print(host('//a/b//c/d-e-f'))
381    print(shell('/w/x/y/z'))
382    print(basename('/as/sd/df/fg/me.txt'))
383    print(dirname('/as/sd/df/fg/me.txt'))
384    print(join('/d', 'g', '/tyty/fgfg'))
385    print('size of . depth all: ', get_size('.'))
386    print('size of . depth   1: ', get_size('.', 1))
387    print('size of . depth   2: ', get_size('.', 2))
388    print('size of . as human : ', get_humanize_size('.'))
389    windows = True
390    print(host('/a/b/c/d-e-f'))
391    print(host('//a/b//c/d-e-f'))
392    print(shell('/w/x/y/z'))
393    print(shell('w:/x/y/z'))
394    print(basename('x:/sd/df/fg/me.txt'))
395    print(dirname('x:/sd/df/fg/me.txt'))
396    print(join('s:/d/e\\f/g', '/h', '/tyty/zxzx', '\\mm\\nn/p'))
Note: See TracBrowser for help on using the repository browser.