mirrors/poky
1
0
Fork 0
mirror of https://git.yoctoproject.org/poky synced 2026-06-04 02:00:04 +00:00
Code Issues Projects Releases Wiki Activity

classes/lib: Update to use python3 command pipeline decoding

Browse Source 
In python3, strings are unicode by default. We need to encode/decode
from command pipelines and other places where we interface with the
real world using the correct locales. This patch updates various
call sites to use the correct encoding/decodings.

(From OE-Core rev: bb4685af1bffe17b3aa92a6d21398f38a44ea874)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
2016-05-20 11:17:05 +01:00
parent 297438e965
commit a7309d5790
20 changed files with 110 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
meta/classes/buildhistory.bbclass
+4 -8
View File
@@ -233,7 +233,7 @@ python buildhistory_emit_pkghistory() {
key = item[0]
if key.endswith('_' + pkg):
key = key[:-len(pkg)-1]
pkgdata[key] = item[1].decode('utf-8').decode('string_escape')
pkgdata[key] = item[1]
pkge = pkgdata.get('PKGE', '0')
pkgv = pkgdata['PKGV']
@@ -288,14 +288,12 @@ python buildhistory_emit_pkghistory() {
def write_recipehistory(rcpinfo, d):
import codecs
bb.debug(2, "Writing recipe history")
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
infofile = os.path.join(pkghistdir, "latest")
with codecs.open(infofile, "w", encoding='utf8') as f:
with open(infofile, "w") as f:
if rcpinfo.pe != "0":
f.write(u"PE = %s\n" % rcpinfo.pe)
f.write(u"PV = %s\n" % rcpinfo.pv)
@@ -305,8 +303,6 @@ def write_recipehistory(rcpinfo, d):
def write_pkghistory(pkginfo, d):
import codecs
bb.debug(2, "Writing package history for package %s" % pkginfo.name)
pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
@@ -316,7 +312,7 @@ def write_pkghistory(pkginfo, d):
bb.utils.mkdirhier(pkgpath)
infofile = os.path.join(pkgpath, "latest")
with codecs.open(infofile, "w", encoding='utf8') as f:
with open(infofile, "w") as f:
if pkginfo.pe != "0":
f.write(u"PE = %s\n" % pkginfo.pe)
f.write(u"PV = %s\n" % pkginfo.pv)
@@ -349,7 +345,7 @@ def write_pkghistory(pkginfo, d):
filevarpath = os.path.join(pkgpath, "latest.%s" % filevar)
val = pkginfo.filevars[filevar]
if val:
with codecs.open(filevarpath, "w", encoding='utf8') as f:
with open(filevarpath, "w") as f:
f.write(val)
else:
if os.path.exists(filevarpath):
meta/classes/chrpath.bbclass
+2
View File
@@ -10,6 +10,8 @@ def process_file_linux(cmd, fpath, rootdir, baseprefix, tmpdir, d):
if p.returncode != 0:
return
err = err.decode('utf-8')
# Handle RUNPATH as well as RPATH
err = err.replace("RUNPATH=","RPATH=")
# Throw away everything other than the rpath list
meta/classes/externalsrc.bbclass
+1 -1
View File
@@ -145,7 +145,7 @@ def srctree_hash_files(d):
env = os.environ.copy()
env['GIT_INDEX_FILE'] = tmp_index.name
subprocess.check_output(['git', 'add', '.'], cwd=s_dir, env=env)
sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env)
sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8")
with open(oe_hash_file, 'w') as fobj:
fobj.write(sha1)
ret = oe_hash_file + ':True'
meta/classes/insane.bbclass
+4 -4
View File
@@ -399,7 +399,7 @@ def package_qa_check_unsafe_references_in_binaries(path, name, d, elf, messages)
sysroot_path_usr = sysroot_path + exec_prefix
try:
ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read()
ldd_output = bb.process.Popen(["prelink-rtld", "--root", sysroot_path, path], stdout=sub.PIPE).stdout.read().decode("utf-8")
except bb.process.CmdError:
error_msg = pn + ": prelink-rtld aborted when processing %s" % path
package_qa_handle_error("unsafe-references-in-binaries", error_msg, d)
@@ -986,12 +986,12 @@ def package_qa_check_expanded_d(path,name,d,elf,messages):
return sane
def package_qa_check_encoding(keys, encode, d):
def check_encoding(key,enc):
def check_encoding(key, enc):
sane = True
value = d.getVar(key, True)
if value:
try:
s = unicode(value, enc)
s = value.encode(enc)
except UnicodeDecodeError as e:
error_msg = "%s has non %s characters" % (key,enc)
sane = False
@@ -1217,7 +1217,7 @@ Missing inherit gettext?""" % (gt, config))
try:
flag = "WARNING: unrecognized options:"
log = os.path.join(d.getVar('B', True), 'config.log')
output = subprocess.check_output(['grep', '-F', flag, log]).replace(', ', ' ')
output = subprocess.check_output(['grep', '-F', flag, log]).decode("utf-8").replace(', ', ' ')
options = set()
for line in output.splitlines():
options |= set(line.partition(flag)[2].split())
meta/classes/libc-package.bbclass
+3
View File
@@ -150,6 +150,7 @@ python package_do_split_gconvs () {
c_re = re.compile('^copy "(.*)"')
i_re = re.compile('^include "(\w+)".*')
for l in f.readlines():
l = l.decode("latin-1")
m = c_re.match(l) or i_re.match(l)
if m:
dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1)))
@@ -171,6 +172,7 @@ python package_do_split_gconvs () {
c_re = re.compile('^copy "(.*)"')
i_re = re.compile('^include "(\w+)".*')
for l in f.readlines():
l = l.decode("latin-1")
m = c_re.match(l) or i_re.match(l)
if m:
dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1)))
@@ -191,6 +193,7 @@ python package_do_split_gconvs () {
c_re = re.compile('^copy "(.*)"')
i_re = re.compile('^include "(\w+)".*')
for l in f.readlines():
l = l.decode("latin-1")
m = c_re.match(l) or i_re.match(l)
if m:
dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1))
meta/classes/package.bbclass
+3 -3
View File
@@ -63,7 +63,7 @@ def legitimize_package_name(s):
def fixutf(m):
cp = m.group(1)
if cp:
return ('\u%s' % cp).decode('unicode_escape').encode('utf-8')
return ('\\u%s' % cp).encode('latin-1').decode('unicode_escape')
# Handle unicode codepoints encoded as <U0123>, as in glibc locale files.
s = re.sub('<U([0-9A-Fa-f]{1,4})>', fixutf, s)
@@ -1259,8 +1259,8 @@ python emit_pkgdata() {
def write_if_exists(f, pkg, var):
def encode(str):
import codecs
c = codecs.getencoder("string_escape")
return c(str)[0]
c = codecs.getencoder("unicode_escape")
return c(str)[0].decode("latin1")
val = d.getVar('%s_%s' % (var, pkg), True)
if val:
meta/classes/package_deb.bbclass
+11 -15
View File
@@ -173,7 +173,7 @@ python do_package_deb () {
# Special behavior for description...
if 'DESCRIPTION' in fs:
summary = localdata.getVar('SUMMARY', True) or localdata.getVar('DESCRIPTION', True) or "."
ctrlfile.write('Description: %s\n' % unicode(summary,'utf-8'))
ctrlfile.write('Description: %s\n' % summary)
description = localdata.getVar('DESCRIPTION', True) or "."
description = textwrap.dedent(description).strip()
if '\\n' in description:
@@ -182,29 +182,25 @@ python do_package_deb () {
# We don't limit the width when manually indent, but we do
# need the textwrap.fill() to set the initial_indent and
# subsequent_indent, so set a large width
ctrlfile.write('%s\n' % unicode(textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '),'utf-8'))
ctrlfile.write('%s\n' % textwrap.fill(t, width=100000, initial_indent=' ', subsequent_indent=' '))
else:
# Auto indent
ctrlfile.write('%s\n' % unicode(textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '),'utf-8'))
ctrlfile.write('%s\n' % textwrap.fill(description.strip(), width=74, initial_indent=' ', subsequent_indent=' '))
else:
ctrlfile.write(unicode(c % tuple(pullData(fs, localdata)),'utf-8'))
ctrlfile.write(c % tuple(pullData(fs, localdata)))
except KeyError:
import sys
(type, value, traceback) = sys.exc_info()
bb.utils.unlockfile(lf)
ctrlfile.close()
raise bb.build.FuncFailed("Missing field for deb generation: %s" % value)
except UnicodeDecodeError:
bb.utils.unlockfile(lf)
ctrlfile.close()
raise bb.build.FuncFailed("Non UTF-8 characters found in one of the fields")
# more fields
custom_fields_chunk = get_package_additional_metadata("deb", localdata)
if custom_fields_chunk is not None:
ctrlfile.write(unicode(custom_fields_chunk))
ctrlfile.write(custom_fields_chunk)
ctrlfile.write("\n")
mapping_rename_hook(localdata)
@@ -255,17 +251,17 @@ python do_package_deb () {
rconflicts = bb.utils.explode_dep_versions2(localdata.getVar("RCONFLICTS", True) or "")
debian_cmp_remap(rconflicts)
if rdepends:
ctrlfile.write("Depends: %s\n" % unicode(bb.utils.join_deps(rdepends)))
ctrlfile.write("Depends: %s\n" % bb.utils.join_deps(rdepends))
if rsuggests:
ctrlfile.write("Suggests: %s\n" % unicode(bb.utils.join_deps(rsuggests)))
ctrlfile.write("Suggests: %s\n" % bb.utils.join_deps(rsuggests))
if rrecommends:
ctrlfile.write("Recommends: %s\n" % unicode(bb.utils.join_deps(rrecommends)))
ctrlfile.write("Recommends: %s\n" % bb.utils.join_deps(rrecommends))
if rprovides:
ctrlfile.write("Provides: %s\n" % unicode(bb.utils.join_deps(rprovides)))
ctrlfile.write("Provides: %s\n" % bb.utils.join_deps(rprovides))
if rreplaces:
ctrlfile.write("Replaces: %s\n" % unicode(bb.utils.join_deps(rreplaces)))
ctrlfile.write("Replaces: %s\n" % bb.utils.join_deps(rreplaces))
if rconflicts:
ctrlfile.write("Conflicts: %s\n" % unicode(bb.utils.join_deps(rconflicts)))
ctrlfile.write("Conflicts: %s\n" % bb.utils.join_deps(rconflicts))
ctrlfile.close()
for script in ["preinst", "postinst", "prerm", "postrm"]:
meta/classes/toaster.bbclass
+1
View File
@@ -33,6 +33,7 @@ python toaster_layerinfo_dumpdata() {
def _get_git_branch(layer_path):
branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
branch = branch.decode('utf-8')
branch = branch.replace('refs/heads/', '').rstrip()
return branch