mirror of
https://git.yoctoproject.org/poky
synced 2026-07-27 07:27:12 +00:00
sstate: Switch to ZStandard compressor support
This patch switches the compressor from Gzip to ZStandard for ssate cache files. Zstandard compression provides a significant improvement in decompression speed as well as improvement in compression speed and disk usage over the 'tgz' format in use. Furthermore, its configurable compression level offers a trade-off between time spent compressing sstate cache files and disk space used by those files. The reduced disk usage also contributes to saving network traffic for those sharing their sstate cache with others. Zstandard should therefore be a good choice when: * disk space is at a premium * network speed / resources are limited * the CI server can sstate packages can be created at high compression * less CPU on the build server should be used for sstate decompression (From OE-Core rev: 0710e98f40ec7394567ede1f88f7c7bae306b925) Signed-off-by: Henry Kleynhans <hkleynhans@fb.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Henry Kleynhans
committed by
Richard Purdie
Richard Purdie
parent
ca8ef6bc38
commit
3579782a44
@@ -114,7 +114,7 @@ echo_error () {
|
||||
# * Add .done/.siginfo to the remove list
|
||||
# * Add destination of symlink to the remove list
|
||||
#
|
||||
# $1: output file, others: sstate cache file (.tgz)
|
||||
# $1: output file, others: sstate cache file (.tar.zst)
|
||||
gen_rmlist (){
|
||||
local rmlist_file="$1"
|
||||
shift
|
||||
@@ -131,13 +131,13 @@ gen_rmlist (){
|
||||
dest="`readlink -e $i`"
|
||||
if [ -n "$dest" ]; then
|
||||
echo $dest >> $rmlist_file
|
||||
# Remove the .siginfo when .tgz is removed
|
||||
# Remove the .siginfo when .tar.zst is removed
|
||||
if [ -f "$dest.siginfo" ]; then
|
||||
echo $dest.siginfo >> $rmlist_file
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
# Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
|
||||
# Add the ".tar.zst.done" and ".siginfo.done" (may exist in the future)
|
||||
base_fn="${i##/*/}"
|
||||
t_fn="$base_fn.done"
|
||||
s_fn="$base_fn.siginfo.done"
|
||||
@@ -188,10 +188,10 @@ remove_duplicated () {
|
||||
total_files=`find $cache_dir -name 'sstate*' | wc -l`
|
||||
# Save all the sstate files in a file
|
||||
sstate_files_list=`mktemp` || exit 1
|
||||
find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
|
||||
find $cache_dir -iname 'sstate:*:*:*:*:*:*:*.tar.zst*' >$sstate_files_list
|
||||
|
||||
echo "Figuring out the suffixes in the sstate cache dir ... "
|
||||
sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
|
||||
sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $sstate_files_list | sort -u`"
|
||||
echo "Done"
|
||||
echo "The following suffixes have been found in the cache dir:"
|
||||
echo $sstate_suffixes
|
||||
@@ -200,10 +200,10 @@ remove_duplicated () {
|
||||
# Using this SSTATE_PKGSPEC definition it's 6th colon separated field
|
||||
# SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
|
||||
for arch in $all_archs; do
|
||||
grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
|
||||
grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list
|
||||
[ $? -eq 0 ] && ava_archs="$ava_archs $arch"
|
||||
# ${builder_arch}_$arch used by toolchain sstate
|
||||
grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
|
||||
grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tar\.zst$" $sstate_files_list
|
||||
[ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
|
||||
done
|
||||
echo "Done"
|
||||
@@ -219,13 +219,13 @@ remove_duplicated () {
|
||||
continue
|
||||
fi
|
||||
# Total number of files including .siginfo and .done files
|
||||
total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null`
|
||||
total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null`
|
||||
total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list | wc -l 2>/dev/null`
|
||||
total_archive_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst$" $sstate_files_list | wc -l 2>/dev/null`
|
||||
# Save the file list to a file, some suffix's file may not exist
|
||||
grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null
|
||||
local deleted_tgz=0
|
||||
grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tar\.zst.*" $sstate_files_list >$list_suffix 2>/dev/null
|
||||
local deleted_archives=0
|
||||
local deleted_files=0
|
||||
for ext in tgz tgz.siginfo tgz.done; do
|
||||
for ext in tar.zst tar.zst.siginfo tar.zst.done; do
|
||||
echo "Figuring out the sstate:xxx_$suffix.$ext ... "
|
||||
# Uniq BPNs
|
||||
file_names=`for arch in $ava_archs ""; do
|
||||
@@ -268,19 +268,19 @@ remove_duplicated () {
|
||||
done
|
||||
done
|
||||
done
|
||||
deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l`
|
||||
deleted_archives=`cat $rm_list.* 2>/dev/null | grep "\.tar\.zst$" | wc -l`
|
||||
deleted_files=`cat $rm_list.* 2>/dev/null | wc -l`
|
||||
[ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.*
|
||||
echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
|
||||
echo "($deleted_archives out of $total_archives_suffix .tar.zst files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)"
|
||||
let total_deleted=$total_deleted+$deleted_files
|
||||
done
|
||||
deleted_tgz=0
|
||||
deleted_archives=0
|
||||
rm_old_list=$remove_listdir/sstate-old-filenames
|
||||
find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
|
||||
[ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
|
||||
find $cache_dir -name 'sstate-*.tar.zst' >$rm_old_list
|
||||
[ -s "$rm_old_list" ] && deleted_archives=`cat $rm_old_list | grep "\.tar\.zst$" | wc -l`
|
||||
[ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
|
||||
[ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
|
||||
echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
|
||||
echo "($deleted_archives or .tar.zst files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
|
||||
let total_deleted=$total_deleted+$deleted_files
|
||||
|
||||
rm -f $list_suffix
|
||||
@@ -289,7 +289,7 @@ remove_duplicated () {
|
||||
read_confirm
|
||||
if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
|
||||
for list in `ls $remove_listdir/`; do
|
||||
echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
|
||||
echo "Removing $list.tar.zst archive (`cat $remove_listdir/$list | wc -w` files) ... "
|
||||
# Remove them one by one to avoid the argument list too long error
|
||||
for i in `cat $remove_listdir/$list`; do
|
||||
rm -f $verbose $i
|
||||
@@ -322,7 +322,7 @@ rm_by_stamps (){
|
||||
find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list
|
||||
|
||||
echo "Figuring out the suffixes in the sstate cache dir ... "
|
||||
local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`"
|
||||
local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tar\.zst.*%\1%g' $cache_list | sort -u`"
|
||||
echo "Done"
|
||||
echo "The following suffixes have been found in the cache dir:"
|
||||
echo $sstate_suffixes
|
||||
|
||||
Reference in New Issue
Block a user