1
0
mirror of https://git.yoctoproject.org/poky synced 2026-07-16 15:57:04 +00:00

bitbake: hashserv: Add gc-mark-stream command for batch hash marking

Implements the `gc-mark-stream` command to allow for marking equivalence entries
in batch, by  making use of stream mode communication to the server.

The aim of this is to improve efficiency by reducing the impact of latency when
marking a high volume of hash entries.

Example usage of the new `gc-mark-stream` command:

```
$ cat << HASHES | \
./bin/bitbake-hashclient --address "ws://localhost:8688/ws" gc-mark-stream "alive"
unihash f37918cc02eb5a520b1aff86faacbc0a38124646
unihash af36b199320e611fbb16f1f277d3ee1d619ca58b
taskhash a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0 method oe.sstatesig.OEOuthashBasic
HASHES
```

(Bitbake rev: c84715f28cd36666ea07a179d91b8c32ea0df8e7)

Signed-off-by: Alexander Marques <c137.marques@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandre Marques
2025-03-12 11:22:04 +00:00
committed by Richard Purdie
parent cb9bff9eac
commit a4f8336982
4 changed files with 124 additions and 0 deletions
+29
View File
@@ -10,6 +10,7 @@ import math
import time
import os
import base64
import json
import hashlib
from . import create_async_client
import bb.asyncrpc
@@ -256,6 +257,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
"backfill-wait": self.handle_backfill_wait,
"remove": self.handle_remove,
"gc-mark": self.handle_gc_mark,
"gc-mark-stream": self.handle_gc_mark_stream,
"gc-sweep": self.handle_gc_sweep,
"gc-status": self.handle_gc_status,
"clean-unused": self.handle_clean_unused,
@@ -583,6 +585,33 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
return {"count": await self.db.gc_mark(mark, condition)}
@permissions(DB_ADMIN_PERM)
async def handle_gc_mark_stream(self, request):
async def handler(line):
try:
decoded_line = json.loads(line)
except json.JSONDecodeError as exc:
raise bb.asyncrpc.InvokeError(
"Could not decode JSONL input '%s'" % line
) from exc
try:
mark = decoded_line["mark"]
condition = decoded_line["where"]
if not isinstance(mark, str):
raise TypeError("Bad mark type %s" % type(mark))
if not isinstance(condition, dict):
raise TypeError("Bad condition type %s" % type(condition))
except KeyError as exc:
raise bb.asyncrpc.InvokeError(
"Input line is missing key '%s' " % exc
) from exc
return json.dumps({"count": await self.db.gc_mark(mark, condition)})
return await self._stream_handler(handler)
@permissions(DB_ADMIN_PERM)
async def handle_gc_sweep(self, request):
mark = request["mark"]