Forward the lab 4 foundation and migrate the lab 5 pipeline to SQLAlchemy

This commit is contained in:
Artyom
2026-11-06 09:12:25 +03:00
parent ae9f60fe1f
commit 1217da5f5e
134 changed files with 8243 additions and 1409 deletions
+9 -2
View File
@@ -77,7 +77,9 @@ def main():
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
if any(out.iterdir()):
p.error("output directory must be empty; choose a new --out")
commit = seed.run(["git", "rev-parse", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
@@ -102,7 +104,12 @@ def main():
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit,
"upload_concurrency": slots,
**seed.over_http(run),
}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")