build(storage): verify pinned source with Go checksum database

This commit is contained in:
2026-11-15 20:42:45 +03:00
parent fba6b6334e
commit 3d5a5ca84a
4 changed files with 143 additions and 6 deletions
+5 -1
View File
@@ -13,7 +13,11 @@ keyset pagination, общий Idempotency-Key, HAProxy и две независ
Нужны Git, Docker с Compose v2 и uv 0.13.7. Python 3.14.7 загружается через uv.
Docker Desktop использует около 6 GiB памяти на Apple M1 (8 CPU, 8 GiB host).
Первичная компиляция MinIO/клиента требует сеть и время; Go parallelism
Первичная компиляция MinIO/клиента требует сеть и время. Исходный ZIP
MinIO с точным module version скачивается восемью потоками HTTP Range,
с ограниченным timeout/числом повторов; затем стандартный `go mod download`
проверяет h1 через Go checksum database перед сборкой. Проверки TLS и
checksum не отключены. Go parallelism
ограничен двумя потоками, зависимости и компилятор используют BuildKit cache.
```sh
+23
View File
@@ -0,0 +1,23 @@
{
"date": "2026-11-15T17:40:32.062021+00:00",
"source_commit": "fba6b6334e32653bf38576ec7e4dbff4e0626034",
"runtime_image_revision": "f449033e15f989aeaf35c776245cb4286e1f0ed5",
"command": "docker compose run --rm --no-deps migrate; repeat",
"migrations": [
{
"name": "001_core.sql",
"checksum": "4f91a73b92b768c48703b1adc1752b8c65f5e9da3da7c1f8b2db6d3dcb3c9efe"
},
{
"name": "002_idempotency.sql",
"checksum": "b24f77687546b4f8d68cabb7009fc2b3db3d138c3c7a1eab32357e41cc0d6b13"
},
{
"name": "003_attachments.sql",
"checksum": "6558f17559d4f96878400915d80a09a1fe5e9bcf7eeb6b0b04794d63ba03f3bd"
}
],
"fresh_postgres_volume": true,
"repeat_idempotent": true,
"result": "PASS"
}
+107
View File
@@ -0,0 +1,107 @@
// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent
// go mod download validates its h1 checksum through the Go checksum database.
package main
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
)
func main() {
const version = "v0.0.0-20251015172955-9e49d5e7a648"
const total int64 = 25342606
const chunk int64 = 512 * 1024
const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip"
directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v"
if err := os.MkdirAll(directory, 0755); err != nil {
panic(err)
}
target := filepath.Join(directory, version+".zip")
if info, err := os.Stat(target); err == nil && info.Size() == total {
fmt.Println("Source ZIP cached; Go will validate checksum")
return
}
client := &http.Client{Timeout: 90 * time.Second}
jobs := make(chan int64)
errors := make(chan error, 64)
var group sync.WaitGroup
for worker := 0; worker < 8; worker++ {
group.Add(1)
go func() {
defer group.Done()
for start := range jobs {
end := min(start+chunk, total) - 1
part := fmt.Sprintf("%s.part-%d", target, start)
if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 {
continue
}
var last error
for attempt := 0; attempt < 4; attempt++ {
req, _ := http.NewRequest("GET", source, nil)
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
response, err := client.Do(req)
if err != nil {
last = fmt.Errorf("source range request failed")
continue
}
expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total)
if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected {
response.Body.Close()
last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode)
continue
}
data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2))
response.Body.Close()
if err != nil || int64(len(data)) != end-start+1 {
last = fmt.Errorf("incomplete source range")
continue
}
if err := os.WriteFile(part, data, 0644); err != nil {
last = err
continue
}
last = nil
fmt.Printf("Source part %d verified\n", start/chunk+1)
break
}
if last != nil {
errors <- last
}
}
}()
}
for start := int64(0); start < total; start += chunk {
jobs <- start
}
close(jobs)
group.Wait()
close(errors)
for err := range errors {
panic(err)
}
out, err := os.Create(target + ".partial")
if err != nil {
panic(err)
}
for start := int64(0); start < total; start += chunk {
part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start))
if err != nil {
panic(err)
}
if _, err := io.Copy(out, part); err != nil {
panic(err)
}
part.Close()
}
if err := out.Close(); err != nil {
panic(err)
}
if err := os.Rename(target+".partial", target); err != nil {
panic(err)
}
}
+8 -5
View File
@@ -3,11 +3,12 @@ ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local
FROM toolchain AS minio-build
WORKDIR /source
RUN curl --http1.1 -fsSL --connect-timeout 10 --max-time 1800 --retry 2 \
--speed-time 60 --speed-limit 1024 \
https://codeload.github.com/minio/minio/tar.gz/9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a \
-o /tmp/source.tgz && \
tar xzf /tmp/source.tgz --strip-components=1 && rm /tmp/source.tgz
COPY infra/fetch-source.go /tmp/fetch-source.go
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go run /tmp/fetch-source.go && \
go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && \
cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/
RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
--mount=type=cache,target=/root/.cache/go-build,sharing=locked \
go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio .
@@ -21,6 +22,7 @@ RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \
go build -trimpath -ldflags='-s -w' -o /out/mc .
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server
LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE
COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint
@@ -31,6 +33,7 @@ EXPOSE 9000
HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"]
FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap
LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z
COPY --from=mc-build /out/mc /usr/local/bin/mc
COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE
COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap