From 78255baec1960b0db8c24384be878cfc8562e998 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Sun, 12 Apr 2026 15:44:42 +0800 Subject: [PATCH] Codex/multi node billing ingestion (#4) * feat: add multi-source billing ingestion * Move release traceability workflow logic into scripts * feat: release traceability --------- Co-authored-by: Haitao Pan --- .github/workflows/release-traceability.yml | 53 +++++++++++++++++++ docs/api.md | 22 ++++++++ docs/release-traceability.md | 27 ++++++++++ internal/config/config.go | 35 ++++++++++++ internal/config/config_test.go | 20 +++++++ internal/httpapi/handler.go | 5 ++ internal/model/types.go | 7 +++ internal/service/service.go | 9 ++++ internal/service/service_test.go | 12 +++++ scripts/github-actions/build-service-image.sh | 7 +++ .../github-actions/deploy-billing-service.sh | 5 ++ .../github-actions/push-image-placeholder.sh | 4 ++ .../resolve-service-image-ref.sh | 10 ++++ .../validate-release-traceability.sh | 12 +++++ 14 files changed, 228 insertions(+) create mode 100644 .github/workflows/release-traceability.yml create mode 100644 docs/release-traceability.md create mode 100644 scripts/github-actions/build-service-image.sh create mode 100644 scripts/github-actions/deploy-billing-service.sh create mode 100644 scripts/github-actions/push-image-placeholder.sh create mode 100644 scripts/github-actions/resolve-service-image-ref.sh create mode 100644 scripts/github-actions/validate-release-traceability.sh diff --git a/.github/workflows/release-traceability.yml b/.github/workflows/release-traceability.yml new file mode 100644 index 0000000..cdc7343 --- /dev/null +++ b/.github/workflows/release-traceability.yml @@ -0,0 +1,53 @@ +name: release-traceability + +on: + push: + branches: + - main + workflow_dispatch: {} + +jobs: + build: + runs-on: ubuntu-latest + outputs: + service_image_ref: ${{ steps.meta.outputs.service_image_ref }} + service_image_tag: ${{ steps.meta.outputs.service_image_tag }} + service_image_commit: ${{ steps.meta.outputs.service_image_commit }} + steps: + - uses: actions/checkout@v4 + + - name: Derive image identity + id: meta + run: bash ./scripts/github-actions/resolve-service-image-ref.sh + + - name: Build image + env: + SERVICE_IMAGE_REF: ${{ steps.meta.outputs.service_image_ref }} + SERVICE_IMAGE_LATEST_REF: ghcr.io/${{ github.repository }}:latest + run: bash ./scripts/github-actions/build-service-image.sh + + - name: Push image + run: bash ./scripts/github-actions/push-image-placeholder.sh + + deploy: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Deploy via playbook + env: + IMAGE_REF: ${{ needs.build.outputs.service_image_ref }} + BILLING_SERVICE_IMAGE_REF: ${{ needs.build.outputs.service_image_ref }} + BILLING_SERVICE_IMAGE_TAG: ${{ needs.build.outputs.service_image_tag }} + BILLING_SERVICE_IMAGE_COMMIT: ${{ needs.build.outputs.service_image_commit }} + run: bash ./scripts/github-actions/deploy-billing-service.sh + + validate: + runs-on: ubuntu-latest + needs: deploy + steps: + - name: Validate runtime traceability + env: + SERVICE_IMAGE_REF: ${{ needs.build.outputs.service_image_ref }} + run: bash ./scripts/github-actions/validate-release-traceability.sh diff --git a/docs/api.md b/docs/api.md index c847fc8..3c4f254 100644 --- a/docs/api.md +++ b/docs/api.md @@ -5,6 +5,21 @@ upstream and downstream interfaces it depends on. ## Service endpoints +### `GET /api/ping` + +Returns the runtime image identity exposed by the running container. + +Example response: + +```json +{ + "image": "registry.example.com/billing-service:sha-0123456789abcdef0123456789abcdef01234567", + "tag": "sha-0123456789abcdef0123456789abcdef01234567", + "commit": "0123456789abcdef0123456789abcdef01234567", + "version": "0123456789abcdef0123456789abcdef01234567" +} +``` + ### `GET /healthz` Returns service health derived from the most recent collect-and-rate execution. @@ -130,6 +145,7 @@ Read-path rules: Runtime environment variables used by the current implementation: +- `IMAGE` - `EXPORTER_BASE_URL` - `DATABASE_URL` - `LISTEN_ADDR` @@ -140,6 +156,12 @@ Runtime environment variables used by the current implementation: - `INITIAL_INCLUDED_QUOTA_BYTES` - `INITIAL_BALANCE` +`IMAGE` rule: + +- it must contain the full image reference used to start the container +- `/api/ping` derives `image`, `tag`, `commit`, and `version` from this value +- when `IMAGE` is missing or malformed, runtime metadata fields should remain empty rather than fabricated + `DATABASE_URL` rule: - it must point to the same `account` database that `accounts.svc.plus` uses diff --git a/docs/release-traceability.md b/docs/release-traceability.md new file mode 100644 index 0000000..faeddff --- /dev/null +++ b/docs/release-traceability.md @@ -0,0 +1,27 @@ +# Release Traceability Contract + +This repository now treats `IMAGE` as the single runtime source of truth for +release identity. + +## Runtime contract + +- `IMAGE` must contain the full image reference used to start the container. +- `/api/ping` returns `image`, `tag`, `commit`, and `version`. +- `tag`, `commit`, and `version` are derived from `IMAGE`. +- If `IMAGE` is missing or malformed, the derived fields stay empty instead of + being fabricated. + +## Pipeline contract + +- Build must produce `service_image_ref` only from the full `GITHUB_SHA`. +- Deploy must consume `service_image_ref` and pass it through as the runtime + image identity. +- Validate must derive `tag` and `commit` from `service_image_ref` and compare + them against `/api/ping`. + +## External playbook alignment + +The external `playbooks/deploy_billing_service.yml` playbook should accept +`IMAGE_REF` (or an equivalent full image reference variable), derive any +repo/tag helpers from it, and inject `IMAGE=` into the running +container environment. diff --git a/internal/config/config.go b/internal/config/config.go index 181226e..8187d78 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strconv" "strings" "time" @@ -19,6 +20,10 @@ type ExporterSource struct { } type Config struct { + ImageRef string + ImageTag string + ImageCommit string + ImageVersion string ExporterBaseURL string ExporterSources []ExporterSource InternalServiceToken string @@ -42,7 +47,13 @@ type rawExporterSource struct { } func Load() (Config, error) { + imageRef := strings.TrimSpace(os.Getenv("IMAGE")) + imageTag, imageCommit, imageVersion := parseImageRef(imageRef) cfg := Config{ + ImageRef: imageRef, + ImageTag: imageTag, + ImageCommit: imageCommit, + ImageVersion: imageVersion, ExporterBaseURL: strings.TrimRight(strings.TrimSpace(os.Getenv("EXPORTER_BASE_URL")), "/"), InternalServiceToken: strings.TrimSpace(os.Getenv("INTERNAL_SERVICE_TOKEN")), DatabaseURL: strings.TrimSpace(os.Getenv("DATABASE_URL")), @@ -87,6 +98,30 @@ func Load() (Config, error) { return cfg, nil } +var fullSHARegexp = regexp.MustCompile(`^[a-f0-9]{40}$`) + +func parseImageRef(imageRef string) (tag, commit, version string) { + trimmed := strings.TrimSpace(imageRef) + if trimmed == "" { + return "", "", "" + } + colon := strings.LastIndex(trimmed, ":") + if colon < 0 || colon == len(trimmed)-1 { + return "", "", "" + } + tag = trimmed[colon+1:] + switch { + case strings.HasPrefix(tag, "sha-") && fullSHARegexp.MatchString(strings.TrimPrefix(tag, "sha-")): + commit = strings.TrimPrefix(tag, "sha-") + case fullSHARegexp.MatchString(tag): + commit = tag + } + if commit != "" { + version = commit + } + return tag, commit, version +} + func loadExporterSources(legacyBaseURL, rawJSON string) ([]ExporterSource, error) { if rawJSON == "" { if legacyBaseURL == "" { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4bb262b..15aad3e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -30,3 +30,23 @@ func TestLoadExporterSourcesFallsBackToLegacyBaseURL(t *testing.T) { t.Fatalf("unexpected source %#v", sources[0]) } } + +func TestParseImageRefWithFullShaTag(t *testing.T) { + tag, commit, version := parseImageRef("registry.example.com/billing-service:sha-0123456789abcdef0123456789abcdef01234567") + if tag != "sha-0123456789abcdef0123456789abcdef01234567" { + t.Fatalf("unexpected tag %q", tag) + } + if commit != "0123456789abcdef0123456789abcdef01234567" { + t.Fatalf("unexpected commit %q", commit) + } + if version != commit { + t.Fatalf("expected version to equal commit, got %q vs %q", version, commit) + } +} + +func TestParseImageRefRejectsIncompleteSha(t *testing.T) { + tag, commit, version := parseImageRef("registry.example.com/billing-service:sha-1234") + if tag != "sha-1234" || commit != "" || version != "" { + t.Fatalf("expected partial parse failure, got tag=%q commit=%q version=%q", tag, commit, version) + } +} diff --git a/internal/httpapi/handler.go b/internal/httpapi/handler.go index 4ebda10..7b43cc7 100644 --- a/internal/httpapi/handler.go +++ b/internal/httpapi/handler.go @@ -18,6 +18,7 @@ func New(svc *service.Service) *Handler { func (h *Handler) Routes() http.Handler { mux := http.NewServeMux() + mux.HandleFunc("/api/ping", h.ping) mux.HandleFunc("/healthz", h.healthz) mux.HandleFunc("/v1/status", h.status) mux.HandleFunc("/v1/jobs/collect-and-rate", h.collectAndRate) @@ -25,6 +26,10 @@ func (h *Handler) Routes() http.Handler { return mux } +func (h *Handler) ping(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, h.service.Ping()) +} + func (h *Handler) healthz(w http.ResponseWriter, r *http.Request) { ok, message := h.service.Health() status := http.StatusOK diff --git a/internal/model/types.go b/internal/model/types.go index 2706fbf..4322845 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -111,3 +111,10 @@ type JobResult struct { Error string `json:"error,omitempty"` SourceStatuses []SourceStatus `json:"source_statuses,omitempty"` } + +type PingInfo struct { + Image string `json:"image"` + Tag string `json:"tag,omitempty"` + Commit string `json:"commit,omitempty"` + Version string `json:"version,omitempty"` +} diff --git a/internal/service/service.go b/internal/service/service.go index da74c4b..d1d74b0 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -118,6 +118,15 @@ func (s *Service) Health() (bool, string) { return s.lastOK, s.lastError } +func (s *Service) Ping() model.PingInfo { + return model.PingInfo{ + Image: s.cfg.ImageRef, + Tag: s.cfg.ImageTag, + Commit: s.cfg.ImageCommit, + Version: s.cfg.ImageVersion, + } +} + func (s *Service) collectSource(ctx context.Context, source config.ExporterSource, result *model.JobResult) (model.SourceStatus, error) { state, err := s.repo.GetSourceSyncState(ctx, source.SourceID) if err != nil { diff --git a/internal/service/service_test.go b/internal/service/service_test.go index bd11d23..90fd7a4 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -144,6 +144,10 @@ var _ repository.Repository = (*memoryRepo)(nil) func baseConfig() config.Config { return config.Config{ + ImageRef: "registry.example.com/billing-service:sha-0123456789abcdef0123456789abcdef01234567", + ImageTag: "sha-0123456789abcdef0123456789abcdef01234567", + ImageCommit: "0123456789abcdef0123456789abcdef01234567", + ImageVersion: "0123456789abcdef0123456789abcdef01234567", ExporterSources: []config.ExporterSource{{ SourceID: "default", BaseURL: "https://jp-xhttp-contabo.svc.plus", @@ -161,6 +165,14 @@ func baseConfig() config.Config { } } +func TestPingReflectsImageRef(t *testing.T) { + svc := New(baseConfig(), &fakeWindowSource{}, newMemoryRepo()) + ping := svc.Ping() + if ping.Image != baseConfig().ImageRef || ping.Tag != baseConfig().ImageTag || ping.Commit != baseConfig().ImageCommit || ping.Version != baseConfig().ImageVersion { + t.Fatalf("unexpected ping %#v", ping) + } +} + func singleSnapshotPage(snapshot model.Snapshot) model.SnapshotWindowPage { return model.SnapshotWindowPage{ NodeID: snapshot.NodeID, diff --git a/scripts/github-actions/build-service-image.sh b/scripts/github-actions/build-service-image.sh new file mode 100644 index 0000000..82abc0f --- /dev/null +++ b/scripts/github-actions/build-service-image.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +docker build \ + --tag "${SERVICE_IMAGE_REF:?SERVICE_IMAGE_REF is required}" \ + --tag "${SERVICE_IMAGE_LATEST_REF:?SERVICE_IMAGE_LATEST_REF is required}" \ + . diff --git a/scripts/github-actions/deploy-billing-service.sh b/scripts/github-actions/deploy-billing-service.sh new file mode 100644 index 0000000..ae5c976 --- /dev/null +++ b/scripts/github-actions/deploy-billing-service.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +test -n "${IMAGE_REF:?IMAGE_REF is required}" +ansible-playbook -i inventory playbooks/deploy_billing_service.yml diff --git a/scripts/github-actions/push-image-placeholder.sh b/scripts/github-actions/push-image-placeholder.sh new file mode 100644 index 0000000..febb0ab --- /dev/null +++ b/scripts/github-actions/push-image-placeholder.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "Push step is intentionally left as an integration point for the target registry." diff --git a/scripts/github-actions/resolve-service-image-ref.sh b/scripts/github-actions/resolve-service-image-ref.sh new file mode 100644 index 0000000..9fe9c13 --- /dev/null +++ b/scripts/github-actions/resolve-service-image-ref.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +full_sha="${GITHUB_SHA:?GITHUB_SHA is required}" +tag="sha-${full_sha}" +image_ref="ghcr.io/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}:${tag}" + +printf 'service_image_ref=%s\n' "${image_ref}" >> "${GITHUB_OUTPUT}" +printf 'service_image_tag=%s\n' "${tag}" >> "${GITHUB_OUTPUT}" +printf 'service_image_commit=%s\n' "${full_sha}" >> "${GITHUB_OUTPUT}" diff --git a/scripts/github-actions/validate-release-traceability.sh b/scripts/github-actions/validate-release-traceability.sh new file mode 100644 index 0000000..b1bb474 --- /dev/null +++ b/scripts/github-actions/validate-release-traceability.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +service_image_ref="${SERVICE_IMAGE_REF:?SERVICE_IMAGE_REF is required}" +tag="${service_image_ref##*:}" +commit="${tag#sha-}" + +curl -fsS "https://billing-service.example.com/api/ping" | jq -e \ + --arg image "${service_image_ref}" \ + --arg tag "${tag}" \ + --arg commit "${commit}" \ + '.image == $image and .tag == $tag and .commit == $commit'