From 88f20165db9bed11e3d1d815c4a3a3056c2ef30d Mon Sep 17 00:00:00 2001 From: k3nny Date: Thu, 11 Jun 2026 00:27:28 +0200 Subject: [PATCH] feat(cli)!: subcommand CLI, graph tree mode, local include resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGES: - `glint ` removed; use `glint check ` - `--graph ` removed; use `glint graph [mode]` - `--graph-out` renamed to `--out` on `glint graph` feat(cli): ruff-style subcommands — `glint check` and `glint graph [mode]` feat(graph): `glint graph tree` — terminal job tree with context annotations feat(graph): context flags (--branch/--tag/--source/--var) on `glint graph` feat(resolver): recursive local include resolution from disk fix(resolver): extends unknown base emits warning instead of fatal error fix(model): script/before_script/after_script accept block scalar string form test(linter): Samba project CI fixtures as integration tests chore(build): fix .gitignore to not exclude cmd/glint/ directory docs: update CHANGELOG, README, ROADMAP for v0.2.0 Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 4 +- CHANGELOG.md | 29 +- README.md | 82 ++- ROADMAP.md | 66 +- Taskfile.yml | 34 +- cmd/glint/main.go | 246 ++++++- internal/graph/includes.go | 279 +++++-- internal/graph/tree.go | 113 +++ internal/linter/keywords.go | 2 +- internal/linter/linter.go | 14 +- internal/linter/samba_test.go | 89 +++ internal/model/pipeline.go | 8 +- internal/resolver/extends.go | 28 +- internal/resolver/includes.go | 147 ++-- .../.gitlab-ci-coverage-runners.yml | 4 + samba-testdata/.gitlab-ci-coverage.yml | 12 + samba-testdata/.gitlab-ci-default-runners.yml | 30 + samba-testdata/.gitlab-ci-default.yml | 10 + samba-testdata/.gitlab-ci-main.yml | 688 ++++++++++++++++++ samba-testdata/.gitlab-ci-private.yml | 5 + samba-testdata/.gitlab-ci.yml | 2 + samba-testdata/.gitleaks.toml | 16 + samba-testdata/bootstrap/.gitlab-ci.yml | 122 ++++ 23 files changed, 1772 insertions(+), 258 deletions(-) create mode 100644 internal/graph/tree.go create mode 100644 internal/linter/samba_test.go create mode 100644 samba-testdata/.gitlab-ci-coverage-runners.yml create mode 100644 samba-testdata/.gitlab-ci-coverage.yml create mode 100644 samba-testdata/.gitlab-ci-default-runners.yml create mode 100644 samba-testdata/.gitlab-ci-default.yml create mode 100644 samba-testdata/.gitlab-ci-main.yml create mode 100644 samba-testdata/.gitlab-ci-private.yml create mode 100644 samba-testdata/.gitlab-ci.yml create mode 100644 samba-testdata/.gitleaks.toml create mode 100644 samba-testdata/bootstrap/.gitlab-ci.yml diff --git a/.gitignore b/.gitignore index 1b3b1ed..2e394c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Compiled binaries -glint -glint.exe +/glint +/glint.exe dist/ bin/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c9cb6d5..336ca49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,37 @@ This project uses [Semantic Versioning](https://semver.org). ## [Unreleased] +## [0.2.0] - 2026-06-11 + ### Added -- **Cross-platform release builds** — two new Taskfile tasks for producing tagged release binaries: +- **Subcommand CLI** — reworked interface inspired by [ruff](https://docs.astral.sh/ruff/): + - `glint check ` — lint a pipeline (replaces bare `glint `) + - `glint graph [mode] ` — visualise the pipeline (replaces `--graph` flag) + - Graph modes: no-arg (tree + includes), `tree`, `includes`, `pipeline`, `all` + - Per-command `--help` with ruff-style layout: `Arguments:`, `Options:` (flag declaration on its own line, description below), `[env: ...]` / `[default: ...]` / `[possible values: ...]` metadata, `Examples:` section + +- **`glint graph tree`** — jobs displayed as a terminal directory tree grouped by stage (like the `tree` command); job-type annotations (`[manual]`, `[delayed]`, `[trigger]`) when no context is set; evaluated-state annotations (`[skipped]`, `[manual]`) when a context is provided via `--branch` / `--tag` / `--source` + +- **Context flags on `glint graph`** — `--branch`, `--tag`, `--source`, `--var` are now available on `glint graph` as well as `glint check` + +- **Local include resolution** — `include: local:` entries are now read from disk, recursively resolved, and merged into the pipeline before linting; enables cross-file `extends:` and `needs:` validation for multi-file pipelines + +- **Cross-platform release builds** — two Taskfile tasks for tagged release binaries: - `task build-windows` — cross-compiles for Windows x64; output: `glint-.exe` - `task build-linux` — cross-compiles for Linux x64; output: `glint--linux-amd64` - - Both tasks enforce that the current commit carries an exact git tag (`git describe --tags --exact-match`); they abort with a clear error otherwise + - Both tasks require an exact git tag on the current commit + +### Fixed + +- **`extends:` unknown base no longer fatal** — when a base job referenced by `extends:` does not exist, glint now emits a resolver warning and skips extends resolution for that job rather than aborting with exit code 2; linting continues on the job's own fields +- **`script: |` (block scalar) support** — jobs using a multiline block scalar for `script:`, `before_script:`, or `after_script:` are now parsed correctly; previously caused false-positive "missing script" errors + +### Changed + +- **`glint ` removed** — use `glint check ` +- **`--graph ` removed** — replaced by `glint graph [mode]` +- **`--graph-out` renamed to `--out`** — now a flag on `glint graph` (`glint graph pipeline --out `) ## [0.1.0] - 2026-06-07 diff --git a/README.md b/README.md index 8d63b98..c7e877a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # glint [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) -[![Release](https://img.shields.io/badge/release-v0.1.0-blue.svg)](CHANGELOG.md) +[![Release](https://img.shields.io/badge/release-v0.2.0-blue.svg)](CHANGELOG.md) > **Disclaimer:** This tool was built through iterative AI-assisted development with [Claude](https://claude.ai). It is experimental, incomplete, and not intended for production use. Coverage of GitLab CI keywords is best-effort and may lag behind GitLab's evolving spec. Use it at your own discretion — no correctness guarantees are made. Contributions and bug reports are welcome. @@ -18,8 +18,11 @@ A local tool to validate and lint `.gitlab-ci.yml` pipelines without needing a G - **Remote project includes** — fetches `include: project:` templates from the GitLab API so extends/needs can be validated against the full merged pipeline - **CI/CD catalog components** — resolves `include: component:` references from the GitLab CI/CD Catalog; public components work without a token - **Deprecation warnings** — flags `only`/`except` usage in favour of `rules` -- **Graph output** — emits Mermaid diagrams for the include dependency tree and the pipeline jobs layout (DAG or classic stage ordering) -- **Context simulation** — pass `--branch`, `--tag`, or `--source` to see which jobs would be active, manual, or skipped for a specific branch push, tag, or pipeline event; evaluates `rules:if:` expressions and `only`/`except` filters +- **Local include resolution** — `include: local:` entries are read from disk and recursively merged before linting, so multi-file pipelines are fully validated +- **Graph output** — `glint graph` prints a job tree (stages → jobs) to the terminal; `glint graph includes` emits a Mermaid include dependency diagram; `glint graph pipeline` renders a GitLab CI-style PNG/SVG +- **Context simulation** — pass `--branch`, `--tag`, or `--source` to `glint check` or `glint graph` to see which jobs would be active, manual, or skipped for a specific pipeline event; evaluates `rules:if:` expressions and `only`/`except` filters + +See [ROADMAP.md](ROADMAP.md) for planned improvements. ## Requirements @@ -42,8 +45,20 @@ task build ## Usage +``` +glint [OPTIONS] + +Commands: + check Lint a pipeline file — exits 0 (clean) or 1 (errors found) + graph Visualise the pipeline as a job tree or Mermaid graph +``` + +Run `glint --help` for command-specific options and examples. + +### `glint check` + ```bash -glint [options] +glint check .gitlab-ci.yml ``` Exits `0` when no errors are found, `1` when at least one error is reported. @@ -55,16 +70,16 @@ Provide a token so `glint` can fetch them: ```bash # personal access token (read_api scope) -GITLAB_TOKEN=glpat-xxxx glint .gitlab-ci.yml +GITLAB_TOKEN=glpat-xxxx glint check .gitlab-ci.yml # CI/CD job token (when running inside a pipeline) -CI_JOB_TOKEN=$CI_JOB_TOKEN glint .gitlab-ci.yml +CI_JOB_TOKEN=$CI_JOB_TOKEN glint check .gitlab-ci.yml # self-hosted GitLab -GITLAB_TOKEN=glpat-xxxx GITLAB_URL=https://gitlab.example.com glint .gitlab-ci.yml +GITLAB_TOKEN=glpat-xxxx GITLAB_URL=https://gitlab.example.com glint check .gitlab-ci.yml # or via flags -glint --token glpat-xxxx --gitlab-url https://gitlab.example.com .gitlab-ci.yml +glint check --token glpat-xxxx --gitlab-url https://gitlab.example.com .gitlab-ci.yml ``` **Project includes** require a token; without one they are skipped with a @@ -104,26 +119,36 @@ GitLab at runtime. Jobs in fetched components may use `$[[ inputs.xxx ]]` placeholders in fields like `stage`; `glint` skips those fields rather than producing false positive errors. -### Graph output +### `glint graph` -Pass `--graph` to visualise the pipeline instead of running lint rules. +Visualise the pipeline. Without a mode word, prints a job tree and the include +dependency graph separated by `---`. ```bash -# Include dependency graph (which files include which) → Mermaid to stdout -glint --graph includes .gitlab-ci.yml > includes.mmd +# Default: job tree + include dependency graph +glint graph .gitlab-ci.yml -# GitLab-like pipeline layout → PNG (or SVG fallback) written to --graph-out dir -glint --graph pipeline .gitlab-ci.yml +# Job tree only (stages → jobs, like the tree command) +glint graph tree .gitlab-ci.yml + +# Include dependency graph → Mermaid flowchart to stdout +glint graph includes .gitlab-ci.yml > includes.mmd + +# GitLab-like pipeline layout → PNG (or SVG fallback) written to --out dir +glint graph pipeline .gitlab-ci.yml # prints the output file path, e.g.: glint-out/pipeline-20260607-143022.png -# Both at once: Mermaid to stdout + pipeline file path to stderr -glint --graph all .gitlab-ci.yml > includes.mmd +# Mermaid to stdout + pipeline file path to stderr +glint graph all .gitlab-ci.yml > includes.mmd -# Custom output directory -glint --graph pipeline --graph-out /tmp/graphs .gitlab-ci.yml +# Custom output directory (pipeline mode) +glint graph pipeline --out /tmp/graphs .gitlab-ci.yml ``` -**Include graph** (`--graph includes`) — [Mermaid](https://mermaid.js.org) flowchart written to stdout. +**Job tree** (`graph tree`) — stages as branches, jobs as leaves. Jobs with +`when: manual`, `when: delayed`, or `trigger:` are annotated in brackets. + +**Include graph** (`graph includes`) — [Mermaid](https://mermaid.js.org) flowchart written to stdout. Pipe to a `.mmd` file or paste into [mermaid.live](https://mermaid.live). One node per include entry, colour-coded by type: - Orange (bold): the main pipeline file @@ -133,8 +158,8 @@ One node per include entry, colour-coded by type: - Grey: `remote:` URL includes - Light orange: GitLab-provided `template:` includes -**Pipeline graph** (`--graph pipeline`) — GitLab CI-style SVG rendered to a timestamped file -in the `--graph-out` directory (default: `glint-out/`). Converted to PNG automatically +**Pipeline graph** (`graph pipeline`) — GitLab CI-style SVG rendered to a timestamped file +in the `--out` directory (default: `glint-out/`). Converted to PNG automatically when `rsvg-convert`, `inkscape`, or `magick` is available; falls back to SVG otherwise. Jobs are colour-coded by type: - Blue (`#1f75cb`): regular jobs @@ -147,21 +172,22 @@ Classic mode draws L-shaped or straight connectors between stage columns otherwi ### Context simulation -Pass `--branch`, `--tag`, or `--source` to see which jobs would run for a given -pipeline event. The pipeline is still fully linted; context output is printed first. +Pass `--branch`, `--tag`, or `--source` to `glint check` to see which jobs +would run for a given pipeline event. The pipeline is still fully linted; +context output is printed first. ```bash # What runs on a push to develop? -glint --branch develop .gitlab-ci.yml +glint check --branch develop .gitlab-ci.yml # What runs when a v1.2.0 tag is pushed? -glint --tag v1.2.0 .gitlab-ci.yml +glint check --tag v1.2.0 .gitlab-ci.yml # Merge request pipeline -glint --source merge_request_event .gitlab-ci.yml +glint check --source merge_request_event .gitlab-ci.yml # Arbitrary variable overrides (repeatable) -glint --branch main --var DEPLOY_ENV=production .gitlab-ci.yml +glint check --branch main --var DEPLOY_ENV=production .gitlab-ci.yml ``` **Evaluated:** @@ -267,7 +293,7 @@ OK: .gitlab-ci.yml — no issues found (5 jobs, 3 stages) | ERROR | Circular dependency detected in `needs:` graph | | ERROR | `dependencies:` references a job that does not exist | | ERROR | `dependencies:` references a job in the same or a later stage | -| ERROR | `extends:` references an unknown job | +| WARNING | `extends:` references an unknown base job (resolver warning; extends chain skipped for that job) | | ERROR | Cycle detected in `extends:` graph | ### Hidden jobs (templates) diff --git a/ROADMAP.md b/ROADMAP.md index 6e3735a..30e6a02 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,58 +4,26 @@ This document tracks planned improvements to `glint`. Items are grouped by theme --- -## Context-aware validation +## Context-aware validation — ✓ single-context shipped in v0.2.0 -Pipelines in Git Flow, Trunk-Based Development, or any branching strategy are rarely uniform: jobs activate or skip based on `$CI_COMMIT_BRANCH`, `$CI_COMMIT_TAG`, `$CI_PIPELINE_SOURCE`, and similar runtime variables. Today `glint` validates structure but cannot tell which jobs are actually reachable for a given context. - -The plan is to make the execution context injectable so the linter can evaluate `rules:if:` / `only` / `except` conditions and report per-context reachability. - -**CLI surface** +Single-context simulation is fully implemented. Pass `--branch`, `--tag`, `--source`, or `--var` to either `glint check` or `glint graph`; jobs are evaluated and shown as active / manual / skipped. ```bash -# Push to develop -glint --branch develop .gitlab-ci.yml - -# Tag push (v1.2.0) — sets CI_COMMIT_TAG and clears CI_COMMIT_BRANCH -glint --tag v1.2.0 .gitlab-ci.yml - -# Merge request pipeline -glint --source merge_request_event \ - --var CI_MERGE_REQUEST_TARGET_BRANCH_NAME=main \ - .gitlab-ci.yml - -# Explicit variable overrides for anything not covered by the shortcuts -glint --var CI_COMMIT_BRANCH=feat/my-feature \ - --var CI_ENVIRONMENT_NAME=staging \ - .gitlab-ci.yml - -# Simulate multiple contexts in one run (print per-context job tables) -glint --context branch=main \ - --context branch=develop \ - --context tag=v1.0.0 \ - .gitlab-ci.yml +# shipped: single-context simulation +glint check --branch develop .gitlab-ci.yml +glint check --tag v1.2.0 .gitlab-ci.yml +glint check --source merge_request_event --var CI_MERGE_REQUEST_TARGET_BRANCH_NAME=main .gitlab-ci.yml +glint graph tree --branch main .gitlab-ci.yml # tree annotated with [skipped] / [manual] ``` -**What context injection enables** +**Remaining work** -- Each job is resolved to **active** / **manual** / **skipped** for the given context -- Warn when the entire pipeline would produce zero runnable jobs (common mistake when a `workflow:rules:` block is too restrictive) -- Lint only the active job subset — skip `needs:` / `dependencies:` cross-checks for jobs that never co-execute in that context -- `--context` multi-simulation: print a table showing which jobs activate per context, making it easy to audit Git Flow rules across branches and tags at once - -**Expression evaluator scope** - -GitLab's `rules:if:` expression language will be implemented incrementally: - -| Priority | Operators / features | -|----------|----------------------| -| 1 (MVP) | `==`, `!=`, `null` check, `&&`, `\|\|`, `!`, parentheses | -| 2 | Regex match `=~` / `!~` with `/pattern/` literals | -| 3 | `$CI_COMMIT_BRANCH =~ /^feat\//`, anchored patterns | -| 4 | `only: branches / tags / merge_requests` shorthand mapping | -| 5 | `changes:` path glob evaluation against a real or mock file tree | - -Predefined variables populated automatically from `--branch` / `--tag` / `--source` shortcuts: `CI_COMMIT_BRANCH`, `CI_COMMIT_TAG`, `CI_COMMIT_REF_NAME`, `CI_COMMIT_REF_SLUG`, `CI_PIPELINE_SOURCE`, `CI_DEFAULT_BRANCH` (defaulting to `main`). +- **Multi-context simulation** — run multiple contexts in one invocation and print a comparison table: + ```bash + glint check --context branch=main --context branch=develop --context tag=v1.0.0 .gitlab-ci.yml + ``` +- **Context-scoped linting** — skip `needs:`/`dependencies:` cross-checks for jobs that are statically unreachable in the given context +- **`rules:changes:` evaluation** — path glob evaluation against the local git tree (expression evaluator priority 5) --- @@ -78,7 +46,7 @@ The current rule set covers the most common sources of broken pipelines. These a ## Include resolution -- **`include: local:`** full resolution — parse and merge locally-referenced YAML files the same way remote project includes are handled; enables cross-file `extends:` and `needs:` validation for monorepo setups +- ~~**`include: local:`** full resolution~~ — ✓ shipped in v0.2.0; local files are read from disk, recursively resolved, and merged before linting - **`include: remote:`** (URL) — fetch and merge plain HTTP/HTTPS URLs (no auth required) - **Recursive include depth limit** — guard against include cycles across files - **Offline mode / cache** — persist fetched remote templates to a local cache directory; `--offline` flag to skip network calls and use only cached copies @@ -99,8 +67,9 @@ Right now the only output is plain-text findings. Structured output enables inte ## Pipeline graph improvements -The SVG renderer covers the basic layout. These would bring it closer to GitLab's full interactive view. +The SVG renderer and terminal tree cover the basic layout. These would bring it closer to GitLab's full interactive view. +- ~~**Terminal job tree**~~ — ✓ shipped in v0.2.0 as `glint graph tree`; stages as branches, jobs as leaves, context-aware annotations - **Multi-job connector accuracy** — draw one connector per job pair rather than one per stage pair in classic mode, so pipelines with uneven columns look correct - **Job tooltip / detail panel** — embed a hidden `` and `<desc>` per chip so SVG viewers show `stage`, `when`, `image`, and `needs` on hover - **`when: on_failure` visual distinction** — dashed border or distinct icon for failure-path jobs @@ -136,5 +105,6 @@ The SVG renderer covers the basic layout. These would bring it closer to GitLab' - **Structured rule IDs** — assign a stable short ID to every rule (e.g. `GS001`) so suppression, documentation, and SARIF output are stable across versions - **`--explain <rule-id>`** — print the rule description, rationale, and an example fix - ~~**Semantic versioning and first release**~~ — shipped as `v0.1.0` (2026-06-07) +- ~~**Subcommand CLI**~~ — shipped as `v0.2.0` (2026-06-11); `glint check` / `glint graph [mode]` with ruff-style `--help` - **Changelog automation** — generate release notes from Conventional Commits via `git-cliff` or similar - **Fuzz testing** — add a `go test -fuzz` target for the YAML parser to harden it against malformed input diff --git a/Taskfile.yml b/Taskfile.yml index 92f0271..7e87efc 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,33 +27,39 @@ tasks: desc: Run glint against all testdata fixtures deps: [build] cmds: - - cmd: ./{{.BINARY}} testdata/valid.yml + - cmd: ./{{.BINARY}} check testdata/valid.yml ignore_error: false - - cmd: ./{{.BINARY}} testdata/extends.yml + - cmd: ./{{.BINARY}} check testdata/extends.yml ignore_error: false - - cmd: ./{{.BINARY}} testdata/keywords_valid.yml + - cmd: ./{{.BINARY}} check testdata/keywords_valid.yml ignore_error: false - - cmd: ./{{.BINARY}} testdata/invalid.yml + - cmd: ./{{.BINARY}} check testdata/invalid.yml ignore_error: true - - cmd: ./{{.BINARY}} testdata/needs.yml + - cmd: ./{{.BINARY}} check testdata/needs.yml ignore_error: true - - cmd: ./{{.BINARY}} testdata/needs_cycle.yml + - cmd: ./{{.BINARY}} check testdata/needs_cycle.yml ignore_error: true - - cmd: ./{{.BINARY}} testdata/keywords_invalid.yml + - cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml ignore_error: true - - cmd: ./{{.BINARY}} testdata/includes_project.yml + - cmd: ./{{.BINARY}} check testdata/includes_project.yml ignore_error: false - - cmd: ./{{.BINARY}} testdata/includes_component.yml + - cmd: ./{{.BINARY}} check testdata/includes_component.yml ignore_error: false - - cmd: ./{{.BINARY}} testdata/context_rules.yml + - cmd: ./{{.BINARY}} check testdata/context_rules.yml ignore_error: false - - cmd: ./{{.BINARY}} --branch main testdata/context_rules.yml + - cmd: ./{{.BINARY}} check --branch main testdata/context_rules.yml ignore_error: false - - cmd: ./{{.BINARY}} --branch develop testdata/context_rules.yml + - cmd: ./{{.BINARY}} check --branch develop testdata/context_rules.yml ignore_error: false - - cmd: ./{{.BINARY}} --branch feat/my-feature testdata/context_rules.yml + - cmd: ./{{.BINARY}} check --branch feat/my-feature testdata/context_rules.yml ignore_error: false - - cmd: ./{{.BINARY}} --tag v1.0.0 testdata/context_rules.yml + - cmd: ./{{.BINARY}} check --tag v1.0.0 testdata/context_rules.yml + ignore_error: false + - cmd: ./{{.BINARY}} check samba-testdata/.gitlab-ci.yml + ignore_error: false + - cmd: ./{{.BINARY}} check samba-testdata/.gitlab-ci-coverage.yml + ignore_error: false + - cmd: ./{{.BINARY}} check samba-testdata/.gitlab-ci-private.yml ignore_error: false lint-go: diff --git a/cmd/glint/main.go b/cmd/glint/main.go index c7d7063..94a457f 100644 --- a/cmd/glint/main.go +++ b/cmd/glint/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "path/filepath" "sort" "strings" @@ -15,6 +16,38 @@ import ( "git.k3nny.fr/glint/internal/resolver" ) +const globalUsage = `glint: Lint and visualise GitLab CI pipelines locally. + +Usage: glint [OPTIONS] <COMMAND> + +Commands: + check Lint a pipeline file — exits 0 (clean) or 1 (errors found) + graph Visualise the pipeline as a job tree or Mermaid graph + +Options: + -h, --help Print help + +For help with a specific command, see: ` + "`glint <command> --help`" + `. +` + +func main() { + if len(os.Args) < 2 { + fmt.Fprint(os.Stderr, globalUsage) + os.Exit(2) + } + switch os.Args[1] { + case "check": + cmdCheck(os.Args[2:]) + case "graph": + cmdGraph(os.Args[2:]) + case "-h", "--help", "help": + fmt.Fprint(os.Stderr, globalUsage) + default: + fmt.Fprintf(os.Stderr, "glint: unknown command %q\n\n%s", os.Args[1], globalUsage) + os.Exit(2) + } +} + // multiFlag allows a flag to be specified multiple times. type multiFlag []string @@ -24,29 +57,70 @@ func (f *multiFlag) Set(v string) error { return nil } -func main() { - var ( - token = flag.String("token", "", "GitLab personal access token (overrides GITLAB_TOKEN)") - gitlabURL = flag.String("gitlab-url", "", "GitLab instance URL (overrides CI_SERVER_URL / GITLAB_URL)") - graphMode = flag.String("graph", "", "graph mode: includes | pipeline | all") - graphOut = flag.String("graph-out", "glint-out", "output directory for pipeline graph files") - branch = flag.String("branch", "", "simulate a branch push (sets CI_COMMIT_BRANCH, …)") - tag = flag.String("tag", "", "simulate a tag push (sets CI_COMMIT_TAG, …)") - source = flag.String("source", "", "set CI_PIPELINE_SOURCE") - vars multiFlag - ) - flag.Var(&vars, "var", "set a CI variable as KEY=VALUE; repeatable") - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: glint [options] <pipeline.yml>\n\n") - flag.PrintDefaults() - } - flag.Parse() +func cmdCheck(args []string) { + fs := flag.NewFlagSet("glint check", flag.ExitOnError) + token := fs.String("token", "", "GitLab personal access token (overrides GITLAB_TOKEN)") + gitlabURL := fs.String("gitlab-url", "", "GitLab instance URL (overrides CI_SERVER_URL / GITLAB_URL)") + branch := fs.String("branch", "", "simulate a branch push (sets CI_COMMIT_BRANCH, …)") + tag := fs.String("tag", "", "simulate a tag push (sets CI_COMMIT_TAG, …)") + source := fs.String("source", "", "set CI_PIPELINE_SOURCE") + var vars multiFlag + fs.Var(&vars, "var", "set a CI variable as KEY=VALUE; repeatable") + fs.Usage = func() { + fmt.Fprint(os.Stderr, `Lint a GitLab CI pipeline file. - if flag.NArg() != 1 { - flag.Usage() +Resolves local includes and extends chains, then runs all lint rules. +Exits 0 when no errors are found, 1 when at least one error is reported. + +Usage: glint check [OPTIONS] <PIPELINE> + +Arguments: + <PIPELINE> Path to the .gitlab-ci.yml file to lint + +Options: + --token <TOKEN> + GitLab personal access token. Required to fetch project: includes; + component: includes are attempted unauthenticated. + [env: GITLAB_TOKEN | CI_JOB_TOKEN | GITLAB_PRIVATE_TOKEN] + + --gitlab-url <URL> + GitLab instance URL. + [env: CI_SERVER_URL | GITLAB_URL] [default: https://gitlab.com] + + --branch <NAME> + Simulate a branch push. Populates: CI_COMMIT_BRANCH, + CI_COMMIT_REF_NAME, CI_COMMIT_REF_SLUG, CI_PIPELINE_SOURCE=push. + + --tag <NAME> + Simulate a tag push. Populates: CI_COMMIT_TAG, CI_COMMIT_REF_NAME, + CI_COMMIT_REF_SLUG, CI_PIPELINE_SOURCE=push. Clears CI_COMMIT_BRANCH. + + --source <EVENT> + Override CI_PIPELINE_SOURCE. + [possible values: push, merge_request_event, schedule, web, api] + + --var <KEY=VALUE> + Set or override a CI variable. Takes precedence over --branch, --tag, + and --source. Repeatable. + + -h, --help + Print help + +Examples: + glint check .gitlab-ci.yml + glint check --branch main .gitlab-ci.yml + GITLAB_TOKEN=glpat-xxxx glint check .gitlab-ci.yml + glint check --token glpat-xxxx --gitlab-url https://gitlab.example.com .gitlab-ci.yml + glint check --branch main --var DEPLOY_ENV=production .gitlab-ci.yml +`) + } + _ = fs.Parse(args) + + if fs.NArg() != 1 { + fs.Usage() os.Exit(2) } - path := flag.Arg(0) + path := fs.Arg(0) cfg := fetcher.AutoConfig().WithOverrides(*gitlabURL, *token) @@ -56,19 +130,19 @@ func main() { os.Exit(2) } - warnings := resolver.ResolveIncludes(p, cfg) + rootDir := filepath.Dir(filepath.Clean(path)) + warnings, _ := resolver.ResolveIncludes(p, cfg, rootDir) for _, w := range warnings { fmt.Fprintf(os.Stderr, "[WARNING] include %s\n", w) } - if err := resolver.Resolve(p); err != nil { + extWarnings, err := resolver.Resolve(p) + if err != nil { fmt.Fprintf(os.Stderr, "error: resolving extends: %v\n", err) os.Exit(2) } - - if *graphMode != "" { - runGraph(p, path, *graphMode, *graphOut) - return + for _, w := range extWarnings { + fmt.Fprintf(os.Stderr, "[WARNING] job %q extends unknown job %q; extends chain skipped\n", w.Job, w.Base) } ctx := cicontext.New(*branch, *tag, *source, vars) @@ -85,11 +159,8 @@ func main() { } } - jobCount := len(p.Jobs) - stageCount := len(p.Stages) - if len(findings) == 0 { - fmt.Printf("OK: %s — no issues found (%d job(s), %d stage(s))\n", path, jobCount, stageCount) + fmt.Printf("OK: %s — no issues found (%d job(s), %d stage(s))\n", path, len(p.Jobs), len(p.Stages)) } else { errCount := 0 for _, f := range findings { @@ -105,28 +176,127 @@ func main() { } } -func runGraph(p *model.Pipeline, path, mode, outDir string) { +var knownGraphModes = map[string]bool{ + "tree": true, "includes": true, "pipeline": true, "all": true, +} + +func cmdGraph(args []string) { + // Optional mode word must come before any flags or the file path. + mode := "default" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") && knownGraphModes[args[0]] { + mode = args[0] + args = args[1:] + } + + fs := flag.NewFlagSet("glint graph", flag.ExitOnError) + token := fs.String("token", "", "GitLab personal access token (overrides GITLAB_TOKEN)") + gitlabURL := fs.String("gitlab-url", "", "GitLab instance URL (overrides CI_SERVER_URL / GITLAB_URL)") + out := fs.String("out", "glint-out", "output directory for Mermaid graph files (pipeline mode)") + fs.Usage = func() { + fmt.Fprint(os.Stderr, `Visualise the pipeline as a job tree and/or Mermaid graph. + +Usage: glint graph [MODE] [OPTIONS] <PIPELINE> + +Arguments: + [MODE] Graph mode; must appear before options [default: tree+includes] + [possible values: tree, includes, pipeline, all] + <PIPELINE> Path to the .gitlab-ci.yml file + +Options: + --out <DIR> + Output directory for rendered graph files. + Used by the pipeline and all modes only. [default: glint-out] + + --token <TOKEN> + GitLab personal access token. Used to fetch remote project: includes + when building the include dependency graph. + [env: GITLAB_TOKEN | CI_JOB_TOKEN | GITLAB_PRIVATE_TOKEN] + + --gitlab-url <URL> + GitLab instance URL. + [env: CI_SERVER_URL | GITLAB_URL] [default: https://gitlab.com] + + --branch <NAME> + Simulate a branch push. Jobs in tree output are annotated with their + evaluated state ([skipped] or [manual]; no tag means active). + Populates: CI_COMMIT_BRANCH, CI_COMMIT_REF_NAME, CI_COMMIT_REF_SLUG, + CI_PIPELINE_SOURCE=push. + + --tag <NAME> + Simulate a tag push. Populates: CI_COMMIT_TAG, CI_COMMIT_REF_NAME, + CI_COMMIT_REF_SLUG, CI_PIPELINE_SOURCE=push. Clears CI_COMMIT_BRANCH. + + --source <EVENT> + Override CI_PIPELINE_SOURCE. + [possible values: push, merge_request_event, schedule, web, api] + + --var <KEY=VALUE> + Set or override a CI variable. Repeatable. + + -h, --help + Print help + +Examples: + glint graph .gitlab-ci.yml + glint graph tree .gitlab-ci.yml + glint graph tree --branch main .gitlab-ci.yml + glint graph includes .gitlab-ci.yml > includes.mmd + glint graph pipeline .gitlab-ci.yml + glint graph pipeline --out /tmp/graphs .gitlab-ci.yml + glint graph all .gitlab-ci.yml > includes.mmd +`) + } + branch := fs.String("branch", "", "simulate a branch push (sets CI_COMMIT_BRANCH, …)") + tag := fs.String("tag", "", "simulate a tag push (sets CI_COMMIT_TAG, …)") + source := fs.String("source", "", "set CI_PIPELINE_SOURCE") + var vars multiFlag + fs.Var(&vars, "var", "set a CI variable as KEY=VALUE; repeatable") + _ = fs.Parse(args) + + if fs.NArg() != 1 { + fs.Usage() + os.Exit(2) + } + path := fs.Arg(0) + + cfg := fetcher.AutoConfig().WithOverrides(*gitlabURL, *token) + + p, err := model.Parse(path) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(2) + } + + rootDir := filepath.Dir(filepath.Clean(path)) + resolver.ResolveIncludes(p, cfg, rootDir) //nolint:errcheck + resolver.Resolve(p) //nolint:errcheck + + ctx := cicontext.New(*branch, *tag, *source, vars) + switch mode { + case "default": + fmt.Print(graph.Tree(p, ctx)) + fmt.Println("---") + fmt.Print(graph.Includes(path, p.Include, cfg)) + case "tree": + fmt.Print(graph.Tree(p, ctx)) case "includes": - fmt.Print(graph.Includes(path, p.Include)) + fmt.Print(graph.Includes(path, p.Include, cfg)) case "pipeline": - outPath, err := graph.RenderPipeline(p, outDir) + outPath, err := graph.RenderPipeline(p, *out) if err != nil { fmt.Fprintf(os.Stderr, "error: rendering pipeline graph: %v\n", err) os.Exit(2) } fmt.Println(outPath) case "all": - fmt.Print(graph.Includes(path, p.Include)) - outPath, err := graph.RenderPipeline(p, outDir) + fmt.Print(graph.Includes(path, p.Include, cfg)) + outPath, err := graph.RenderPipeline(p, *out) if err != nil { fmt.Fprintf(os.Stderr, "error: rendering pipeline graph: %v\n", err) os.Exit(2) } fmt.Fprintln(os.Stderr, outPath) - default: - fmt.Fprintf(os.Stderr, "error: unknown --graph value %q; use: includes, pipeline, all\n", mode) - os.Exit(2) } } diff --git a/internal/graph/includes.go b/internal/graph/includes.go index 6adef5d..d15901c 100644 --- a/internal/graph/includes.go +++ b/internal/graph/includes.go @@ -2,12 +2,183 @@ package graph import ( "fmt" + "os" + "path/filepath" "strings" + + "git.k3nny.fr/glint/internal/fetcher" + "git.k3nny.fr/glint/internal/model" ) -// Includes returns a Mermaid flowchart showing include file dependencies. -// sourcePath is the path to the main pipeline file; rawIncludes is Pipeline.Include. -func Includes(sourcePath string, rawIncludes []any) string { +// Includes returns a Mermaid flowchart of the full include dependency tree. +// It recurses into project:, component:, and local: includes to expose +// transitive dependencies. Includes that cannot be fetched are shown but not expanded. +func Includes(sourcePath string, rawIncludes []any, cfg fetcher.GitLabConfig) string { + b := &treeBuilder{ + visited: map[string]bool{}, + cfg: cfg, + baseDir: filepath.Dir(sourcePath), + } + root := &treeNode{id: "root", label: mermaidLabel(sourcePath), class: "main"} + b.buildChildren(root, rawIncludes) + return renderTree(root) +} + +type treeNode struct { + id string + label string + class string + children []*treeNode +} + +// treeBuilder accumulates state while recursively traversing include entries. +type treeBuilder struct { + counter int + visited map[string]bool // prevents infinite loops on circular includes + cfg fetcher.GitLabConfig + baseDir string // directory of the current pipeline file; changes for local includes +} + +func (b *treeBuilder) nextID() string { + b.counter++ + return fmt.Sprintf("inc%d", b.counter) +} + +func (b *treeBuilder) buildChildren(parent *treeNode, rawIncludes []any) { + for _, entry := range rawIncludes { + for _, child := range b.parseEntry(entry) { + parent.children = append(parent.children, child) + } + } +} + +func (b *treeBuilder) parseEntry(entry any) []*treeNode { + switch v := entry.(type) { + case string: + node := &treeNode{id: b.nextID(), label: "local: " + mermaidLabel(v), class: "local"} + b.recurseLocal(node, v) + return []*treeNode{node} + case map[string]any: + return b.parseMap(v) + } + return nil +} + +func (b *treeBuilder) parseMap(m map[string]any) []*treeNode { + if comp, ok := m["component"].(string); ok { + node := &treeNode{ + id: b.nextID(), + label: "component:<br>" + mermaidLabel(comp), + class: "component", + } + b.recurseComponent(node, comp) + return []*treeNode{node} + } + if proj, ok := m["project"].(string); ok { + ref, _ := m["ref"].(string) + if ref == "" { + ref = "HEAD" + } + files := includeFileList(m["file"]) + if len(files) == 0 { + return []*treeNode{{ + id: b.nextID(), + label: fmt.Sprintf("project: %s @ %s", mermaidLabel(proj), ref), + class: "project", + }} + } + var nodes []*treeNode + for _, f := range files { + node := &treeNode{ + id: b.nextID(), + label: fmt.Sprintf("project: %s<br>%s @ %s", mermaidLabel(proj), mermaidLabel(f), ref), + class: "project", + } + b.recurseProject(node, proj, f, ref) + nodes = append(nodes, node) + } + return nodes + } + if local, ok := m["local"].(string); ok { + node := &treeNode{id: b.nextID(), label: "local: " + mermaidLabel(local), class: "local"} + b.recurseLocal(node, local) + return []*treeNode{node} + } + if remote, ok := m["remote"].(string); ok { + return []*treeNode{{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}} + } + if tmpl, ok := m["template"].(string); ok { + return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}} + } + return nil +} + +func (b *treeBuilder) recurseLocal(node *treeNode, path string) { + absPath := filepath.Join(b.baseDir, path) + key := "local:" + absPath + if b.visited[key] { + return + } + b.visited[key] = true + + data, err := os.ReadFile(absPath) + if err != nil { + return + } + p, err := model.ParseBytes(data) + if err != nil || len(p.Include) == 0 { + return + } + orig := b.baseDir + b.baseDir = filepath.Dir(absPath) + b.buildChildren(node, p.Include) + b.baseDir = orig +} + +func (b *treeBuilder) recurseProject(node *treeNode, project, filePath, ref string) { + key := fmt.Sprintf("project:%s:%s@%s", project, filePath, ref) + if b.visited[key] || !b.cfg.HasToken() || filePath == "" { + return + } + b.visited[key] = true + + data, err := b.cfg.FetchFile(project, filePath, ref) + if err != nil { + return + } + p, err := model.ParseBytes(data) + if err != nil || len(p.Include) == 0 { + return + } + b.buildChildren(node, p.Include) +} + +func (b *treeBuilder) recurseComponent(node *treeNode, ref string) { + if strings.ContainsRune(ref, '$') { + return + } + key := "component:" + ref + if b.visited[key] { + return + } + b.visited[key] = true + + host, project, component, version, err := parseComponentRef(ref) + if err != nil { + return + } + data, err := fetchComponentFile(b.cfg.ForHost(host), project, component, version) + if err != nil { + return + } + p, err := model.ParseBytes(data) + if err != nil || len(p.Include) == 0 { + return + } + b.buildChildren(node, p.Include) +} + +func renderTree(root *treeNode) string { var sb strings.Builder w := func(s string) { sb.WriteString(s + "\n") } wf := func(f string, a ...any) { fmt.Fprintf(&sb, f+"\n", a...) } @@ -23,84 +194,52 @@ func Includes(sourcePath string, rawIncludes []any) string { w(" classDef remote fill:#868686,stroke:#686868,color:#fff") w(" classDef template fill:#fc6d26,stroke:#e56b1f,color:#fff") w("") - wf(" root[\"%s\"]:::main", mermaidLabel(sourcePath)) - if len(rawIncludes) == 0 { - return sb.String() - } - - w("") - - counter := 0 - for _, entry := range rawIncludes { - for _, n := range parseIncludeEntry(entry, &counter) { - wf(" %s[\"%s\"]:::%s", n.id, n.label, n.class) - wf(" root --> %s", n.id) + var emit func(n *treeNode) + emit = func(n *treeNode) { + wf(" %s[\"%s\"]:::%s", n.id, n.label, n.class) + for _, child := range n.children { + emit(child) + wf(" %s --> %s", n.id, child.id) } } + emit(root) return sb.String() } -type incNode struct { - id, label, class string +// parseComponentRef parses a CI/CD component reference of the form +// <host>/<project-path>/<component-name>@<version>. +func parseComponentRef(ref string) (host, project, component, version string, err error) { + atIdx := strings.LastIndex(ref, "@") + if atIdx < 0 || atIdx == len(ref)-1 { + err = fmt.Errorf("component reference %q must include a version", ref) + return + } + version = ref[atIdx+1:] + path := ref[:atIdx] + parts := strings.Split(path, "/") + if len(parts) < 3 { + err = fmt.Errorf("component reference %q must be <host>/<project>/<component>@<version>", ref) + return + } + host = parts[0] + component = parts[len(parts)-1] + project = strings.Join(parts[1:len(parts)-1], "/") + return } -func parseIncludeEntry(entry any, counter *int) []incNode { - newID := func() string { - *counter++ - return fmt.Sprintf("inc%d", *counter) +// fetchComponentFile fetches a component's template YAML, trying the single-file +// layout first and the directory layout as fallback. +func fetchComponentFile(cfg fetcher.GitLabConfig, project, component, version string) ([]byte, error) { + if data, err := cfg.FetchFile(project, "templates/"+component+".yml", version); err == nil { + return data, nil } - switch v := entry.(type) { - case string: - return []incNode{{id: newID(), label: mermaidLabel(v), class: "local"}} - case map[string]any: - return parseIncludeMap(v, newID) + data, err := cfg.FetchFile(project, "templates/"+component+"/template.yml", version) + if err != nil { + return nil, fmt.Errorf("component %s/%s@%s not found", project, component, version) } - return nil -} - -func parseIncludeMap(m map[string]any, newID func() string) []incNode { - if comp, ok := m["component"].(string); ok { - return []incNode{{ - id: newID(), - label: "component:<br>" + mermaidLabel(comp), - class: "component", - }} - } - if proj, ok := m["project"].(string); ok { - ref, _ := m["ref"].(string) - if ref == "" { - ref = "HEAD" - } - files := includeFileList(m["file"]) - if len(files) == 0 { - return []incNode{{ - id: newID(), - label: fmt.Sprintf("project: %s @ %s", mermaidLabel(proj), ref), - class: "project", - }} - } - var nodes []incNode - for _, f := range files { - nodes = append(nodes, incNode{ - id: newID(), - label: fmt.Sprintf("project: %s<br>%s @ %s", mermaidLabel(proj), mermaidLabel(f), ref), - class: "project", - }) - } - return nodes - } - if local, ok := m["local"].(string); ok { - return []incNode{{id: newID(), label: "local: " + mermaidLabel(local), class: "local"}} - } - if remote, ok := m["remote"].(string); ok { - return []incNode{{id: newID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}} - } - if tmpl, ok := m["template"].(string); ok { - return []incNode{{id: newID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}} - } - return nil + return data, nil } func includeFileList(v any) []string { diff --git a/internal/graph/tree.go b/internal/graph/tree.go new file mode 100644 index 0000000..200dc09 --- /dev/null +++ b/internal/graph/tree.go @@ -0,0 +1,113 @@ +package graph + +import ( + "sort" + "strings" + + "git.k3nny.fr/glint/internal/cicontext" + "git.k3nny.fr/glint/internal/model" +) + +// Tree returns a tree-command-style text representation of pipeline jobs +// grouped by stage. +// +// Hidden template jobs (names starting with ".") are excluded. +// When ctx is non-nil and non-empty, each job is annotated with its evaluated +// state ([skipped] or [manual]); active jobs carry no annotation. +// Without a context, job-type annotations ([manual], [delayed], [trigger]) are +// shown instead. +func Tree(p *model.Pipeline, ctx *cicontext.Context) string { + var sb strings.Builder + + // Collect visible jobs. + var visible []string + for name := range p.Jobs { + if !strings.HasPrefix(name, ".") { + visible = append(visible, name) + } + } + sort.Strings(visible) + + // Group by stage. + byStage := make(map[string][]string) + for _, name := range visible { + stage := p.Jobs[name].Stage + if stage == "" { + stage = "test" + } + byStage[stage] = append(byStage[stage], name) + } + + // Build ordered stage list: declared first, then extras. + seen := make(map[string]bool) + var stages []string + for _, s := range p.Stages { + if len(byStage[s]) > 0 && !seen[s] { + stages = append(stages, s) + seen[s] = true + } + } + for _, name := range visible { + stage := p.Jobs[name].Stage + if stage == "" { + stage = "test" + } + if !seen[stage] { + stages = append(stages, stage) + seen[stage] = true + } + } + + sb.WriteString("pipeline\n") + + for si, stage := range stages { + lastStage := si == len(stages)-1 + stagePrefix, childPrefix := branchChars(lastStage) + + sb.WriteString(stagePrefix + stage + "\n") + + jobs := byStage[stage] + sort.Strings(jobs) + for ji, name := range jobs { + lastJob := ji == len(jobs)-1 + jobBranch, _ := branchChars(lastJob) + sb.WriteString(childPrefix + jobBranch + jobLabel(p.Jobs[name], name, ctx) + "\n") + } + } + + return sb.String() +} + +func branchChars(last bool) (branch, continuation string) { + if last { + return "└── ", " " + } + return "├── ", "│ " +} + +func jobLabel(job model.Job, name string, ctx *cicontext.Context) string { + if ctx != nil && !ctx.IsEmpty() { + switch cicontext.EvalJob(job, ctx) { + case cicontext.JobSkipped: + return name + " [skipped]" + case cicontext.JobManual: + return name + " [manual]" + } + return name + } + // No context: annotate by job type. + var tags []string + if job.When == "manual" { + tags = append(tags, "manual") + } + if job.When == "delayed" { + tags = append(tags, "delayed") + } + if job.Trigger != nil { + tags = append(tags, "trigger") + } + if len(tags) == 0 { + return name + } + return name + " [" + strings.Join(tags, ", ") + "]" +} diff --git a/internal/linter/keywords.go b/internal/linter/keywords.go index 17f1885..e21d739 100644 --- a/internal/linter/keywords.go +++ b/internal/linter/keywords.go @@ -264,7 +264,7 @@ func checkTrigger(name string, job model.Job) []Finding { return nil } var findings []Finding - if len(job.Script) > 0 { + if scriptNonEmpty(job.Script) { findings = append(findings, Finding{ Severity: Error, Job: name, diff --git a/internal/linter/linter.go b/internal/linter/linter.go index 3878641..1330838 100644 --- a/internal/linter/linter.go +++ b/internal/linter/linter.go @@ -88,7 +88,7 @@ func checkJob(name string, job model.Job, stageSet map[string]bool) []Finding { // After extends resolution, a job with no script/run is an error. // Exceptions: trigger jobs, pages jobs (use pages: keyword), and template jobs. - hasScript := len(job.Script) > 0 || job.Run != nil + hasScript := scriptNonEmpty(job.Script) || job.Run != nil if !isTemplate && !isTrigger && job.Pages == nil && !hasScript { findings = append(findings, Finding{ Severity: Error, @@ -139,3 +139,15 @@ func checkJob(name string, job model.Job, stageSet map[string]bool) []Finding { return findings } + +// scriptNonEmpty reports whether a script/before_script/after_script field +// (which may be a []any list or a plain string) is non-empty. +func scriptNonEmpty(v any) bool { + switch s := v.(type) { + case []any: + return len(s) > 0 + case string: + return s != "" + } + return false +} diff --git a/internal/linter/samba_test.go b/internal/linter/samba_test.go new file mode 100644 index 0000000..364de74 --- /dev/null +++ b/internal/linter/samba_test.go @@ -0,0 +1,89 @@ +package linter_test + +import ( + "path/filepath" + "testing" + + "git.k3nny.fr/glint/internal/fetcher" + "git.k3nny.fr/glint/internal/linter" + "git.k3nny.fr/glint/internal/model" + "git.k3nny.fr/glint/internal/resolver" +) + +// TestSambaCI verifies that the Samba project's .gitlab-ci.yml (a real-world +// pipeline that is valid on GitLab) produces no Error findings. +// These files exercise local include resolution and multi-level extends chains. +func TestSambaCI(t *testing.T) { + entryPoint := "../../samba-testdata/.gitlab-ci.yml" + + p, err := model.Parse(entryPoint) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + rootDir := filepath.Dir(filepath.Clean(entryPoint)) + incWarnings, _ := resolver.ResolveIncludes(p, fetcher.GitLabConfig{}, rootDir) + for _, w := range incWarnings { + t.Logf("include warning: %s", w) + } + + extWarnings, err := resolver.Resolve(p) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + for _, w := range extWarnings { + t.Logf("extends warning: job %q extends unknown %q", w.Job, w.Base) + } + + findings := linter.Lint(p) + + for _, f := range findings { + if f.Severity == linter.Error { + t.Errorf("unexpected error finding on valid Samba CI: %s", f) + } + } +} + +// TestSambaCIEntryFiles verifies all of the Samba entry-point files +// (files that can each act as the top-level CI file) lint without errors. +func TestSambaCIEntryFiles(t *testing.T) { + entryPoints := []struct { + name string + path string + }{ + {"default", "../../samba-testdata/.gitlab-ci.yml"}, + {"coverage", "../../samba-testdata/.gitlab-ci-coverage.yml"}, + {"private", "../../samba-testdata/.gitlab-ci-private.yml"}, + } + + for _, tc := range entryPoints { + tc := tc + t.Run(tc.name, func(t *testing.T) { + p, err := model.Parse(tc.path) + if err != nil { + t.Fatalf("Parse: %v", err) + } + + rootDir := filepath.Dir(filepath.Clean(tc.path)) + incWarnings, _ := resolver.ResolveIncludes(p, fetcher.GitLabConfig{}, rootDir) + for _, w := range incWarnings { + t.Logf("include warning: %s", w) + } + + extWarnings, err := resolver.Resolve(p) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + for _, w := range extWarnings { + t.Logf("extends warning: job %q extends unknown %q", w.Job, w.Base) + } + + findings := linter.Lint(p) + for _, f := range findings { + if f.Severity == linter.Error { + t.Errorf("unexpected error finding: %s", f) + } + } + }) + } +} diff --git a/internal/model/pipeline.go b/internal/model/pipeline.go index fbf2c76..0aeb4a3 100644 --- a/internal/model/pipeline.go +++ b/internal/model/pipeline.go @@ -31,10 +31,10 @@ type Workflow struct { type Job struct { Name string // set by parser, not from YAML Stage string `yaml:"stage"` - Script []string `yaml:"script"` - Run any `yaml:"run"` // alternative to script (CI steps) - BeforeScript []string `yaml:"before_script"` - AfterScript []string `yaml:"after_script"` + Script any `yaml:"script"` // []string or string (block scalar) + Run any `yaml:"run"` // alternative to script (CI steps) + BeforeScript any `yaml:"before_script"` // []string or string + AfterScript any `yaml:"after_script"` // []string or string Image any `yaml:"image"` Services []any `yaml:"services"` Variables map[string]string `yaml:"variables"` diff --git a/internal/resolver/extends.go b/internal/resolver/extends.go index fdfc69d..7e3b638 100644 --- a/internal/resolver/extends.go +++ b/internal/resolver/extends.go @@ -10,33 +10,45 @@ import ( // Resolve resolves all extends: references in p.Jobs in place. // Jobs are merged depth-first so that base definitions are resolved before // derived ones. Mutates p.Jobs with the fully merged Job structs. -func Resolve(p *model.Pipeline) error { +// +// The first return value lists extends references whose base job could not be +// found (e.g. it lives in a remote include that was not fetched). Those jobs +// are left unmerged but still passed to the linter. A non-nil error is only +// returned for unrecoverable situations such as circular dependencies. +func Resolve(p *model.Pipeline) ([]ExtendWarning, error) { + var extWarnings []ExtendWarning + // Build extends graph: jobName -> ordered list of base job names. extendsGraph := make(map[string][]string, len(p.Jobs)) for name, job := range p.Jobs { bases, err := parseExtends(job.Extends) if err != nil { - return fmt.Errorf("job %q: invalid extends: %w", name, err) + return extWarnings, fmt.Errorf("job %q: invalid extends: %w", name, err) } if len(bases) == 0 { continue } + skip := false for _, base := range bases { if _, ok := p.RawJobs[base]; !ok { - return fmt.Errorf("job %q extends unknown job %q", name, base) + extWarnings = append(extWarnings, ExtendWarning{Job: name, Base: base}) + skip = true } } + if skip { + continue // leave this job unmerged; still linted with its own fields + } extendsGraph[name] = bases } if len(extendsGraph) == 0 { - return nil + return extWarnings, nil } // Topological sort — bases must be resolved before derived jobs. order, err := topoSort(extendsGraph) if err != nil { - return err + return extWarnings, err } // resolved holds the final merged raw map for each processed job. @@ -61,17 +73,17 @@ func Resolve(p *model.Pipeline) error { // Re-decode the merged map into a Job struct. data, err := yaml.Marshal(merged) if err != nil { - return fmt.Errorf("job %q: re-encoding merged definition: %w", name, err) + return extWarnings, fmt.Errorf("job %q: re-encoding merged definition: %w", name, err) } var j model.Job if err := yaml.Unmarshal(data, &j); err != nil { - return fmt.Errorf("job %q: re-decoding merged definition: %w", name, err) + return extWarnings, fmt.Errorf("job %q: re-decoding merged definition: %w", name, err) } j.Name = name p.Jobs[name] = j } - return nil + return extWarnings, nil } // parseExtends normalises the extends field (string or []any) into []string. diff --git a/internal/resolver/includes.go b/internal/resolver/includes.go index 77e05d8..dbd3fff 100644 --- a/internal/resolver/includes.go +++ b/internal/resolver/includes.go @@ -2,18 +2,18 @@ package resolver import ( "fmt" + "os" + "path/filepath" "strings" "git.k3nny.fr/glint/internal/fetcher" "git.k3nny.fr/glint/internal/model" ) -// IncludeWarning describes a remote include entry that could not be resolved. +// IncludeWarning describes an include entry that could not be resolved. // These are surfaced to the user as [WARNING] lines before the lint findings. type IncludeWarning struct { - // Label is a short human-readable identifier shown in the warning message, - // e.g. "project my-group/templates:/ci.yml@main" or - // "component gitlab.com/components/golang/build@v1.0". + // Label is a short human-readable identifier shown in the warning message. Label string Err error // nil when Skipped is true Skipped bool // no token available — skipped without attempting a network call @@ -30,46 +30,114 @@ func (w IncludeWarning) String() string { w.Label, w.Err) } +// ExtendWarning describes an extends: reference whose base job could not be +// found. This typically means the base is in a remote include that was not +// fetched (e.g. no token). The job's extends chain is skipped; the job itself +// is still linted with whatever fields it directly defines. +type ExtendWarning struct { + Job string // job that declares the extends + Base string // the unknown base job name +} + // ResolveIncludes processes the pipeline's include: block. // -// Project includes (include: project: ...) require authentication and are -// skipped with a warning when no token is configured. +// rootDir is the repository root directory used to resolve local: includes. +// In practice this is the directory of the top-level pipeline file being linted. // -// Component includes (include: component: ...) attempt the fetch -// unauthenticated first, so public CI/CD catalog components work without a -// token. A warning is emitted when the fetch fails. +// Local includes are read from disk and merged recursively. +// Project includes require authentication and are skipped with a warning when +// no token is configured. Component includes are attempted unauthenticated. +// Remote and template includes are silently skipped (resolved by GitLab at runtime). // -// Non-resolvable include forms (local, remote, template) are silently skipped. -func ResolveIncludes(p *model.Pipeline, cfg fetcher.GitLabConfig) []IncludeWarning { +// The second return value carries extends warnings discovered while recursively +// processing included files (forwarded from resolver.Resolve calls). +func ResolveIncludes(p *model.Pipeline, cfg fetcher.GitLabConfig, rootDir string) ([]IncludeWarning, []ExtendWarning) { + visited := map[string]bool{} + return resolveIncludes(p, p.Include, cfg, rootDir, visited) +} + +// resolveIncludes is the recursive core of ResolveIncludes. +func resolveIncludes(p *model.Pipeline, includes []any, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { var warnings []IncludeWarning - for _, inc := range p.Include { + var extWarnings []ExtendWarning + + for _, inc := range includes { entry, ok := normaliseInclude(inc) if !ok { continue } if project, _ := entry["project"].(string); project != "" { - warnings = append(warnings, resolveProjectInclude(p, entry, project, cfg)...) + w, ew := resolveProjectInclude(p, entry, project, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) continue } if compRef, _ := entry["component"].(string); compRef != "" { - if w, ok := resolveComponentInclude(p, compRef, cfg); ok { + w, ew, hadErr := resolveComponentInclude(p, compRef, cfg, rootDir, visited) + if hadErr { warnings = append(warnings, w) } + extWarnings = append(extWarnings, ew...) continue } - // local, remote, template — resolved by GitLab at runtime, skip silently. + if local, _ := entry["local"].(string); local != "" { + w, ew := resolveLocalInclude(p, local, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) + continue + } + + // remote, template — resolved by GitLab at runtime, skip silently. } - return warnings + return warnings, extWarnings +} + +// resolveLocalInclude reads a local file from disk (paths are always relative +// to the repository root, with or without a leading slash), recursively +// resolves its own includes, and merges it into p. +func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { + relPath := strings.TrimPrefix(rawPath, "/") + absPath := filepath.Join(rootDir, relPath) + label := "local " + rawPath + + if visited[absPath] { + return nil, nil + } + visited[absPath] = true + + data, err := os.ReadFile(absPath) + if err != nil { + return []IncludeWarning{{Label: label, Err: err}}, nil + } + + included, err := model.ParseBytes(data) + if err != nil { + return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil + } + + // Recursively resolve the included file's own includes first, merging + // everything into `included` before we merge it into the parent `p`. + var warnings []IncludeWarning + var extWarnings []ExtendWarning + if len(included.Include) > 0 { + w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) + } + + mergeIncluded(p, included) + return warnings, extWarnings } // resolveProjectInclude fetches all files listed under a single project: entry // and merges them into p. -func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig) []IncludeWarning { +func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { ref, _ := entry["ref"].(string) var warnings []IncludeWarning + var extWarnings []ExtendWarning for _, filePath := range includeFiles(entry) { label := fmt.Sprintf("project %s:%s", project, filePath) @@ -94,56 +162,58 @@ func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project stri continue } + if len(included.Include) > 0 { + w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) + } + mergeIncluded(p, included) } - return warnings + return warnings, extWarnings } // resolveComponentInclude fetches a CI/CD catalog component and merges it into p. -// Returns (warning, true) if something went wrong; (zero, false) on success. -func resolveComponentInclude(p *model.Pipeline, ref string, cfg fetcher.GitLabConfig) (IncludeWarning, bool) { +// Returns (warning, extWarnings, true) if something went wrong; (zero, nil, false) on success. +func resolveComponentInclude(p *model.Pipeline, ref string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) (IncludeWarning, []ExtendWarning, bool) { label := "component " + ref - // Variable interpolation (e.g. $CI_SERVER_FQDN) cannot be resolved locally. if strings.ContainsRune(ref, '$') { return IncludeWarning{ Label: label, Err: fmt.Errorf("component reference contains a CI variable that cannot be resolved at lint time"), - }, true + }, nil, true } host, project, component, version, err := parseComponentRef(ref) if err != nil { - return IncludeWarning{Label: label, Err: err}, true + return IncludeWarning{Label: label, Err: err}, nil, true } hostCfg := cfg.ForHost(host) data, err := fetchComponentFile(hostCfg, project, component, version) if err != nil { - return IncludeWarning{Label: label, Err: err}, true + return IncludeWarning{Label: label, Err: err}, nil, true } included, err := model.ParseBytes(data) if err != nil { - return IncludeWarning{Label: label, Err: fmt.Errorf("parsing component YAML: %w", err)}, true + return IncludeWarning{Label: label, Err: fmt.Errorf("parsing component YAML: %w", err)}, nil, true + } + + var extWarnings []ExtendWarning + if len(included.Include) > 0 { + _, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited) + extWarnings = append(extWarnings, ew...) } mergeIncluded(p, included) - return IncludeWarning{}, false + return IncludeWarning{}, extWarnings, false } // parseComponentRef parses a CI/CD component reference of the form: // // <host>/<project-path>/<component-name>@<version> -// -// The host is the GitLab instance FQDN. The last path segment before @ is the -// component name; everything between host and component name is the project path. -// -// Examples: -// -// gitlab.com/components/golang/build@v1.0.0 -// gitlab.example.com/my-org/ci-templates/lint@main -// gitlab.com/components/secret-detection/secret-detection@~latest func parseComponentRef(ref string) (host, project, component, version string, err error) { atIdx := strings.LastIndex(ref, "@") if atIdx < 0 || atIdx == len(ref)-1 { @@ -154,7 +224,6 @@ func parseComponentRef(ref string) (host, project, component, version string, er path := ref[:atIdx] parts := strings.Split(path, "/") - // Minimum: host + at least one project segment + component = 3 parts. if len(parts) < 3 { err = fmt.Errorf("component reference %q must be <host>/<project>/<component>@<version>", ref) return @@ -167,9 +236,6 @@ func parseComponentRef(ref string) (host, project, component, version string, er } // fetchComponentFile fetches a component's template YAML from a GitLab project. -// GitLab supports two layouts; both are attempted: -// - templates/<component>.yml (single-file component) -// - templates/<component>/template.yml (directory component) func fetchComponentFile(cfg fetcher.GitLabConfig, project, component, version string) ([]byte, error) { primary := "templates/" + component + ".yml" data, err := cfg.FetchFile(project, primary, version) @@ -180,14 +246,12 @@ func fetchComponentFile(cfg fetcher.GitLabConfig, project, component, version st fallback := "templates/" + component + "/template.yml" data2, err2 := cfg.FetchFile(project, fallback, version) if err2 != nil { - // Return the primary error — it refers to the canonical path. return nil, fmt.Errorf("%v (also tried %s: %v)", err, fallback, err2) } return data2, nil } // normaliseInclude converts a raw include: list element to a string-keyed map. -// An include: value can be a plain string (shorthand local) or a map. func normaliseInclude(raw any) (map[string]any, bool) { switch v := raw.(type) { case map[string]any: @@ -199,7 +263,6 @@ func normaliseInclude(raw any) (map[string]any, bool) { } // includeFiles returns the list of file paths from an include entry. -// The file: key may be a single string or a list of strings. func includeFiles(entry map[string]any) []string { raw, ok := entry["file"] if !ok { diff --git a/samba-testdata/.gitlab-ci-coverage-runners.yml b/samba-testdata/.gitlab-ci-coverage-runners.yml new file mode 100644 index 0000000..331c5d2 --- /dev/null +++ b/samba-testdata/.gitlab-ci-coverage-runners.yml @@ -0,0 +1,4 @@ +include: + - /.gitlab-ci-default-runners.yml + +# Currently we're happy with the defaults diff --git a/samba-testdata/.gitlab-ci-coverage.yml b/samba-testdata/.gitlab-ci-coverage.yml new file mode 100644 index 0000000..8a9ded8 --- /dev/null +++ b/samba-testdata/.gitlab-ci-coverage.yml @@ -0,0 +1,12 @@ +# This is just used for the scheduled pipelines in the +# https://gitlab.com/samba-team/samba configuration +# + +variables: + SAMBA_CI_FLAVOR: "coverage" + # "--enable-coverage" or "" + SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage" + +include: + - /.gitlab-ci-coverage-runners.yml + - /.gitlab-ci-main.yml diff --git a/samba-testdata/.gitlab-ci-default-runners.yml b/samba-testdata/.gitlab-ci-default-runners.yml new file mode 100644 index 0000000..bdc504a --- /dev/null +++ b/samba-testdata/.gitlab-ci-default-runners.yml @@ -0,0 +1,30 @@ +# From https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html +# +# ... +# +# Runner Tag vCPUs Memory Storage +# saas-linux-small-amd64 2 8 GB 25 GB +# +# Our current private runner 'docker', 'samba-ci-private', 'shared' and +# 'ubuntu2204'. It runs with an ubuntu2204 kernel (5.15) and provides an +# ext4 filesystem, 2 CPU and 4 GB (shared tag) 8G (samba-ci-private tag) RAM. +# + +.shared_runner_build: + # We use saas-linux-small-amd64 shared runners by default. + # We avoid adding explicit tags for them in order + # to work with potential changes in future + # + # In order to generate valid yaml, we define a dummy variable... + variables: + SAMBA_SHARED_RUNNER_BUILD_DUMMY_VARIABLE: shared_runner_build + +.shared_runner_test: + # We use saas-linux-small-amd64 shared runners by default. + extends: .shared_runner_build + +.private_runner_test: + # We use our private runner only for special tests + tags: + - docker + - samba-ci-private diff --git a/samba-testdata/.gitlab-ci-default.yml b/samba-testdata/.gitlab-ci-default.yml new file mode 100644 index 0000000..e608918 --- /dev/null +++ b/samba-testdata/.gitlab-ci-default.yml @@ -0,0 +1,10 @@ +variables: + SAMBA_CI_FLAVOR: "default" + # "--enable-coverage" or "" + # See .gitlab-ci-coverage.yml + SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "" + AUTOBUILD_SKIP_SAMBA_O3: "0" + +include: + - /.gitlab-ci-default-runners.yml + - /.gitlab-ci-main.yml diff --git a/samba-testdata/.gitlab-ci-main.yml b/samba-testdata/.gitlab-ci-main.yml new file mode 100644 index 0000000..978165b --- /dev/null +++ b/samba-testdata/.gitlab-ci-main.yml @@ -0,0 +1,688 @@ +# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options + +# Stages explained +# +# images: Build the images with the bootstrap script +# build_first: Build a few things first to find silly errors (fast job) +# (don't pay for 35 machines until something compiles) +# build: The main parallel job +# (keep these to 1hour as we are billed per hour) +# test_only: Tests using the build from prior stages, these typically +# have an explicit dependency defined to a specific build job, +# which means that start as soon as the build job finished. +# test_private: Like test_only, but running on private runners +# report: Code coverage reporting + +stages: + - images + - build_first + - build + - test_only + - test_private + - report + +variables: + # We want to be resilient to runner failures + ARTIFACT_DOWNLOAD_ATTEMPTS: "3" + EXECUTOR_JOB_SECTION_ATTEMPTS: "3" + GET_SOURCES_ATTEMPTS: "3" + RESTORE_CACHE_ATTEMPTS: "3" + # + GIT_STRATEGY: fetch + GIT_DEPTH: "3" + # + # Use GZip by default, it is fast and is good enough. Other options include --xz + + SAMBA_TESTBASE_TAR_OPTIONS: -z + + # + # we run autobuild.py inside a samba CI docker image located on gitlab's registry + # overwrite this variable if you want use your own image registry. + # + # Or better ask for access to the shared development repository, see + # https://wiki.samba.org/index.php/Samba_CI_on_gitlab#Getting_Access + # + SAMBA_CI_CONTAINER_REGISTRY: registry.gitlab.com/samba-team/devel/samba + # + # Set this to the contents of bootstrap/sha1sum.txt + # which is generated by bootstrap/template.py --render + # + SAMBA_CI_CONTAINER_TAG: e494a8092a6d0e794223f56ddb2ffbaf76402cf6 + # + # We use the ubuntu2204 image as default as + # it matches what we have on atb-devel-224 + # + SAMBA_CI_CONTAINER_IMAGE: ubuntu2204 + # + # The following images are available + # Please see the samba-o3 sections at the end of this file! + # We should run that for each available image + # + SAMBA_CI_CONTAINER_IMAGE_ubuntu2204: ubuntu2204 + SAMBA_CI_CONTAINER_IMAGE_ubuntu2404: ubuntu2404 + SAMBA_CI_CONTAINER_IMAGE_ubuntu2604: ubuntu2604 + SAMBA_CI_CONTAINER_IMAGE_debian11: debian11 + SAMBA_CI_CONTAINER_IMAGE_debian11_32bit: debian11-32bit + SAMBA_CI_CONTAINER_IMAGE_debian12: debian12 + SAMBA_CI_CONTAINER_IMAGE_opensuse160: opensuse160 + SAMBA_CI_CONTAINER_IMAGE_rocky8: rocky8 + SAMBA_CI_CONTAINER_IMAGE_centos9s: centos9s + SAMBA_CI_CONTAINER_IMAGE_fedora43: fedora43 + +include: + # The image creation details are specified in a separate file + # See bootstrap/README.md for details + - 'bootstrap/.gitlab-ci.yml' + +.shared_runner_build_image: + extends: .shared_runner_build + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE} + image: ${SAMBA_CI_CONTAINER_REGISTRY}/samba-ci-${SAMBA_CI_JOB_IMAGE}:${SAMBA_CI_CONTAINER_TAG} + +.shared_template: + extends: .shared_runner_build_image + # All Samba jobs are interruptible, this avoids burning CPU when a + # newer branch is pushed. + interruptible: true + timeout: 2h + + # Otherwise we run twice, once on push and once on MR + # https://forum.gitlab.com/t/new-rules-syntax-and-detached-pipelines/37292 + rules: + - if: $CI_MERGE_REQUEST_ID + when: never + - when: on_success + + variables: + AUTOBUILD_JOB_NAME: $CI_JOB_NAME + stage: build + cache: + key: ccache.${CI_JOB_NAME}.${SAMBA_CI_JOB_IMAGE}.${SAMBA_CI_FLAVOR} + paths: + - ccache + + # This is overridden in many cases, but ensures none of the other + # main jobs start until and unless this build finishes. However + # this also ensures we do not download artifacts from any build + # unless we specifically depend on it, saving bandwidth + + needs: + - job: samba-def-build + artifacts: false + + before_script: + - uname -a + - ls -l /sys/module/ + - ls -l /sys/kernel/security/ + - if [ -e /sys/kernel/security/lsm ]; then cat /sys/kernel/security/lsm ; echo; fi + - if [ -e /proc/config.gz ]; then sudo zcat /proc/config.gz; echo; fi + - lsb_release -a + - cat /etc/os-release + - id + - cat /proc/self/status + - lscpu + - cat /proc/cpuinfo + - mount + - df -h + - cat /proc/swaps + - free -h + # ld will fail if coverage enabled, force link ld to ld.bfd + - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then sudo ln -sf $(which ld.bfd) $(which ld); fi + # See bootstrap/.gitlab-ci.yml how to generate a new image + - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]" + - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]" + - echo "SAMBA_CI_JOB_IMAGE[${SAMBA_CI_JOB_IMAGE}]" + - echo "CI_JOB_IMAGE[${CI_JOB_IMAGE}]" + - bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt + - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt + - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt + - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt + - diff -u bootstrap/sha1sum.txt /sha1sum.txt + - echo "${CI_COMMIT_SHA} ${CI_COMMIT_TITLE}" > /tmp/commit.txt + - export CCACHE_BASEDIR="${PWD}" + - export CCACHE_DIR="${PWD}/ccache" && mkdir -pv "$CCACHE_DIR" + - export CC="ccache cc" + - export CXX="ccache c++" + - ccache -z -M 500M + - ccache -s + # We are already running .gitlab-ci directives from this repo, remove additional checks that break our CI + - git config --global --add safe.directory '*' + after_script: + - mount + - df -h + - cat /proc/swaps + - free -h + - CCACHE_BASEDIR="${PWD}" CCACHE_DIR="${PWD}/ccache" ccache -s -c + artifacts: + expire_in: 1 week + paths: + - "*.stdout" + - "*.stderr" + - "*.info" + - public + - system-info.txt + retry: + max: 2 + when: + - runner_system_failure + - stuck_or_timeout_failure + - api_failure + - runner_unsupported + - stale_schedule + - archived_failure + - scheduler_failure + - data_integrity_failure + + script: + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase + +# Ensure when adding a new job below that you also add it to +# the dependencies for 'pages' below for the code coverage page +# generation. + +others: + extends: .shared_template + script: + - script/autobuild.py pidl $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/pidl + - script/autobuild.py replace $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/replace + - script/autobuild.py talloc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/talloc + - script/autobuild.py tdb $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/tdb + - script/autobuild.py tevent $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/tevent + - script/autobuild.py samba-xc $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/samba-xc + - script/autobuild.py docs-xml $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase/docs-xml + - make -C coverity + +.shared_template_build_only: + extends: .shared_template + timeout: 2h + needs: + artifacts: + expire_in: 1 week + paths: + - "*.stdout" + - "*.stderr" + - "*.info" + - system-info.txt + - samba-testbase.tar + script: + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase + # On success we need to pack everything into an artifacts file + # which needs to be in the git checkout. + # As tar doesn't handle hardlink of read-only files, + # we remember the acls and add write permissions + # before creating the archive. The consumer will apply + # the acls again. + - cp -a /sha1sum.txt /builds/samba-testbase/image-sha1sum.txt + - cp -a /tmp/commit.txt /builds/samba-testbase/commit.txt + - ln -s /builds/samba-testbase/${AUTOBUILD_JOB_NAME}/ /builds/samba-testbase/build_subdir_link + - pushd /builds && getfacl -R samba-testbase > samba-testbase.acl.dump && popd + - chmod -R +w /builds/samba-testbase + - mv /builds/samba-testbase.acl.dump /builds/samba-testbase/ + - tar $SAMBA_TESTBASE_TAR_OPTIONS -cf samba-testbase.tar /builds/samba-testbase + - ls -la samba-testbase.tar + - sha1sum samba-testbase.tar + +.shared_template_test_only: + extends: + - .shared_template + - .shared_runner_test + stage: test_only + script: + # Print the Kerberos version to check we ended up with the right one + # in the runner. We do not have configure output to recognize it + # otherwise. + - if [ -x "$(command -v krb5-config)" ]; then krb5-config --version; fi + # We unpack the artifacts file created by the .shared_template_build_only + # run we depend on + - ls -la samba-testbase.tar + - sha1sum samba-testbase.tar + - tar $SAMBA_TESTBASE_TAR_OPTIONS -xf samba-testbase.tar -C / + - diff -u /builds/samba-testbase/image-sha1sum.txt /sha1sum.txt + - diff -u /builds/samba-testbase/commit.txt /tmp/commit.txt + - mv /builds/samba-testbase/samba-testbase.acl.dump /builds/samba-testbase.acl.dump + - pushd /builds && setfacl --restore=/builds/samba-testbase.acl.dump && popd + - ls -la /builds/samba-testbase/ + - ls -la /builds/samba-testbase/build_subdir_link + - ls -la /builds/samba-testbase/build_subdir_link/ + - if [ -n "$SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE" ]; then find /builds/samba-testbase/build_subdir_link/ -type d -printf "'%p'\n" | xargs chmod u+w; fi + - ls -la /builds/samba-testbase/build_subdir_link/ + # gitlab predefines CI_JOB_NAME for each job. The gitlab job usually matches the + # autobuild name, which means we can define a default template that runs most autobuild jobs + - script/autobuild.py $AUTOBUILD_JOB_NAME $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE --skip-dependencies --verbose --nocleanup --keeplogs --tail --full-testbase /builds/samba-testbase + +samba-def-build: + extends: .shared_template_build_only + stage: build_first + +.needs_samba-def-build: + extends: .shared_template_test_only + needs: + - job: samba-def-build + artifacts: true + - job: samba-codecheck + +samba-mit-build: + extends: .shared_template_build_only + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora43} + stage: build_first + +.needs_samba-mit-build: + extends: .shared_template_test_only + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora43} + needs: + - job: samba-mit-build + artifacts: true + - job: samba-codecheck + +samba-h5l-build: + extends: .shared_template_build_only + +.needs_samba-h5l-build: + extends: .shared_template_test_only + needs: + - job: samba-h5l-build + artifacts: true + +samba-without-smb1-build: + extends: .shared_template_build_only + +.needs_samba-without-smb1-build: + extends: .shared_template_test_only + needs: + - job: samba-without-smb1-build + artifacts: true + +samba-nt4-build: + extends: .shared_template_build_only + +.needs_samba-nt4-build: + extends: .shared_template_test_only + needs: + - job: samba-nt4-build + artifacts: true + +samba-no-opath-build: + extends: .shared_template_build_only + +.needs_samba-no-opath-build: + extends: .shared_template_test_only + needs: + - job: samba-no-opath-build + artifacts: true + +samba: + extends: .shared_template + +samba-mitkrb5: + extends: .shared_template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora43} + +samba-minimal-smbd: + extends: .shared_template + +samba-nopython: + extends: .shared_template + +samba-admem: + extends: .needs_samba-def-build + +samba-ad-dc-2: + extends: .needs_samba-def-build + +samba-ad-dc-3: + extends: .needs_samba-def-build + +samba-ad-dc-4a: + extends: .needs_samba-def-build + +samba-ad-dc-4b: + extends: .needs_samba-def-build + +samba-ad-dc-5: + extends: .needs_samba-def-build + +samba-ad-dc-6: + extends: .needs_samba-def-build + +samba-ad-back1: + extends: .needs_samba-def-build + +samba-ad-back2: + extends: .needs_samba-def-build + +samba-schemaupgrade: + extends: .needs_samba-def-build + +samba-libs: + extends: .shared_template + +samba-fuzz: + extends: .shared_template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2404} + +ctdb: + extends: .shared_template + +samba-ctdb: + extends: .shared_template + +samba-ad-dc-ntvfs: + extends: .needs_samba-def-build + +samba-admem-mit: + extends: .needs_samba-mit-build + +samba-addc-mit-4a: + extends: .needs_samba-mit-build + +samba-addc-mit-4b: + extends: .needs_samba-mit-build + +# This task is run first to ensure we compile before we start the +# main run as it is the fastest full compile of Samba. +samba-fips: + extends: .shared_template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora43} + +samba-codecheck: + extends: .shared_template + needs: + stage: build_first + +.private_test_only: + extends: .private_runner_test + stage: test_private + rules: + # See above, to avoid a duplicate CI on the MR (these rules override the others) + - if: $CI_MERGE_REQUEST_ID + when: never + + # These jobs are only run if the gitlab repo has private runners available. + # To enable private jobs, you must add the following var and value to + # your gitlab repo by navigating to: + # settings -> CI/CD -> Environment variables + - if: $SUPPORT_PRIVATE_TEST == "yes" + +.needs_ext4_support: + # All runners provide an ext4 filesystem + # + # Note: we don't use + # extends: .shared_template_test_only + # as that somehow resets the needs section + # and generates problems for something + # like this (which is used below) + # + # .needs_samba-SOME-build-ext4: + # extends: + # - .needs_samba-SOME-build + # - .needs_ext4_support + # + # So we only set stage again instead... + stage: test_only + +.needs_5_15_kernel: + # Our private runners are based on + # ubuntu2204 with a 5.15 kernel. + # + # And they also provide an ext4 filesystem + extends: .private_test_only + +.needs_samba-def-build-ext4: + extends: + - .needs_samba-def-build + - .needs_ext4_support + +.needs_samba-mit-build-ext4: + extends: + - .needs_samba-mit-build + - .needs_ext4_support + +.needs_samba-h5l-build-ext4: + extends: + - .needs_samba-h5l-build + - .needs_ext4_support + +.needs_samba-without-smb1-build-5_15: + # Currently this doesn't strictly + # require a kernel >= 5.15, but only + # ext4 support. + # + # But we want to make sure that + # our private runners keep working + # and at least do a single job. + # + # In future we'll be able to run + # tests with io_uring in this + # setup, which will requires a + # 5.15 kernel in order to be useful. + extends: + - .needs_samba-without-smb1-build + - .needs_5_15_kernel + +.needs_samba-nt4-build-ext4: + extends: + - .needs_samba-nt4-build + - .needs_ext4_support + +.needs_samba-no-opath-build-ext4: + extends: + - .needs_samba-no-opath-build + - .needs_ext4_support + +samba-fileserver: + extends: .needs_samba-h5l-build-ext4 + +samba-fileserver-without-smb1: + extends: .needs_samba-without-smb1-build + +# This is a full build without the AD DC so we test the build with MIT +# Kerberos from the default system (Ubuntu 22.04 at this stage). +# Runtime behaviour checked via the ktest (static ccache and keytab) +# environment +samba-ktest-mit: + extends: .shared_template + +samba-ad-dc-1: + extends: .needs_samba-def-build-ext4 + +samba-nt4: + extends: .needs_samba-nt4-build-ext4 + +samba-addc-mit-1: + extends: .needs_samba-mit-build-ext4 + +samba-no-opath1: + extends: .needs_samba-no-opath-build-ext4 + +samba-no-opath2: + extends: .needs_samba-no-opath-build-ext4 + +# 'pages' is a special job which can publish artifacts in `public` dir to gitlab pages +pages: + extends: .shared_runner_build_image + stage: report + dependencies: # tell gitlab to download artifacts for these jobs + - others + - samba + - samba-mitkrb5 + - samba-admem + - samba-ad-dc-2 + - samba-ad-dc-3 + - samba-ad-dc-4a + - samba-ad-dc-4b + - samba-ad-dc-5 + - samba-ad-dc-6 + - samba-libs + - samba-minimal-smbd + - samba-nopython + - samba-fuzz + # - ctdb # TODO + - samba-ctdb + - samba-ad-dc-ntvfs + - samba-admem-mit + - samba-addc-mit-4a + - samba-addc-mit-4b + - samba-ad-back1 + - samba-ad-back2 + - samba-fileserver + - samba-fileserver-without-smb1 + - samba-ad-dc-1 + - samba-nt4 + - samba-schemaupgrade + - samba-addc-mit-1 + - samba-fips + - samba-no-opath1 + - samba-no-opath2 + - ubuntu2204-samba-o3 + script: + - ls -la *.info + - ./configure.developer + - make -j + - ls -la *.info + - lcov $(ls *.info | xargs -I{} echo -n "-a {} ") -o all.info + - ls -la *.info + - genhtml all.info --ignore-errors source --output-directory public --prefix=$(pwd) --title "coverage report for $CI_COMMIT_REF_NAME $CI_COMMIT_SHORT_SHA" + artifacts: + expire_in: 30 days + paths: + - public + only: + variables: + - $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "--enable-coverage" + +# Coverity Scan +coverity: + extends: .shared_runner_build_image + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse160} + stage: build + script: + - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz + - tar xf /tmp/coverity_tool.tgz + - ./configure.developer --with-cluster-support + - cov-analysis-linux64-*/bin/coverity capture --dir cov-int --project-dir ./ + - tar czf cov-int.tar.gz cov-int + - curl + --form token=$COVERITY_SCAN_TOKEN + --form email=$COVERITY_SCAN_EMAIL + --form file=@cov-int.tar.gz + --form version="`git describe --tags`" + --form description="CI build" + https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME + only: + refs: + - master + - schedules + variables: + - $COVERITY_SCAN_TOKEN != null + - $COVERITY_SCAN_PROJECT_NAME != null + - $COVERITY_SCAN_EMAIL != null + artifacts: + expire_in: 1 week + when: on_failure + paths: + - cov-int/*.txt + +debian11-samba-32bit: + extends: .shared_template + variables: + AUTOBUILD_JOB_NAME: samba-32bit + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian11_32bit} + +# +# We build samba-o3 on all supported distributions +# + +# This job, which matches the main CI, needs to still do coverage so +# we show the coverage on the "none" environment tests +# +# We want --enable-coverage specified here otherwise we will have a +# different set of build options on the coverage build and can fail +# when -O3 gets combined with --enable-coverage in the scheduled +# builds. + +ubuntu2204-samba-o3: + extends: .shared_template + variables: + AUTOBUILD_JOB_NAME: samba-o3 + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2204} + SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE: "--enable-coverage" + rules: + # See above, to avoid a duplicate CI on the MR (these rules override the others) + - if: $CI_MERGE_REQUEST_ID + when: never + # do not run o3 builds (which run a lot of VMs) if told not to + # (this uses the same variable as autobuild.py) + - if: $AUTOBUILD_SKIP_SAMBA_O3 == "1" + when: never + - when: on_success + +# All other jobs do not want code coverage. +.samba-o3-template: + extends: .shared_template + variables: + AUTOBUILD_JOB_NAME: samba-o3 + rules: + # See above, to avoid a duplicate CI on the MR (these rules override the others) + - if: $CI_MERGE_REQUEST_ID + when: never + # do not run o3 builds (which run a lot of VMs) if told not to + # (this uses the same variable as autobuild.py) + - if: $AUTOBUILD_SKIP_SAMBA_O3 == "1" + when: never + # do not run o3 for coverage since they are using different images + - if: $SAMBA_CI_AUTOBUILD_ENABLE_COVERAGE == "" + +ubuntu2404-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2404} + +ubuntu2604-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_ubuntu2604} + +debian11-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian11} + +debian12-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_debian12} + +opensuse160-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_opensuse160} + +rocky8-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_rocky8} + +centos9s-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_centos9s} + +fedora43-samba-o3: + extends: .samba-o3-template + variables: + SAMBA_CI_JOB_IMAGE: ${SAMBA_CI_CONTAINER_IMAGE_fedora43} + +# +# Keep the samba-o3 sections at the end ... +# diff --git a/samba-testdata/.gitlab-ci-private.yml b/samba-testdata/.gitlab-ci-private.yml new file mode 100644 index 0000000..edd0b6c --- /dev/null +++ b/samba-testdata/.gitlab-ci-private.yml @@ -0,0 +1,5 @@ +# This is just a legacy alias used by the +# https://gitlab.com/samba-team/devel/samba configuration +# +include: + - '/.gitlab-ci.yml' diff --git a/samba-testdata/.gitlab-ci.yml b/samba-testdata/.gitlab-ci.yml new file mode 100644 index 0000000..9110d6f --- /dev/null +++ b/samba-testdata/.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + - /.gitlab-ci-default.yml diff --git a/samba-testdata/.gitleaks.toml b/samba-testdata/.gitleaks.toml new file mode 100644 index 0000000..721d0d9 --- /dev/null +++ b/samba-testdata/.gitleaks.toml @@ -0,0 +1,16 @@ +# +# GitLeaks Repo Specific Configuration +# +# This allowlist is used to help Red Hat ignore false positives during its code +# scans. + +[allowlist] + paths = [ + '''docs-xml/manpages/smbstatus.1.xml''', + '''selftests/*''', + '''source3/script/tests/*''', + '''source4/dsdb/tests/*''', + '''source4/torture/*''', + '''testprogs/blackbox/*''', + '''tests/*''', + ] diff --git a/samba-testdata/bootstrap/.gitlab-ci.yml b/samba-testdata/bootstrap/.gitlab-ci.yml new file mode 100644 index 0000000..9da9d74 --- /dev/null +++ b/samba-testdata/bootstrap/.gitlab-ci.yml @@ -0,0 +1,122 @@ +--- +.build_image_template: + image: quay.io/podman/stable:latest + stage: images + tags: + # We need to make sure we only use gitlab.com + # runners and not our own runners, as our current runners + # don't allow 'docker build ...' to run. + - saas-linux-small-amd64 + variables: + SAMBA_CI_IS_BROKEN_IMAGE: "no" + SAMBA_CI_TEST_JOB: "samba-o3" + SAMBA_CI_PLATFORM: "linux/amd64" + before_script: + # install prerequisites + - dnf install -qy diffutils + # Ensure we are generating correct the container + - uname -a + - cat /etc/os-release + - echo "SAMBA_CI_CONTAINER_REGISTRY[${SAMBA_CI_CONTAINER_REGISTRY}]" + - echo "SAMBA_CI_CONTAINER_TAG[${SAMBA_CI_CONTAINER_TAG}]" + - echo "SAMBA_CI_IS_BROKEN_IMAGE[${SAMBA_CI_IS_BROKEN_IMAGE}]" + - echo "SAMBA_CI_REBUILD_IMAGES[${SAMBA_CI_REBUILD_IMAGES}]" + - echo "SAMBA_CI_REBUILD_BROKEN_IMAGES[${SAMBA_CI_REBUILD_BROKEN_IMAGES}]" + - echo "GITLAB_USER_LOGIN[${GITLAB_USER_LOGIN}]" + - echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt + - diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt + script: | + set -xueo pipefail + ci_image_name=samba-ci-${CI_JOB_NAME} + podman build --platform ${SAMBA_CI_PLATFORM} --tag ${ci_image_name} --build-arg SHA1SUM=${SAMBA_CI_CONTAINER_TAG} bootstrap/generated-dists/${CI_JOB_NAME} + ci_image_path="${SAMBA_CI_CONTAINER_REGISTRY}/${ci_image_name}" + timestamp=$(date +%Y%m%d%H%M%S) + container_hash=$(podman image inspect --format='{{ .Id }}' ${ci_image_name} | cut -c 1-9) + timestamp_tag=${SAMBA_CI_CONTAINER_TAG}-${timestamp}-${GITLAB_USER_LOGIN}-${container_hash} + samba_repo_root=/home/samba/samba + # Ensure we are generating the correct container that we expect to be in + echo "${SAMBA_CI_CONTAINER_TAG}" > /tmp/sha1sum-tag.txt + diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt + podman run --volume $(pwd):${samba_repo_root} --workdir ${samba_repo_root} ${ci_image_name} \ + /bin/bash -c "echo \"${SAMBA_CI_CONTAINER_TAG}\" > /tmp/sha1sum-tag.txt; diff -u bootstrap/sha1sum.txt /tmp/sha1sum-tag.txt" + podman run --volume $(pwd):${samba_repo_root} --workdir ${samba_repo_root} ${ci_image_name} \ + diff -u bootstrap/sha1sum.txt /sha1sum.txt + podman run --volume $(pwd):${samba_repo_root} --workdir ${samba_repo_root} ${ci_image_name} \ + bootstrap/template.py --sha1sum > /tmp/sha1sum-template.txt + diff -u bootstrap/sha1sum.txt /tmp/sha1sum-template.txt + # run smoke test with samba-o3 or samba-fuzz + podman run --volume $(pwd):/src:ro ${ci_image_name} \ + /bin/bash -c "git config --global --add safe.directory /src/.git && git clone /src samba && cd samba && export PKG_CONFIG_PATH=/usr/lib64/compat-gnutls34/pkgconfig:/usr/lib64/compat-nettle32/pkgconfig && script/autobuild.py ${SAMBA_CI_TEST_JOB} --verbose --nocleanup --keeplogs --tail --testbase /tmp/samba-testbase" + podman tag ${ci_image_name} ${ci_image_path}:${SAMBA_CI_CONTAINER_TAG} + podman tag ${ci_image_name} ${ci_image_path}:${timestamp_tag} + # We build all images, but only upload is it's not marked as broken + test x"${SAMBA_CI_IS_BROKEN_IMAGE}" = x"yes" || { \ + podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY; \ + podman push ${ci_image_path}:${SAMBA_CI_CONTAINER_TAG}; \ + podman push ${ci_image_path}:${timestamp_tag}; \ + } + echo "Success for ${ci_image_path}:${timestamp_tag}" + test x"${SAMBA_CI_IS_BROKEN_IMAGE}" = x"no" || { \ + echo "The image ${CI_JOB_NAME} is marked as broken and should have failed!"; \ + echo "Replace .build_image_template_force_broken with .build_image_template!"; \ + echo "Add a .samba-o3-template section at the end of the main .gitlab-ci.yml!"; \ + /bin/false; \ + } + only: + variables: + # + # You need a custom pipeline which passes + # SAMBA_CI_REBUILD_IMAGES="yes". + # + # https://gitlab.com/samba-team/devel/samba/pipelines/new + # + - $SAMBA_CI_REBUILD_IMAGES == "yes" + +.build_image_template_force_broken: + extends: .build_image_template + variables: + SAMBA_CI_IS_BROKEN_IMAGE: "yes" + only: + variables: + # + # You need a custom pipeline which passes + # SAMBA_CI_REBUILD_BROKEN_IMAGES="yes" + # in order to build broken images for debugging + # + # https://gitlab.com/samba-team/devel/samba/pipelines/new + # + - $SAMBA_CI_REBUILD_BROKEN_IMAGES == "yes" + +ubuntu2204: + extends: .build_image_template + +ubuntu2404: + extends: .build_image_template + +ubuntu2604: + extends: .build_image_template + +debian11: + extends: .build_image_template + +debian12: + extends: .build_image_template + +fedora43: + extends: .build_image_template + +debian11-32bit: + extends: .build_image_template + variables: + SAMBA_CI_TEST_JOB: "samba-32bit" + SAMBA_CI_PLATFORM: "linux/i386" + +rocky8: + extends: .build_image_template + +centos9s: + extends: .build_image_template + +opensuse160: + extends: .build_image_template +