From ff0d9b51f3158fd4ffbfb0bf9de61380dfabb92a Mon Sep 17 00:00:00 2001 From: k3nny Date: Sun, 7 Jun 2026 20:13:03 +0200 Subject: [PATCH] feat(gitlab-sim): :rocket: ajout graph --- .gitignore | 3 + CHANGELOG.md | 19 ++ CLAUDE.md | 234 +++++++++++++++----- LICENSE | 185 ++++++++++++++++ README.md | 112 +++++++++- Taskfile.yml | 4 + internal/fetcher/gitlab.go | 152 +++++++++++++ internal/graph/includes.go | 127 +++++++++++ internal/graph/pipeline.go | 172 +++++++++++++++ internal/graph/render.go | 363 ++++++++++++++++++++++++++++++++ internal/linter/linter.go | 6 +- internal/model/parser.go | 4 + internal/resolver/includes.go | 245 +++++++++++++++++++++ testdata/includes_component.yml | 30 +++ testdata/includes_project.yml | 33 +++ 15 files changed, 1638 insertions(+), 51 deletions(-) create mode 100644 LICENSE create mode 100644 internal/fetcher/gitlab.go create mode 100644 internal/graph/includes.go create mode 100644 internal/graph/pipeline.go create mode 100644 internal/graph/render.go create mode 100644 internal/resolver/includes.go create mode 100644 testdata/includes_component.yml create mode 100644 testdata/includes_project.yml diff --git a/.gitignore b/.gitignore index 0b950ca..c7c3ba7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ gitlab-sim.exe dist/ bin/ +# Graph output directory +gitlab-sim-out/ + # Go test & coverage artifacts *.test *.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed6547..d161f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,25 @@ This project does not yet use semantic versioning; entries are listed under `[Un ### Added +- **Graph output (`--graph`)** — visualises the pipeline instead of running lint rules: + - `--graph includes` — [Mermaid](https://mermaid.js.org) flowchart of include dependencies written to stdout; one node per `include:` entry (project, component, local, remote, template), colour-coded by type; pipe to a `.mmd` file or paste into [mermaid.live](https://mermaid.live) + - `--graph pipeline` — GitLab CI-style SVG/PNG pipeline graph written to a timestamped file in `--graph-out` (default: `gitlab-sim-out/`); jobs rendered as white chip cards with a coloured status indicator (blue: regular, orange: manual, purple: trigger, amber: delayed); DAG mode draws job-to-job Bézier arrows when any job has `needs:`, classic mode draws L-shaped connectors between stage columns; converted to PNG automatically when `rsvg-convert`, `inkscape`, or `magick` is available + - `--graph all` — include Mermaid to stdout, pipeline file path to stderr + - New `internal/graph` package (`includes.go`, `pipeline.go`, `render.go`); no new external dependencies + +- **CI/CD catalog component resolution** — resolves `include: component:` references from the GitLab CI/CD Catalog: + - Reference format: `//@` (host determines which GitLab instance is queried) + - Tries single-file layout (`templates/.yml`) then directory layout (`templates//template.yml`) automatically + - Public catalog components are fetched without authentication (no token required) + - References containing CI variables (e.g. `$CI_SERVER_FQDN`) are skipped with a warning — they cannot be resolved at lint time + - Jobs imported from a component may use `$[[ inputs.xxx ]]` input placeholders in stage names; the stage validation check is skipped for those values rather than producing false positives +- **Remote project include resolution** — fetches `include: project:` templates from the GitLab REST API before linting; jobs from remote templates are merged into the pipeline so `extends:`, `needs:`, and `dependencies:` references can be validated across file boundaries + - Token auto-discovery: `GITLAB_TOKEN` (→ `PRIVATE-TOKEN` header) → `CI_JOB_TOKEN` (→ `JOB-TOKEN` header) → `GITLAB_PRIVATE_TOKEN` + - Instance URL auto-discovery: `--gitlab-url` flag → `CI_SERVER_URL` → `GITLAB_URL` → `https://gitlab.com` + - `--token` and `--gitlab-url` CLI flags for explicit overrides + - `file:` accepts both string and list-of-strings forms + - Project includes require a token; they are skipped with a `WARNING` when none is configured + - Component includes attempt the fetch unauthenticated first; a `WARNING` is emitted only on failure - **Comprehensive keyword validation** — checks for all major GitLab CI YAML keywords based on the official docs: - `when` valid values: `on_success`, `on_failure`, `always`, `manual`, `delayed`, `never` - `start_in` only allowed when `when: delayed`; error if set without it diff --git a/CLAUDE.md b/CLAUDE.md index b747c15..087dd5e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,47 +1,187 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -gitlab-sim is a tool aiming to simulate and lint gitlab-ci pipelines locally and produce a graph. - -## Commit Message Guidelines - -**IMPORTANT: This project uses [Conventional Commits](https://www.conventionalcommits.org/) format.** - -All commit messages must follow this format: -``` -(): - -[optional body] - -[optional footer(s)] -``` - -**Types:** -- `feat`: A new feature -- `fix`: A bug fix -- `docs`: Documentation only changes -- `refactor`: Code change that neither fixes a bug nor adds a feature -- `test`: Adding missing tests or correcting existing tests -- `chore`: Changes to build process or auxiliary tools -- `perf`: Performance improvements -- `style`: Code style changes (formatting, missing semicolons, etc.) - -**Scopes (commonly used):** -- `auth`: Authentication/authorization changes -- `security`: Security-related changes -- `gui`: Web GUI changes -- `api`: API changes -- `readme`: README.md changes -- `claude`: CLAUDE.md changes -- `core`: Core library changes - -**Breaking Changes:** -Add `!` after type/scope for breaking changes: -- `feat(api)!: remove deprecated endpoint` - -**Note:** Always include a scope in parentheses, even for documentation changes. - -When Claude Code creates commits, it will automatically follow this format. +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +`gitlab-sim` is a CLI tool that validates and lints `.gitlab-ci.yml` pipelines locally, without a GitLab server. It resolves `extends:` inheritance, fetches remote `include: project:` templates and `include: component:` catalog entries, then runs a set of lint rules over the fully-merged pipeline. + +**Goals:** catch misconfigured pipelines early in a developer's workflow, before pushing to GitLab. Eventual goal: produce a visual graph of the pipeline DAG. + +**Non-goals:** full GitLab CI emulation, job execution, secret resolution, or runner management. + +--- + +## Architecture + +### Data flow + +``` +model.Parse(path) // two-pass YAML → *Pipeline + └─ resolver.ResolveIncludes // fetch project: / component: includes, merge + └─ resolver.Resolve // resolve extends: inheritance chains + └─ linter.Lint // run all lint rules → []Finding +``` + +This order is intentional and must be preserved: includes must be merged before `extends:` resolution so that jobs defined in remote templates are available as base templates; `extends:` must be resolved before linting so that derived jobs carry their full merged definition. + +### Package responsibilities + +| Package | Role | +|---|---| +| `internal/model` | Data structures (`Pipeline`, `Job`, `Rule`, …) and YAML parser | +| `internal/linter` | Lint rules; each rule returns `[]Finding` | +| `internal/resolver` | `extends:` resolution and `include:` merging | +| `internal/fetcher` | GitLab REST API client (token auth, file fetch) | +| `internal/graph` | Mermaid graph generators (include dependencies, pipeline jobs) | +| `cmd/gitlab-sim` | CLI entrypoint, flag parsing, output formatting | + +### Two-pass YAML parser (`internal/model/parser.go`) + +`ParseBytes` runs two `yaml.Unmarshal` passes over the same input: + +1. **Raw pass** — into `map[string]yaml.Node` to collect every top-level key without type assumptions. +2. **Typed pass** — into `*Pipeline` to decode reserved keys (`stages`, `variables`, `default`, `include`, `workflow`, `spec`) into typed structs. + +Keys that survive the raw pass and are not in `ReservedKeys` are decoded individually as `Job` structs. This approach handles the open-ended job-name namespace without requiring a catch-all map in the typed struct. + +**When adding a new top-level reserved key:** add it to `model.ReservedKeys` and add the corresponding typed field to `Pipeline`. + +### `any` fields in `Job` + +Many `Job` fields (e.g. `Artifacts`, `Cache`, `Image`, `Trigger`) are typed as `any` because GitLab CI allows both scalar and map forms. The linter type-asserts these at check time. Follow the existing pattern in `internal/linter/keywords.go`: + +```go +switch v := job.Artifacts.(type) { +case map[string]any: + // map form +case nil: + // not set +} +``` + +Never change an `any` field to a concrete struct type unless the GitLab CI spec guarantees only one form — doing so will silently drop the other form during YAML decode. + +--- + +## Adding a New Lint Rule + +1. Add the check function to the appropriate file in `internal/linter/`: + - **Per-job keyword constraint** → `keywords.go`, called from `checkJobKeywords` + - **Cross-job graph rule** (needs, deps, extends) → dedicated file (`needs.go`, `dependencies.go`) + - **Pipeline-level rule** → `linter.go`, called from `Lint` + +2. Return `[]Finding` with the correct `Severity` (`Error` or `Warning`) and a `Job` field set when the finding is job-scoped, empty for pipeline-level. + +3. Add a testdata fixture that triggers the new rule, and add it to the `validate` task in `Taskfile.yml`. + +4. Document the rule in the lint rules table in `README.md`. + +### Finding severity guide + +| Severity | Use when | +|---|---| +| `Error` | The pipeline will definitely fail or behave incorrectly | +| `Warning` | Deprecated usage, best-practice deviation, or a condition that *may* be wrong | + +--- + +## Adding a New Model Field + +1. Add the field to `model.Job` with the correct `yaml:` tag. +2. If the field has constrained values, add a validity map to `internal/linter/keywords.go` (e.g. `validJobWhen`). +3. Add the check to `checkJobKeywords` and call it from there. +4. Add both a valid and an invalid fixture case to `testdata/keywords_valid.yml` / `testdata/keywords_invalid.yml`. + +--- + +## Go Code Style + +- **Format:** `gofmt` (enforced via `go vet` in the `ci` task). Never commit unformatted code. +- **Imports:** group stdlib / external / internal with a blank line between groups. +- **Errors:** always wrap with context using `fmt.Errorf("doing X: %w", err)`. Use `%w` (not `%s`) to preserve the error chain. +- **No panics** in library code (`internal/`). Panics are acceptable only in `main()` for truly unrecoverable startup failures — prefer returning errors. +- **No global mutable state.** All configuration flows through function parameters or structs (see `fetcher.GitLabConfig`). +- **Exported symbols** must have a doc comment. Unexported helpers do not need one unless the logic is non-obvious. +- **Table-driven tests** are the default style for unit tests. Group cases in a `[]struct{ name, input, want }` slice and range over them. +- **Dependencies:** prefer the standard library. The only current external dependency is `gopkg.in/yaml.v3`; keep it that way unless there is a compelling reason. +- `go.sum` must always be committed alongside `go.mod`. + +--- + +## Commit Message Guidelines + +**IMPORTANT: This project uses [Conventional Commits](https://www.conventionalcommits.org/) format.** + +All commit messages must follow this format: +``` +(): + +[optional body] + +[optional footer(s)] +``` + +**Types:** +- `feat`: A new feature +- `fix`: A bug fix +- `docs`: Documentation only changes +- `refactor`: Code change that neither fixes a bug nor adds a feature +- `test`: Adding missing tests or correcting existing tests +- `chore`: Changes to build process or auxiliary tools +- `perf`: Performance improvements +- `style`: Code style changes (formatting, missing semicolons, etc.) + +**Scopes:** +- `linter`: changes to lint rules (`internal/linter/`) +- `model`: data structures or YAML parser (`internal/model/`) +- `resolver`: extends/include resolution (`internal/resolver/`) +- `fetcher`: GitLab API client (`internal/fetcher/`) +- `graph`: Mermaid graph generators (`internal/graph/`) +- `cli`: CLI entrypoint (`cmd/gitlab-sim/`) +- `testdata`: fixture files only +- `docs`: README, CHANGELOG, or other documentation +- `build`: Taskfile, go.mod, go.sum, CI config +- `claude`: CLAUDE.md changes + +**Breaking Changes:** +Add `!` after type/scope for breaking changes (e.g. changed CLI flags, removed output fields): +``` +feat(cli)!: rename --token to --api-token +``` + +**Note:** Always include a scope in parentheses, even for documentation changes. + +--- + +## Development Workflow + +```bash +task ci # full check: vet → test → build → validate (run before every commit) +task build # compile the binary +task test # run Go unit tests +task lint-go # go vet only +task validate # run the binary against all testdata fixtures +task clean # remove build artifacts +``` + +`task ci` must pass before any commit is created. + +### Testdata fixtures + +Every fixture in `testdata/` is run by `task validate`. Files whose expected behaviour is *clean* (exit 0) are listed with `ignore_error: false`; files that are expected to produce errors (exit 1) use `ignore_error: true`. Keep both categories; do not use `ignore_error: true` for a fixture that is supposed to be clean. + +--- + +## Environment Variables (for manual testing) + +| Variable | Purpose | +|---|---| +| `GITLAB_TOKEN` | Personal access token (`read_api` scope) | +| `CI_JOB_TOKEN` | CI/CD job token (when running inside a pipeline) | +| `GITLAB_PRIVATE_TOKEN` | Legacy PAT name (lowest priority) | +| `CI_SERVER_URL` | GitLab instance URL (takes precedence over `GITLAB_URL`) | +| `GITLAB_URL` | GitLab instance URL fallback | + +Token resolution order (first non-empty wins): `GITLAB_TOKEN` → `CI_JOB_TOKEN` → `GITLAB_PRIVATE_TOKEN`. +URL resolution order: `--gitlab-url` flag → `CI_SERVER_URL` → `GITLAB_URL` → `https://gitlab.com`. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8c1756b --- /dev/null +++ b/LICENSE @@ -0,0 +1,185 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship made available under + the License, as indicated by a copyright notice that is included in + or attached to the work (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean, as submitted to the Licensor for inclusion + in the Work by the copyright owner or by an individual or Legal Entity + authorized to submit on behalf of the copyright owner. For the purposes + of this definition, "submitted" means any form of electronic, verbal, + or written communication sent to the Licensor or its representatives, + including but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are managed + by, or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or designated in writing by the copyright owner as "Not a + Contribution." + + "Contributor" shall mean Licensor and any Legal Entity on behalf of + whom a Contribution has been received by the Licensor and included + within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by the combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a cross-claim + or counterclaim in a lawsuit) alleging that the Work or any + Contributor's patent claim(s) is infringed by making, using, selling, + offering for sale, importing, or otherwise transferring the Work, the + License granted to You under this section shall terminate as of the + date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative + Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, You must include a readable copy of the + attribution notices contained within such NOTICE file, in + at least one of the following places: within a NOTICE text + file distributed as part of the Derivative Works; within + the Source form or documentation, if provided along with the + Derivative Works; or, within a display generated by the + Derivative Works, if and wherever such third-party notices + normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. + You may add Your own attribution notices within Derivative + Works that You distribute, alongside or as an addendum to + the NOTICE text from the Work, provided that such additional + attribution notices cannot be construed as modifying the + License. + + You may add Your own license statement for Your modifications and + may provide additional grant of rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the + Contribution, either on an original basis or as modifications to a + Contribution. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any conditions of TITLE, + NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR + PURPOSE. You are solely responsible for determining the + appropriateness of using or reproducing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or exemplary damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or all other + commercial damages or losses), even if such Contributor has been + advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2026 K3nnyfr (alex@k3nny.fr) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 65dd449..ec1bc79 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # gitlab-sim +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) + +> **Disclaimer:** This tool was built through iterative AI-assisted development with [Claude](https://claude.ai). It is experimental, incomplete, and not intended for production use. Coverage of GitLab CI keywords is best-effort and may lag behind GitLab's evolving spec. Use it at your own discretion — no correctness guarantees are made. Contributions and bug reports are welcome. + A local tool to validate and lint `.gitlab-ci.yml` pipelines without needing a GitLab server. ## Features @@ -8,7 +12,12 @@ A local tool to validate and lint `.gitlab-ci.yml` pipelines without needing a G - **Stage validation** — every job's `stage` must be declared in `stages` - **`extends:` resolution** — resolves single and multi-level template inheritance before linting, so derived jobs are evaluated against their fully merged definition - **`needs:` DAG validation** — checks that `needs:` references exist, respect stage ordering, and contain no circular dependencies +- **`dependencies:` validation** — checks that artifact dependency references exist and are in earlier stages +- **Keyword validation** — validates constraints on `when`, `parallel`, `retry`, `allow_failure`, `trigger`, `artifacts`, `cache`, `release`, `environment`, `coverage`, `rules`, and more +- **Remote project includes** — fetches `include: project:` templates from the GitLab API so extends/needs can be validated against the full merged pipeline +- **CI/CD catalog components** — resolves `include: component:` references from the GitLab CI/CD Catalog; public components work without a token - **Deprecation warnings** — flags `only`/`except` usage in favour of `rules` +- **Graph output** — emits Mermaid diagrams for the include dependency tree and the pipeline jobs layout (DAG or classic stage ordering) ## Requirements @@ -32,11 +41,108 @@ task build ## Usage ```bash -gitlab-sim +gitlab-sim [options] ``` Exits `0` when no errors are found, `1` when at least one error is reported. +### Remote project includes + +Pipelines that include templates from other GitLab projects are supported. +Provide a token so `gitlab-sim` can fetch them: + +```bash +# personal access token (read_api scope) +GITLAB_TOKEN=glpat-xxxx gitlab-sim .gitlab-ci.yml + +# CI/CD job token (when running inside a pipeline) +CI_JOB_TOKEN=$CI_JOB_TOKEN gitlab-sim .gitlab-ci.yml + +# self-hosted GitLab +GITLAB_TOKEN=glpat-xxxx GITLAB_URL=https://gitlab.example.com gitlab-sim .gitlab-ci.yml + +# or via flags +gitlab-sim --token glpat-xxxx --gitlab-url https://gitlab.example.com .gitlab-ci.yml +``` + +**Project includes** require a token; without one they are skipped with a +warning and the rest of the pipeline is linted as-is. + +**Component includes** (`include: component: ...`) attempt the fetch +unauthenticated, so public [CI/CD Catalog](https://gitlab.com/explore/catalog) +components work without a token. A warning is emitted if the fetch fails. + +Token resolution order (first non-empty wins): + +| Source | Header used | +|--------|-------------| +| `--token` flag / `GITLAB_TOKEN` | `PRIVATE-TOKEN` | +| `CI_JOB_TOKEN` | `JOB-TOKEN` | +| `GITLAB_PRIVATE_TOKEN` | `PRIVATE-TOKEN` | + +Instance URL resolution order: `--gitlab-url` flag → `CI_SERVER_URL` → +`GITLAB_URL` → `https://gitlab.com` + +### Component reference format + +``` +//@ + +gitlab.com/components/secret-detection/secret-detection@v0.1.0 +gitlab.com/my-org/ci-catalog/lint@main +gitlab.example.com/platform/components/build@~latest +``` + +The component file is looked up in order: +1. `templates/.yml` (single-file layout) +2. `templates//template.yml` (directory layout) + +Component input parameters (`with:`) are not validated — they are resolved by +GitLab at runtime. Jobs in fetched components may use `$[[ inputs.xxx ]]` +placeholders in fields like `stage`; `gitlab-sim` skips those fields rather +than producing false positive errors. + +### Graph output + +Pass `--graph` to visualise the pipeline instead of running lint rules. + +```bash +# Include dependency graph (which files include which) → Mermaid to stdout +gitlab-sim --graph includes .gitlab-ci.yml > includes.mmd + +# GitLab-like pipeline layout → PNG (or SVG fallback) written to --graph-out dir +gitlab-sim --graph pipeline .gitlab-ci.yml +# prints the output file path, e.g.: gitlab-sim-out/pipeline-20260607-143022.png + +# Both at once: Mermaid to stdout + pipeline file path to stderr +gitlab-sim --graph all .gitlab-ci.yml > includes.mmd + +# Custom output directory +gitlab-sim --graph pipeline --graph-out /tmp/graphs .gitlab-ci.yml +``` + +**Include graph** (`--graph includes`) — [Mermaid](https://mermaid.js.org) flowchart written to stdout. +Pipe to a `.mmd` file or paste into [mermaid.live](https://mermaid.live). +One node per include entry, colour-coded by type: +- Orange (bold): the main pipeline file +- Purple: `project:` includes +- Green: `component:` includes +- Blue: `local:` includes +- Grey: `remote:` URL includes +- Light orange: GitLab-provided `template:` includes + +**Pipeline graph** (`--graph pipeline`) — GitLab CI-style SVG rendered to a timestamped file +in the `--graph-out` directory (default: `gitlab-sim-out/`). Converted to PNG automatically +when `rsvg-convert`, `inkscape`, or `magick` is available; falls back to SVG otherwise. +Jobs are colour-coded by type: +- Blue (`#1f75cb`): regular jobs +- Orange (`#fc6d26`): `when: manual` jobs +- Purple (`#6b4fbb`): `trigger:` jobs +- Amber (`#fca326`): `when: delayed` jobs + +DAG mode (job-to-job Bézier arrows) activates automatically when any job has a `needs:` list. +Classic mode draws L-shaped or straight connectors between stage columns otherwise. + ### Example output ``` @@ -135,9 +241,11 @@ task clean # remove build artifacts . ├── cmd/gitlab-sim/ # CLI entrypoint ├── internal/ +│ ├── fetcher/ # GitLab API client (project include fetching) +│ ├── graph/ # Mermaid and SVG/PNG graph generators │ ├── linter/ # lint rules and findings │ ├── model/ # pipeline data structures and YAML parser -│ └── resolver/ # extends: resolution +│ └── resolver/ # extends: resolution and project include merging ├── testdata/ # sample pipelines used for manual validation ├── Taskfile.yml └── go.mod diff --git a/Taskfile.yml b/Taskfile.yml index 3b06b59..5fda2dd 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -41,6 +41,10 @@ tasks: ignore_error: true - cmd: ./{{.BINARY}} testdata/keywords_invalid.yml ignore_error: true + - cmd: ./{{.BINARY}} testdata/includes_project.yml + ignore_error: false + - cmd: ./{{.BINARY}} testdata/includes_component.yml + ignore_error: false lint-go: desc: Run go vet on all packages diff --git a/internal/fetcher/gitlab.go b/internal/fetcher/gitlab.go new file mode 100644 index 0000000..e48cc25 --- /dev/null +++ b/internal/fetcher/gitlab.go @@ -0,0 +1,152 @@ +// Package fetcher retrieves remote GitLab CI YAML files via the GitLab REST API. +package fetcher + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +// TokenSource describes where a token was found, which determines the correct +// authentication header to use with the GitLab API. +type TokenSource int + +const ( + TokenNone TokenSource = iota + TokenPrivate // GITLAB_TOKEN or GITLAB_PRIVATE_TOKEN → PRIVATE-TOKEN header + TokenJobToken // CI_JOB_TOKEN → JOB-TOKEN header +) + +// GitLabConfig holds everything needed to reach a GitLab instance. +type GitLabConfig struct { + BaseURL string + Token string + Source TokenSource +} + +// AutoConfig builds a GitLabConfig from environment variables. +// +// Base URL resolution order: CI_SERVER_URL → GITLAB_URL → https://gitlab.com +// Token resolution order: GITLAB_TOKEN → CI_JOB_TOKEN → GITLAB_PRIVATE_TOKEN +func AutoConfig() GitLabConfig { + baseURL := firstNonEmpty( + os.Getenv("CI_SERVER_URL"), + os.Getenv("GITLAB_URL"), + "https://gitlab.com", + ) + + cfg := GitLabConfig{BaseURL: strings.TrimRight(baseURL, "/")} + + if t := os.Getenv("GITLAB_TOKEN"); t != "" { + cfg.Token = t + cfg.Source = TokenPrivate + } else if t := os.Getenv("CI_JOB_TOKEN"); t != "" { + cfg.Token = t + cfg.Source = TokenJobToken + } else if t := os.Getenv("GITLAB_PRIVATE_TOKEN"); t != "" { + cfg.Token = t + cfg.Source = TokenPrivate + } + + return cfg +} + +// WithOverrides returns a copy of cfg with non-empty overrides applied. +func (cfg GitLabConfig) WithOverrides(baseURL, token string) GitLabConfig { + if baseURL != "" { + cfg.BaseURL = strings.TrimRight(baseURL, "/") + } + if token != "" { + cfg.Token = token + cfg.Source = TokenPrivate + } + return cfg +} + +// ForHost returns a copy of cfg with BaseURL set to https://. +// Used when a CI/CD component reference specifies a different GitLab host. +func (cfg GitLabConfig) ForHost(host string) GitLabConfig { + cfg.BaseURL = "https://" + host + return cfg +} + +// HasToken reports whether the config carries a usable token. +func (cfg GitLabConfig) HasToken() bool { return cfg.Token != "" } + +// FetchFile downloads a single file from a GitLab project repository. +// If no token is configured the request is sent unauthenticated (works for +// public projects and GitLab CI/CD catalog components). +// +// - project — namespace/project path, e.g. "my-group/my-project" +// - filePath — repository file path, e.g. "/templates/ci.yml" +// - ref — branch, tag, or commit SHA; empty string defaults to HEAD +func (cfg GitLabConfig) FetchFile(project, filePath, ref string) ([]byte, error) { + encodedProject := url.PathEscape(project) + encodedFile := url.PathEscape(strings.TrimPrefix(filePath, "/")) + + apiURL := fmt.Sprintf("%s/api/v4/projects/%s/repository/files/%s/raw", + cfg.BaseURL, encodedProject, encodedFile) + + if ref == "" { + ref = "HEAD" + } + + req, err := http.NewRequest(http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("building request: %w", err) + } + + q := req.URL.Query() + q.Set("ref", ref) + req.URL.RawQuery = q.Encode() + + // Add auth header only when a token is available. + if cfg.Token != "" { + switch cfg.Source { + case TokenJobToken: + req.Header.Set("JOB-TOKEN", cfg.Token) + default: + req.Header.Set("PRIVATE-TOKEN", cfg.Token) + } + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", apiURL, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + msg := strings.TrimSpace(string(body)) + switch resp.StatusCode { + case http.StatusUnauthorized: + if cfg.Token == "" { + return nil, fmt.Errorf("authentication required (401) — set GITLAB_TOKEN or CI_JOB_TOKEN") + } + return nil, fmt.Errorf("authentication failed (401) — check your token") + case http.StatusNotFound: + return nil, fmt.Errorf("not found (404): %s@%s in project %s — check the project path, file path, and token permissions", filePath, ref, project) + default: + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, msg) + } + } + + return body, nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} diff --git a/internal/graph/includes.go b/internal/graph/includes.go new file mode 100644 index 0000000..6adef5d --- /dev/null +++ b/internal/graph/includes.go @@ -0,0 +1,127 @@ +package graph + +import ( + "fmt" + "strings" +) + +// Includes returns a Mermaid flowchart showing include file dependencies. +// sourcePath is the path to the main pipeline file; rawIncludes is Pipeline.Include. +func Includes(sourcePath string, rawIncludes []any) string { + var sb strings.Builder + w := func(s string) { sb.WriteString(s + "\n") } + wf := func(f string, a ...any) { fmt.Fprintf(&sb, f+"\n", a...) } + + w("---") + w("title: Include Dependencies") + w("---") + w("flowchart LR") + w(" classDef main fill:#e24329,stroke:#c73a1e,color:#fff,font-weight:bold") + w(" classDef project fill:#6b4fbb,stroke:#5a3fa0,color:#fff") + w(" classDef component fill:#1aaa55,stroke:#148a43,color:#fff") + w(" classDef local fill:#428fdc,stroke:#1068bf,color:#fff") + w(" classDef remote fill:#868686,stroke:#686868,color:#fff") + w(" classDef template fill:#fc6d26,stroke:#e56b1f,color:#fff") + w("") + wf(" root[\"%s\"]:::main", mermaidLabel(sourcePath)) + + if len(rawIncludes) == 0 { + return sb.String() + } + + w("") + + counter := 0 + for _, entry := range rawIncludes { + for _, n := range parseIncludeEntry(entry, &counter) { + wf(" %s[\"%s\"]:::%s", n.id, n.label, n.class) + wf(" root --> %s", n.id) + } + } + + return sb.String() +} + +type incNode struct { + id, label, class string +} + +func parseIncludeEntry(entry any, counter *int) []incNode { + newID := func() string { + *counter++ + return fmt.Sprintf("inc%d", *counter) + } + switch v := entry.(type) { + case string: + return []incNode{{id: newID(), label: mermaidLabel(v), class: "local"}} + case map[string]any: + return parseIncludeMap(v, newID) + } + return nil +} + +func parseIncludeMap(m map[string]any, newID func() string) []incNode { + if comp, ok := m["component"].(string); ok { + return []incNode{{ + id: newID(), + label: "component:
" + mermaidLabel(comp), + class: "component", + }} + } + if proj, ok := m["project"].(string); ok { + ref, _ := m["ref"].(string) + if ref == "" { + ref = "HEAD" + } + files := includeFileList(m["file"]) + if len(files) == 0 { + return []incNode{{ + id: newID(), + label: fmt.Sprintf("project: %s @ %s", mermaidLabel(proj), ref), + class: "project", + }} + } + var nodes []incNode + for _, f := range files { + nodes = append(nodes, incNode{ + id: newID(), + label: fmt.Sprintf("project: %s
%s @ %s", mermaidLabel(proj), mermaidLabel(f), ref), + class: "project", + }) + } + return nodes + } + if local, ok := m["local"].(string); ok { + return []incNode{{id: newID(), label: "local: " + mermaidLabel(local), class: "local"}} + } + if remote, ok := m["remote"].(string); ok { + return []incNode{{id: newID(), label: "remote:
" + mermaidLabel(remote), class: "remote"}} + } + if tmpl, ok := m["template"].(string); ok { + return []incNode{{id: newID(), label: "template:
" + mermaidLabel(tmpl), class: "template"}} + } + return nil +} + +func includeFileList(v any) []string { + switch f := v.(type) { + case string: + return []string{f} + case []any: + var out []string + for _, item := range f { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + return out + } + return nil +} + +// mermaidLabel escapes s for use inside a Mermaid double-quoted node label. +func mermaidLabel(s string) string { + s = strings.ReplaceAll(s, `"`, `'`) + s = strings.ReplaceAll(s, `#`, `#`) + return s +} diff --git a/internal/graph/pipeline.go b/internal/graph/pipeline.go new file mode 100644 index 0000000..7a3d548 --- /dev/null +++ b/internal/graph/pipeline.go @@ -0,0 +1,172 @@ +package graph + +import ( + "fmt" + "sort" + "strings" + + "git.k3nny.fr/gitlab-sim/internal/model" +) + +// Pipeline returns a Mermaid flowchart of pipeline jobs grouped by stage. +// +// Hidden template jobs (names starting with ".") are excluded. +// When any job has a needs: list, DAG mode is used and individual needs arrows +// are drawn. Otherwise, classic mode draws stage-ordering arrows between +// stage subgraphs. +func Pipeline(p *model.Pipeline) string { + var sb strings.Builder + w := func(s string) { sb.WriteString(s + "\n") } + wf := func(f string, a ...any) { fmt.Fprintf(&sb, f+"\n", a...) } + + w("---") + w("title: Pipeline") + w("---") + w("flowchart LR") + w(" classDef regular fill:#428fdc,stroke:#1068bf,color:#fff") + w(" classDef manual fill:#fc6d26,stroke:#e56b1f,color:#fff") + w(" classDef trigger fill:#6b4fbb,stroke:#5a3fa0,color:#fff") + w(" classDef delayed fill:#fca326,stroke:#d98a1e,color:#333") + w("") + + // Collect visible (non-template) job names; sort for stable output. + var visible []string + for name := range p.Jobs { + if !strings.HasPrefix(name, ".") { + visible = append(visible, name) + } + } + sort.Strings(visible) + + if len(visible) == 0 { + w(" empty[\"no jobs defined\"]") + return sb.String() + } + + // Assign stable numeric IDs so graph output is deterministic. + jobID := make(map[string]string, len(visible)) + for i, name := range visible { + jobID[name] = fmt.Sprintf("j%d", i+1) + } + + // Group jobs by stage; fall back to "test" (GitLab default) when unset. + byStage := make(map[string][]string) + for _, name := range visible { + stage := p.Jobs[name].Stage + if stage == "" { + stage = "test" + } + byStage[stage] = append(byStage[stage], name) + } + + // Build ordered stage list: declared stages first, then undeclared extras. + seen := make(map[string]bool) + var stages []string + for _, s := range p.Stages { + if len(byStage[s]) > 0 && !seen[s] { + stages = append(stages, s) + seen[s] = true + } + } + for _, name := range visible { + stage := p.Jobs[name].Stage + if stage == "" { + stage = "test" + } + if !seen[stage] { + stages = append(stages, stage) + seen[stage] = true + } + } + + // Emit one subgraph per stage. + for _, stage := range stages { + jobs := byStage[stage] + if len(jobs) == 0 { + continue + } + sort.Strings(jobs) + wf(" subgraph %s[\"%s\"]", stageID(stage), stage) + for _, name := range jobs { + wf(" %s[\"%s\"]:::%s", jobID[name], mermaidLabel(name), jobClass(p.Jobs[name])) + } + w(" end") + w("") + } + + // DAG mode: at least one visible job has a needs: list. + dagMode := false + for _, name := range visible { + if len(p.Jobs[name].Needs) > 0 { + dagMode = true + break + } + } + + if dagMode { + w(" %% DAG: needs dependencies") + for _, name := range visible { + for _, need := range p.Jobs[name].Needs { + dep := needsJobName(need) + if dep == "" { + continue + } + if depID, ok := jobID[dep]; ok { + wf(" %s --> %s", depID, jobID[name]) + } + } + } + } else if len(stages) > 1 { + w(" %% Classic: stage ordering") + ids := make([]string, len(stages)) + for i, s := range stages { + ids[i] = stageID(s) + } + wf(" %s", strings.Join(ids, " --> ")) + } + + return sb.String() +} + +func stageID(s string) string { return "stage_" + sanitizeID(s) } + +func sanitizeID(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +func jobClass(job model.Job) string { + if job.Trigger != nil { + return "trigger" + } + switch job.When { + case "manual": + return "manual" + case "delayed": + return "delayed" + } + return "regular" +} + +func needsJobName(need any) string { + switch v := need.(type) { + case string: + return v + case map[string]any: + if _, cross := v["pipeline"]; cross { + return "" // cross-pipeline needs have no local counterpart + } + if j, ok := v["job"].(string); ok { + return j + } + } + return "" +} diff --git a/internal/graph/render.go b/internal/graph/render.go new file mode 100644 index 0000000..365b8b2 --- /dev/null +++ b/internal/graph/render.go @@ -0,0 +1,363 @@ +package graph + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + "git.k3nny.fr/gitlab-sim/internal/model" +) + +// Layout constants (pixels) – tuned to resemble GitLab's full pipeline graph view. +const ( + chipW = 178 // job chip width + chipH = 34 // job chip height + chipGap = 6 // vertical gap between chips in a column + iconCX = 14 // status circle: x offset from chip left edge to circle centre + iconR = 7 // status circle radius (14 px diameter) + textLeft = 27 // job name text: x offset from chip left edge + labelH = 30 // stage-name label area height (text + bottom gap) + topPad = 40 // outer top padding + sidePad = 40 // outer left / right padding + stageGap = 50 // horizontal gap between stage columns (connector space) + botPad = 48 // outer bottom padding (legend lives here) +) + +type svgPt struct{ x, y int } + +// RenderPipeline writes a PNG (or SVG fallback) file with a GitLab CI-style +// pipeline layout and returns the path to the generated file. +func RenderPipeline(p *model.Pipeline, outDir string) (string, error) { + if err := os.MkdirAll(outDir, 0o755); err != nil { + return "", fmt.Errorf("creating output directory %s: %w", outDir, err) + } + + svg, err := pipelineSVG(p) + if err != nil { + return "", err + } + + ts := time.Now().Format("20060102-150405") + svgPath := filepath.Join(outDir, "pipeline-"+ts+".svg") + if err := os.WriteFile(svgPath, []byte(svg), 0o644); err != nil { + return "", fmt.Errorf("writing SVG: %w", err) + } + + pngPath := filepath.Join(outDir, "pipeline-"+ts+".png") + if convertToPNG(svgPath, pngPath) { + _ = os.Remove(svgPath) + return pngPath, nil + } + return svgPath, nil +} + +// convertToPNG tries rsvg-convert, Inkscape, magick, and convert (not on Windows). +func convertToPNG(svgPath, pngPath string) bool { + type cand struct { + args []string + skipWin bool + } + for _, c := range []cand{ + {[]string{"rsvg-convert", "--output", pngPath, svgPath}, false}, + {[]string{"inkscape", "--export-filename=" + pngPath, svgPath}, false}, + {[]string{"magick", svgPath, pngPath}, false}, + {[]string{"convert", svgPath, pngPath}, true}, + } { + if c.skipWin && runtime.GOOS == "windows" { + continue + } + if bin, err := exec.LookPath(c.args[0]); err == nil { + if exec.Command(bin, c.args[1:]...).Run() == nil { + return true + } + } + } + return false +} + +func pipelineSVG(p *model.Pipeline) (string, error) { + // Collect visible (non-template) job names in sorted order. + var visible []string + for name := range p.Jobs { + if !strings.HasPrefix(name, ".") { + visible = append(visible, name) + } + } + sort.Strings(visible) + + if len(visible) == 0 { + return svgEmpty(), nil + } + + // Group by stage; fall back to "test" (GitLab default) when stage is unset. + byStage := make(map[string][]string) + for _, name := range visible { + s := p.Jobs[name].Stage + if s == "" { + s = "test" + } + byStage[s] = append(byStage[s], name) + } + + // Build ordered stage list: declared stages first, then undeclared extras. + seen := make(map[string]bool) + var stages []string + for _, s := range p.Stages { + if len(byStage[s]) > 0 && !seen[s] { + stages = append(stages, s) + seen[s] = true + } + } + for _, name := range visible { + s := p.Jobs[name].Stage + if s == "" { + s = "test" + } + if !seen[s] { + stages = append(stages, s) + seen[s] = true + } + } + + // Compute per-stage column heights and job anchor points for connectors. + colH := make([]int, len(stages)) // height of the chip stack (no label) + colCtY := make([]int, len(stages)) // y centre of the chip stack + maxColH := 0 + rightMid := make(map[string]svgPt) + leftMid := make(map[string]svgPt) + + for i, stage := range stages { + jobs := byStage[stage] + sort.Strings(jobs) + n := len(jobs) + h := n*chipH + max(0, n-1)*chipGap + colH[i] = h + if h > maxColH { + maxColH = h + } + cx := sidePad + i*(chipW+stageGap) + for j, name := range jobs { + chy := topPad + labelH + j*(chipH+chipGap) + rightMid[name] = svgPt{cx + chipW, chy + chipH/2} + leftMid[name] = svgPt{cx, chy + chipH/2} + } + colCtY[i] = topPad + labelH + h/2 + } + + svgW := sidePad*2 + len(stages)*chipW + max(0, len(stages)-1)*stageGap + svgH := topPad + labelH + maxColH + botPad + + // DAG mode: any visible job with a needs: list triggers job-to-job arrows. + dagMode := false + for _, name := range visible { + if len(p.Jobs[name].Needs) > 0 { + dagMode = true + break + } + } + + var sb strings.Builder + w := func(s string) { sb.WriteString(s + "\n") } + wf := func(f string, a ...any) { fmt.Fprintf(&sb, f+"\n", a...) } + + // ── Document ────────────────────────────────────────────────────────────── + wf(``, + svgW, svgH, svgW, svgH) + + w(` `) + // Subtle drop shadow for job chips. + w(` `) + w(` `) + w(` `) + w(` `) + + // White page background. + wf(` `, svgW, svgH) + + // ── Stage columns ───────────────────────────────────────────────────────── + for i, stage := range stages { + cx := sidePad + i*(chipW+stageGap) + jobs := byStage[stage] + sort.Strings(jobs) + + // Stage name – small, gray, uppercase, centered above the chip stack. + wf(` %s`, + cx+chipW/2, topPad+13, svgEsc(strings.ToUpper(stage))) + + // Subtle separator line below stage name. + wf(` `, + cx, topPad+21, cx+chipW, topPad+21) + + // Job chips. + for j, name := range jobs { + job := p.Jobs[name] + chy := topPad + labelH + j*(chipH+chipGap) + color := chipColor(job) + + // Chip card (white, rounded, subtle border + shadow). + wf(` `, + cx, chy, chipW, chipH) + + // Colored status indicator circle. + wf(` `, + cx+iconCX, chy+chipH/2, iconR, color) + + // Icon symbol inside the circle. + drawChipIcon(&sb, job, cx+iconCX, chy+chipH/2) + + // Job name text. + wf(` %s`, + cx+textLeft, chy+chipH/2, svgEsc(svgTrunc(name, 20))) + } + } + + // ── Connectors ──────────────────────────────────────────────────────────── + const connStroke = "#dbdbdb" + + if dagMode { + // Job-to-job bezier curves from needs:. + for _, name := range visible { + for _, need := range p.Jobs[name].Needs { + dep := needsJobName(need) + if dep == "" { + continue + } + src, okS := rightMid[dep] + dst, okD := leftMid[name] + if !okS || !okD { + continue + } + cpX := (src.x + dst.x) / 2 + wf(` `, + src.x, src.y, cpX, src.y, cpX, dst.y, dst.x-7, dst.y, connStroke) + // Small right-pointing triangle arrowhead. + wf(` `, + dst.x-7, dst.y-4, dst.x, dst.y, dst.x-7, dst.y+4, connStroke) + } + } + } else { + // Classic: one connector per adjacent stage pair. + for i := 0; i < len(stages)-1; i++ { + x1 := sidePad + i*(chipW+stageGap) + chipW + x2 := sidePad + (i+1)*(chipW+stageGap) + y1 := colCtY[i] + y2 := colCtY[i+1] + midX := (x1 + x2) / 2 + + if y1 == y2 { + // Straight horizontal line. + wf(` `, + x1, y1, x2-7, y2, connStroke) + } else { + // L-shaped elbow: right → vertical → right. + wf(` `, + x1, y1, midX, y1, midX, y2, x2-7, y2, connStroke) + } + // Arrowhead at the destination. + wf(` `, + x2-7, y2-4, x2, y2, x2-7, y2+4, connStroke) + } + } + + // ── Legend ──────────────────────────────────────────────────────────────── + legend := []struct{ color, label string }{ + {"#1f75cb", "regular"}, + {"#fc6d26", "manual"}, + {"#6b4fbb", "trigger"}, + {"#fca326", "delayed"}, + } + const legendItemW = 82 + legendY := topPad + labelH + maxColH + botPad/2 + lx0 := (svgW - len(legend)*legendItemW) / 2 + for k, item := range legend { + lx := lx0 + k*legendItemW + wf(` `, lx+6, legendY, item.color) + wf(` %s`, + lx+16, legendY, item.label) + } + + w(``) + return sb.String(), nil +} + +// drawChipIcon writes an SVG symbol inside the status circle to help identify +// the job type at a glance. +func drawChipIcon(sb *strings.Builder, job model.Job, cx, cy int) { + if job.Trigger != nil { + // Right-pointing chevron for trigger jobs. + fmt.Fprintf(sb, " \n", + cx-3, cy-3, cx+3, cy, cx-3, cy+3) + return + } + switch job.When { + case "manual": + // Filled play triangle. + fmt.Fprintf(sb, " \n", + cx-3, cy-4, cx+5, cy, cx-3, cy+4) + case "delayed": + // Clock hands: vertical + horizontal. + fmt.Fprintf(sb, " \n", cx, cy) + fmt.Fprintf(sb, " \n", + cx, cy, cx, cy-3) + fmt.Fprintf(sb, " \n", + cx, cy, cx+2, cy+1) + default: + // Regular job: small white checkmark outline. + fmt.Fprintf(sb, " \n", + cx-3, cy, cx-1, cy+3, cx+4, cy-3) + } +} + +func chipColor(job model.Job) string { + if job.Trigger != nil { + return "#6b4fbb" + } + switch job.When { + case "manual": + return "#fc6d26" + case "delayed": + return "#fca326" + } + return "#1f75cb" +} + +func svgEsc(s string) string { + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + return s +} + +func svgTrunc(s string, maxLen int) string { + runes := []rune(s) + if len(runes) <= maxLen { + return s + } + return string(runes[:maxLen-1]) + "…" +} + +func svgEmpty() string { + const w, h = 300, 80 + return fmt.Sprintf( + ``+ + ``+ + `no jobs defined`+ + ``, + w, h, w, h, w/2, h/2) +} diff --git a/internal/linter/linter.go b/internal/linter/linter.go index 60d1cd8..021d86d 100644 --- a/internal/linter/linter.go +++ b/internal/linter/linter.go @@ -97,8 +97,10 @@ func checkJob(name string, job model.Job, stageSet map[string]bool) []Finding { }) } - // stage must exist in declared stages (if stages were declared) - if job.Stage != "" && len(stageSet) > 0 && !stageSet[job.Stage] { + // stage must exist in declared stages (if stages were declared). + // Skip when the stage value is a component input placeholder ($[[ inputs.xxx ]]): + // those are resolved server-side and cannot be validated locally. + if job.Stage != "" && !strings.Contains(job.Stage, "$[[") && len(stageSet) > 0 && !stageSet[job.Stage] { findings = append(findings, Finding{ Severity: Error, Job: name, diff --git a/internal/model/parser.go b/internal/model/parser.go index 90faa25..bf6de02 100644 --- a/internal/model/parser.go +++ b/internal/model/parser.go @@ -13,7 +13,11 @@ func Parse(path string) (*Pipeline, error) { if err != nil { return nil, fmt.Errorf("reading file: %w", err) } + return ParseBytes(data) +} +// ParseBytes parses YAML from an in-memory byte slice. +func ParseBytes(data []byte) (*Pipeline, error) { // First pass: decode into a raw map to extract job keys. var raw map[string]yaml.Node if err := yaml.Unmarshal(data, &raw); err != nil { diff --git a/internal/resolver/includes.go b/internal/resolver/includes.go new file mode 100644 index 0000000..a2f974b --- /dev/null +++ b/internal/resolver/includes.go @@ -0,0 +1,245 @@ +package resolver + +import ( + "fmt" + "strings" + + "git.k3nny.fr/gitlab-sim/internal/fetcher" + "git.k3nny.fr/gitlab-sim/internal/model" +) + +// IncludeWarning describes a remote include entry that could not be resolved. +// These are surfaced to the user as [WARNING] lines before the lint findings. +type IncludeWarning struct { + // Label is a short human-readable identifier shown in the warning message, + // e.g. "project my-group/templates:/ci.yml@main" or + // "component gitlab.com/components/golang/build@v1.0". + Label string + Err error // nil when Skipped is true + Skipped bool // no token available — skipped without attempting a network call +} + +func (w IncludeWarning) String() string { + if w.Skipped { + return fmt.Sprintf( + "%s: skipped — no GitLab token found (set GITLAB_TOKEN, CI_JOB_TOKEN, or GITLAB_PRIVATE_TOKEN); findings may be incomplete", + w.Label) + } + return fmt.Sprintf( + "%s: could not be fetched — %v; findings may be incomplete or contain false positives", + w.Label, w.Err) +} + +// ResolveIncludes processes the pipeline's include: block. +// +// Project includes (include: project: ...) require authentication and are +// skipped with a warning when no token is configured. +// +// Component includes (include: component: ...) attempt the fetch +// unauthenticated first, so public CI/CD catalog components work without a +// token. A warning is emitted when the fetch fails. +// +// Non-resolvable include forms (local, remote, template) are silently skipped. +func ResolveIncludes(p *model.Pipeline, cfg fetcher.GitLabConfig) []IncludeWarning { + var warnings []IncludeWarning + for _, inc := range p.Include { + entry, ok := normaliseInclude(inc) + if !ok { + continue + } + + if project, _ := entry["project"].(string); project != "" { + warnings = append(warnings, resolveProjectInclude(p, entry, project, cfg)...) + continue + } + + if compRef, _ := entry["component"].(string); compRef != "" { + if w, ok := resolveComponentInclude(p, compRef, cfg); ok { + warnings = append(warnings, w) + } + continue + } + + // local, remote, template — resolved by GitLab at runtime, skip silently. + } + return warnings +} + +// resolveProjectInclude fetches all files listed under a single project: entry +// and merges them into p. +func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig) []IncludeWarning { + ref, _ := entry["ref"].(string) + var warnings []IncludeWarning + + for _, filePath := range includeFiles(entry) { + label := fmt.Sprintf("project %s:%s", project, filePath) + if ref != "" { + label += "@" + ref + } + + if !cfg.HasToken() { + warnings = append(warnings, IncludeWarning{Label: label, Skipped: true}) + continue + } + + data, err := cfg.FetchFile(project, filePath, ref) + if err != nil { + warnings = append(warnings, IncludeWarning{Label: label, Err: err}) + continue + } + + included, err := model.ParseBytes(data) + if err != nil { + warnings = append(warnings, IncludeWarning{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}) + continue + } + + mergeIncluded(p, included) + } + return warnings +} + +// resolveComponentInclude fetches a CI/CD catalog component and merges it into p. +// Returns (warning, true) if something went wrong; (zero, false) on success. +func resolveComponentInclude(p *model.Pipeline, ref string, cfg fetcher.GitLabConfig) (IncludeWarning, bool) { + label := "component " + ref + + // Variable interpolation (e.g. $CI_SERVER_FQDN) cannot be resolved locally. + if strings.ContainsRune(ref, '$') { + return IncludeWarning{ + Label: label, + Err: fmt.Errorf("component reference contains a CI variable that cannot be resolved at lint time"), + }, true + } + + host, project, component, version, err := parseComponentRef(ref) + if err != nil { + return IncludeWarning{Label: label, Err: err}, true + } + + hostCfg := cfg.ForHost(host) + data, err := fetchComponentFile(hostCfg, project, component, version) + if err != nil { + return IncludeWarning{Label: label, Err: err}, true + } + + included, err := model.ParseBytes(data) + if err != nil { + return IncludeWarning{Label: label, Err: fmt.Errorf("parsing component YAML: %w", err)}, true + } + + mergeIncluded(p, included) + return IncludeWarning{}, false +} + +// parseComponentRef parses a CI/CD component reference of the form: +// +// //@ +// +// The host is the GitLab instance FQDN. The last path segment before @ is the +// component name; everything between host and component name is the project path. +// +// Examples: +// +// gitlab.com/components/golang/build@v1.0.0 +// gitlab.example.com/my-org/ci-templates/lint@main +// gitlab.com/components/secret-detection/secret-detection@~latest +func parseComponentRef(ref string) (host, project, component, version string, err error) { + atIdx := strings.LastIndex(ref, "@") + if atIdx < 0 || atIdx == len(ref)-1 { + err = fmt.Errorf("component reference %q must include a version (e.g. @v1.0 or @main)", ref) + return + } + version = ref[atIdx+1:] + path := ref[:atIdx] + + parts := strings.Split(path, "/") + // Minimum: host + at least one project segment + component = 3 parts. + if len(parts) < 3 { + err = fmt.Errorf("component reference %q must be //@", ref) + return + } + + host = parts[0] + component = parts[len(parts)-1] + project = strings.Join(parts[1:len(parts)-1], "/") + return +} + +// fetchComponentFile fetches a component's template YAML from a GitLab project. +// GitLab supports two layouts; both are attempted: +// - templates/.yml (single-file component) +// - templates//template.yml (directory component) +func fetchComponentFile(cfg fetcher.GitLabConfig, project, component, version string) ([]byte, error) { + primary := "templates/" + component + ".yml" + data, err := cfg.FetchFile(project, primary, version) + if err == nil { + return data, nil + } + + fallback := "templates/" + component + "/template.yml" + data2, err2 := cfg.FetchFile(project, fallback, version) + if err2 != nil { + // Return the primary error — it refers to the canonical path. + return nil, fmt.Errorf("%v (also tried %s: %v)", err, fallback, err2) + } + return data2, nil +} + +// normaliseInclude converts a raw include: list element to a string-keyed map. +// An include: value can be a plain string (shorthand local) or a map. +func normaliseInclude(raw any) (map[string]any, bool) { + switch v := raw.(type) { + case map[string]any: + return v, true + case string: + return map[string]any{"local": v}, true + } + return nil, false +} + +// includeFiles returns the list of file paths from an include entry. +// The file: key may be a single string or a list of strings. +func includeFiles(entry map[string]any) []string { + raw, ok := entry["file"] + if !ok { + return nil + } + switch v := raw.(type) { + case string: + return []string{v} + case []any: + var paths []string + for _, item := range v { + if s, ok := item.(string); ok { + paths = append(paths, s) + } + } + return paths + } + return nil +} + +// mergeIncluded copies jobs and stages from src into dst. +// dst (the main pipeline) always wins when a key already exists. +func mergeIncluded(dst, src *model.Pipeline) { + stageSet := make(map[string]bool, len(dst.Stages)) + for _, s := range dst.Stages { + stageSet[s] = true + } + for _, s := range src.Stages { + if !stageSet[s] { + dst.Stages = append(dst.Stages, s) + stageSet[s] = true + } + } + + for name, job := range src.Jobs { + if _, exists := dst.Jobs[name]; !exists { + dst.Jobs[name] = job + if raw, ok := src.RawJobs[name]; ok { + dst.RawJobs[name] = raw + } + } + } +} diff --git a/testdata/includes_component.yml b/testdata/includes_component.yml new file mode 100644 index 0000000..ce47406 --- /dev/null +++ b/testdata/includes_component.yml @@ -0,0 +1,30 @@ +stages: + - test + - deploy + +# CI/CD catalog components. Without network access or a token the fetches will +# fail and gitlab-sim emits a WARNING for each, then lints the local jobs normally. +include: + # Standard single-segment component path (file layout: templates/sast.yml) + - component: gitlab.com/components/sast/sast@latest + with: + stage: test + + # Multi-segment project path (file layout: templates/build.yml) + - component: gitlab.com/my-org/ci-templates/build@v2.0.0 + + # Directory-based component (file layout: templates/lint/template.yml) + - component: gitlab.example.com/platform/components/lint@main + + # Component using a CI variable in the host — cannot be resolved at lint time. + - component: $CI_SERVER_FQDN/internal/ci-catalog/deploy@v1.0 + + # Mixed includes: project include alongside component + - project: my-org/shared-ci + ref: main + file: /templates/common.yml + +deploy-job: + stage: deploy + script: + - echo "deploying" diff --git a/testdata/includes_project.yml b/testdata/includes_project.yml new file mode 100644 index 0000000..25614a3 --- /dev/null +++ b/testdata/includes_project.yml @@ -0,0 +1,33 @@ +stages: + - build + - test + +# This pipeline includes templates from another GitLab project. +# Without a token, gitlab-sim warns but still lints the local jobs. +include: + - project: my-group/ci-templates + ref: main + file: /templates/build.yml + + - project: my-group/ci-templates + ref: main + file: + - /templates/test-base.yml + - /templates/security.yml + + # These are ignored by the fetcher (not project: includes) + - local: .gitlab/common.yml + - template: Auto-DevOps.gitlab-ci.yml + +build-job: + stage: build + # extends: .build-template # would reference a job from the remote include + script: + - go build ./... + +test-job: + stage: test + script: + - go test ./... + needs: + - build-job