diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ee067..fdc2634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ This project uses [Semantic Versioning](https://semver.org). ### Fixed +- **`include: remote:` URL includes are now fetched and merged** — glint fetches plain HTTPS URLs in `include: remote:` entries (no authentication), parses the resulting YAML, merges its jobs into the pipeline, and recursively resolves any sub-includes the remote file itself declares. Unreachable or unparseable URLs emit a `[WARNING]` and lint continues on the rest of the pipeline. The `glint graph includes` command now expands remote nodes with their jobs and sub-include tree, matching the behaviour of local and project includes. + - **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph. - **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured). diff --git a/Taskfile.yml b/Taskfile.yml index b92307c..fb0253c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -41,6 +41,8 @@ tasks: ignore_error: true - cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml ignore_error: true + - cmd: ./{{.BINARY}} check testdata/includes_remote.yml + ignore_error: false - cmd: ./{{.BINARY}} check testdata/includes_project.yml ignore_error: false - cmd: ./{{.BINARY}} check testdata/includes_component.yml diff --git a/internal/fetcher/gitlab.go b/internal/fetcher/gitlab.go index e48cc25..bb52024 100644 --- a/internal/fetcher/gitlab.go +++ b/internal/fetcher/gitlab.go @@ -142,6 +142,24 @@ func (cfg GitLabConfig) FetchFile(project, filePath, ref string) ([]byte, error) return body, nil } +// FetchURL downloads the content at a plain HTTPS URL without authentication. +// Used for include: remote: entries which are public by definition. +func FetchURL(rawURL string) ([]byte, error) { + resp, err := http.Get(rawURL) //nolint:noctx + if err != nil { + return nil, fmt.Errorf("GET %s: %w", rawURL, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading body: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: status %d", rawURL, resp.StatusCode) + } + return body, nil +} + func firstNonEmpty(values ...string) string { for _, v := range values { if v != "" { diff --git a/internal/graph/includes.go b/internal/graph/includes.go index b98e012..5c3e861 100644 --- a/internal/graph/includes.go +++ b/internal/graph/includes.go @@ -114,7 +114,9 @@ func (b *treeBuilder) parseMap(m map[string]any) []*treeNode { return []*treeNode{node} } if remote, ok := m["remote"].(string); ok { - return []*treeNode{{id: b.nextID(), label: "remote:
" + mermaidLabel(remote), class: "remote"}} + node := &treeNode{id: b.nextID(), label: "remote:
" + mermaidLabel(remote), class: "remote"} + b.recurseRemote(node, remote) + return []*treeNode{node} } if tmpl, ok := m["template"].(string); ok { return []*treeNode{{id: b.nextID(), label: "template:
" + mermaidLabel(tmpl), class: "template"}} @@ -170,6 +172,28 @@ func (b *treeBuilder) recurseProject(node *treeNode, project, filePath, ref stri b.buildChildren(node, p.Include) } +func (b *treeBuilder) recurseRemote(node *treeNode, rawURL string) { + key := "remote:" + rawURL + if b.visited[key] { + return + } + b.visited[key] = true + + data, err := fetcher.FetchURL(rawURL) + if err != nil { + return + } + p, err := model.ParseBytes(data) + if err != nil { + return + } + node.jobs = jobNames(p) + if len(p.Include) == 0 { + return + } + b.buildChildren(node, p.Include) +} + func (b *treeBuilder) recurseComponent(node *treeNode, ref string) { if strings.ContainsRune(ref, '$') { return diff --git a/internal/resolver/includes.go b/internal/resolver/includes.go index 998e276..87130c0 100644 --- a/internal/resolver/includes.go +++ b/internal/resolver/includes.go @@ -90,7 +90,14 @@ func resolveIncludes(p *model.Pipeline, includes []any, cfg fetcher.GitLabConfig continue } - // remote, template — resolved by GitLab at runtime, skip silently. + if remote, _ := entry["remote"].(string); remote != "" { + w, ew := resolveRemoteInclude(p, remote, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) + continue + } + + // template — resolved by GitLab at runtime, skip silently. } return warnings, extWarnings } @@ -133,6 +140,39 @@ func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabCo return warnings, extWarnings } +// resolveRemoteInclude fetches a plain HTTPS URL, parses it as CI YAML, and +// merges it into p. Sub-includes of the fetched file are resolved recursively. +func resolveRemoteInclude(p *model.Pipeline, rawURL string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { + label := "remote " + rawURL + + if visited[rawURL] { + return nil, nil + } + visited[rawURL] = true + + data, err := fetcher.FetchURL(rawURL) + if err != nil { + return []IncludeWarning{{Label: label, Err: err}}, nil + } + + included, err := model.ParseBytes(data) + if err != nil { + return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil + } + included.SetJobOrigin(rawURL) + + var warnings []IncludeWarning + var extWarnings []ExtendWarning + if len(included.Include) > 0 { + w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited) + warnings = append(warnings, w...) + extWarnings = append(extWarnings, ew...) + } + + mergeIncluded(p, included) + return warnings, extWarnings +} + // resolveProjectInclude fetches all files listed under a single project: entry // and merges them into p. func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { diff --git a/testdata/includes_remote.yml b/testdata/includes_remote.yml new file mode 100644 index 0000000..8a593f8 --- /dev/null +++ b/testdata/includes_remote.yml @@ -0,0 +1,14 @@ +--- +# Exercises include: remote: URL fetching and sub-include recursion. +# The remote file is a real public GitLab CI template; it may contain its own +# includes which should also be resolved. Exit code is 0 (warnings allowed). +stages: + - test + +include: + - remote: https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml + +local-job: + stage: test + script: + - echo "local job alongside remote include"