feat(resolver,graph): fetch and resolve include: remote: HTTPS URLs
release / Build and publish release (push) Successful in 1m14s

Remote includes (include: remote: https://...) were previously skipped
silently in the resolver and rendered as unexpanded leaf nodes in the
graph.

Changes:
- fetcher.FetchURL: new shared unauthenticated HTTP GET helper
- resolver: resolveRemoteInclude fetches the URL, parses YAML, sets job
  origin to the URL string, recursively resolves sub-includes, and emits
  a warning on failure (lint continues on the rest of the pipeline)
- graph: recurseRemote fetches the URL, captures direct job names, and
  recurses into sub-includes so remote nodes expand like local ones

Adds testdata/includes_remote.yml fixture.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 21:42:15 +02:00
parent d34c39927d
commit b21a7d60dc
6 changed files with 102 additions and 2 deletions
+2
View File
@@ -15,6 +15,8 @@ This project uses [Semantic Versioning](https://semver.org).
### Fixed ### Fixed
- **`include: remote:` URL includes are now fetched and merged** — glint fetches plain HTTPS URLs in `include: remote:` entries (no authentication), parses the resulting YAML, merges its jobs into the pipeline, and recursively resolves any sub-includes the remote file itself declares. Unreachable or unparseable URLs emit a `[WARNING]` and lint continues on the rest of the pipeline. The `glint graph includes` command now expands remote nodes with their jobs and sub-include tree, matching the behaviour of local and project includes.
- **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph. - **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph.
- **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured). - **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured).
+2
View File
@@ -41,6 +41,8 @@ tasks:
ignore_error: true ignore_error: true
- cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml - cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml
ignore_error: true ignore_error: true
- cmd: ./{{.BINARY}} check testdata/includes_remote.yml
ignore_error: false
- cmd: ./{{.BINARY}} check testdata/includes_project.yml - cmd: ./{{.BINARY}} check testdata/includes_project.yml
ignore_error: false ignore_error: false
- cmd: ./{{.BINARY}} check testdata/includes_component.yml - cmd: ./{{.BINARY}} check testdata/includes_component.yml
+18
View File
@@ -142,6 +142,24 @@ func (cfg GitLabConfig) FetchFile(project, filePath, ref string) ([]byte, error)
return body, nil return body, nil
} }
// FetchURL downloads the content at a plain HTTPS URL without authentication.
// Used for include: remote: entries which are public by definition.
func FetchURL(rawURL string) ([]byte, error) {
resp, err := http.Get(rawURL) //nolint:noctx
if err != nil {
return nil, fmt.Errorf("GET %s: %w", rawURL, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("GET %s: status %d", rawURL, resp.StatusCode)
}
return body, nil
}
func firstNonEmpty(values ...string) string { func firstNonEmpty(values ...string) string {
for _, v := range values { for _, v := range values {
if v != "" { if v != "" {
+25 -1
View File
@@ -114,7 +114,9 @@ func (b *treeBuilder) parseMap(m map[string]any) []*treeNode {
return []*treeNode{node} return []*treeNode{node}
} }
if remote, ok := m["remote"].(string); ok { if remote, ok := m["remote"].(string); ok {
return []*treeNode{{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}} node := &treeNode{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}
b.recurseRemote(node, remote)
return []*treeNode{node}
} }
if tmpl, ok := m["template"].(string); ok { if tmpl, ok := m["template"].(string); ok {
return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}} return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}}
@@ -170,6 +172,28 @@ func (b *treeBuilder) recurseProject(node *treeNode, project, filePath, ref stri
b.buildChildren(node, p.Include) b.buildChildren(node, p.Include)
} }
func (b *treeBuilder) recurseRemote(node *treeNode, rawURL string) {
key := "remote:" + rawURL
if b.visited[key] {
return
}
b.visited[key] = true
data, err := fetcher.FetchURL(rawURL)
if err != nil {
return
}
p, err := model.ParseBytes(data)
if err != nil {
return
}
node.jobs = jobNames(p)
if len(p.Include) == 0 {
return
}
b.buildChildren(node, p.Include)
}
func (b *treeBuilder) recurseComponent(node *treeNode, ref string) { func (b *treeBuilder) recurseComponent(node *treeNode, ref string) {
if strings.ContainsRune(ref, '$') { if strings.ContainsRune(ref, '$') {
return return
+41 -1
View File
@@ -90,7 +90,14 @@ func resolveIncludes(p *model.Pipeline, includes []any, cfg fetcher.GitLabConfig
continue continue
} }
// remote, template — resolved by GitLab at runtime, skip silently. if remote, _ := entry["remote"].(string); remote != "" {
w, ew := resolveRemoteInclude(p, remote, cfg, rootDir, visited)
warnings = append(warnings, w...)
extWarnings = append(extWarnings, ew...)
continue
}
// template — resolved by GitLab at runtime, skip silently.
} }
return warnings, extWarnings return warnings, extWarnings
} }
@@ -133,6 +140,39 @@ func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabCo
return warnings, extWarnings return warnings, extWarnings
} }
// resolveRemoteInclude fetches a plain HTTPS URL, parses it as CI YAML, and
// merges it into p. Sub-includes of the fetched file are resolved recursively.
func resolveRemoteInclude(p *model.Pipeline, rawURL string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
label := "remote " + rawURL
if visited[rawURL] {
return nil, nil
}
visited[rawURL] = true
data, err := fetcher.FetchURL(rawURL)
if err != nil {
return []IncludeWarning{{Label: label, Err: err}}, nil
}
included, err := model.ParseBytes(data)
if err != nil {
return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil
}
included.SetJobOrigin(rawURL)
var warnings []IncludeWarning
var extWarnings []ExtendWarning
if len(included.Include) > 0 {
w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited)
warnings = append(warnings, w...)
extWarnings = append(extWarnings, ew...)
}
mergeIncluded(p, included)
return warnings, extWarnings
}
// resolveProjectInclude fetches all files listed under a single project: entry // resolveProjectInclude fetches all files listed under a single project: entry
// and merges them into p. // and merges them into p.
func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) { func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
+14
View File
@@ -0,0 +1,14 @@
---
# Exercises include: remote: URL fetching and sub-include recursion.
# The remote file is a real public GitLab CI template; it may contain its own
# includes which should also be resolved. Exit code is 0 (warnings allowed).
stages:
- test
include:
- remote: https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
local-job:
stage: test
script:
- echo "local job alongside remote include"