feat(resolver,graph): fetch and resolve include: remote: HTTPS URLs
release / Build and publish release (push) Successful in 1m14s
release / Build and publish release (push) Successful in 1m14s
Remote includes (include: remote: https://...) were previously skipped silently in the resolver and rendered as unexpanded leaf nodes in the graph. Changes: - fetcher.FetchURL: new shared unauthenticated HTTP GET helper - resolver: resolveRemoteInclude fetches the URL, parses YAML, sets job origin to the URL string, recursively resolves sub-includes, and emits a warning on failure (lint continues on the rest of the pipeline) - graph: recurseRemote fetches the URL, captures direct job names, and recurses into sub-includes so remote nodes expand like local ones Adds testdata/includes_remote.yml fixture. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,8 @@ This project uses [Semantic Versioning](https://semver.org).
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **`include: remote:` URL includes are now fetched and merged** — glint fetches plain HTTPS URLs in `include: remote:` entries (no authentication), parses the resulting YAML, merges its jobs into the pipeline, and recursively resolves any sub-includes the remote file itself declares. Unreachable or unparseable URLs emit a `[WARNING]` and lint continues on the rest of the pipeline. The `glint graph includes` command now expands remote nodes with their jobs and sub-include tree, matching the behaviour of local and project includes.
|
||||||
|
|
||||||
- **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph.
|
- **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph.
|
||||||
|
|
||||||
- **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured).
|
- **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured).
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ tasks:
|
|||||||
ignore_error: true
|
ignore_error: true
|
||||||
- cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml
|
- cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml
|
||||||
ignore_error: true
|
ignore_error: true
|
||||||
|
- cmd: ./{{.BINARY}} check testdata/includes_remote.yml
|
||||||
|
ignore_error: false
|
||||||
- cmd: ./{{.BINARY}} check testdata/includes_project.yml
|
- cmd: ./{{.BINARY}} check testdata/includes_project.yml
|
||||||
ignore_error: false
|
ignore_error: false
|
||||||
- cmd: ./{{.BINARY}} check testdata/includes_component.yml
|
- cmd: ./{{.BINARY}} check testdata/includes_component.yml
|
||||||
|
|||||||
@@ -142,6 +142,24 @@ func (cfg GitLabConfig) FetchFile(project, filePath, ref string) ([]byte, error)
|
|||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchURL downloads the content at a plain HTTPS URL without authentication.
|
||||||
|
// Used for include: remote: entries which are public by definition.
|
||||||
|
func FetchURL(rawURL string) ([]byte, error) {
|
||||||
|
resp, err := http.Get(rawURL) //nolint:noctx
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("GET %s: %w", rawURL, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading body: %w", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("GET %s: status %d", rawURL, resp.StatusCode)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
func firstNonEmpty(values ...string) string {
|
func firstNonEmpty(values ...string) string {
|
||||||
for _, v := range values {
|
for _, v := range values {
|
||||||
if v != "" {
|
if v != "" {
|
||||||
|
|||||||
@@ -114,7 +114,9 @@ func (b *treeBuilder) parseMap(m map[string]any) []*treeNode {
|
|||||||
return []*treeNode{node}
|
return []*treeNode{node}
|
||||||
}
|
}
|
||||||
if remote, ok := m["remote"].(string); ok {
|
if remote, ok := m["remote"].(string); ok {
|
||||||
return []*treeNode{{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}}
|
node := &treeNode{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}
|
||||||
|
b.recurseRemote(node, remote)
|
||||||
|
return []*treeNode{node}
|
||||||
}
|
}
|
||||||
if tmpl, ok := m["template"].(string); ok {
|
if tmpl, ok := m["template"].(string); ok {
|
||||||
return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}}
|
return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}}
|
||||||
@@ -170,6 +172,28 @@ func (b *treeBuilder) recurseProject(node *treeNode, project, filePath, ref stri
|
|||||||
b.buildChildren(node, p.Include)
|
b.buildChildren(node, p.Include)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *treeBuilder) recurseRemote(node *treeNode, rawURL string) {
|
||||||
|
key := "remote:" + rawURL
|
||||||
|
if b.visited[key] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.visited[key] = true
|
||||||
|
|
||||||
|
data, err := fetcher.FetchURL(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p, err := model.ParseBytes(data)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node.jobs = jobNames(p)
|
||||||
|
if len(p.Include) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.buildChildren(node, p.Include)
|
||||||
|
}
|
||||||
|
|
||||||
func (b *treeBuilder) recurseComponent(node *treeNode, ref string) {
|
func (b *treeBuilder) recurseComponent(node *treeNode, ref string) {
|
||||||
if strings.ContainsRune(ref, '$') {
|
if strings.ContainsRune(ref, '$') {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -90,7 +90,14 @@ func resolveIncludes(p *model.Pipeline, includes []any, cfg fetcher.GitLabConfig
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// remote, template — resolved by GitLab at runtime, skip silently.
|
if remote, _ := entry["remote"].(string); remote != "" {
|
||||||
|
w, ew := resolveRemoteInclude(p, remote, cfg, rootDir, visited)
|
||||||
|
warnings = append(warnings, w...)
|
||||||
|
extWarnings = append(extWarnings, ew...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// template — resolved by GitLab at runtime, skip silently.
|
||||||
}
|
}
|
||||||
return warnings, extWarnings
|
return warnings, extWarnings
|
||||||
}
|
}
|
||||||
@@ -133,6 +140,39 @@ func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabCo
|
|||||||
return warnings, extWarnings
|
return warnings, extWarnings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveRemoteInclude fetches a plain HTTPS URL, parses it as CI YAML, and
|
||||||
|
// merges it into p. Sub-includes of the fetched file are resolved recursively.
|
||||||
|
func resolveRemoteInclude(p *model.Pipeline, rawURL string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
||||||
|
label := "remote " + rawURL
|
||||||
|
|
||||||
|
if visited[rawURL] {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
visited[rawURL] = true
|
||||||
|
|
||||||
|
data, err := fetcher.FetchURL(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return []IncludeWarning{{Label: label, Err: err}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
included, err := model.ParseBytes(data)
|
||||||
|
if err != nil {
|
||||||
|
return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil
|
||||||
|
}
|
||||||
|
included.SetJobOrigin(rawURL)
|
||||||
|
|
||||||
|
var warnings []IncludeWarning
|
||||||
|
var extWarnings []ExtendWarning
|
||||||
|
if len(included.Include) > 0 {
|
||||||
|
w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited)
|
||||||
|
warnings = append(warnings, w...)
|
||||||
|
extWarnings = append(extWarnings, ew...)
|
||||||
|
}
|
||||||
|
|
||||||
|
mergeIncluded(p, included)
|
||||||
|
return warnings, extWarnings
|
||||||
|
}
|
||||||
|
|
||||||
// resolveProjectInclude fetches all files listed under a single project: entry
|
// resolveProjectInclude fetches all files listed under a single project: entry
|
||||||
// and merges them into p.
|
// and merges them into p.
|
||||||
func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
||||||
|
|||||||
Vendored
+14
@@ -0,0 +1,14 @@
|
|||||||
|
---
|
||||||
|
# Exercises include: remote: URL fetching and sub-include recursion.
|
||||||
|
# The remote file is a real public GitLab CI template; it may contain its own
|
||||||
|
# includes which should also be resolved. Exit code is 0 (warnings allowed).
|
||||||
|
stages:
|
||||||
|
- test
|
||||||
|
|
||||||
|
include:
|
||||||
|
- remote: https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
|
||||||
|
|
||||||
|
local-job:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- echo "local job alongside remote include"
|
||||||
Reference in New Issue
Block a user