Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b21a7d60dc | |||
| d34c39927d | |||
| a303f63a5e | |||
| a962c996c1 |
@@ -7,8 +7,20 @@ This project uses [Semantic Versioning](https://semver.org).
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **File and line numbers on findings** — every finding now includes the source file and line where the job is defined, e.g. `[ERROR] job "deploy" (src/deploy.yml:14): …`. For jobs that come from local or fetched includes the file reflects the include source. Pipeline-level findings (workflow rules, missing stages) reference the root pipeline file.
|
||||||
|
|
||||||
|
- **`glint graph includes` shows jobs per file** — each node in the Mermaid include dependency graph now shows the jobs defined directly in that file. Jobs are rendered as rounded nodes (`(name)`) in a distinct light-purple style, connected with dashed arrows (`-.->`) to distinguish ownership from the include hierarchy (solid `-->` arrows). The root pipeline file always shows its direct jobs; local and fetched project/component nodes show theirs when the file can be read.
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **`include: remote:` URL includes are now fetched and merged** — glint fetches plain HTTPS URLs in `include: remote:` entries (no authentication), parses the resulting YAML, merges its jobs into the pipeline, and recursively resolves any sub-includes the remote file itself declares. Unreachable or unparseable URLs emit a `[WARNING]` and lint continues on the rest of the pipeline. The `glint graph includes` command now expands remote nodes with their jobs and sub-include tree, matching the behaviour of local and project includes.
|
||||||
|
|
||||||
|
- **`needs: optional: true` downgraded to warning** — a `needs:` entry that carries `optional: true` and references a job not present in the pipeline now emits `[WARNING]` instead of `[ERROR]`. GitLab CI silently skips such dependencies at runtime (the job is absent when its include was not triggered), so the finding was a false positive. Non-optional missing needs remain errors. Optional missing deps are also excluded from the cycle-detection graph.
|
||||||
|
|
||||||
|
- **`extends:` jobs with missing script downgraded to warning** — a job that declares `extends:` but has no `script` after resolution now emits `[WARNING]` instead of `[ERROR]`. The script may legitimately come from a base job in a remote include that could not be fetched at lint time (e.g. no token configured).
|
||||||
|
|
||||||
- **Variable map form now parses correctly** — `variables:` entries that use the extended `{value, description, options}` form (GitLab CI 13.7+) no longer cause `yaml: cannot unmarshal !!map into string`. Both `Pipeline.Variables` and per-job `Variables` now accept either plain strings or map-form declarations.
|
- **Variable map form now parses correctly** — `variables:` entries that use the extended `{value, description, options}` form (GitLab CI 13.7+) no longer cause `yaml: cannot unmarshal !!map into string`. Both `Pipeline.Variables` and per-job `Variables` now accept either plain strings or map-form declarations.
|
||||||
- **`default.image` map form now parses correctly** — `default: image: {name: ..., pull_policy: ...}` used to cause `yaml: cannot unmarshal !!map into string`; `DefaultConfig.Image` is now typed as `any` to match `Job.Image`.
|
- **`default.image` map form now parses correctly** — `default: image: {name: ..., pull_policy: ...}` used to cause `yaml: cannot unmarshal !!map into string`; `DefaultConfig.Image` is now typed as `any` to match `Job.Image`.
|
||||||
- **`default.before_script` / `default.after_script` now accept both list and scalar forms** — previously `DefaultConfig.BeforeScript` and `DefaultConfig.AfterScript` were `[]string`, causing a parse error when the field was written as a block scalar string. They are now typed as `any` to match the corresponding `Job` fields.
|
- **`default.before_script` / `default.after_script` now accept both list and scalar forms** — previously `DefaultConfig.BeforeScript` and `DefaultConfig.AfterScript` were `[]string`, causing a parse error when the field was written as a block scalar string. They are now typed as `any` to match the corresponding `Job` fields.
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ tasks:
|
|||||||
ignore_error: true
|
ignore_error: true
|
||||||
- cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml
|
- cmd: ./{{.BINARY}} check testdata/keywords_invalid.yml
|
||||||
ignore_error: true
|
ignore_error: true
|
||||||
|
- cmd: ./{{.BINARY}} check testdata/includes_remote.yml
|
||||||
|
ignore_error: false
|
||||||
- cmd: ./{{.BINARY}} check testdata/includes_project.yml
|
- cmd: ./{{.BINARY}} check testdata/includes_project.yml
|
||||||
ignore_error: false
|
ignore_error: false
|
||||||
- cmd: ./{{.BINARY}} check testdata/includes_component.yml
|
- cmd: ./{{.BINARY}} check testdata/includes_component.yml
|
||||||
|
|||||||
@@ -142,6 +142,24 @@ func (cfg GitLabConfig) FetchFile(project, filePath, ref string) ([]byte, error)
|
|||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FetchURL downloads the content at a plain HTTPS URL without authentication.
|
||||||
|
// Used for include: remote: entries which are public by definition.
|
||||||
|
func FetchURL(rawURL string) ([]byte, error) {
|
||||||
|
resp, err := http.Get(rawURL) //nolint:noctx
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("GET %s: %w", rawURL, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading body: %w", err)
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("GET %s: status %d", rawURL, resp.StatusCode)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
func firstNonEmpty(values ...string) string {
|
func firstNonEmpty(values ...string) string {
|
||||||
for _, v := range values {
|
for _, v := range values {
|
||||||
if v != "" {
|
if v != "" {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"git.k3nny.fr/glint/internal/fetcher"
|
"git.k3nny.fr/glint/internal/fetcher"
|
||||||
@@ -11,6 +12,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Includes returns a Mermaid flowchart of the full include dependency tree.
|
// Includes returns a Mermaid flowchart of the full include dependency tree.
|
||||||
|
// Each node shows the jobs defined directly in that file, connected with
|
||||||
|
// dashed arrows (solid arrows represent the include hierarchy itself).
|
||||||
// It recurses into project:, component:, and local: includes to expose
|
// It recurses into project:, component:, and local: includes to expose
|
||||||
// transitive dependencies. Includes that cannot be fetched are shown but not expanded.
|
// transitive dependencies. Includes that cannot be fetched are shown but not expanded.
|
||||||
func Includes(sourcePath string, rawIncludes []any, cfg fetcher.GitLabConfig) string {
|
func Includes(sourcePath string, rawIncludes []any, cfg fetcher.GitLabConfig) string {
|
||||||
@@ -19,7 +22,12 @@ func Includes(sourcePath string, rawIncludes []any, cfg fetcher.GitLabConfig) st
|
|||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
baseDir: filepath.Dir(sourcePath),
|
baseDir: filepath.Dir(sourcePath),
|
||||||
}
|
}
|
||||||
root := &treeNode{id: "root", label: mermaidLabel(sourcePath), class: "main"}
|
root := &treeNode{
|
||||||
|
id: "root",
|
||||||
|
label: mermaidLabel(sourcePath),
|
||||||
|
class: "main",
|
||||||
|
jobs: directJobs(sourcePath),
|
||||||
|
}
|
||||||
b.buildChildren(root, rawIncludes)
|
b.buildChildren(root, rawIncludes)
|
||||||
return renderTree(root)
|
return renderTree(root)
|
||||||
}
|
}
|
||||||
@@ -29,6 +37,7 @@ type treeNode struct {
|
|||||||
label string
|
label string
|
||||||
class string
|
class string
|
||||||
children []*treeNode
|
children []*treeNode
|
||||||
|
jobs []string // job names defined directly in this file
|
||||||
}
|
}
|
||||||
|
|
||||||
// treeBuilder accumulates state while recursively traversing include entries.
|
// treeBuilder accumulates state while recursively traversing include entries.
|
||||||
@@ -105,7 +114,9 @@ func (b *treeBuilder) parseMap(m map[string]any) []*treeNode {
|
|||||||
return []*treeNode{node}
|
return []*treeNode{node}
|
||||||
}
|
}
|
||||||
if remote, ok := m["remote"].(string); ok {
|
if remote, ok := m["remote"].(string); ok {
|
||||||
return []*treeNode{{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}}
|
node := &treeNode{id: b.nextID(), label: "remote:<br>" + mermaidLabel(remote), class: "remote"}
|
||||||
|
b.recurseRemote(node, remote)
|
||||||
|
return []*treeNode{node}
|
||||||
}
|
}
|
||||||
if tmpl, ok := m["template"].(string); ok {
|
if tmpl, ok := m["template"].(string); ok {
|
||||||
return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}}
|
return []*treeNode{{id: b.nextID(), label: "template:<br>" + mermaidLabel(tmpl), class: "template"}}
|
||||||
@@ -126,7 +137,11 @@ func (b *treeBuilder) recurseLocal(node *treeNode, path string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p, err := model.ParseBytes(data)
|
p, err := model.ParseBytes(data)
|
||||||
if err != nil || len(p.Include) == 0 {
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node.jobs = jobNames(p)
|
||||||
|
if len(p.Include) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
orig := b.baseDir
|
orig := b.baseDir
|
||||||
@@ -147,7 +162,33 @@ func (b *treeBuilder) recurseProject(node *treeNode, project, filePath, ref stri
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p, err := model.ParseBytes(data)
|
p, err := model.ParseBytes(data)
|
||||||
if err != nil || len(p.Include) == 0 {
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node.jobs = jobNames(p)
|
||||||
|
if len(p.Include) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.buildChildren(node, p.Include)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *treeBuilder) recurseRemote(node *treeNode, rawURL string) {
|
||||||
|
key := "remote:" + rawURL
|
||||||
|
if b.visited[key] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.visited[key] = true
|
||||||
|
|
||||||
|
data, err := fetcher.FetchURL(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p, err := model.ParseBytes(data)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node.jobs = jobNames(p)
|
||||||
|
if len(p.Include) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
b.buildChildren(node, p.Include)
|
b.buildChildren(node, p.Include)
|
||||||
@@ -172,7 +213,11 @@ func (b *treeBuilder) recurseComponent(node *treeNode, ref string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p, err := model.ParseBytes(data)
|
p, err := model.ParseBytes(data)
|
||||||
if err != nil || len(p.Include) == 0 {
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
node.jobs = jobNames(p)
|
||||||
|
if len(p.Include) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
b.buildChildren(node, p.Include)
|
b.buildChildren(node, p.Include)
|
||||||
@@ -193,11 +238,19 @@ func renderTree(root *treeNode) string {
|
|||||||
w(" classDef local fill:#428fdc,stroke:#1068bf,color:#fff")
|
w(" classDef local fill:#428fdc,stroke:#1068bf,color:#fff")
|
||||||
w(" classDef remote fill:#868686,stroke:#686868,color:#fff")
|
w(" classDef remote fill:#868686,stroke:#686868,color:#fff")
|
||||||
w(" classDef template fill:#fc6d26,stroke:#e56b1f,color:#fff")
|
w(" classDef template fill:#fc6d26,stroke:#e56b1f,color:#fff")
|
||||||
|
w(" classDef job fill:#f5f5ff,stroke:#7175a0,color:#333")
|
||||||
w("")
|
w("")
|
||||||
|
|
||||||
|
jobCounter := 0
|
||||||
var emit func(n *treeNode)
|
var emit func(n *treeNode)
|
||||||
emit = func(n *treeNode) {
|
emit = func(n *treeNode) {
|
||||||
wf(" %s[\"%s\"]:::%s", n.id, n.label, n.class)
|
wf(" %s[\"%s\"]:::%s", n.id, n.label, n.class)
|
||||||
|
for _, jobName := range n.jobs {
|
||||||
|
jobCounter++
|
||||||
|
jobID := fmt.Sprintf("job%d", jobCounter)
|
||||||
|
wf(" %s(\"%s\"):::job", jobID, mermaidLabel(jobName))
|
||||||
|
wf(" %s -.-> %s", n.id, jobID)
|
||||||
|
}
|
||||||
for _, child := range n.children {
|
for _, child := range n.children {
|
||||||
emit(child)
|
emit(child)
|
||||||
wf(" %s --> %s", n.id, child.id)
|
wf(" %s --> %s", n.id, child.id)
|
||||||
@@ -208,6 +261,29 @@ func renderTree(root *treeNode) string {
|
|||||||
return sb.String()
|
return sb.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// directJobs parses a single pipeline file (without include resolution) and
|
||||||
|
// returns the sorted list of job names defined directly in it.
|
||||||
|
func directJobs(path string) []string {
|
||||||
|
p, err := model.Parse(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return jobNames(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobNames returns a sorted slice of all job names defined in p.
|
||||||
|
func jobNames(p *model.Pipeline) []string {
|
||||||
|
if len(p.Jobs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(p.Jobs))
|
||||||
|
for name := range p.Jobs {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
// parseComponentRef parses a CI/CD component reference of the form
|
// parseComponentRef parses a CI/CD component reference of the form
|
||||||
// <host>/<project-path>/<component-name>@<version>.
|
// <host>/<project-path>/<component-name>@<version>.
|
||||||
func parseComponentRef(ref string) (host, project, component, version string, err error) {
|
func parseComponentRef(ref string) (host, project, component, version string, err error) {
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ func checkDependencies(p *model.Pipeline) []Finding {
|
|||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: Error,
|
||||||
Job: name,
|
Job: name,
|
||||||
|
File: job.File,
|
||||||
|
Line: job.Line,
|
||||||
Message: fmt.Sprintf("'dependencies' references unknown job %q", dep),
|
Message: fmt.Sprintf("'dependencies' references unknown job %q", dep),
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
@@ -34,6 +36,8 @@ func checkDependencies(p *model.Pipeline) []Finding {
|
|||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: Error,
|
||||||
Job: name,
|
Job: name,
|
||||||
|
File: job.File,
|
||||||
|
Line: job.Line,
|
||||||
Message: fmt.Sprintf("'dependencies' job %q must be in an earlier stage (in %q, current job is in %q)", dep, depJob.Stage, job.Stage),
|
Message: fmt.Sprintf("'dependencies' job %q must be in an earlier stage (in %q, current job is in %q)", dep, depJob.Stage, job.Stage),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,12 +17,25 @@ const (
|
|||||||
type Finding struct {
|
type Finding struct {
|
||||||
Severity Severity
|
Severity Severity
|
||||||
Job string // empty for pipeline-level findings
|
Job string // empty for pipeline-level findings
|
||||||
|
File string // source file where the finding originates
|
||||||
|
Line int // line number in File (0 = unknown)
|
||||||
Message string
|
Message string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f Finding) String() string {
|
func (f Finding) String() string {
|
||||||
|
loc := ""
|
||||||
|
if f.File != "" {
|
||||||
|
if f.Line > 0 {
|
||||||
|
loc = fmt.Sprintf(" (%s:%d)", f.File, f.Line)
|
||||||
|
} else {
|
||||||
|
loc = fmt.Sprintf(" (%s)", f.File)
|
||||||
|
}
|
||||||
|
}
|
||||||
if f.Job != "" {
|
if f.Job != "" {
|
||||||
return fmt.Sprintf("[%s] job %q: %s", f.Severity, f.Job, f.Message)
|
return fmt.Sprintf("[%s] job %q%s: %s", f.Severity, f.Job, loc, f.Message)
|
||||||
|
}
|
||||||
|
if loc != "" {
|
||||||
|
return fmt.Sprintf("[%s]%s: %s", f.Severity, loc, f.Message)
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("[%s] %s", f.Severity, f.Message)
|
return fmt.Sprintf("[%s] %s", f.Severity, f.Message)
|
||||||
}
|
}
|
||||||
@@ -43,6 +56,7 @@ func checkStages(p *model.Pipeline) []Finding {
|
|||||||
if len(p.Stages) == 0 {
|
if len(p.Stages) == 0 {
|
||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Warning,
|
Severity: Warning,
|
||||||
|
File: p.SourceFile,
|
||||||
Message: "no stages defined; GitLab will use default stages (build, test, deploy)",
|
Message: "no stages defined; GitLab will use default stages (build, test, deploy)",
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -58,6 +72,7 @@ func checkWorkflow(p *model.Pipeline) []Finding {
|
|||||||
if rule.When != "" && !validWorkflowRuleWhen[rule.When] {
|
if rule.When != "" && !validWorkflowRuleWhen[rule.When] {
|
||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: Error,
|
||||||
|
File: p.SourceFile,
|
||||||
Message: fmt.Sprintf("workflow.rules[%d].when has invalid value %q; valid: always, never", i, rule.When),
|
Message: fmt.Sprintf("workflow.rules[%d].when has invalid value %q; valid: always, never", i, rule.When),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -88,10 +103,16 @@ func checkJob(name string, job model.Job, stageSet map[string]bool) []Finding {
|
|||||||
|
|
||||||
// After extends resolution, a job with no script/run is an error.
|
// After extends resolution, a job with no script/run is an error.
|
||||||
// Exceptions: trigger jobs, pages jobs (use pages: keyword), and template jobs.
|
// Exceptions: trigger jobs, pages jobs (use pages: keyword), and template jobs.
|
||||||
|
// When the job has extends:, the script may come from a base that couldn't be
|
||||||
|
// fetched (e.g. a remote include without a token), so downgrade to warning.
|
||||||
hasScript := scriptNonEmpty(job.Script) || job.Run != nil
|
hasScript := scriptNonEmpty(job.Script) || job.Run != nil
|
||||||
if !isTemplate && !isTrigger && job.Pages == nil && !hasScript {
|
if !isTemplate && !isTrigger && job.Pages == nil && !hasScript {
|
||||||
|
sev := Error
|
||||||
|
if job.Extends != nil {
|
||||||
|
sev = Warning
|
||||||
|
}
|
||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: sev,
|
||||||
Job: name,
|
Job: name,
|
||||||
Message: "missing required field 'script' (or 'run')",
|
Message: "missing required field 'script' (or 'run')",
|
||||||
})
|
})
|
||||||
@@ -137,6 +158,13 @@ func checkJob(name string, job model.Job, stageSet map[string]bool) []Finding {
|
|||||||
|
|
||||||
findings = append(findings, checkJobKeywords(name, job)...)
|
findings = append(findings, checkJobKeywords(name, job)...)
|
||||||
|
|
||||||
|
// Attach source location to every job-scoped finding collected above.
|
||||||
|
for i := range findings {
|
||||||
|
if findings[i].Job != "" && findings[i].File == "" {
|
||||||
|
findings[i].File = job.File
|
||||||
|
findings[i].Line = job.Line
|
||||||
|
}
|
||||||
|
}
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+42
-19
@@ -6,6 +6,12 @@ import (
|
|||||||
"git.k3nny.fr/glint/internal/model"
|
"git.k3nny.fr/glint/internal/model"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// needEntry is a parsed element from a job's needs: list.
|
||||||
|
type needEntry struct {
|
||||||
|
job string
|
||||||
|
optional bool // true when the needs entry carries optional: true
|
||||||
|
}
|
||||||
|
|
||||||
func checkNeeds(p *model.Pipeline) []Finding {
|
func checkNeeds(p *model.Pipeline) []Finding {
|
||||||
var findings []Finding
|
var findings []Finding
|
||||||
|
|
||||||
@@ -15,7 +21,7 @@ func checkNeeds(p *model.Pipeline) []Finding {
|
|||||||
stageIndex[s] = i
|
stageIndex[s] = i
|
||||||
}
|
}
|
||||||
|
|
||||||
// needsGraph maps each job to the list of jobs it depends on.
|
// needsGraph maps each job to the jobs it depends on (existing jobs only).
|
||||||
// Used for cycle detection after individual checks.
|
// Used for cycle detection after individual checks.
|
||||||
needsGraph := make(map[string][]string)
|
needsGraph := make(map[string][]string)
|
||||||
|
|
||||||
@@ -24,22 +30,33 @@ func checkNeeds(p *model.Pipeline) []Finding {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
neededNames := parseNeedJobNames(job.Needs)
|
entries := parseNeedEntries(job.Needs)
|
||||||
needsGraph[name] = neededNames
|
|
||||||
|
|
||||||
jobStageIdx, jobHasStage := stageIndex[job.Stage]
|
jobStageIdx, jobHasStage := stageIndex[job.Stage]
|
||||||
|
|
||||||
for _, needed := range neededNames {
|
for _, entry := range entries {
|
||||||
neededJob, exists := p.Jobs[needed]
|
neededJob, exists := p.Jobs[entry.job]
|
||||||
if !exists {
|
if !exists {
|
||||||
|
// optional: true means GitLab CI will silently skip the
|
||||||
|
// dependency when the job is absent (e.g. from a conditional
|
||||||
|
// include). Downgrade to warning so users are informed without
|
||||||
|
// failing the lint.
|
||||||
|
sev := Error
|
||||||
|
if entry.optional {
|
||||||
|
sev = Warning
|
||||||
|
}
|
||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: sev,
|
||||||
Job: name,
|
Job: name,
|
||||||
Message: fmt.Sprintf("needs unknown job %q", needed),
|
File: job.File,
|
||||||
|
Line: job.Line,
|
||||||
|
Message: fmt.Sprintf("needs unknown job %q", entry.job),
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add to the cycle-detection graph only when the dep exists.
|
||||||
|
needsGraph[name] = append(needsGraph[name], entry.job)
|
||||||
|
|
||||||
// A job cannot need a job in a later stage.
|
// A job cannot need a job in a later stage.
|
||||||
if len(p.Stages) > 0 && jobHasStage && neededJob.Stage != "" {
|
if len(p.Stages) > 0 && jobHasStage && neededJob.Stage != "" {
|
||||||
neededStageIdx, neededHasStage := stageIndex[neededJob.Stage]
|
neededStageIdx, neededHasStage := stageIndex[neededJob.Stage]
|
||||||
@@ -47,9 +64,11 @@ func checkNeeds(p *model.Pipeline) []Finding {
|
|||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: Error,
|
||||||
Job: name,
|
Job: name,
|
||||||
|
File: job.File,
|
||||||
|
Line: job.Line,
|
||||||
Message: fmt.Sprintf(
|
Message: fmt.Sprintf(
|
||||||
"needs %q which is in a later stage (%q after %q)",
|
"needs %q which is in a later stage (%q after %q)",
|
||||||
needed, neededJob.Stage, job.Stage,
|
entry.job, neededJob.Stage, job.Stage,
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -57,32 +76,33 @@ func checkNeeds(p *model.Pipeline) []Finding {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
findings = append(findings, detectNeedsCycles(needsGraph)...)
|
findings = append(findings, detectNeedsCycles(needsGraph, p.Jobs)...)
|
||||||
return findings
|
return findings
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseNeedJobNames extracts job names from a needs: list.
|
// parseNeedEntries extracts needs entries from a needs: list, preserving the
|
||||||
// Each element is either a plain string or a map with a "job" key.
|
// optional flag. Each element is a plain string (job name) or a map with a
|
||||||
// Cross-pipeline needs (maps with a "pipeline" key) are skipped.
|
// "job" key. Cross-pipeline needs (maps with a "pipeline" key) are skipped.
|
||||||
func parseNeedJobNames(needs []any) []string {
|
func parseNeedEntries(needs []any) []needEntry {
|
||||||
var names []string
|
var entries []needEntry
|
||||||
for _, n := range needs {
|
for _, n := range needs {
|
||||||
switch v := n.(type) {
|
switch v := n.(type) {
|
||||||
case string:
|
case string:
|
||||||
names = append(names, v)
|
entries = append(entries, needEntry{job: v})
|
||||||
case map[string]any:
|
case map[string]any:
|
||||||
if _, crossPipeline := v["pipeline"]; crossPipeline {
|
if _, crossPipeline := v["pipeline"]; crossPipeline {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if job, ok := v["job"].(string); ok {
|
if job, ok := v["job"].(string); ok {
|
||||||
names = append(names, job)
|
optional, _ := v["optional"].(bool)
|
||||||
|
entries = append(entries, needEntry{job: job, optional: optional})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return names
|
return entries
|
||||||
}
|
}
|
||||||
|
|
||||||
func detectNeedsCycles(graph map[string][]string) []Finding {
|
func detectNeedsCycles(graph map[string][]string, jobs map[string]model.Job) []Finding {
|
||||||
const (
|
const (
|
||||||
unvisited = 0
|
unvisited = 0
|
||||||
visiting = 1
|
visiting = 1
|
||||||
@@ -101,9 +121,12 @@ func detectNeedsCycles(graph map[string][]string) []Finding {
|
|||||||
case visiting:
|
case visiting:
|
||||||
if !reported[name] {
|
if !reported[name] {
|
||||||
reported[name] = true
|
reported[name] = true
|
||||||
|
j := jobs[name]
|
||||||
findings = append(findings, Finding{
|
findings = append(findings, Finding{
|
||||||
Severity: Error,
|
Severity: Error,
|
||||||
Job: name,
|
Job: name,
|
||||||
|
File: j.File,
|
||||||
|
Line: j.Line,
|
||||||
Message: fmt.Sprintf("circular dependency in needs: %v → %s", path, name),
|
Message: fmt.Sprintf("circular dependency in needs: %v → %s", path, name),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,14 +13,22 @@ func Parse(path string) (*Pipeline, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading file: %w", err)
|
return nil, fmt.Errorf("reading file: %w", err)
|
||||||
}
|
}
|
||||||
return ParseBytes(data)
|
p, err := ParseBytes(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
p.SourceFile = path
|
||||||
|
p.SetJobOrigin(path)
|
||||||
|
return p, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseBytes parses YAML from an in-memory byte slice.
|
// ParseBytes parses YAML from an in-memory byte slice.
|
||||||
func ParseBytes(data []byte) (*Pipeline, error) {
|
func ParseBytes(data []byte) (*Pipeline, error) {
|
||||||
// First pass: decode into a raw map to extract job keys.
|
// First pass: parse into a yaml.Node document to extract job keys with
|
||||||
var raw map[string]yaml.Node
|
// their exact source line numbers (key nodes carry the line, value nodes
|
||||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
// carry the body we decode into Job / map[string]any).
|
||||||
|
var doc yaml.Node
|
||||||
|
if err := yaml.Unmarshal(data, &doc); err != nil {
|
||||||
return nil, fmt.Errorf("parsing YAML: %w", err)
|
return nil, fmt.Errorf("parsing YAML: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,21 +40,36 @@ func ParseBytes(data []byte) (*Pipeline, error) {
|
|||||||
|
|
||||||
p.Jobs = make(map[string]Job)
|
p.Jobs = make(map[string]Job)
|
||||||
p.RawJobs = make(map[string]map[string]any)
|
p.RawJobs = make(map[string]map[string]any)
|
||||||
for key, node := range raw {
|
|
||||||
|
if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 {
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
root := doc.Content[0]
|
||||||
|
if root.Kind != yaml.MappingNode {
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk root mapping in key/value pairs.
|
||||||
|
for i := 0; i+1 < len(root.Content); i += 2 {
|
||||||
|
keyNode := root.Content[i]
|
||||||
|
valNode := root.Content[i+1]
|
||||||
|
key := keyNode.Value
|
||||||
if ReservedKeys[key] {
|
if ReservedKeys[key] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
var rawMap map[string]any
|
var rawMap map[string]any
|
||||||
if err := node.Decode(&rawMap); err != nil {
|
if err := valNode.Decode(&rawMap); err != nil {
|
||||||
return nil, fmt.Errorf("parsing raw job %q: %w", key, err)
|
return nil, fmt.Errorf("parsing raw job %q: %w", key, err)
|
||||||
}
|
}
|
||||||
p.RawJobs[key] = rawMap
|
p.RawJobs[key] = rawMap
|
||||||
|
|
||||||
var j Job
|
var j Job
|
||||||
if err := node.Decode(&j); err != nil {
|
if err := valNode.Decode(&j); err != nil {
|
||||||
return nil, fmt.Errorf("parsing job %q: %w", key, err)
|
return nil, fmt.Errorf("parsing job %q: %w", key, err)
|
||||||
}
|
}
|
||||||
j.Name = key
|
j.Name = key
|
||||||
|
j.Line = keyNode.Line // exact line of the job name key
|
||||||
p.Jobs[key] = j
|
p.Jobs[key] = j
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,16 +3,28 @@ package model
|
|||||||
// Pipeline represents the top-level structure of a .gitlab-ci.yml file.
|
// Pipeline represents the top-level structure of a .gitlab-ci.yml file.
|
||||||
// Unknown top-level keys are collected into Jobs.
|
// Unknown top-level keys are collected into Jobs.
|
||||||
type Pipeline struct {
|
type Pipeline struct {
|
||||||
Stages []string `yaml:"stages"`
|
SourceFile string // path of the root pipeline file; set by Parse
|
||||||
Variables map[string]any `yaml:"variables"` // string or {value,description,options} map
|
Stages []string `yaml:"stages"`
|
||||||
Default *DefaultConfig `yaml:"default"`
|
Variables map[string]any `yaml:"variables"` // string or {value,description,options} map
|
||||||
Include []any `yaml:"include"`
|
Default *DefaultConfig `yaml:"default"`
|
||||||
Workflow *Workflow `yaml:"workflow"`
|
Include []any `yaml:"include"`
|
||||||
|
Workflow *Workflow `yaml:"workflow"`
|
||||||
// Jobs holds every non-reserved top-level key (i.e. job definitions).
|
// Jobs holds every non-reserved top-level key (i.e. job definitions).
|
||||||
Jobs map[string]Job `yaml:"-"`
|
Jobs map[string]Job `yaml:"-"`
|
||||||
RawJobs map[string]map[string]any `yaml:"-"` // pre-resolution raw maps, used by the resolver
|
RawJobs map[string]map[string]any `yaml:"-"` // pre-resolution raw maps, used by the resolver
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetJobOrigin sets the File field on all jobs that don't already have one.
|
||||||
|
// Called after ParseBytes to record which file each job came from.
|
||||||
|
func (p *Pipeline) SetJobOrigin(file string) {
|
||||||
|
for name, j := range p.Jobs {
|
||||||
|
if j.File == "" {
|
||||||
|
j.File = file
|
||||||
|
}
|
||||||
|
p.Jobs[name] = j
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type DefaultConfig struct {
|
type DefaultConfig struct {
|
||||||
Image any `yaml:"image"` // string or {name,pull_policy,...} map
|
Image any `yaml:"image"` // string or {name,pull_policy,...} map
|
||||||
BeforeScript any `yaml:"before_script"` // []string or string (block scalar)
|
BeforeScript any `yaml:"before_script"` // []string or string (block scalar)
|
||||||
@@ -29,7 +41,9 @@ type Workflow struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
Name string // set by parser, not from YAML
|
Name string // set by parser, not from YAML
|
||||||
|
File string // source file; set by Parse / resolver
|
||||||
|
Line int // line of the job key in its source file; set by parser
|
||||||
Stage string `yaml:"stage"`
|
Stage string `yaml:"stage"`
|
||||||
Script any `yaml:"script"` // []string or string (block scalar)
|
Script any `yaml:"script"` // []string or string (block scalar)
|
||||||
Run any `yaml:"run"` // alternative to script (CI steps)
|
Run any `yaml:"run"` // alternative to script (CI steps)
|
||||||
|
|||||||
@@ -90,7 +90,14 @@ func resolveIncludes(p *model.Pipeline, includes []any, cfg fetcher.GitLabConfig
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// remote, template — resolved by GitLab at runtime, skip silently.
|
if remote, _ := entry["remote"].(string); remote != "" {
|
||||||
|
w, ew := resolveRemoteInclude(p, remote, cfg, rootDir, visited)
|
||||||
|
warnings = append(warnings, w...)
|
||||||
|
extWarnings = append(extWarnings, ew...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// template — resolved by GitLab at runtime, skip silently.
|
||||||
}
|
}
|
||||||
return warnings, extWarnings
|
return warnings, extWarnings
|
||||||
}
|
}
|
||||||
@@ -117,6 +124,7 @@ func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabCo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil
|
return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil
|
||||||
}
|
}
|
||||||
|
included.SetJobOrigin(absPath)
|
||||||
|
|
||||||
// Recursively resolve the included file's own includes first, merging
|
// Recursively resolve the included file's own includes first, merging
|
||||||
// everything into `included` before we merge it into the parent `p`.
|
// everything into `included` before we merge it into the parent `p`.
|
||||||
@@ -132,6 +140,39 @@ func resolveLocalInclude(p *model.Pipeline, rawPath string, cfg fetcher.GitLabCo
|
|||||||
return warnings, extWarnings
|
return warnings, extWarnings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveRemoteInclude fetches a plain HTTPS URL, parses it as CI YAML, and
|
||||||
|
// merges it into p. Sub-includes of the fetched file are resolved recursively.
|
||||||
|
func resolveRemoteInclude(p *model.Pipeline, rawURL string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
||||||
|
label := "remote " + rawURL
|
||||||
|
|
||||||
|
if visited[rawURL] {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
visited[rawURL] = true
|
||||||
|
|
||||||
|
data, err := fetcher.FetchURL(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return []IncludeWarning{{Label: label, Err: err}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
included, err := model.ParseBytes(data)
|
||||||
|
if err != nil {
|
||||||
|
return []IncludeWarning{{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)}}, nil
|
||||||
|
}
|
||||||
|
included.SetJobOrigin(rawURL)
|
||||||
|
|
||||||
|
var warnings []IncludeWarning
|
||||||
|
var extWarnings []ExtendWarning
|
||||||
|
if len(included.Include) > 0 {
|
||||||
|
w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited)
|
||||||
|
warnings = append(warnings, w...)
|
||||||
|
extWarnings = append(extWarnings, ew...)
|
||||||
|
}
|
||||||
|
|
||||||
|
mergeIncluded(p, included)
|
||||||
|
return warnings, extWarnings
|
||||||
|
}
|
||||||
|
|
||||||
// resolveProjectInclude fetches all files listed under a single project: entry
|
// resolveProjectInclude fetches all files listed under a single project: entry
|
||||||
// and merges them into p.
|
// and merges them into p.
|
||||||
func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project string, cfg fetcher.GitLabConfig, rootDir string, visited map[string]bool) ([]IncludeWarning, []ExtendWarning) {
|
||||||
@@ -161,6 +202,7 @@ func resolveProjectInclude(p *model.Pipeline, entry map[string]any, project stri
|
|||||||
warnings = append(warnings, IncludeWarning{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)})
|
warnings = append(warnings, IncludeWarning{Label: label, Err: fmt.Errorf("parsing YAML: %w", err)})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
included.SetJobOrigin(label)
|
||||||
|
|
||||||
if len(included.Include) > 0 {
|
if len(included.Include) > 0 {
|
||||||
w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited)
|
w, ew := resolveIncludes(included, included.Include, cfg, rootDir, visited)
|
||||||
@@ -200,6 +242,7 @@ func resolveComponentInclude(p *model.Pipeline, ref string, cfg fetcher.GitLabCo
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return IncludeWarning{Label: label, Err: fmt.Errorf("parsing component YAML: %w", err)}, nil, true
|
return IncludeWarning{Label: label, Err: fmt.Errorf("parsing component YAML: %w", err)}, nil, true
|
||||||
}
|
}
|
||||||
|
included.SetJobOrigin(label)
|
||||||
|
|
||||||
var extWarnings []ExtendWarning
|
var extWarnings []ExtendWarning
|
||||||
if len(included.Include) > 0 {
|
if len(included.Include) > 0 {
|
||||||
|
|||||||
Vendored
+14
@@ -0,0 +1,14 @@
|
|||||||
|
---
|
||||||
|
# Exercises include: remote: URL fetching and sub-include recursion.
|
||||||
|
# The remote file is a real public GitLab CI template; it may contain its own
|
||||||
|
# includes which should also be resolved. Exit code is 0 (warnings allowed).
|
||||||
|
stages:
|
||||||
|
- test
|
||||||
|
|
||||||
|
include:
|
||||||
|
- remote: https://gitlab.com/gitlab-org/gitlab/-/raw/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
|
||||||
|
|
||||||
|
local-job:
|
||||||
|
stage: test
|
||||||
|
script:
|
||||||
|
- echo "local job alongside remote include"
|
||||||
Vendored
+9
@@ -38,3 +38,12 @@ missing-needs-job:
|
|||||||
- echo "bad"
|
- echo "bad"
|
||||||
needs:
|
needs:
|
||||||
- nonexistent-job # ERROR: job doesn't exist
|
- nonexistent-job # ERROR: job doesn't exist
|
||||||
|
|
||||||
|
# optional: true — missing dep should be WARNING not ERROR
|
||||||
|
optional-needs-job:
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- echo "I depend on something that may not exist"
|
||||||
|
needs:
|
||||||
|
- job: nonexistent-optional-job
|
||||||
|
optional: true
|
||||||
|
|||||||
Reference in New Issue
Block a user