package main import ( "bufio" "context" "encoding/csv" "encoding/json" "flag" "fmt" "io" "net/url" "os" "os/exec" "slices" "sort" "strconv" "strings" "time" "unicode" "github.com/google/go-github/v63/github" "gopkg.in/yaml.v3" ) // Config represents the image-scanning configuration and has a 1:1 relationship // with its configuration file. type Config struct { ReportFile string `yaml:"report_file"` ReportDir string `yaml:"report_dir"` ImagesFile string `yaml:"images_file"` ImagesTeamsFile string `yaml:"images_teams_file"` SourcesTeamsFile string `yaml:"sources_teams_file"` EOLFile string `yaml:"eol_file"` Repo struct { GitUrl string `yaml:"git_url"` Owner string `yaml:"owner"` Repo string `yaml:"repo"` Dashboard struct { Title string `yaml:"title"` Label string `yaml:"label"` } `yaml:"dashboard"` Issue struct { Title string `yaml:"title"` Label string `yaml:"label"` LabelPrefix string `yaml:"label_prefix"` TeamPrefix string `yaml:"team_prefix"` CriticalLabel string `yaml:"critical_label"` MirroredImageLabel string `yaml:"mirrored_image_label"` RancherImageLabel string `yaml:"rancher_image_label"` } `yaml:"issue"` IssueFailure struct { Title string `yaml:"title"` Label string `yaml:"label"` } `yaml:"issue_failure"` } `yaml:"repo"` CVES struct { Status struct { Default string `yaml:"default"` NotAffected string `yaml:"not_affected"` Affected string `yaml:"affected"` Fixed string `yaml:"fixed"` } `yaml:"status"` Justification struct { Jus1 string `yaml:"jus_1"` Jus2 string `yaml:"jus_2"` Jus3 string `yaml:"jus_3"` Jus4 string `yaml:"jus_4"` Jus5 string `yaml:"jus_5"` Jus6 string `yaml:"jus_6"` } `yaml:"justification"` } `yaml:"cves"` Labels struct { Teams []string `yaml:"teams"` } `yaml:"labels"` Images struct { Tier1 []string `yaml:"tier1"` } `yaml:"images"` Scanner struct { Tool string `yaml:"tool"` Scanners string `yaml:"scanners"` Format string `yaml:"format"` Report string `yaml:"report"` Severity string `yaml:"severity"` } `yaml:"scanner"` } // EOLConfig represents the configuration file that holds information about EOL // (end-of-life) images. type EOLConfig struct { Images []string `yaml:"images"` } // RepoIssue contains only the needed data from the GitHub issue. type RepoIssue struct { Number int State string Title string Body string Labels []string URL string Match bool // Used to match the issue to a scan report. CreatedAt time.Time ClosedAt time.Time } type RepoIssues map[string]RepoIssue // The CVE CSV database file is an array of string arrays, where each field // matches a column in the file. type CvesRecords [][]string // Image is the main structure of image-scanning that holds all the data about // the scan of an image, its metadata with the source labels, the // vulnerabilities and data about the issue in GitHub. type Image struct { Image string EOL bool SourceLabels []string ReleaseLabels []string Issue RepoIssue IssueID int IssueTitle string IssueBody string IssueLabels []string Mirrored bool ScanFailed bool ScanFailureReason string UnsupportedBaseImage bool VulnCount map[string]map[string]int Metadata struct { OS struct { Family string Name string EOL bool } Image struct { Labels struct { Source string URL string } } } Vulnerabilities []Vulnerability } type Images []Image type TeamsFile map[string][]string // Vulnerability contains all the info about a vulnerability. Most of the fields // match the data that comes from Trivy's scan report, other fields are // populated with data from the CVE database file. type Vulnerability struct { Key string SoftKey string Target string Class string Type string Status string Justification string Title string VulnerabilityID string PublishedDate string PkgName string InstalledVersion string FixedVersion string PrimaryURL string Severity string } // VulnScanReport holds Trivy's scan report, in JSON format, of an image. type VulnScanReport struct { Metadata struct { OS struct { Family string `json:"Family"` Name string `json:"Name"` EOSL bool `json:"EOSL"` } `json:"OS"` ImageConfig struct { Config struct { Labels struct { Source string `json:"org.opencontainers.image.source"` URL string `json:"org.opencontainers.image.url"` } `json:"Labels"` } `json:"config"` } `json:"ImageConfig"` } `json:"Metadata"` Results []struct { Target string `json:"Target"` Class string `json:"Class"` Type string `json:"Type"` Vulnerabilities []struct { VulnerabilityID string `json:"VulnerabilityID"` Title string `json:"Title"` PkgName string `json:"PkgName"` InstalledVersion string `json:"InstalledVersion"` FixedVersion string `json:"FixedVersion"` PrimaryURL string `json:"PrimaryURL"` Severity string `json:"Severity"` PublishedDate string `json:"PublishedDate"` } `json:"Vulnerabilities"` } `json:"Results"` } const BASE_OS_PKG = "os-pkgs" const UPSTREAM = "upstream" // Constant for the columns of the CVE CSV database file. const ( COL_IMAGE int = iota COL_RELEASE COL_PACKAGE_NAME COL_PACKAGE_VERSION COL_TYPE COL_VULNERABILITY_ID COL_SEVERITY COL_URL COL_TARGET COL_PATCHED_VERSION COL_MIRRORED COL_STATUS COL_JUSTIFICATION COL_KEY COL_SOFTKEY ) var CLIENT *github.Client var CTX context.Context var CONFIG Config var EOLCONFIG EOLConfig var FLAG_REPO_TOKEN string var FLAG_DRY_RUN bool var FLAG_REPORT bool var FLAG_DASHBOARD bool var FLAG_CONFIG_FILE string // Function that adds all the related issue labels to the images. func addLabels(image Image, imagesTeams TeamsFile, sourcesTeams TeamsFile) Image { var teamLabels []string var foundTeam bool i := image // Add default cve report label. labels := []string{CONFIG.Repo.Issue.Label} // Add critical CVE label. if i.VulnCount["total"]["CRITICAL"] > 0 { labels = append(labels, prefixLabels( []string{CONFIG.Repo.Issue.CriticalLabel}, CONFIG.Repo.Issue.LabelPrefix)...) } // Add the original sources labels. labels = append(labels, prefixLabels(i.SourceLabels, CONFIG.Repo.Issue.LabelPrefix)...) // Add the releases labels. labels = append(labels, prefixLabels(i.ReleaseLabels, CONFIG.Repo.Issue.LabelPrefix)...) // Add the Rancher made or mirrored image label if i.Mirrored { labels = append(labels, prefixLabels( []string{CONFIG.Repo.Issue.MirroredImageLabel}, CONFIG.Repo.Issue.LabelPrefix)...) } else { labels = append(labels, prefixLabels( []string{CONFIG.Repo.Issue.RancherImageLabel}, CONFIG.Repo.Issue.LabelPrefix)...) } // Retrieve the upstream labels from the issue related to the // image. upstreamLabels, upstreamTeamLabels := filterUpstreamLabels(i.IssueLabels) // Check if the image has a direct team ownership and add the // label. teams := imagesTeams[strings.Split(i.Image, ":")[0]] if len(teams) > 0 { teamLabels = append(teamLabels, teams...) foundTeam = true } // Otherwise, check if the upstream issue has a team label // assigned. if !foundTeam { if len(upstreamTeamLabels) > 0 { teamLabels = append(teamLabels, upstreamTeamLabels...) foundTeam = true } } // Otherwise, we use a heuristics to assign the team ownership // based on the sources labels. if !foundTeam { for team, sources := range sourcesTeams { for _, source := range sources { if hasLabel(source, i.SourceLabels) { teamLabels = append(teamLabels, team) foundTeam = true } } } } // if no team was found, assign the default unassigned label. if !foundTeam { teamLabels = append(teamLabels, "unassigned") foundTeam = true } // Prefix the teams labels with team/ and add them. labels = append(labels, prefixLabels(teamLabels, CONFIG.Repo.Issue.TeamPrefix)...) // Re-add the filtered upstream labels. labels = append(labels, upstreamLabels...) sortedUniqLabels := slices.Clone(labels) sort.Strings(sortedUniqLabels) i.IssueLabels = slices.Compact(sortedUniqLabels) return i } func checkErrAndExit(e error) { if e != nil { fmt.Println(e) os.Exit(1) } } func cliFlags() { flag.StringVar(&FLAG_REPO_TOKEN, "token", "GH_TOKEN", "Name (not value) of the env var containing a GitHub token to access the repo") flag.BoolVar(&FLAG_DRY_RUN, "dry-run", false, "No issues will be created/modified in dry-run mode, only the scanner will run") flag.BoolVar(&FLAG_REPORT, "report", false, "Generate only the CVE reports") flag.BoolVar(&FLAG_DASHBOARD, "dashboard", false, "Generate and update the dashboard GH issue") flag.StringVar(&FLAG_CONFIG_FILE, "config-file", "config.yml", "Configuration file") flag.Parse() } func cveDBRead(report string) CvesRecords { var cves CvesRecords fmt.Println("Reading CVE database") content, err := os.Open(report) // If we fail to open the database file, we still continue with the // scan, as it will create a new file. if err != nil { fmt.Printf("Failed to open CVE database file %s - %s\n", report, err) return cves } reader := csv.NewReader(content) record, err := reader.Read() checkErrAndExit(err) // If the first record isn't the CSV file header, then we reset the // reader, otherwise we skip it. if record[COL_IMAGE] != "image" { _, err = content.Seek(0, io.SeekStart) checkErrAndExit(err) } for { record, err := reader.Read() if err == io.EOF { break } checkErrAndExit(err) key, softKey := generateCveKey(record[COL_IMAGE], record[COL_TARGET], record[COL_VULNERABILITY_ID], record[COL_PACKAGE_NAME], record[COL_PACKAGE_VERSION]) record = append(record, key, softKey) cves = append(cves, record) } return cves } func cveDBSave(report string, images Images) error { var csvFile [][]string fmt.Println("Saving CVE database") // Reminder to keep a save default permissions for the database file. out, err := os.OpenFile(report, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640) checkErrAndExit(err) defer out.Close() reportCsv := csv.NewWriter(out) row := []string{"image", "release", "package_name", "package_version", "type", "vulnerability_id", "severity", "url", "target", "patched_version", "mirrored", "status", "justification"} csvFile = append(csvFile, row) for _, image := range images { for _, vuln := range image.Vulnerabilities { row = []string{image.Image, strings.ReplaceAll(strings.Join(image.ReleaseLabels, ","), "release/", ""), vuln.PkgName, vuln.InstalledVersion, vuln.Type, vuln.VulnerabilityID, vuln.Severity, vuln.PrimaryURL, vuln.Target, vuln.FixedVersion, strconv.FormatBool(image.Mirrored), vuln.Status, vuln.Justification} csvFile = append(csvFile, row) } } err = reportCsv.WriteAll(csvFile) checkErrAndExit(err) return err } func cveScan(images Images, cveRecords CvesRecords) Images { var issues RepoIssues var scannedImages Images var imagesTeams, sourcesTeams TeamsFile fmt.Println("Starting the scan of the images") if !FLAG_DRY_RUN { repoAuth() issues = repoIssuesGet("all", []string{CONFIG.Repo.Issue.Label}, CONFIG.Repo.Issue.Title) imagesTeams = readTeamsFile(CONFIG.ImagesTeamsFile) sourcesTeams = readTeamsFile(CONFIG.SourcesTeamsFile) } for _, i := range images { image := i if imageIsEOL(image.Image) { fmt.Printf("Skipping image %s - it's marked as EOL (in %s)\n", image.Image, CONFIG.EOLFile) image.EOL = true continue } fmt.Printf("Scanning image %s\n", image.Image) // TODO: Add check to verify if the scanner binary exists or // not, if not, it should fail and exit. out, err := exec.Command(CONFIG.Scanner.Tool, "--quiet", "image", "--vex", "repo", "--scanners", CONFIG.Scanner.Scanners, "--format", CONFIG.Scanner.Format, "--output", CONFIG.Scanner.Report, "--severity", CONFIG.Scanner.Severity, image.Image).CombinedOutput() if err != nil { fmt.Printf("Failed to scan image %s - %s\n%s---\n", image.Image, err.Error(), string(out)) image.ScanFailed = true image.ScanFailureReason = sanitizeMarkdownTable(string(out)) scannedImages = append(scannedImages, image) continue } image = parseVulnScanReport(image, CONFIG.Scanner.Report) _ = os.Remove(CONFIG.Scanner.Report) if !FLAG_DRY_RUN { if len(image.Vulnerabilities) > 0 { image = recordCVEStatus(image, cveRecords) // Update vuln count in case a recorded CVE // status is found above. image.VulnCount = imageVulnCount(image) if strings.Contains(image.Image, "/mirrored-") { image.Mirrored = true } image.IssueBody = generateIssueContent(image) image, issues = matchImageIssue(image, issues) image = addLabels(image, imagesTeams, sourcesTeams) publish := hasPublishLabel(image.IssueLabels) if image.IssueID == 0 && publish { image.IssueTitle = fmt.Sprintf("%s %s", CONFIG.Repo.Issue.Title, image.Image) _, _ = repoIssueCreateUpdateClose("create", image) } else if publish && issueContentDiffers(image) { _, _ = repoIssueCreateUpdateClose("update", image) } else if (!publish) && (image.IssueID > 0) && (image.Issue.State == "open") { _, _ = repoIssueCreateUpdateClose("close", image) } } scannedImages = append(scannedImages, image) } } if !FLAG_DRY_RUN { for _, v := range issues { if !v.Match && v.State == "open" { i := Image{} i.IssueID = v.Number _, _ = repoIssueCreateUpdateClose("close", i) } } scanStatusIssue(scannedImages) } return scannedImages } // Function that reviews the labels retrieved from an upstream issue and, // through some heuristics, defines which cve and team related labels will be // keept and returned. The function that calls this one can then define which // labels it wants to use or not. func filterUpstreamLabels(labels []string) ([]string, []string) { var cveLabels, teamLabels []string for _, label := range labels { if label == CONFIG.Repo.Issue.CriticalLabel { // Ignore `cve/critical` label, because it needs to be // recalculated. continue } else if strings.HasPrefix(label, "cve/release/") { // Ignore `cve/release/` label, because their source of // truth is the scanned image file. continue } else if label == CONFIG.Repo.Issue.Label { // Keep the default `cve-report` label. cveLabels = append(cveLabels, label) } else if strings.HasPrefix(label, CONFIG.Repo.Issue.LabelPrefix) { // At this stage, after the first checks, if it contains // the `cve/` label, we keep it. cveLabels = append(cveLabels, label) } else if strings.HasPrefix(label, CONFIG.Repo.Issue.TeamPrefix) { // Only keep a `team/` label if it exists in the config // file in `labels.teams`. for _, team := range CONFIG.Labels.Teams { if (CONFIG.Repo.Issue.TeamPrefix + team) == label { teamLabels = append(teamLabels, label) } } } } return cveLabels, teamLabels } // Function to generate the CVE key of a vulnerability. The CVE key is later // used to match a CVE in the database with a CVE from a scan, so we can recover // the history of the CVE and other details from the database. Currently, two // keys are generated: // // - key: it's a strict key with the same fields used in the first // implementation of image-scanning in Python: // - image (in the format of image:version), target (that contains the // image:version and the binary or layer where the vulnerability is), the CVE // ID and the name of the package that has the vulnerability. // // - soft key: a more relaxed version of key where we remove any reference to // the specific version of the image: // - image (in the format of the image name only), target (that contains the // image name only and the binary or layer where the vulnerability is), the // CVE ID and the name of the package that has the vulnerability. // // The fields are separated by a sequece of 3 dashes `---`. // // The soft key allows to match the same CVE across different versions of the // same image. func generateCveKey(image string, target string, vulnID string, pkgName string, pkgVersion string) (string, string) { fieldSeparator := "---" key := strings.Join([]string{image, target, vulnID, pkgName, pkgVersion}, fieldSeparator) imageName := strings.Split(image, ":")[0] targetImageName := strings.ReplaceAll(target, image, imageName) softKey := strings.Join([]string{imageName, targetImageName, vulnID, pkgName, pkgVersion}, fieldSeparator) return key, softKey } // Function that generates the contents of the issue in Markdown format. func generateIssueContent(image Image) string { var deps, basePkgs, falsePositive, upstream string cveHeader := "| Vulnerability ID | Package Name | Installed Version | Fixed Version |" + " Severity | Target |\n| - | - | - | - | - | - |\n" cveHeaderUpstream := "| Vulnerability ID | Package Name | Installed Version |" + " Fixed Version | Severity | Target |\n|" + " - | - | - | - | - | - |\n" cveHeaderFalsePositive := "| Vulnerability ID | Package Name | Installed Version |" + " Fixed Version | Severity | Target | Justification |\n|" + " - | - | - | - | - | - | - |\n" body := fmt.Sprintf("## CVE scan report for image `%s`\n%s%s\n---\n", image.Image, generateIssueMetadata(image), generateIssueTotalVulns(image)) // TODO: Sort vulnerabilitiess per severity and then CVE. for _, vuln := range image.Vulnerabilities { target := vuln.Target if vuln.Class == BASE_OS_PKG { target = fmt.Sprintf("%s (base OS pkg)", vuln.PkgName) } if vuln.Status == CONFIG.CVES.Status.NotAffected { falsePositive = fmt.Sprintf("%s| %s | %s | %s | %s | %s | %s | %s |\n", falsePositive, vuln.VulnerabilityID, vuln.PkgName, vuln.InstalledVersion, vuln.FixedVersion, vuln.Severity, target, vuln.Justification) } else if vuln.Status == CONFIG.CVES.Status.Affected && vuln.Justification == CONFIG.CVES.Justification.Jus6 { upstream = fmt.Sprintf("%s| %s | %s | %s | %s | %s | %s | %s |\n", upstream, vuln.VulnerabilityID, vuln.PkgName, vuln.InstalledVersion, vuln.FixedVersion, vuln.Severity, target, vuln.Justification) } else if vuln.Class == BASE_OS_PKG { basePkgs = fmt.Sprintf("%s| %s | %s | %s | %s | %s | %s |\n", basePkgs, vuln.VulnerabilityID, vuln.PkgName, vuln.InstalledVersion, vuln.FixedVersion, vuln.Severity, target) } else { deps = fmt.Sprintf("%s| %s | %s | %s | %s | %s | %s |\n", deps, vuln.VulnerabilityID, vuln.PkgName, vuln.InstalledVersion, vuln.FixedVersion, vuln.Severity, target) } } if len(deps) > 0 { body = body + fmt.Sprintf("## CVEs in dependencies\n\n%s%s\n\n", cveHeader, deps) } if len(basePkgs) > 0 { body = body + fmt.Sprintf("## CVEs in base OS packages\n\n%s%s\n\n", cveHeader, basePkgs) } if len(falsePositive) > 0 { body = body + fmt.Sprintf("## False positives CVEs\n\n%s%s\n\n", cveHeaderFalsePositive, falsePositive) } if len(upstream) > 0 { body = body + fmt.Sprintf("## Upstream CVEs\n\n%s%s\n\n", cveHeaderUpstream, upstream) } return body } func generateIssueMetadata(image Image) string { baseImageOS := strings.Join([]string{image.Metadata.OS.Family, image.Metadata.OS.Name}, ":") // In case the scanner wasn't able to identify the base image OS and // version. if len(baseImageOS) == 1 { baseImageOS = "Unknown" } baseImageEOL := "Yes, this base image is still maintained" if image.Metadata.OS.EOL { baseImageEOL = ":warning: This base image is EOL, please update it :warning:" } baseImageMirrored := "This is a Rancher made image" if image.Mirrored { baseImageMirrored = "This is an upstream mirrored image (not made by Rancher)" } body := fmt.Sprintf("### Image metadata\n\n| Image | %s |\n| - | - |\n", image.Image) body = body + fmt.Sprintf("| Image source | %s |\n", imageSource(image.Metadata.Image.Labels.Source)) body = body + fmt.Sprintf("| Base image version | %s |\n", baseImageOS) body = body + fmt.Sprintf("| Base image maintained? | %s |\n", baseImageEOL) body = body + fmt.Sprintf("| Mirrored image? | %s |\n\n---\n", baseImageMirrored) return body } func generateIssueTotalVulns(image Image) string { line := "### Severity summary\n\n| Severity | Total (applicable¹) |" + " Upstream² | False positive³ |\n| - | - | - | - |\n" severities := strings.Split(CONFIG.Scanner.Severity, ",") c := image.VulnCount for _, s := range severities { total := c["total"][s] - c[s][UPSTREAM] - c[s][CONFIG.CVES.Status.NotAffected] line = line + fmt.Sprintf("| %s | %s | %s | %s |\n", s, strconv.Itoa(total), strconv.Itoa(c[s][UPSTREAM]), strconv.Itoa(c[s][CONFIG.CVES.Status.NotAffected])) } total := c["total"]["total"] - c["total"][UPSTREAM] - c["total"][CONFIG.CVES.Status.NotAffected] line = line + fmt.Sprintf("| Total | %s | %s | %s |\n", strconv.Itoa(total), strconv.Itoa(c["total"][UPSTREAM]), strconv.Itoa(c["total"][CONFIG.CVES.Status.NotAffected])) line = line + "\n¹ The total applicable vulnerabilities exclude upstream and false positive issues," + " it considers issues with the status `" + CONFIG.CVES.Status.Default + "`" + " and `" + CONFIG.CVES.Status.Affected + "`.\n" + "² Upstream are applicable vulnerabilities too, but counted separately and consider" + " issues with status `" + CONFIG.CVES.Status.Affected + "` and justification" + " `" + UPSTREAM + "`.\n" + "³ False positives do not count and are issues with status `" + CONFIG.CVES.Status.NotAffected + "` and that contain a valid justification.\n" return line } func generateStatusIssueContent(images Images) string { var body, scanFailed, unsupportedImages string scanFailedHeader := "### Images not scanned due to failure\n\n| Image | Reason |\n| - | - |\n" unsupportedImagesHeader := "### Images not scanned due to unsupported base image\n\n| Image |\n| - |\n" for _, image := range images { if image.ScanFailed { scanFailed = fmt.Sprintf("%s| %s | %s |\n", scanFailed, image.Image, image.ScanFailureReason) } else if image.UnsupportedBaseImage { unsupportedImages = fmt.Sprintf("%s| %s | - |\n", unsupportedImages, image.Image) } } if len(scanFailed) > 0 { body = body + fmt.Sprintf("%s%s\n\n", scanFailedHeader, scanFailed) } if len(unsupportedImages) > 0 { body = body + fmt.Sprintf("%s%s\n\n", unsupportedImagesHeader, unsupportedImages) } return body } // Function that checks if a string (a label) is present on a string array (of // labels). func hasLabel(label string, labels []string) bool { for _, l := range labels { if label == l { return true } } return false } // Function that checks and controls if a given set of labels contain specific // labels that allow the creation of an upstream (GitHub) issue for the // respective image. func hasPublishLabel(labels []string) bool { if labelsHasSuffix("-head", labels) { // Currently we create issues for images that are in `-head` // releases of Rancher, Harvester and Longhorn. // Example: `cve/release//vX.Y-head` return true } if labelsContains("harvester/master", labels) || labelsContains("longhorn/master", labels) { // Otherwise, check if the image comes from the master release of // Harvester or Longhorn. return true } if labelsContains("release/rke2", labels) { // Or if the image comes from RKE2's standalone releases. return true } return false } func imageIsEOL(image string) bool { for _, i := range EOLCONFIG.Images { if image == i { return true } } return false } // Function that receives the source label, from the scanned image, and // generates a Markdown style link pointing to the repository where the image is // build. func imageSource(source string) string { var s string s = strings.TrimSuffix(source, ".git") if len(s) == 0 { s = "Not identified" } else if strings.HasPrefix(s, CONFIG.Repo.GitUrl) { repo := strings.TrimPrefix(s, CONFIG.Repo.GitUrl) s = fmt.Sprintf("[%s](%s)", repo, s) } else { _, err := url.ParseRequestURI(s) if err == nil { s = fmt.Sprintf("[source repo](%s)", s) } else { s = "Not identified" } } return s } func imageVulnCount(image Image) map[string]map[string]int { count := make(map[string]map[string]int) count["total"] = make(map[string]int) for _, vuln := range image.Vulnerabilities { if _, key := count[vuln.Severity]; !key { count[vuln.Severity] = make(map[string]int) } if vuln.Status == CONFIG.CVES.Status.Affected && vuln.Justification == CONFIG.CVES.Justification.Jus6 { count[vuln.Severity][UPSTREAM]++ count["total"][UPSTREAM]++ } else { count[vuln.Severity][vuln.Status]++ count["total"][vuln.Status]++ } count["total"][vuln.Severity]++ } count["total"]["total"] = len(image.Vulnerabilities) return count } func issueContentDiffers(image Image) bool { if image.Issue.State == "closed" { return true } if image.IssueBody != image.Issue.Body { return true } slice1 := slices.Clone(image.IssueLabels) sort.Strings(slice1) slice1 = slices.Compact(slice1) slice2 := slices.Clone(image.Issue.Labels) sort.Strings(slice2) slice2 = slices.Compact(slice2) if slices.Compare(slice1, slice2) != 0 { return true } return false } func labelsContains(str string, labels []string) bool { for _, label := range labels { if strings.Contains(label, str) { return true } } return false } func labelsHasSuffix(str string, labels []string) bool { for _, label := range labels { if strings.HasSuffix(label, str) { return true } } return false } func loadConfigs() { configContent, err := os.ReadFile(FLAG_CONFIG_FILE) checkErrAndExit(err) err = yaml.Unmarshal(configContent, &CONFIG) checkErrAndExit(err) eolConfigContent, err := os.ReadFile(CONFIG.EOLFile) checkErrAndExit(err) err = yaml.Unmarshal(eolConfigContent, &EOLCONFIG) checkErrAndExit(err) } // Function that compares an image with a list of issues to find a match between // them. The match is currently done based on the image's name and the title of // the issue. func matchImageIssue(image Image, issues RepoIssues) (Image, RepoIssues) { im := image is := issues title := fmt.Sprintf("%s %s", CONFIG.Repo.Issue.Title, im.Image) if v, ok := issues[title]; ok { if (v.Title == title) && (len(im.Vulnerabilities) > 0) { v.Match = true is[title] = v im.Issue = v im.IssueLabels = v.Labels im.IssueID = v.Number } } return im, is } func parseIssueLabels(issueLabels []*github.Label) []string { l := []string{} for _, label := range issueLabels { l = append(l, stringify(label.Name)) } return l } func parsePublishedDate(date string) string { parsedDate, err := time.Parse(time.RFC3339, date) if err != nil { return "unavailable" } openDays := strconv.Itoa(int(time.Since(parsedDate).Hours() / 24)) return fmt.Sprintf("%s days", openDays) } // Function to parse the Trivy scan report in JSON format into the structure // VulnScanReport that contains the most important data from the raw Trivy JSON // report format. func parseTrivyJSONReport(report string) VulnScanReport { vulnScanReport := VulnScanReport{} content, err := os.ReadFile(report) if err != nil { fmt.Printf("Failed to read scan report - %s\n", err.Error()) return vulnScanReport } err = json.Unmarshal(content, &vulnScanReport) if err != nil { fmt.Printf("Failed to unmarshall scan report - %s\n", err.Error()) return vulnScanReport } return vulnScanReport } // Function to parse the VulnScanReport structure, that is specific to Trivy's // report format, into the generic Image strucuture, that has more fields and // data. func parseVulnScanReport(image Image, reportName string) Image { report := parseTrivyJSONReport(reportName) i := image i.Metadata.OS.Family = report.Metadata.OS.Family i.Metadata.OS.Name = report.Metadata.OS.Name i.Metadata.OS.EOL = report.Metadata.OS.EOSL i.Metadata.Image.Labels.Source = report.Metadata.ImageConfig.Config.Labels.Source i.Metadata.Image.Labels.URL = report.Metadata.ImageConfig.Config.Labels.URL // If there is no results, then that's because our scanner, currently // Trivy, doesn't properly understand the base image, so we consider it // as an unsupported base image. This mostly happens with scratch // images. if len(report.Results) == 0 { i.UnsupportedBaseImage = true } for _, r := range report.Results { var vuln Vulnerability vuln.Target = r.Target vuln.Class = r.Class vuln.Type = r.Type for _, v := range r.Vulnerabilities { vuln.VulnerabilityID = v.VulnerabilityID vuln.Title = v.Title vuln.PkgName = v.PkgName vuln.InstalledVersion = v.InstalledVersion vuln.VulnerabilityID = v.VulnerabilityID vuln.FixedVersion = v.FixedVersion vuln.PrimaryURL = v.PrimaryURL vuln.Severity = v.Severity vuln.PublishedDate = parsePublishedDate(v.PublishedDate) vuln.Status = CONFIG.CVES.Status.Default vuln.Key, vuln.SoftKey = generateCveKey(i.Image, vuln.Target, vuln.VulnerabilityID, vuln.PkgName, vuln.InstalledVersion) i.Vulnerabilities = append(i.Vulnerabilities, vuln) } i.VulnCount = imageVulnCount(i) } return i } func prefixLabels(labels []string, prefix string) []string { var newLabels []string for _, label := range labels { if !strings.HasPrefix(label, prefix) { newLabels = append(newLabels, (prefix + label)) } else { newLabels = append(newLabels, label) } } return newLabels } // Function that finds if the vulnerabilities of a given image have a recorded // CVE status (history) based on the CVE key. Currently, the CVE history is // composed of the status of the vulnerability, see cves.status in config.yml, // and any jusitifcatons that were added to the specific vulnerability. func recordCVEStatus(image Image, cveRecords CvesRecords) Image { i := image for k, v := range i.Vulnerabilities { for _, cve := range cveRecords { // Currently we match the key based on a soft key model, // which allows to more easily propagate the status of a // vulnerability that was triaged for a given image // image_x:version_y to other versions of the same // image, for example, image_x:version_z. We use the // soft key model, because the changes are high that if // a vulnerability was triaged for a specific version of // a certain image, the triaged status will be the same // for other versions of the same image, for example, // when triaging false positives. // If needed, we can switch the match from a soft key to // the more specific image:version match model. See // generateCveKey() for details about how the CVE key is // calculated. if v.SoftKey == cve[COL_SOFTKEY] { i.Vulnerabilities[k].Status = cve[COL_STATUS] i.Vulnerabilities[k].Justification = cve[COL_JUSTIFICATION] } } } return i } // Function to parse the file with the images to scan. The file follows the same // pattern of Rancher's artifact file `rancher-images-sources.txt`. Each line // contains an :, separated by a and the list of sources // labels separated by a comman <,>. // The file is slighly modified when it's generated by image-scanning to also // contain the releases of where the image is found in the format of // />, where can be `rancher` or something. func readImagesFile(imagesFile string) Images { var images Images readFile, err := os.Open(imagesFile) checkErrAndExit(err) defer readFile.Close() fileScanner := bufio.NewScanner(readFile) fileScanner.Split(bufio.ScanLines) for fileScanner.Scan() { line := fileScanner.Text() // Ignore commented lines. if strings.HasPrefix(line, "#") { continue } split := strings.Split(line, " ") image := Image{} image.Image = split[0] // Image has an origin specified. if len(split) > 1 { sources := strings.Split(split[1], ",") for _, source := range sources { if strings.HasPrefix(source, "release/") { // The source represents the release where this // image is found. image.ReleaseLabels = append(image.ReleaseLabels, source) } else { // Source labels might contain a ":" // that specifies the version of a // chart. We don't want the version, so // we remove it. image.SourceLabels = append(image.SourceLabels, strings.Split(source, ":")[0]) } } } images = append(images, image) } return images } // Generic function to parse the files containing the list of images and which // teams own them and the teams with the list sources labels that they own, // respectively, the configurations `images_teams_file` and `sources_teams_file` // from config.yml. Both files follow with the same format: // etc. See each file for // specific details. func readTeamsFile(file string) TeamsFile { teams := make(TeamsFile) readFile, err := os.Open(file) checkErrAndExit(err) defer readFile.Close() fileScanner := bufio.NewScanner(readFile) fileScanner.Split(bufio.ScanLines) for fileScanner.Scan() { line := fileScanner.Text() // Ignore commented lines. if strings.HasPrefix(line, "#") { continue } split := strings.Split(line, " ") field1 := split[0] if len(split) > 1 { field2 := strings.Split(split[1], ",") for _, v := range field2 { teams[field1] = append(teams[field1], v) } } } return teams } func repoAuth() { CTX = context.Background() CLIENT = github.NewClient(nil).WithAuthToken(os.Getenv(FLAG_REPO_TOKEN)) // Dummy request to check if the auth was successful. _, _, err := CLIENT.Users.Get(CTX, "") checkErrAndExit(err) } func repoIssueCreateUpdateClose(action string, image Image) (*github.Issue, error) { var err error var issue *github.Issue var resp *github.Response // TODO: Use response to check the rate limiting and possibly slow down // the requests or use a second GH token. // https://pkg.go.dev/github.com/google/go-github/v61/github#Response issueReq := &github.IssueRequest{ Body: github.String(image.IssueBody), Labels: &image.IssueLabels, State: github.String("open"), } if action == "create" { issueReq.Title = github.String(image.IssueTitle) issue, resp, err = CLIENT.Issues.Create(CTX, CONFIG.Repo.Owner, CONFIG.Repo.Repo, issueReq) } else if action == "update" { issue, resp, err = CLIENT.Issues.Edit(CTX, CONFIG.Repo.Owner, CONFIG.Repo.Repo, image.IssueID, issueReq) } else if action == "close" { issueReq = &github.IssueRequest{ State: github.String("closed"), } issue, resp, err = CLIENT.Issues.Edit(CTX, CONFIG.Repo.Owner, CONFIG.Repo.Repo, image.IssueID, issueReq) } if err != nil { fmt.Printf("Failed to %s issue - %s\n", action, err.Error()) _, err1 := err.(*github.RateLimitError) _, err2 := err.(*github.AbuseRateLimitError) if err1 || err2 { fmt.Printf("Rate limit error - %+v\n", resp.Rate) } } else { fmt.Printf("%s issue %s\n", action, stringify(issue.HTMLURL)) } // TODO: Future improvement - 10 seconds seems to be a good option to // avoid hiting the secondary rate limit, because there is no good way // to determine if we are close to hit it or not. We tested with 7 // seconds, but still hit the rate limit almost when the scans was about // to complete. We could also use a second GH API token, but we would // still need to do a sleep. If instead we wait the recommended time // that is returned by the GH API, then the scanner can be on an idle // loop for almost one hour, maybe even longer. // Information about the secondary rate limit is available in // https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#about-secondary-rate-limits. time.Sleep(10 * time.Second) return issue, err } func repoIssuesGet(state string, labels []string, titleFilter string) RepoIssues { repoIssues := make(RepoIssues) opt := &github.IssueListByRepoOptions{ State: state, Labels: labels, ListOptions: github.ListOptions{PerPage: 100}, } for { issues, resp, err := CLIENT.Issues.ListByRepo(CTX, CONFIG.Repo.Owner, CONFIG.Repo.Repo, opt) checkErrAndExit(err) for _, i := range issues { title := stringify(i.Title) if !strings.HasPrefix(title, titleFilter) { continue } issue := RepoIssue{} issue.Number = *i.Number issue.State = stringify(i.State) issue.Title = title issue.Body = stringify(i.Body) issue.URL = stringify(i.HTMLURL) issue.Labels = parseIssueLabels(i.Labels) issue.Match = false if issue.State == "closed" { issue.ClosedAt = i.ClosedAt.Time } else { issue.CreatedAt = i.CreatedAt.Time } for _, label := range i.Labels { if !slices.Contains(issue.Labels, stringify(label.Name)) { issue.Labels = append(issue.Labels, stringify(label.Name)) } } // TODO: Future improvement - here we can check, before // directly adding to the the map, if there is more than // one issue with the same title, in case someone // manually created issues in the repo. repoIssues[issue.Title] = issue } if resp.NextPage == 0 { break } opt.Page = resp.NextPage } return repoIssues } // Function that sanitizes some chars that can break the formatting of a // Markdown table. func sanitizeMarkdownTable(str string) string { safeString := strings.ReplaceAll(str, "\n", "
") safeString = strings.ReplaceAll(safeString, "|", "") safeString = strings.Map(func(r rune) rune { if unicode.IsPrint(r) { return r } return -1 }, safeString) return safeString } func scanStatusIssue(images Images) { i := Image{} i.IssueTitle = CONFIG.Repo.IssueFailure.Title i.IssueBody = generateStatusIssueContent(images) i.IssueLabels = append(i.IssueLabels, CONFIG.Repo.IssueFailure.Label) statusIssue := repoIssuesGet("all", i.IssueLabels, i.IssueTitle) if v, ok := statusIssue[i.IssueTitle]; ok { i.IssueID = v.Number _, _ = repoIssueCreateUpdateClose("update", i) } else { _, _ = repoIssueCreateUpdateClose("create", i) } } // Removes the double quotes that are added by github.Stringify(). func stringify(s interface{}) string { str := github.Stringify(s) if len(str) > 2 { return str[1 : len(str)-1] } return "" } func main() { var cveRecords CvesRecords var images Images cliFlags() if FLAG_DRY_RUN { fmt.Println("Starting image-scanning in dry-run mode") } else { fmt.Println("Starting image-scanning") } loadConfigs() cveRecords = cveDBRead(CONFIG.ReportFile) images = readImagesFile(CONFIG.ImagesFile) if FLAG_REPORT { mainGenerateReports(cveRecords, images) return } if FLAG_DASHBOARD { mainGenerateDashboard(cveRecords, images) return } images = cveScan(images, cveRecords) if FLAG_DRY_RUN { fmt.Println("Finished dry-run scan. CVE database and issues were not modified.") return } cveDBSave(CONFIG.ReportFile, images) fmt.Println("image-scanning finished successfully") }