fix: emit valid SPDX license expressions when enriching

README.md

@@ -133,6 +133,8 @@ parlay ecosystems repo https://github.com/open-policy-agent/conftest
 
 parlay enriches components and packages with their license information from ecosyste.ms on a best-effort basis. It prefers the license data of the package version at hand; however, it may not always be possible to retrieve the license for a specific version (see [ecosyste.ms issue here](https://github.com/ecosyste-ms/packages/issues/1027) for more info). In this case, parlay will fall back to enriching with the license data of the package's latest release. In rare cases — where the licensing model of a package changed over time — this may result in license data inaccuracies.
 
+Licenses are resolved against the SPDX license list. A license string that is not valid SPDX — registries such as Maven accept any string — is captured as a `LicenseRef-` with the original text recorded, so that the documents parlay emits stay valid. Where a package carries several licenses they are emitted as a choice between them (`(MIT OR Apache-2.0)`), following the convention that a package listing several licenses may be used under any one of them.
+
 
 ## Enriching with Snyk
 

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/CycloneDX/cyclonedx-go v0.9.2
 	github.com/deepmap/oapi-codegen v1.12.4
 	github.com/edoardottt/depsdev v0.0.3
+	github.com/github/go-spdx/v2 v2.7.0
 	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/jarcoal/httpmock v1.3.0
@@ -14,7 +15,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/zerolog v1.29.1
 	github.com/spdx/tools-golang v0.5.4-0.20240304222056-8baafa1a79c4
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.10.0
 )
@@ -37,7 +38,7 @@ require (
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/text v0.14.0 // indirect

go.sum

@@ -59,7 +59,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -79,6 +79,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ=
+github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -210,12 +212,12 @@ github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk=
 github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
 github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
@@ -254,6 +256,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

internal/utils/cdx.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"cmp"
+
 	cdx "github.com/CycloneDX/cyclonedx-go"
 )
 
@@ -27,3 +29,36 @@ func DiscoverCDXComponents(bom *cdx.BOM) []*cdx.Component {
 	}
 	return comps
 }
+
+// CDXLicenses renders licenses as a CycloneDX license choice list. Anything not
+// usable as an SPDX identifier is named instead of identified, since CycloneDX
+// cannot mix an expression and named licenses in one list. Naming is the only
+// escape hatch CycloneDX offers, so a choice between licenses degrades to a
+// plain list, which consumers may read as all of them applying at once.
+func CDXLicenses(licenses []License) *cdx.Licenses {
+	if len(licenses) == 0 {
+		return nil
+	}
+
+	ids := make([]string, 0, len(licenses))
+	for _, l := range licenses {
+		if l.Raw != "" {
+			ids = nil
+			break
+		}
+		ids = append(ids, l.SPDXID)
+	}
+	if ids != nil {
+		return &cdx.Licenses{{Expression: LicenseExpression(ids)}}
+	}
+
+	out := make(cdx.Licenses, 0, len(licenses))
+	for _, l := range licenses {
+		if l.IsIdentifier() {
+			out = append(out, cdx.LicenseChoice{License: &cdx.License{ID: l.SPDXID}})
+			continue
+		}
+		out = append(out, cdx.LicenseChoice{License: &cdx.License{Name: cmp.Or(l.Raw, l.SPDXID)}})
+	}
+	return &out
+}

internal/utils/cdx_test.go

@@ -31,3 +31,56 @@ func TestDiscoverCDXComponents(t *testing.T) {
 
 	assert.Equal(len(result), 3)
 }
+
+func TestCDXLicenses(t *testing.T) {
+	for name, tc := range map[string]struct {
+		in   []utils.License
+		want *cdx.Licenses
+	}{
+		"nothing in, nothing out": {
+			in:   nil,
+			want: nil,
+		},
+		"all valid becomes one expression": {
+			in:   []utils.License{{SPDXID: "MIT"}, {SPDXID: "Apache-2.0"}},
+			want: &cdx.Licenses{{Expression: "(MIT OR Apache-2.0)"}},
+		},
+		"any invalid falls back to a list": {
+			in: []utils.License{
+				{SPDXID: "MIT"},
+				{SPDXID: "LicenseRef-Eclipse-Public-License-1.0", Raw: "Eclipse Public License 1.0"},
+			},
+			want: &cdx.Licenses{
+				{License: &cdx.License{ID: "MIT"}},
+				{License: &cdx.License{Name: "Eclipse Public License 1.0"}},
+			},
+		},
+		// CycloneDX validates license ids against a closed list, which carries
+		// only the six grandfathered "+" identifiers, so an "or later" license
+		// has to be named rather than identified.
+		"or later is named, it is not a CycloneDX license id": {
+			in: []utils.License{
+				{SPDXID: "Apache-2.0+"},
+				{SPDXID: "LicenseRef-Proprietary", Raw: "Proprietary"},
+			},
+			want: &cdx.Licenses{
+				{License: &cdx.License{Name: "Apache-2.0+"}},
+				{License: &cdx.License{Name: "Proprietary"}},
+			},
+		},
+		"a grandfathered or later is still a license id": {
+			in: []utils.License{
+				{SPDXID: "GPL-2.0+"},
+				{SPDXID: "LicenseRef-Proprietary", Raw: "Proprietary"},
+			},
+			want: &cdx.Licenses{
+				{License: &cdx.License{ID: "GPL-2.0+"}},
+				{License: &cdx.License{Name: "Proprietary"}},
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, tc.want, utils.CDXLicenses(tc.in))
+		})
+	}
+}

internal/utils/license.go

@@ -0,0 +1,173 @@
+/*
+ * © 2023 Snyk Limited All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"fmt"
+	"hash/fnv"
+	"regexp"
+	"strings"
+
+	"github.com/github/go-spdx/v2/spdxexp"
+)
+
+// License is one license string from ecosyste.ms, resolved against the SPDX
+// license list.
+type License struct {
+	// SPDXID is the normalized SPDX expression, or a "LicenseRef-" identifier
+	// when Raw is not valid SPDX.
+	SPDXID string
+
+	// Raw is the original ecosyste.ms string, set only when it is not valid
+	// SPDX and had to be captured as a LicenseRef.
+	Raw string
+}
+
+// ClassifyLicenses resolves each license string against the SPDX license list.
+// ecosyste.ms passes package metadata through as-is, and registries such as
+// Maven accept any string, so the result is not always valid SPDX. Strings that
+// are not get turned into LicenseRef- identifiers, so that the documents parlay
+// emits still validate. See https://github.com/snyk/parlay/issues/80.
+func ClassifyLicenses(licenses []string) []License {
+	out := make([]License, 0, len(licenses))
+
+	// Keyed on the resolved license rather than the incoming string, so that
+	// strings differing only in case or punctuation collapse into one entry.
+	seen := make(map[License]bool, len(licenses))
+	add := func(l License) {
+		if seen[l] {
+			return
+		}
+		seen[l] = true
+		out = append(out, l)
+	}
+
+	for _, raw := range licenses {
+		raw = strings.TrimSpace(raw)
+		if raw == "" {
+			continue
+		}
+
+		// Both only carry meaning on their own, and neither can be joined into
+		// an expression, so a package saying nothing keeps saying nothing.
+		if raw == "NONE" || raw == "NOASSERTION" {
+			continue
+		}
+
+		// Refs are rejected so that a ref arriving from ecosyste.ms is captured
+		// as extracted licensing info here, rather than dangling undefined.
+		normalized, invalid := spdxexp.ValidateAndNormalizeLicensesWithOptions(
+			[]string{raw},
+			spdxexp.ValidateLicensesOptions{FailAllLicenseRefs: true, FailAllDocumentRefs: true},
+		)
+		if len(invalid) == 0 && len(normalized) == 1 {
+			add(License{SPDXID: normalized[0]})
+			continue
+		}
+
+		id := licenseRefID(raw)
+		if id == "" {
+			// ponytail: nothing left to identify the license by, so drop it
+			// rather than emit a meaningless shared ref.
+			continue
+		}
+		add(License{SPDXID: id, Raw: raw})
+	}
+
+	return out
+}
+
+// A license name carrying its version after a comma, "The Apache Software
+// License, Version 2.0" being the one Maven hands out most, is one license
+// rather than two. Nothing else about a fragment says whether the comma before
+// it separates licenses or belongs to a name, so only a version is rejoined.
+var licenseVersionFragment = regexp.MustCompile(`(?i)^\s*(v(er(sion)?)?)?\.?\s*\d[\d.]*\s*$`)
+
+// SplitLicenseList splits the comma separated license list ecosyste.ms holds
+// for a package version.
+//
+// ponytail: only version fragments are rejoined, so a name split across a
+// comma any other way still ends up as two licenses. Recognising those needs
+// the license list itself, which is a bigger heuristic than the one case that
+// actually shows up.
+func SplitLicenseList(licenses string) []string {
+	parts := strings.Split(licenses, ",")
+
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		if len(out) > 0 && licenseVersionFragment.MatchString(part) {
+			out[len(out)-1] += "," + part
+			continue
+		}
+		out = append(out, part)
+	}
+	return out
+}
+
+// LicenseExpression joins license identifiers into one SPDX license expression.
+func LicenseExpression(ids []string) string {
+	if len(ids) < 2 {
+		return strings.Join(ids, "")
+	}
+	return "(" + strings.Join(ids, " OR ") + ")"
+}
+
+// An idstring is limited to letters, numbers, "." and "-", so a run of
+// anything else collapses to a single "-".
+var licenseRefStrip = regexp.MustCompile(`[^A-Za-z0-9.]+`)
+
+// licenseRefID builds a "LicenseRef-[idstring]" identifier from a license
+// string. Two strings differing only in stripped characters land on the same
+// identifier; DisambiguateLicenseRef separates them where that matters.
+func licenseRefID(raw string) string {
+	// An incoming ref keeps its name rather than gaining a second prefix.
+	id := strings.Trim(licenseRefStrip.ReplaceAllString(strings.TrimPrefix(raw, "LicenseRef-"), "-"), "-")
+	if id == "" {
+		return ""
+	}
+	return "LicenseRef-" + id
+}
+
+// cdxOrLaterIDs are the "or later" licenses CycloneDX still lists as license
+// identifiers. SPDX allows a "+" suffix on any identifier, but CycloneDX
+// validates ids against a closed list that only kept these six.
+var cdxOrLaterIDs = map[string]bool{
+	"GPL-1.0+": true, "GPL-2.0+": true, "GPL-3.0+": true,
+	"LGPL-2.0+": true, "LGPL-2.1+": true, "LGPL-3.0+": true,
+}
+
+// IsIdentifier reports whether the license is a single SPDX license identifier,
+// the only form CycloneDX can carry in a license id field.
+func (l License) IsIdentifier() bool {
+	if l.Raw != "" || strings.ContainsAny(l.SPDXID, " ()") {
+		return false
+	}
+	if strings.HasSuffix(l.SPDXID, "+") {
+		return cdxOrLaterIDs[l.SPDXID]
+	}
+	return true
+}
+
+// DisambiguateLicenseRef derives a distinct identifier for a license string
+// that sanitizes down to an identifier already taken by a different string.
+// Without it the two would share one extracted licensing info entry, which
+// would put packages under a license they are not under.
+func DisambiguateLicenseRef(id, raw string) string {
+	h := fnv.New32a()
+	h.Write([]byte(raw))
+	return fmt.Sprintf("%s-%08x", id, h.Sum32())
+}