Add `excludes` to `license resolve` config (#117)
diff --git a/README.md b/README.md
index 55d81d3..405d1d0 100644
--- a/README.md
+++ b/README.md
@@ -769,6 +769,9 @@
version: dependency-version # <19>
license: Apache-2.0 # <20>
threshold: 75 # <21>
+ excludes: # <22>
+ - name: dependency-name # the same format as <18>
+ version: dependency-version # the same format as <19>
```
1. The `header` section is configurations for source codes license header.
@@ -788,10 +791,11 @@
15. The `dependency` section is configurations for resolving dependencies' licenses.
16. The `files` are the files that declare the dependencies of a project, typically, `go.mod` in Go project, `pom.xml` in maven project, and `package.json` in NodeJS project. If it's a relative path, it's relative to the `.licenserc.yaml`.
17. Declare the licenses which cannot be identified by this tool.
-18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project.
-19. The `version` of the dependency, it's locked, preventing license changed between different versions.
+18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project. You can use file pattern as described in [the doc](https://pkg.go.dev/path/filepath#Match).
+19. The `version` of the dependency, comma seperated string (such as `1.0,2.0,3.0`), if this is empty, it means all versions of the dependency.
20. The [SPDX ID](https://spdx.org/licenses/) of the dependency license.
21. The minimum percentage of the file that must contain license text for identifying a license, default is `75`.
+22. The dependencies that should be excluded when analyzing the licenses, this is useful when you declare the dependencies in `pom.xml` with `compile` scope but don't distribute them in package. (Note that non-`compile` scope dependencies are automatically excluded so you don't need to put them here).
**NOTE**: When the `SPDX-ID` is Apache-2.0 and the owner is Apache Software foundation, the content would be [a dedicated license](https://www.apache.org/legal/src-headers.html#headers) specified by the ASF, otherwise, the license would be [the standard one](https://www.apache.org/foundation/license-faq.html#Apply-My-Software).
diff --git a/pkg/deps/config.go b/pkg/deps/config.go
index d3a77ed..9993bad 100644
--- a/pkg/deps/config.go
+++ b/pkg/deps/config.go
@@ -20,6 +20,7 @@
import (
"os"
"path/filepath"
+ "strings"
)
// DefaultCoverageThreshold is the minimum percentage of the file
@@ -31,6 +32,7 @@
Threshold int `yaml:"threshold"`
Files []string `yaml:"files"`
Licenses []*ConfigDepLicense `yaml:"licenses"`
+ Excludes []Exclude `yaml:"excludes"`
}
type ConfigDepLicense struct {
@@ -39,6 +41,11 @@
License string `yaml:"license"`
}
+type Exclude struct {
+ Name string `yaml:"name"`
+ Version string `yaml:"version"`
+}
+
func (config *ConfigDeps) Finalize(configFile string) error {
configFileAbsPath, err := filepath.Abs(configFile)
if err != nil {
@@ -58,3 +65,37 @@
return nil
}
+
+func (config *ConfigDeps) GetUserConfiguredLicense(name, version string) (string, bool) {
+ for _, license := range config.Licenses {
+ if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return license.License, true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return license.License, true
+ }
+ }
+ }
+ return "", false
+}
+
+func (config *ConfigDeps) IsExcluded(name, version string) bool {
+ for _, license := range config.Excludes {
+ if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/pkg/deps/golang.go b/pkg/deps/golang.go
index 842be06..700329e 100644
--- a/pkg/deps/golang.go
+++ b/pkg/deps/golang.go
@@ -27,7 +27,6 @@
"os/exec"
"path/filepath"
"regexp"
- "strings"
"github.com/apache/skywalking-eyes/internal/logger"
"github.com/apache/skywalking-eyes/pkg/license"
@@ -86,17 +85,16 @@
func (resolver *GoModResolver) ResolvePackages(modules []*packages.Module, config *ConfigDeps, report *Report) error {
for _, module := range modules {
func() {
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == module.Path && version == module.Version {
- report.Resolve(&Result{
- Dependency: module.Path,
- LicenseSpdxID: l.License,
- Version: module.Version,
- })
- return
- }
- }
+ if config.IsExcluded(module.Path, module.Version) {
+ return
+ }
+ if l, ok := config.GetUserConfiguredLicense(module.Path, module.Version); ok {
+ report.Resolve(&Result{
+ Dependency: module.Path,
+ LicenseSpdxID: l,
+ Version: module.Version,
+ })
+ return
}
err := resolver.ResolvePackageLicense(config, module, report)
if err != nil {
diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go
index 28c5c08..7220823 100644
--- a/pkg/deps/maven.go
+++ b/pkg/deps/maven.go
@@ -58,14 +58,14 @@
return err
}
- deps, err := resolver.LoadDependencies()
+ deps, err := resolver.LoadDependencies(config)
if err != nil {
// attempt to download dependencies
if err = resolver.DownloadDeps(); err != nil {
return fmt.Errorf("dependencies download error")
}
// load again
- deps, err = resolver.LoadDependencies()
+ deps, err = resolver.LoadDependencies(config)
if err != nil {
return err
}
@@ -125,7 +125,7 @@
return install.Run()
}
-func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) {
+func (resolver *MavenPomResolver) LoadDependencies(config *ConfigDeps) ([]*Dependency, error) {
buf := bytes.NewBuffer(nil)
cmd := exec.Command(resolver.maven, "dependency:tree") // #nosec G204
@@ -138,7 +138,7 @@
return nil, err
}
- deps := LoadDependencies(buf.Bytes())
+ deps := LoadDependencies(buf.Bytes(), config)
return deps, nil
}
@@ -146,24 +146,20 @@
func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency, config *ConfigDeps, report *Report) error {
for _, dep := range deps {
func() {
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == fmt.Sprintf("%s:%s", strings.Join(dep.GroupID, "."), dep.ArtifactID) && version == dep.Version {
- report.Resolve(&Result{
- Dependency: dep.String(),
- LicenseSpdxID: l.License,
- Version: dep.Version,
- })
- return
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(dep.Name(), dep.Version); ok {
+ report.Resolve(&Result{
+ Dependency: dep.Name(),
+ LicenseSpdxID: l,
+ Version: dep.Version,
+ })
+ return
}
state := NotFound
err := resolver.ResolveLicense(config, &state, dep, report)
if err != nil {
- logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.String(), state.String())
+ logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.Name(), state.String())
report.Skip(&Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseSpdxID: Unknown,
Version: dep.Version,
})
@@ -177,7 +173,7 @@
func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state *State, dep *Dependency, report *Report) error {
result1, err1 := resolver.ResolveJar(config, state, filepath.Join(resolver.repo, dep.Path(), dep.Jar()), dep.Version)
if result1 != nil {
- result1.Dependency = dep.String()
+ result1.Dependency = dep.Name()
report.Resolve(result1)
return nil
}
@@ -188,7 +184,7 @@
return nil
}
- return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.String(), err1, err2)
+ return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.Name(), err1, err2)
}
// ResolveLicenseFromPom search for license in the pom file, which may appear in the header comments or in license element of xml
@@ -202,7 +198,7 @@
if pom != nil && len(pom.Licenses) != 0 {
return &Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseFilePath: pomFile,
LicenseContent: pom.Raw(),
LicenseSpdxID: pom.AllLicenses(config),
@@ -215,7 +211,7 @@
return nil, err
} else if headerComments != "" {
*state |= FoundLicenseInPomHeader
- return resolver.IdentifyLicense(config, pomFile, dep.String(), headerComments, dep.Version)
+ return resolver.IdentifyLicense(config, pomFile, dep.Name(), headerComments, dep.Version)
}
return nil, fmt.Errorf("not found in pom file")
@@ -287,7 +283,7 @@
return reMaybeLicense.MatchString(content)
}
-func LoadDependencies(data []byte) []*Dependency {
+func LoadDependencies(data []byte, config *ConfigDeps) []*Dependency {
depsTree := LoadDependenciesTree(data)
cnt := 0
@@ -299,6 +295,9 @@
queue := []*Dependency{}
for _, depTree := range depsTree {
+ if config.IsExcluded(depTree.Name(), depTree.Version) {
+ continue
+ }
queue = append(queue, depTree)
for len(queue) > 0 {
dep := queue[0]
@@ -326,9 +325,8 @@
deps := make([]*Dependency, 0, len(rawDeps))
for _, rawDep := range rawDeps {
- gid := strings.Split(string(rawDep[reFind.SubexpIndex("gid")]), ".")
dep := &Dependency{
- GroupID: gid,
+ GroupID: string(rawDep[reFind.SubexpIndex("gid")]),
ArtifactID: string(rawDep[reFind.SubexpIndex("aid")]),
Packaging: string(rawDep[reFind.SubexpIndex("packaging")]),
Version: string(rawDep[reFind.SubexpIndex("version")]),
@@ -406,9 +404,8 @@
}
type Dependency struct {
- GroupID []string
- ArtifactID, Version, Packaging, Scope string
- TransitiveDeps []*Dependency
+ GroupID, ArtifactID, Version, Packaging, Scope string
+ TransitiveDeps []*Dependency
}
func (dep *Dependency) Clone() *Dependency {
@@ -430,7 +427,7 @@
}
func (dep *Dependency) Path() string {
- return fmt.Sprintf("%v/%v/%v", strings.Join(dep.GroupID, "/"), dep.ArtifactID, dep.Version)
+ return fmt.Sprintf("%v/%v/%v", strings.ReplaceAll(dep.GroupID, ".", "/"), dep.ArtifactID, dep.Version)
}
func (dep *Dependency) Pom() string {
@@ -441,24 +438,8 @@
return fmt.Sprintf("%v-%v.jar", dep.ArtifactID, dep.Version)
}
-func (dep *Dependency) String() string {
- buf := bytes.NewBuffer(nil)
- w := bufio.NewWriter(buf)
-
- _, err := w.WriteString(fmt.Sprintf("%v:%v", strings.Join(dep.GroupID, "."), dep.ArtifactID))
- if err != nil {
- logger.Log.Error(err)
- }
-
- for _, tDep := range dep.TransitiveDeps {
- _, err = w.WriteString(fmt.Sprintf("\n\t%v", tDep))
- if err != nil {
- logger.Log.Error(err)
- }
- }
-
- _ = w.Flush()
- return buf.String()
+func (dep *Dependency) Name() string {
+ return fmt.Sprintf("%v:%v", dep.GroupID, dep.ArtifactID)
}
// PomFile is used to extract license from the pom.xml file
diff --git a/pkg/deps/npm.go b/pkg/deps/npm.go
index 4cdb28f..c1d97dd 100644
--- a/pkg/deps/npm.go
+++ b/pkg/deps/npm.go
@@ -190,7 +190,7 @@
Dependency: pkgName,
}
// resolve from the package.json file
- if err := resolver.ResolvePkgFile(result, pkgPath, config.Licenses); err != nil {
+ if err := resolver.ResolvePkgFile(result, pkgPath, config); err != nil {
result.ResolveErrors = append(result.ResolveErrors, err)
}
@@ -203,7 +203,7 @@
}
// ResolvePkgFile tries to find and parse the package.json file to capture the license field
-func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, licenses []*ConfigDepLicense) error {
+func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, config *ConfigDeps) error {
expectedPkgFile := filepath.Join(pkgPath, PkgFileName)
packageInfo, err := resolver.ParsePkgFile(expectedPkgFile)
if err != nil {
@@ -211,13 +211,9 @@
}
result.Version = packageInfo.Version
- for _, l := range licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == packageInfo.Name && version == packageInfo.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(packageInfo.Name, packageInfo.Version); ok {
+ result.LicenseSpdxID = l
+ return nil
}
if lcs, ok := resolver.ResolveLicenseField(packageInfo.License); ok {
@@ -287,13 +283,9 @@
if result.LicenseSpdxID != "" {
return nil
}
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == info.Name() && version == result.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(info.Name(), result.Version); ok {
+ result.LicenseSpdxID = l
+ return nil
}
identifier, err := license.Identify(string(content), config.Threshold)
if err != nil {
diff --git a/pkg/deps/result.go b/pkg/deps/result.go
index 95a318b..66f87df 100644
--- a/pkg/deps/result.go
+++ b/pkg/deps/result.go
@@ -20,6 +20,7 @@
import (
"fmt"
"math"
+ "sort"
"strings"
)
@@ -56,6 +57,13 @@
}
func (report *Report) String() string {
+ sort.SliceStable(report.Resolved, func(i, j int) bool {
+ return report.Resolved[i].Dependency < report.Resolved[j].Dependency
+ })
+ sort.SliceStable(report.Skipped, func(i, j int) bool {
+ return report.Skipped[i].Dependency < report.Skipped[j].Dependency
+ })
+
dWidth, lWidth, vWidth := .0, .0, .0
for _, r := range report.Skipped {
dWidth = math.Max(float64(len(r.Dependency)), dWidth)
