pkg/bufman/bufpkg/bufmodule/bufmoduleref/bufmoduleref.go

// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bufmoduleref

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"
)

import (
	"go.uber.org/multierr"
)

import (
	"github.com/apache/dubbo-kubernetes/pkg/bufman/bufpkg/buflock"
	modulev1alpha1 "github.com/apache/dubbo-kubernetes/pkg/bufman/gen/proto/go/module/v1alpha1"
	"github.com/apache/dubbo-kubernetes/pkg/bufman/pkg/manifest"
	"github.com/apache/dubbo-kubernetes/pkg/bufman/pkg/storage"
	"github.com/apache/dubbo-kubernetes/pkg/bufman/pkg/uuidutil"
)

const (
	// Main is the default reference used if no other reference is specified.
	Main = "main"
)

// FileInfo contains module file info.
type FileInfo interface {
	// Path is the path of the file relative to the root it is contained within.
	// This will be normalized, validated and never empty,
	// This will be unique within a given Image.
	Path() string
	// ExternalPath returns the path that identifies this file externally.
	//
	// This will be unnormalized.
	// Never empty. Falls back to Path if there is not an external path.
	//
	// Example:
	//	 Assume we had the input path /foo/bar which is a local directory.
	//   Path: one/one.proto
	//   RootDirPath: proto
	//   ExternalPath: /foo/bar/proto/one/one.proto
	ExternalPath() string
	// IsImport returns true if this file is an import.
	IsImport() bool
	// ModuleIdentity is the module that this file came from.
	//
	// Note this *can* be nil if we did not build from a named module.
	// All code must assume this can be nil.
	// Note that nil checking should work since the backing type is always a pointer.
	ModuleIdentity() ModuleIdentity
	// Commit is the commit for the module that this file came from.
	//
	// This will only be set if ModuleIdentity is set, but may not be set
	// even if ModuleIdentity is set, that is commit is optional information
	// even if we know what module this file came from.
	Commit() string
	// WithIsImport returns this FileInfo with the given IsImport value.
	WithIsImport(isImport bool) FileInfo

	isFileInfo()
}

// NewFileInfo returns a new FileInfo.
//
// TODO: we should make moduleIdentity and commit options.
// TODO: we don't validate commit
func NewFileInfo(
	path string,
	externalPath string,
	isImport bool,
	moduleIdentity ModuleIdentity,
	commit string,
) (FileInfo, error) {
	return newFileInfo(
		path,
		externalPath,
		isImport,
		moduleIdentity,
		commit,
	)
}

// ModuleOwner is a module owner.
//
// It just contains remote, owner.
//
// This is shared by ModuleIdentity.
type ModuleOwner interface {
	Remote() string
	Owner() string

	isModuleOwner()
}

// NewModuleOwner returns a new ModuleOwner.
func NewModuleOwner(
	remote string,
	owner string,
) (ModuleOwner, error) {
	return newModuleOwner(remote, owner)
}

// ModuleOwnerForString returns a new ModuleOwner for the given string.
//
// This parses the path in the form remote/owner.
func ModuleOwnerForString(path string) (ModuleOwner, error) {
	slashSplit := strings.Split(path, "/")
	if len(slashSplit) != 2 {
		return nil, newInvalidModuleOwnerStringError(path)
	}
	remote := strings.TrimSpace(slashSplit[0])
	if remote == "" {
		return nil, newInvalidModuleIdentityStringError(path)
	}
	owner := strings.TrimSpace(slashSplit[1])
	if owner == "" {
		return nil, newInvalidModuleIdentityStringError(path)
	}
	return NewModuleOwner(remote, owner)
}

// ModuleIdentity is a module identity.
//
// It just contains remote, owner, repository.
//
// This is shared by ModuleReference and ModulePin.
type ModuleIdentity interface {
	ModuleOwner

	Repository() string

	// IdentityString is the string remote/owner/repository.
	IdentityString() string

	isModuleIdentity()
}

// NewModuleIdentity returns a new ModuleIdentity.
func NewModuleIdentity(
	remote string,
	owner string,
	repository string,
) (ModuleIdentity, error) {
	return newModuleIdentity(remote, owner, repository)
}

// ModuleIdentityForString returns a new ModuleIdentity for the given string.
//
// This parses the path in the form remote/owner/repository
//
// TODO: we may want to add a special error if we detect / or @ as this may be a common mistake.
func ModuleIdentityForString(path string) (ModuleIdentity, error) {
	remote, owner, repository, err := parseModuleIdentityComponents(path)
	if err != nil {
		return nil, err
	}
	return NewModuleIdentity(remote, owner, repository)
}

// ModuleReference is a module reference.
//
// It references either a branch, tag, or a commit.
// Note that since commits belong to branches, we can deduce
// the branch from the commit when resolving.
type ModuleReference interface {
	ModuleIdentity

	// Prints either remote/owner/repository:{branch,commit}
	// If the reference is equal to MainBranch, prints remote/owner/repository.
	fmt.Stringer

	// Either branch, tag, or commit
	Reference() string

	isModuleReference()
}

// NewModuleReference returns a new validated ModuleReference.
func NewModuleReference(
	remote string,
	owner string,
	repository string,
	reference string,
) (ModuleReference, error) {
	return newModuleReference(remote, owner, repository, reference)
}

// NewModuleReferenceForProto returns a new ModuleReference for the given proto ModuleReference.
func NewModuleReferenceForProto(protoModuleReference *modulev1alpha1.ModuleReference) (ModuleReference, error) {
	return newModuleReferenceForProto(protoModuleReference)
}

// NewModuleReferencesForProtos maps the Protobuf equivalent into the internal representation.
func NewModuleReferencesForProtos(protoModuleReferences ...*modulev1alpha1.ModuleReference) ([]ModuleReference, error) {
	if len(protoModuleReferences) == 0 {
		return nil, nil
	}
	moduleReferences := make([]ModuleReference, len(protoModuleReferences))
	for i, protoModuleReference := range protoModuleReferences {
		moduleReference, err := NewModuleReferenceForProto(protoModuleReference)
		if err != nil {
			return nil, err
		}
		moduleReferences[i] = moduleReference
	}
	return moduleReferences, nil
}

// NewProtoModuleReferenceForModuleReference returns a new proto ModuleReference for the given ModuleReference.
func NewProtoModuleReferenceForModuleReference(moduleReference ModuleReference) *modulev1alpha1.ModuleReference {
	return newProtoModuleReferenceForModuleReference(moduleReference)
}

// NewProtoModuleReferencesForModuleReferences maps the given module references into the protobuf representation.
func NewProtoModuleReferencesForModuleReferences(moduleReferences ...ModuleReference) []*modulev1alpha1.ModuleReference {
	if len(moduleReferences) == 0 {
		return nil
	}
	protoModuleReferences := make([]*modulev1alpha1.ModuleReference, len(moduleReferences))
	for i, moduleReference := range moduleReferences {
		protoModuleReferences[i] = NewProtoModuleReferenceForModuleReference(moduleReference)
	}
	return protoModuleReferences
}

// ModuleReferenceForString returns a new ModuleReference for the given string.
// If a branch, commit, draft, or tag is not provided, the "main" branch is used.
//
// This parses the path in the form remote/owner/repository{:branch,:commit,:draft,:tag}.
func ModuleReferenceForString(path string) (ModuleReference, error) {
	remote, owner, repository, reference, err := parseModuleReferenceComponents(path)
	if err != nil {
		return nil, err
	}
	if reference == "" {
		// Default to the main branch if a ':' separator was not specified.
		reference = Main
	}
	return NewModuleReference(remote, owner, repository, reference)
}

// IsCommitModuleReference returns true if the ModuleReference references a commit.
//
// If false, this means the ModuleReference references a branch or tag.
// Branch and tag disambiguation needs to be done server-side.
func IsCommitModuleReference(moduleReference ModuleReference) bool {
	return IsCommitReference(moduleReference.Reference())
}

// IsCommitReference returns whether the provided reference is a commit.
func IsCommitReference(reference string) bool {
	_, err := uuidutil.FromDashless(reference)
	return err == nil
}

// ModulePin is a module pin.
//
// It references a specific point in time of a Module.
//
// Note that a commit does this itself, but we want all this information.
// This is what is stored in a buf.lock file.
type ModulePin interface {
	ModuleIdentity

	// Prints remote/owner/repository:commit, which matches ModuleReference
	fmt.Stringer

	// all of these will be set
	Branch() string
	Commit() string
	Digest() string
	CreateTime() time.Time

	isModulePin()
}

// NewModulePin returns a new validated ModulePin.
func NewModulePin(
	remote string,
	owner string,
	repository string,
	branch string,
	commit string,
	digest string,
	createTime time.Time,
) (ModulePin, error) {
	return newModulePin(remote, owner, repository, branch, commit, digest, createTime)
}

// NewModulePinForProto returns a new ModulePin for the given proto ModulePin.
func NewModulePinForProto(protoModulePin *modulev1alpha1.ModulePin) (ModulePin, error) {
	return newModulePinForProto(protoModulePin)
}

// NewModulePinsForProtos maps the Protobuf equivalent into the internal representation.
func NewModulePinsForProtos(protoModulePins ...*modulev1alpha1.ModulePin) ([]ModulePin, error) {
	if len(protoModulePins) == 0 {
		return nil, nil
	}
	modulePins := make([]ModulePin, len(protoModulePins))
	for i, protoModulePin := range protoModulePins {
		modulePin, err := NewModulePinForProto(protoModulePin)
		if err != nil {
			return nil, err
		}
		modulePins[i] = modulePin
	}
	return modulePins, nil
}

// NewProtoModulePinForModulePin returns a new proto ModulePin for the given ModulePin.
func NewProtoModulePinForModulePin(modulePin ModulePin) *modulev1alpha1.ModulePin {
	return newProtoModulePinForModulePin(modulePin)
}

// NewProtoModulePinsForModulePins maps the given module pins into the protobuf representation.
func NewProtoModulePinsForModulePins(modulePins ...ModulePin) []*modulev1alpha1.ModulePin {
	if len(modulePins) == 0 {
		return nil
	}
	protoModulePins := make([]*modulev1alpha1.ModulePin, len(modulePins))
	for i, modulePin := range modulePins {
		protoModulePins[i] = NewProtoModulePinForModulePin(modulePin)
	}
	return protoModulePins
}

// ValidateModuleReferencesUniqueByIdentity returns an error if the module references contain any duplicates.
//
// This only checks remote, owner, repository.
func ValidateModuleReferencesUniqueByIdentity(moduleReferences []ModuleReference) error {
	seenModuleReferences := make(map[string]struct{})
	for _, moduleReference := range moduleReferences {
		moduleIdentityString := moduleReference.IdentityString()
		if _, ok := seenModuleReferences[moduleIdentityString]; ok {
			return fmt.Errorf("module %s appeared twice", moduleIdentityString)
		}
		seenModuleReferences[moduleIdentityString] = struct{}{}
	}
	return nil
}

// ValidateModulePinsUniqueByIdentity returns an error if the module pins contain any duplicates.
//
// This only checks remote, owner, repository.
func ValidateModulePinsUniqueByIdentity(modulePins []ModulePin) error {
	seenModulePins := make(map[string]struct{})
	for _, modulePin := range modulePins {
		moduleIdentityString := modulePin.IdentityString()
		if _, ok := seenModulePins[moduleIdentityString]; ok {
			return fmt.Errorf("module %s appeared twice", moduleIdentityString)
		}
		seenModulePins[moduleIdentityString] = struct{}{}
	}
	return nil
}

// ValidateModulePinsConsistentDigests verifies that module pins to the same commit don't change digests.
// This is important to avoid MITM issues, where the module digest stored in a buf.lock file doesn't match
// the module pin returned from the BSR.
// Returns an error that fulfills IsDigestChanged if any valid digest changed from the buf.lock file for
// the same dependency commit.
func ValidateModulePinsConsistentDigests(
	ctx context.Context,
	bucket storage.ReadBucket,
	modulePins []ModulePin,
) error {
	currentConfig, err := buflock.ReadConfig(ctx, bucket)
	if err != nil {
		if storage.IsNotExist(err) {
			return nil
		}
		return err
	}
	if len(currentConfig.Dependencies) == 0 {
		return nil
	}
	currentIdentityAndCommitToDigest := make(map[string]string, len(currentConfig.Dependencies))
	for _, dep := range currentConfig.Dependencies {
		// Ignore dependencies with no digest
		if dep.Digest == "" {
			continue
		}
		// Ignore dependencies with an invalid digest.
		// We want to replace these with a valid digest.
		if _, err := manifest.NewDigestFromString(dep.Digest); err != nil {
			continue
		}
		key := fmt.Sprintf("%s/%s/%s:%s", dep.Remote, dep.Owner, dep.Repository, dep.Commit)
		currentIdentityAndCommitToDigest[key] = dep.Digest
	}
	var changedErrors error
	for _, pin := range modulePins {
		if pin.Digest() == "" {
			continue
		}
		if currentDigest, ok := currentIdentityAndCommitToDigest[pin.String()]; ok && currentDigest != pin.Digest() {
			changedErrors = multierr.Append(changedErrors, &digestChangedError{
				currentDigest: currentDigest,
				updatedPin:    pin,
			})
		}
	}
	return changedErrors
}

// ModuleReferenceEqual returns true if a equals b.
func ModuleReferenceEqual(a ModuleReference, b ModuleReference) bool {
	if (a == nil) != (b == nil) {
		return false
	}
	if a == nil {
		return true
	}
	return a.Remote() == b.Remote() &&
		a.Owner() == b.Owner() &&
		a.Repository() == b.Repository() &&
		a.Reference() == b.Reference()
}

// ModulePinEqual returns true if a equals b.
func ModulePinEqual(a ModulePin, b ModulePin) bool {
	if (a == nil) != (b == nil) {
		return false
	}
	if a == nil {
		return true
	}
	return a.Remote() == b.Remote() &&
		a.Owner() == b.Owner() &&
		a.Repository() == b.Repository() &&
		a.Branch() == b.Branch() &&
		a.Commit() == b.Commit() &&
		a.Digest() == b.Digest() &&
		a.CreateTime().Equal(b.CreateTime())
}

// DependencyModulePinsForBucket reads the module dependencies from the lock file in the bucket.
func DependencyModulePinsForBucket(
	ctx context.Context,
	readBucket storage.ReadBucket,
) ([]ModulePin, error) {
	lockFile, err := buflock.ReadConfig(ctx, readBucket)
	if err != nil {
		return nil, fmt.Errorf("failed to read lock file: %w", err)
	}
	modulePins := make([]ModulePin, 0, len(lockFile.Dependencies))
	for _, dep := range lockFile.Dependencies {
		modulePin, err := NewModulePin(
			dep.Remote,
			dep.Owner,
			dep.Repository,
			"",
			dep.Commit,
			dep.Digest,
			time.Time{},
		)
		if err != nil {
			return nil, err
		}
		modulePins = append(modulePins, modulePin)
	}
	// just to be safe
	SortModulePins(modulePins)
	if err := ValidateModulePinsUniqueByIdentity(modulePins); err != nil {
		return nil, err
	}
	return modulePins, nil
}

// PutDependencyModulePinsToBucket writes the module dependencies to the write bucket in the form of a lock file.
func PutDependencyModulePinsToBucket(
	ctx context.Context,
	writeBucket storage.WriteBucket,
	modulePins []ModulePin,
) error {
	if err := ValidateModulePinsUniqueByIdentity(modulePins); err != nil {
		return err
	}
	SortModulePins(modulePins)
	lockFile := &buflock.Config{
		Dependencies: make([]buflock.Dependency, 0, len(modulePins)),
	}
	for _, pin := range modulePins {
		lockFile.Dependencies = append(
			lockFile.Dependencies,
			buflock.Dependency{
				Remote:     pin.Remote(),
				Owner:      pin.Owner(),
				Repository: pin.Repository(),
				Commit:     pin.Commit(),
				Digest:     pin.Digest(),
			},
		)
	}
	return buflock.WriteConfig(ctx, writeBucket, lockFile)
}

// SortFileInfos sorts the FileInfos by Path.
//
// This should be treated as the default sorting mechanism.
func SortFileInfos(fileInfos []FileInfo) {
	if len(fileInfos) == 0 {
		return
	}
	sort.Slice(
		fileInfos,
		func(i int, j int) bool {
			return fileInfos[i].Path() < fileInfos[j].Path()
		},
	)
}

// SortFileInfosByExternalPath sorts the FileInfos by ExternalPath.
func SortFileInfosByExternalPath(fileInfos []FileInfo) {
	if len(fileInfos) == 0 {
		return
	}
	sort.Slice(
		fileInfos,
		func(i int, j int) bool {
			return fileInfos[i].ExternalPath() < fileInfos[j].ExternalPath()
		},
	)
}

// SortModuleReferences sorts the ModuleReferences lexicographically by their identity.
func SortModuleReferences(references []ModuleReference) {
	sort.Slice(references, func(i, j int) bool {
		return references[i].IdentityString() < references[j].IdentityString()
	})
}

// SortModulePins sorts the ModulePins.
func SortModulePins(modulePins []ModulePin) {
	sort.Slice(modulePins, func(i, j int) bool {
		return modulePinLess(modulePins[i], modulePins[j])
	})
}

// IsDigestChanged returns true if the error indicates an unexpected digest change.
func IsDigestChanged(err error) bool {
	var errDigestChanged *digestChangedError
	return errors.As(err, &errDigestChanged)
}

// digestChangedError is returned if module pin digests have changed unexpectedly.
type digestChangedError struct {
	// currentDigest is the digest found in the buf.lock file.
	currentDigest string
	// updatedPin is a module pin with a different digest than currentDigest for the same commit.
	updatedPin ModulePin
}

func (e *digestChangedError) Error() string {
	return fmt.Sprintf(
		"module %s commit %q returned an unexpected digest: local buf.lock=%q, remote=%q",
		e.updatedPin.IdentityString(),
		e.updatedPin.Commit(),
		e.currentDigest,
		e.updatedPin.Digest(),
	)
}