blob: e41c4e4b9964c790173e303f63757813693af1be [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dubbo
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
errors2 "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
client2 "sigs.k8s.io/controller-runtime/pkg/client"
)
const (
// DubboFile is the file used for the serialized form of a function.
DubboFile = "dubbo.yaml"
Dockerfile = "Dockerfile"
// RunDataDir holds transient runtime metadata
// By default it is excluded from source control.
RunDataDir = ".dubbo"
// buildstamp is the name of the file within the run data directory whose
// existence indicates the function has been built, and whose content is
// a fingerprint of the filesystem at the time of the build.
buildstamp = "built"
)
type Dubbo struct {
// Root on disk at which to find/create source and config files.
Root string `yaml:"-"`
// Name of the application.
Name string `yaml:"name,omitempty" jsonschema:"pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"`
// Runtime is the language plus context. java|go etc.
Runtime string `yaml:"runtime,omitempty"`
// Template for the application.
Template string `yaml:"template,omitempty"`
// Optional full OCI image tag in form:
// [registry]/[namespace]/[name]:[tag]
// example:
// quay.io/alice/my.function.name
// Registry is optional and is defaulted to DefaultRegistry
// example:
// alice/my.function.name
// If Image is provided, it overrides the default of concatenating
// "Registry+Name:latest" to derive the Image.
Image string `yaml:"image,omitempty"`
// SHA256 hash of the latest image that has been built
ImageDigest string `yaml:"-"`
// Created time is the moment that creation was successfully completed
// according to the client which is in charge of what constitutes being
// fully "Created" (aka initialized)
Created time.Time `yaml:"created,omitempty"`
// BuildSpec define the build properties for a function
Build BuildSpec `yaml:"build,omitempty"`
// DeploySpec define the deployment properties for a function
Deploy DeploySpec `yaml:"deploy,omitempty"`
}
type Env struct {
Name *string `yaml:"name,omitempty" jsonschema:"pattern=^[-._a-zA-Z][-._a-zA-Z0-9]*$"`
Value *string `yaml:"value,omitempty"`
}
type Label struct {
// Key consist of optional prefix part (ended by '/') and name part
// Prefix part validation pattern: [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*
// Name part validation pattern: ([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]
Key *string `yaml:"key" jsonschema:"pattern=^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\\/)?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$"`
Value *string `yaml:"value,omitempty" jsonschema:"pattern=^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$"`
}
type BuildSpec struct {
// BuilderImages define optional explicit builder images to use by
// builder implementations in lea of the in-code defaults. They key
// is the builder's short name. For example:
// builderImages:
// pack: example.com/user/my-pack-node-builder
BuilderImages map[string]string `yaml:"builderImages,omitempty"`
// Optional list of build-packs to use when building the application
Buildpacks []string `yaml:"buildpacks,omitempty"`
BuildEnvs []Env `json:"buildEnvs,omitempty"`
}
type DeploySpec struct {
Namespace string `yaml:"namespace,omitempty"`
Output string `yaml:"output,omitempty"`
ContainerPort int `yaml:"containerPort,omitempty"`
TargetPort int `yaml:"targetPort,omitempty"`
NodePort int `yaml:"nodePort,omitempty"`
UseProm bool `yaml:"-"`
}
func (f *Dubbo) Validate() error {
if f.Root == "" {
return errors.New("dubbo root path is required")
}
var ctr int
errs := [][]string{
validateOptions(),
}
var b strings.Builder
b.WriteString(fmt.Sprintf("'%v' contains errors:", DubboFile))
for _, ee := range errs {
if len(ee) > 0 {
b.WriteString("\n") // Precede each group of errors with a linebreak
}
for _, e := range ee {
ctr++
b.WriteString("\t" + e)
}
}
if ctr == 0 {
return nil // Return nil if there were no validation errors.
}
return errors.New(b.String())
}
// Initialized returns if the function has been initialized.
// Any errors are considered failure (invalid or inaccessible root, config file, etc).
func (f *Dubbo) Initialized() bool {
return !f.Created.IsZero()
}
// nameFromPath returns the default name for a function derived from a path.
// This consists of the last directory in the given path, if derivable (empty
// paths, paths consisting of all slashes, etc. return the zero value "")
func nameFromPath(path string) string {
pathParts := strings.Split(strings.TrimRight(path, string(os.PathSeparator)), string(os.PathSeparator))
return pathParts[len(pathParts)-1]
/* the above may have edge conditions as it assumes the trailing value
* is a directory name. If errors are encountered, we _may_ need to use the
* inbuilt logic in the std lib and either check if the path indicated is a
* directory (appending slash) and then run:
base := filepath.Base(filepath.Dir(path))
if base == string(os.PathSeparator) || base == "." {
return "" // Consider it underivable: string zero value
}
return base
*/
}
// assertEmptyRoot ensures that the directory is empty enough to be used for
// initializing a new function.
func assertEmptyRoot(path string) (err error) {
// If there exists contentious files (config files for instance), this function may have already been initialized.
files, err := contentiousFilesIn(path)
if err != nil {
return
} else if len(files) > 0 {
return fmt.Errorf("the chosen directory '%v' contains contentious files: %v. Has the Service function already been created? Try either using a different directory, deleting the function if it exists, or manually removing the files", path, files)
}
// Ensure there are no non-hidden files, and again none of the aforementioned contentious files.
empty, err := isEffectivelyEmpty(path)
if err != nil {
return
} else if !empty {
err = errors.New("the directory must be empty of visible files and recognized config files before it can be initialized")
return
}
return
}
// contentiousFiles are files which, if extant, preclude the creation of a
// function rooted in the given directory.
var contentiousFiles = []string{
DubboFile,
".gitignore",
}
// contentiousFilesIn the given directory
func contentiousFilesIn(dir string) (contentious []string, err error) {
files, err := os.ReadDir(dir)
for _, file := range files {
for _, name := range contentiousFiles {
if file.Name() == name {
contentious = append(contentious, name)
}
}
}
return
}
// effectivelyEmpty directories are those which have no visible files
func isEffectivelyEmpty(dir string) (bool, error) {
// Check for any non-hidden files
files, err := os.ReadDir(dir)
if err != nil {
return false, err
}
for _, file := range files {
if !strings.HasPrefix(file.Name(), ".") {
return false, nil
}
}
return true, nil
}
// NewDubboWith defaults as provided.
func NewDubboWith(defaults *Dubbo, init bool) *Dubbo {
if !init {
if defaults.Template == "" {
defaults.Template = DefaultTemplate
}
} else {
defaults.Template = "init"
}
if defaults.Build.BuilderImages == nil {
defaults.Build.BuilderImages = make(map[string]string)
}
return defaults
}
// NewDubbo from a given path.
// Invalid paths, or no function at path are errors.
// Syntactic errors are returned immediately (yaml unmarshal errors).
// Dubbo which are syntactically valid are also then logically validated.
// Dubbo from earlier versions are brought up to current using migrations.
// Migrations are run prior to validators such that validation can omit
// concerning itself with backwards compatibility. Migrators must therefore
// selectively consider the minimal validation necessary to enable migration.
func NewDubbo(path string) (*Dubbo, error) {
var err error
f := &Dubbo{}
f.Build.BuilderImages = make(map[string]string)
// Path defaults to current working directory, and if provided explicitly
// Path must exist and be a directory
if path == "" {
if path, err = os.Getwd(); err != nil {
return f, err
}
}
f.Root = path // path is not persisted, as this is the purview of the FS
// Path must exist and be a directory
fd, err := os.Stat(path)
if err != nil {
return f, err
}
if !fd.IsDir() {
return f, fmt.Errorf("function path must be a directory")
}
// If no dubbo.yaml in directory, return the default function which will
// have f.Initialized() == false
filename := filepath.Join(path, DubboFile)
if _, err = os.Stat(filename); err != nil {
if os.IsNotExist(err) {
err = nil
}
return f, err
}
// Path is valid and dubbo.yaml exists: load it
bb, err := os.ReadFile(filename)
if err != nil {
return f, err
}
err = yaml.Unmarshal(bb, f)
if err != nil {
return f, err
}
return f, nil
}
// buildStamp returns the current (last) build stamp for the function
// at the given path, if it can be found.
func (f *Dubbo) buildStamp() string {
buildstampPath := filepath.Join(f.Root, RunDataDir, buildstamp)
if _, err := os.Stat(buildstampPath); err != nil {
return ""
}
b, err := os.ReadFile(buildstampPath)
if err != nil {
return ""
}
return string(b)
}
func (f *Dubbo) CheckLabels(ns string, client *Client) error {
key := client2.ObjectKey{
Namespace: metav1.NamespaceSystem,
Name: ns,
}
var err error
err = client.KubeCtl.Get(context.Background(), key, &corev1.Namespace{})
if err != nil {
if errors2.IsNotFound(err) {
nsObj := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Namespace: metav1.NamespaceSystem,
Name: ns,
},
}
if err := client.KubeCtl.Create(context.Background(), nsObj); err != nil {
return err
}
return nil
} else {
return fmt.Errorf("failed to check if namespace %v exists: %v", ns, err)
}
}
namespaceSelector := client2.MatchingLabels{
"dubbo-deploy": "enabled",
}
nsList := &corev1.NamespaceList{}
if err = client.KubeCtl.List(context.Background(), nsList, namespaceSelector); err != nil {
if errors2.IsNotFound(err) {
return nil
} else {
return err
}
}
var namespace string
if len(nsList.Items) > 0 {
namespace = nsList.Items[0].Name
}
env := os.Getenv("DUBBO_DEPLOY_NS")
if env != "" {
namespace = env
}
if namespace != "" {
promSelector := client2.MatchingLabels{
"dubbo.apache.org/prometheus": "true",
}
promList := &corev1.ServiceList{}
if err := client.KubeCtl.List(context.Background(), promList, promSelector, client2.InNamespace(namespace)); err != nil {
if errors2.IsNotFound(err) {
return nil
} else {
return err
}
}
if len(promList.Items) > 0 {
f.Deploy.UseProm = true
}
}
return nil
}
// Write aka (save, serialize, marshal) the function to disk at its path.
// Only valid functions can be written.
// In order to retain built status (staleness checks), the file is only
// modified if the structure actually changes.
func (f *Dubbo) Write() (err error) {
if err = f.Validate(); err != nil {
return
}
dubboyamlpath := filepath.Join(f.Root, DubboFile)
var dubbobytes []byte
if dubbobytes, err = yaml.Marshal(f); err != nil {
return
}
if err = os.WriteFile(dubboyamlpath, dubbobytes, 0o644); err != nil {
return
}
return
}
type (
stampOptions struct{ journal bool }
stampOption func(o *stampOptions)
)
// Stamp a function as being built.
//
// This is a performance optimization used when updates to the
// function are known to have no effect on its built container. This
// stamp is checked before certain operations, and if it has been updated,
// the build can be skipped. If in doubt, just use .Write only.
//
// Updates the build stamp at ./dubbo/built (and the log
// at .dubbo/built.log) to reflect the current state of the filesystem.
// Note that the caller should call .Write first to flush any changes to the
// application in-memory to the filesystem prior to calling stamp.
//
// The runtime data directory .dubbo is created in the application root if
// necessary.
func (f *Dubbo) Stamp(oo ...stampOption) (err error) {
options := &stampOptions{}
for _, o := range oo {
o(options)
}
if err = EnsureRunDataDir(f.Root); err != nil {
return
}
// Calculate the hash and a logfile of what comprised it
var hash, log string
if hash, log, err = Fingerprint(f); err != nil {
return
}
// Write out the hash
if err = os.WriteFile(filepath.Join(f.Root, RunDataDir, "built"), []byte(hash), os.ModePerm); err != nil {
return
}
// Write out the logfile, optionally timestamped for retention.
logfileName := "built.log"
if options.journal {
logfileName = timestamp(logfileName)
}
logfile, err := os.Create(filepath.Join(f.Root, RunDataDir, logfileName))
if err != nil {
return
}
defer logfile.Close()
_, err = fmt.Fprintln(logfile, log)
return
}
func (f *Dubbo) EnsureDockerfile(cmd *cobra.Command) (err error) {
dockerfilepath := filepath.Join(f.Root, Dockerfile)
dockerfilebytes, ok := DockerfileByRuntime[f.Runtime]
if !ok {
fmt.Fprintln(cmd.OutOrStdout(), "The runtime of your current project is not one of Java or go. We cannot help you generate a Dockerfile template.")
return
}
if err = os.WriteFile(dockerfilepath, []byte(dockerfilebytes), 0o644); err != nil {
return
}
return
}
// timestamp returns the given string prefixed with a microsecond-precision
// timestamp followed by a dot.
// YYYYMMDDHHMMSS.$nanosecond.$s
func timestamp(s string) string {
t := time.Now()
return fmt.Sprintf("%s.%09d.%s", t.Format("20060102150405"), t.Nanosecond(), s)
}
// Fingerprint the files at a given path. Returns a hash calculated from the
// filenames and modification timestamps of the files within the given root.
// Also returns a logfile consisting of the filenames and modification times
// which contributed to the hash.
// Intended to determine if there were appreciable changes to a function's
// source code, certain directories and files are ignored, such as
// .git and .dubbo.
func Fingerprint(f *Dubbo) (hash, log string, err error) {
h := sha256.New() // Hash builder
l := bytes.Buffer{} // Log buffer
root := f.Root
abs, err := filepath.Abs(root)
if err != nil {
return "", "", err
}
output := f.Deploy.Output
err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if path == root {
return nil
}
// Always ignore .dubbo, .git etc
if info.IsDir() && (info.Name() == RunDataDir || info.Name() == ".git" || info.Name() == ".idea") {
return filepath.SkipDir
}
if info.Name() == DubboFile || info.Name() == Dockerfile || info.Name() == output {
return nil
}
fmt.Fprintf(h, "%v:%v:", path, info.ModTime().UnixNano()) // Write to the Hashed
fmt.Fprintf(&l, "%v:%v\n", path, info.ModTime().UnixNano()) // Write to the Log
return nil
})
return fmt.Sprintf("%x", h.Sum(nil)), l.String(), err
}