blob: 960041521def882a5ae38974bb97245f9705b6b5 [file] [log] [blame]
// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:generate ./COMPILE-PROTOS.sh
// Gnostic is a tool for building better REST APIs through knowledge.
//
// Gnostic reads declarative descriptions of REST APIs that conform
// to the OpenAPI Specification, reports errors, resolves internal
// dependencies, and puts the results in a binary form that can
// be used in any language that is supported by the Protocol Buffer
// tools.
//
// Gnostic models are validated and typed. This allows API tool
// developers to focus on their product and not worry about input
// validation and type checking.
//
// Gnostic calls plugins that implement a variety of API implementation
// and support features including generation of client and server
// support code.
package main
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/golang/protobuf/proto"
"github.com/googleapis/gnostic/OpenAPIv2"
"github.com/googleapis/gnostic/OpenAPIv3"
"github.com/googleapis/gnostic/compiler"
"github.com/googleapis/gnostic/discovery"
"github.com/googleapis/gnostic/jsonwriter"
plugins "github.com/googleapis/gnostic/plugins"
surface "github.com/googleapis/gnostic/surface"
"gopkg.in/yaml.v2"
)
const ( // Source Format
SourceFormatUnknown = 0
SourceFormatOpenAPI2 = 2
SourceFormatOpenAPI3 = 3
SourceFormatDiscovery = 4
)
// Determine the version of an OpenAPI description read from JSON or YAML.
func getOpenAPIVersionFromInfo(info interface{}) int {
m, ok := compiler.UnpackMap(info)
if !ok {
return SourceFormatUnknown
}
swagger, ok := compiler.MapValueForKey(m, "swagger").(string)
if ok && strings.HasPrefix(swagger, "2.0") {
return SourceFormatOpenAPI2
}
openapi, ok := compiler.MapValueForKey(m, "openapi").(string)
if ok && strings.HasPrefix(openapi, "3.0") {
return SourceFormatOpenAPI3
}
kind, ok := compiler.MapValueForKey(m, "kind").(string)
if ok && kind == "discovery#restDescription" {
return SourceFormatDiscovery
}
return SourceFormatUnknown
}
const (
pluginPrefix = "gnostic-"
extensionPrefix = "gnostic-x-"
)
type pluginCall struct {
Name string
Invocation string
}
// Invokes a plugin.
func (p *pluginCall) perform(document proto.Message, sourceFormat int, sourceName string, timePlugins bool) ([]*plugins.Message, error) {
if p.Name != "" {
request := &plugins.Request{}
// Infer the name of the executable by adding the prefix.
executableName := pluginPrefix + p.Name
// Validate invocation string with regular expression.
invocation := p.Invocation
//
// Plugin invocations must consist of
// zero or more comma-separated key=value pairs followed by a path.
// If pairs are present, a colon separates them from the path.
// Keys and values must be alphanumeric strings and may contain
// dashes, underscores, periods, or forward slashes.
// A path can contain any characters other than the separators ',', ':', and '='.
//
invocationRegex := regexp.MustCompile(`^([\w-_\/\.]+=[\w-_\/\.]+(,[\w-_\/\.]+=[\w-_\/\.]+)*:)?[^,:=]+$`)
if !invocationRegex.Match([]byte(p.Invocation)) {
return nil, fmt.Errorf("Invalid invocation of %s: %s", executableName, invocation)
}
invocationParts := strings.Split(p.Invocation, ":")
var outputLocation string
switch len(invocationParts) {
case 1:
outputLocation = invocationParts[0]
case 2:
parameters := strings.Split(invocationParts[0], ",")
for _, keyvalue := range parameters {
pair := strings.Split(keyvalue, "=")
if len(pair) == 2 {
request.Parameters = append(request.Parameters, &plugins.Parameter{Name: pair[0], Value: pair[1]})
}
}
outputLocation = invocationParts[1]
default:
// badly-formed request
outputLocation = invocationParts[len(invocationParts)-1]
}
version := &plugins.Version{}
version.Major = 0
version.Minor = 1
version.Patch = 0
request.CompilerVersion = version
request.OutputPath = outputLocation
request.SourceName = sourceName
switch sourceFormat {
case SourceFormatOpenAPI2:
request.AddModel("openapi.v2.Document", document)
// include experimental API surface model
surfaceModel, err := surface.NewModelFromOpenAPI2(document.(*openapi_v2.Document))
if err == nil {
request.AddModel("surface.v1.Model", surfaceModel)
}
case SourceFormatOpenAPI3:
request.AddModel("openapi.v3.Document", document)
// include experimental API surface model
surfaceModel, err := surface.NewModelFromOpenAPI3(document.(*openapi_v3.Document))
if err == nil {
request.AddModel("surface.v1.Model", surfaceModel)
}
case SourceFormatDiscovery:
request.AddModel("discovery.v1.Document", document)
default:
}
requestBytes, _ := proto.Marshal(request)
cmd := exec.Command(executableName, "-plugin")
cmd.Stdin = bytes.NewReader(requestBytes)
cmd.Stderr = os.Stderr
pluginStartTime := time.Now()
output, err := cmd.Output()
pluginElapsedTime := time.Since(pluginStartTime)
if timePlugins {
fmt.Printf("> %s (%s)\n", executableName, pluginElapsedTime)
}
if err != nil {
return nil, err
}
response := &plugins.Response{}
err = proto.Unmarshal(output, response)
if err != nil {
// Gnostic expects plugins to only write the
// response message to stdout. Be sure that
// any logging messages are written to stderr only.
return nil, errors.New("Invalid plugin response (plugins must write log messages to stderr, not stdout).")
}
plugins.HandleResponse(response, outputLocation)
return response.Messages, nil
}
return nil, nil
}
func isFile(path string) bool {
fileInfo, err := os.Stat(path)
if err != nil {
return false
}
return !fileInfo.IsDir()
}
func isDirectory(path string) bool {
fileInfo, err := os.Stat(path)
if err != nil {
return false
}
return fileInfo.IsDir()
}
// Write bytes to a named file.
// Certain names have special meaning:
// ! writes nothing
// - writes to stdout
// = writes to stderr
// If a directory name is given, the file is written there with
// a name derived from the source and extension arguments.
func writeFile(name string, bytes []byte, source string, extension string) {
var writer io.Writer
if name == "!" {
return
} else if name == "-" {
writer = os.Stdout
} else if name == "=" {
writer = os.Stderr
} else if isDirectory(name) {
base := filepath.Base(source)
// Remove the original source extension.
base = base[0 : len(base)-len(filepath.Ext(base))]
// Build the path that puts the result in the passed-in directory.
filename := name + "/" + base + "." + extension
file, _ := os.Create(filename)
defer file.Close()
writer = file
} else {
file, _ := os.Create(name)
defer file.Close()
writer = file
}
writer.Write(bytes)
if name == "-" || name == "=" {
writer.Write([]byte("\n"))
}
}
// The Gnostic structure holds global state information for gnostic.
type Gnostic struct {
usage string
sourceName string
binaryOutputPath string
textOutputPath string
yamlOutputPath string
jsonOutputPath string
errorOutputPath string
messageOutputPath string
resolveReferences bool
pluginCalls []*pluginCall
extensionHandlers []compiler.ExtensionHandler
sourceFormat int
timePlugins bool
}
// Initialize a structure to store global application state.
func newGnostic() *Gnostic {
g := &Gnostic{}
// Option fields initialize to their default values.
g.usage = `
Usage: gnostic SOURCE [OPTIONS]
SOURCE is the filename or URL of an API description.
Options:
--pb-out=PATH Write a binary proto to the specified location.
--text-out=PATH Write a text proto to the specified location.
--json-out=PATH Write a json API description to the specified location.
--yaml-out=PATH Write a yaml API description to the specified location.
--errors-out=PATH Write compilation errors to the specified location.
--messages-out=PATH Write messages generated by plugins to the specified
location. Messages from all plugin invocations are
written to a single common file.
--PLUGIN-out=PATH Run the plugin named gnostic-PLUGIN and write results
to the specified location.
--PLUGIN Run the plugin named gnostic-PLUGIN but don't write any
results. Used for plugins that return messages only.
PLUGIN must not match any other gnostic option.
--x-EXTENSION Use the extension named gnostic-x-EXTENSION
to process OpenAPI specification extensions.
--resolve-refs Explicitly resolve $ref references.
This could have problems with recursive definitions.
--time-plugins Report plugin runtimes.
`
// Initialize internal structures.
g.pluginCalls = make([]*pluginCall, 0)
g.extensionHandlers = make([]compiler.ExtensionHandler, 0)
return g
}
// Parse command-line options.
func (g *Gnostic) readOptions() {
// plugin processing matches patterns of the form "--PLUGIN-out=PATH" and "--PLUGIN_out=PATH"
pluginRegex := regexp.MustCompile("--(.+)[-_]out=(.+)")
// extension processing matches patterns of the form "--x-EXTENSION"
extensionRegex := regexp.MustCompile("--x-(.+)")
for i, arg := range os.Args {
if i == 0 {
continue // skip the tool name
}
var m [][]byte
if m = pluginRegex.FindSubmatch([]byte(arg)); m != nil {
pluginName := string(m[1])
invocation := string(m[2])
switch pluginName {
case "pb":
g.binaryOutputPath = invocation
case "text":
g.textOutputPath = invocation
case "json":
g.jsonOutputPath = invocation
case "yaml":
g.yamlOutputPath = invocation
case "errors":
g.errorOutputPath = invocation
case "messages":
g.messageOutputPath = invocation
default:
p := &pluginCall{Name: pluginName, Invocation: invocation}
g.pluginCalls = append(g.pluginCalls, p)
}
} else if m = extensionRegex.FindSubmatch([]byte(arg)); m != nil {
extensionName := string(m[1])
extensionHandler := compiler.ExtensionHandler{Name: extensionPrefix + extensionName}
g.extensionHandlers = append(g.extensionHandlers, extensionHandler)
} else if arg == "--resolve-refs" {
g.resolveReferences = true
} else if arg == "--time-plugins" {
g.timePlugins = true
} else if arg[0] == '-' && arg[1] == '-' {
// try letting the option specify a plugin with no output files (or unwanted output files)
// this is useful for calling plugins like linters that only return messages
p := &pluginCall{Name: arg[2:len(arg)], Invocation: "!"}
g.pluginCalls = append(g.pluginCalls, p)
} else if arg[0] == '-' {
fmt.Fprintf(os.Stderr, "Unknown option: %s.\n%s\n", arg, g.usage)
os.Exit(-1)
} else {
g.sourceName = arg
}
}
}
// Validate command-line options.
func (g *Gnostic) validateOptions() {
if g.binaryOutputPath == "" &&
g.textOutputPath == "" &&
g.yamlOutputPath == "" &&
g.jsonOutputPath == "" &&
g.errorOutputPath == "" &&
len(g.pluginCalls) == 0 {
fmt.Fprintf(os.Stderr, "Missing output directives.\n%s\n", g.usage)
os.Exit(-1)
}
if g.sourceName == "" {
fmt.Fprintf(os.Stderr, "No input specified.\n%s\n", g.usage)
os.Exit(-1)
}
// If we get here and the error output is unspecified, write errors to stderr.
if g.errorOutputPath == "" {
g.errorOutputPath = "="
}
}
// Generate an error message to be written to stderr or a file.
func (g *Gnostic) errorBytes(err error) []byte {
return []byte("Errors reading " + g.sourceName + "\n" + err.Error())
}
// Read an OpenAPI description from YAML or JSON.
func (g *Gnostic) readOpenAPIText(bytes []byte) (message proto.Message, err error) {
info, err := compiler.ReadInfoFromBytes(g.sourceName, bytes)
if err != nil {
return nil, err
}
// Determine the OpenAPI version.
g.sourceFormat = getOpenAPIVersionFromInfo(info)
if g.sourceFormat == SourceFormatUnknown {
return nil, errors.New("unable to identify OpenAPI version")
}
// Compile to the proto model.
if g.sourceFormat == SourceFormatOpenAPI2 {
document, err := openapi_v2.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
if err != nil {
return nil, err
}
message = document
} else if g.sourceFormat == SourceFormatOpenAPI3 {
document, err := openapi_v3.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
if err != nil {
return nil, err
}
message = document
} else {
document, err := discovery_v1.NewDocument(info, compiler.NewContextWithExtensions("$root", nil, &g.extensionHandlers))
if err != nil {
return nil, err
}
message = document
}
return message, err
}
// Read an OpenAPI binary file.
func (g *Gnostic) readOpenAPIBinary(data []byte) (message proto.Message, err error) {
// try to read an OpenAPI v3 document
documentV3 := &openapi_v3.Document{}
err = proto.Unmarshal(data, documentV3)
if err == nil && strings.HasPrefix(documentV3.Openapi, "3.0") {
g.sourceFormat = SourceFormatOpenAPI3
return documentV3, nil
}
// if that failed, try to read an OpenAPI v2 document
documentV2 := &openapi_v2.Document{}
err = proto.Unmarshal(data, documentV2)
if err == nil && strings.HasPrefix(documentV2.Swagger, "2.0") {
g.sourceFormat = SourceFormatOpenAPI2
return documentV2, nil
}
// if that failed, try to read a Discovery Format document
discoveryDocument := &discovery_v1.Document{}
err = proto.Unmarshal(data, discoveryDocument)
if err == nil { // && strings.HasPrefix(documentV2.Swagger, "2.0") {
g.sourceFormat = SourceFormatDiscovery
return discoveryDocument, nil
}
return nil, err
}
// Write a binary pb representation.
func (g *Gnostic) writeBinaryOutput(message proto.Message) {
protoBytes, err := proto.Marshal(message)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
defer os.Exit(-1)
} else {
writeFile(g.binaryOutputPath, protoBytes, g.sourceName, "pb")
}
}
// Write a text pb representation.
func (g *Gnostic) writeTextOutput(message proto.Message) {
bytes := []byte(proto.MarshalTextString(message))
writeFile(g.textOutputPath, bytes, g.sourceName, "text")
}
// Write JSON/YAML OpenAPI representations.
func (g *Gnostic) writeJSONYAMLOutput(message proto.Message) {
// Convert the OpenAPI document into an exportable MapSlice.
var rawInfo yaml.MapSlice
var ok bool
var err error
if g.sourceFormat == SourceFormatOpenAPI2 {
document := message.(*openapi_v2.Document)
rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
if !ok {
rawInfo = nil
}
} else if g.sourceFormat == SourceFormatOpenAPI3 {
document := message.(*openapi_v3.Document)
rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
if !ok {
rawInfo = nil
}
} else if g.sourceFormat == SourceFormatDiscovery {
document := message.(*discovery_v1.Document)
rawInfo, ok = document.ToRawInfo().(yaml.MapSlice)
if !ok {
rawInfo = nil
}
}
// Optionally write description in yaml format.
if g.yamlOutputPath != "" {
var bytes []byte
if rawInfo != nil {
bytes, err = yaml.Marshal(rawInfo)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating yaml output %s\n", err.Error())
}
writeFile(g.yamlOutputPath, bytes, g.sourceName, "yaml")
} else {
fmt.Fprintf(os.Stderr, "No yaml output available.\n")
}
}
// Optionally write description in json format.
if g.jsonOutputPath != "" {
var bytes []byte
if rawInfo != nil {
bytes, _ = jsonwriter.Marshal(rawInfo)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating json output %s\n", err.Error())
}
writeFile(g.jsonOutputPath, bytes, g.sourceName, "json")
} else {
fmt.Fprintf(os.Stderr, "No json output available.\n")
}
}
}
// Write messages.
func (g *Gnostic) writeMessagesOutput(message proto.Message) {
protoBytes, err := proto.Marshal(message)
if err != nil {
writeFile(g.messageOutputPath, g.errorBytes(err), g.sourceName, "errors")
defer os.Exit(-1)
} else {
writeFile(g.messageOutputPath, protoBytes, g.sourceName, "messages.pb")
}
}
// Perform all actions specified in the command-line options.
func (g *Gnostic) performActions(message proto.Message) (err error) {
// Optionally resolve internal references.
if g.resolveReferences {
if g.sourceFormat == SourceFormatOpenAPI2 {
document := message.(*openapi_v2.Document)
_, err = document.ResolveReferences(g.sourceName)
} else if g.sourceFormat == SourceFormatOpenAPI3 {
document := message.(*openapi_v3.Document)
_, err = document.ResolveReferences(g.sourceName)
}
if err != nil {
return err
}
}
// Optionally write proto in binary format.
if g.binaryOutputPath != "" {
g.writeBinaryOutput(message)
}
// Optionally write proto in text format.
if g.textOutputPath != "" {
g.writeTextOutput(message)
}
// Optionally write document in yaml and/or json formats.
if g.yamlOutputPath != "" || g.jsonOutputPath != "" {
g.writeJSONYAMLOutput(message)
}
// Call all specified plugins.
messages := make([]*plugins.Message, 0)
for _, p := range g.pluginCalls {
pluginMessages, err := p.perform(message, g.sourceFormat, g.sourceName, g.timePlugins)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
defer os.Exit(-1) // run all plugins, even when some have errors
}
messages = append(messages, pluginMessages...)
}
if g.messageOutputPath != "" {
g.writeMessagesOutput(&plugins.Messages{Messages: messages})
} else {
// Print any messages from the plugins
if len(messages) > 0 {
for _, message := range messages {
fmt.Printf("%+v\n", message)
}
}
}
return nil
}
func (g *Gnostic) main() {
var err error
g.readOptions()
g.validateOptions()
// Read the OpenAPI source.
bytes, err := compiler.ReadBytesForFile(g.sourceName)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
os.Exit(-1)
}
extension := strings.ToLower(filepath.Ext(g.sourceName))
var message proto.Message
if extension == ".json" || extension == ".yaml" {
// Try to read the source as JSON/YAML.
message, err = g.readOpenAPIText(bytes)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
os.Exit(-1)
}
} else if extension == ".pb" {
// Try to read the source as a binary protocol buffer.
message, err = g.readOpenAPIBinary(bytes)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
os.Exit(-1)
}
} else {
err = errors.New("unknown file extension. 'json', 'yaml', and 'pb' are accepted")
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
os.Exit(-1)
}
// Perform actions specified by command options.
err = g.performActions(message)
if err != nil {
writeFile(g.errorOutputPath, g.errorBytes(err), g.sourceName, "errors")
os.Exit(-1)
}
}
func main() {
g := newGnostic()
g.main()
}