feat: tcp route translation (#401)
diff --git a/docs/en/latest/concepts/apisix_route.md b/docs/en/latest/concepts/apisix_route.md
index 8a16659..68799ae 100644
--- a/docs/en/latest/concepts/apisix_route.md
+++ b/docs/en/latest/concepts/apisix_route.md
@@ -240,3 +240,27 @@
servicePort: 8080
websocket: true
```
+
+TCP Route
+---------
+
+apisix-ingress-controller supports the port-based tcp route.
+
+```yaml
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixRoute
+metadata:
+ name: tcp-route
+spec:
+ tcp:
+ - name: tcp-route-rule1
+ match:
+ ingressPort: 9100
+ backend:
+ serviceName: tcp-server
+ servicePort: 8080
+```
+
+The above yaml configuration guides TCP traffic entered to the Ingress proxy server (i.e. [APISIX](https://apisix.apache.org)) port `9100` should be routed to the backend service `tcp-server`.
+
+Note since APISIX doesn't support dynamic listening, so here the `9100` port should be pre-defined in APISIX [configuration](https://github.com/apache/apisix/blob/master/conf/config-default.yaml#L101).
diff --git a/docs/en/latest/references/apisix_route_v2alpha1.md b/docs/en/latest/references/apisix_route_v2alpha1.md
index 1f054d1..b22ec79 100644
--- a/docs/en/latest/references/apisix_route_v2alpha1.md
+++ b/docs/en/latest/references/apisix_route_v2alpha1.md
@@ -44,18 +44,26 @@
| http[].match.exprs[].set | array | Expected expression result set, only used when the operator is `In` or `NotIn`, it's exclusive with `http[].match.exprs[].value`.
| http[].backend | object | The backend service. Deprecated: use http[].backends instead.
| http[].backend.serviceName | string | The backend service name, note the service and ApisixRoute should be created in the same namespace. Cross namespace referencing is not allowed.
-| http[].backend.servicePort | integer or string | The backend service port, can be the port number of the name defined in the service object.
-| http[].backend.resolveGranualrity | string | See [Service Resolve Granularity](#service-resolve-granularity) for the details.
+| http[].backend.servicePort | integer or string | The backend service port, can be the port number or the name defined in the service object.
+| http[].backend.resolveGranularity | string | See [Service Resolve Granularity](#service-resolve-granularity) for the details.
| http[].backends | object | The backend services. When the number of backends more than one, weight based traffic split policy will be applied to shifting traffic between these backends.
| http[].backends[].serviceName | string | The backend service name, note the service and ApisixRoute should be created in the same namespace. Cross namespace referencing is not allowed.
-| http[].backends[].servicePort | integer or string | The backend service port, can be the port number of the name defined in the service object.
-| http[].backends[].resolveGranualrity | string | See [Service Resolve Granularity](#service-resolve-granularity) for the details.
+| http[].backends[].servicePort | integer or string | The backend service port, can be the port number or the name defined in the service object.
+| http[].backends[].resolveGranularity | string | See [Service Resolve Granularity](#service-resolve-granularity) for the details.
| http[].backends[].weight | int | The backend weight, which is critical when shifting traffic between multiple backends, default is `100`. Weight is ignored when there is only one backend.
| http[].plugins | array | A series of APISIX plugins that will be executed once this route rule is matched |
| http[].plugins[].name | string | The plugin name, see [docs](http://apisix.apache.org/docs/apisix/getting-started) for learning the available plugins.
| http[].plugins[].enable | boolean | Whether the plugin is in use |
| http[].plugins[].config | object | The plugin configuration, fields should be same as in APISIX. |
| http[].websocket | boolean | Whether enable websocket proxy. |
+| tcp | array | ApisixRoutes' tcp route rules. |
+| tcp[].name | string (required) | The Route rule name. |
+| tcp[].match | object (required) | The Route match conditions. |
+| tcp[].match.ingressPort | integer (required) | the Ingress proxy server listening port, note since APISIX doesn't support dynamic listening, this port should be defined in [apisix configuration](https://github.com/apache/apisix/blob/master/conf/config-default.yaml#L101).|
+| tcp[].backend | object | The backend service. Deprecated: use http[].backends instead.
+| tcp[].backend.serviceName | string | The backend service name, note the service and ApisixRoute should be created in the same namespace. Cross namespace referencing is not allowed.
+| tcp[].backend.servicePort | integer or string | The backend service port, can be the port number or the name defined in the service object.
+| tcp[].backend.resolveGranularity | string | See [Service Resolve Granularity](#service-resolve-granularity) for the details.
## Expression Operators
diff --git a/pkg/ingress/apisix_route.go b/pkg/ingress/apisix_route.go
index a1b67a2..f6fe291 100644
--- a/pkg/ingress/apisix_route.go
+++ b/pkg/ingress/apisix_route.go
@@ -155,8 +155,9 @@
)
m := &manifest{
- routes: tctx.Routes,
- upstreams: tctx.Upstreams,
+ routes: tctx.Routes,
+ upstreams: tctx.Upstreams,
+ streamRoutes: tctx.StreamRoutes,
}
var (
@@ -187,8 +188,9 @@
}
om := &manifest{
- routes: oldCtx.Routes,
- upstreams: oldCtx.Upstreams,
+ routes: oldCtx.Routes,
+ upstreams: oldCtx.Upstreams,
+ streamRoutes: oldCtx.StreamRoutes,
}
added, updated, deleted = m.diff(om)
}
diff --git a/pkg/ingress/manifest.go b/pkg/ingress/manifest.go
index 8e3034c..42cfb61 100644
--- a/pkg/ingress/manifest.go
+++ b/pkg/ingress/manifest.go
@@ -80,30 +80,60 @@
return
}
+func diffStreamRoutes(olds, news []*apisixv1.StreamRoute) (added, updated, deleted []*apisixv1.StreamRoute) {
+ oldMap := make(map[string]*apisixv1.StreamRoute, len(olds))
+ newMap := make(map[string]*apisixv1.StreamRoute, len(news))
+ for _, sr := range olds {
+ oldMap[sr.ID] = sr
+ }
+ for _, sr := range news {
+ newMap[sr.ID] = sr
+ }
+
+ for _, sr := range news {
+ if ou, ok := oldMap[sr.ID]; !ok {
+ added = append(added, sr)
+ } else if !reflect.DeepEqual(ou, sr) {
+ updated = append(updated, sr)
+ }
+ }
+ for _, sr := range olds {
+ if _, ok := newMap[sr.ID]; !ok {
+ deleted = append(deleted, sr)
+ }
+ }
+ return
+}
+
type manifest struct {
- routes []*apisixv1.Route
- upstreams []*apisixv1.Upstream
+ routes []*apisixv1.Route
+ upstreams []*apisixv1.Upstream
+ streamRoutes []*apisixv1.StreamRoute
}
func (m *manifest) diff(om *manifest) (added, updated, deleted *manifest) {
ar, ur, dr := diffRoutes(om.routes, m.routes)
au, uu, du := diffUpstreams(om.upstreams, m.upstreams)
- if ar != nil || au != nil {
+ asr, usr, dsr := diffStreamRoutes(om.streamRoutes, m.streamRoutes)
+ if ar != nil || au != nil || asr != nil {
added = &manifest{
- routes: ar,
- upstreams: au,
+ routes: ar,
+ upstreams: au,
+ streamRoutes: asr,
}
}
- if ur != nil || uu != nil {
+ if ur != nil || uu != nil || usr != nil {
updated = &manifest{
- routes: ur,
- upstreams: uu,
+ routes: ur,
+ upstreams: uu,
+ streamRoutes: usr,
}
}
- if dr != nil || du != nil {
+ if dr != nil || du != nil || dsr != nil {
deleted = &manifest{
- routes: dr,
- upstreams: du,
+ routes: dr,
+ upstreams: du,
+ streamRoutes: dsr,
}
}
return
@@ -124,6 +154,11 @@
merr = multierror.Append(merr, err)
}
}
+ for _, sr := range deleted.streamRoutes {
+ if err := c.apisix.Cluster(clusterName).StreamRoute().Delete(ctx, sr); err != nil {
+ merr = multierror.Append(merr, err)
+ }
+ }
}
if added != nil {
// Should create upstreams firstly due to the dependencies.
@@ -137,6 +172,11 @@
merr = multierror.Append(merr, err)
}
}
+ for _, sr := range added.streamRoutes {
+ if _, err := c.apisix.Cluster(clusterName).StreamRoute().Create(ctx, sr); err != nil {
+ merr = multierror.Append(merr, err)
+ }
+ }
}
if updated != nil {
for _, r := range updated.upstreams {
@@ -149,6 +189,11 @@
merr = multierror.Append(merr, err)
}
}
+ for _, sr := range updated.streamRoutes {
+ if _, err := c.apisix.Cluster(clusterName).StreamRoute().Create(ctx, sr); err != nil {
+ merr = multierror.Append(merr, err)
+ }
+ }
}
if merr != nil {
return merr
diff --git a/pkg/ingress/manifest_test.go b/pkg/ingress/manifest_test.go
index 9d568e6..3ad5e3a 100644
--- a/pkg/ingress/manifest_test.go
+++ b/pkg/ingress/manifest_test.go
@@ -75,6 +75,51 @@
assert.Equal(t, deleted[0].ID, "2")
}
+func TestDiffStreamRoutes(t *testing.T) {
+ news := []*apisixv1.StreamRoute{
+ {
+ ID: "1",
+ },
+ {
+ ID: "3",
+ ServerPort: 8080,
+ },
+ }
+ added, updated, deleted := diffStreamRoutes(nil, news)
+ assert.Nil(t, updated)
+ assert.Nil(t, deleted)
+ assert.Len(t, added, 2)
+ assert.Equal(t, added[0].ID, "1")
+ assert.Equal(t, added[1].ID, "3")
+ assert.Equal(t, added[1].ServerPort, int32(8080))
+
+ olds := []*apisixv1.StreamRoute{
+ {
+ ID: "2",
+ },
+ {
+ ID: "3",
+ ServerPort: 8081,
+ },
+ }
+ added, updated, deleted = diffStreamRoutes(olds, nil)
+ assert.Nil(t, updated)
+ assert.Nil(t, added)
+ assert.Len(t, deleted, 2)
+ assert.Equal(t, deleted[0].ID, "2")
+ assert.Equal(t, deleted[1].ID, "3")
+ assert.Equal(t, deleted[1].ServerPort, int32(8081))
+
+ added, updated, deleted = diffStreamRoutes(olds, news)
+ assert.Len(t, added, 1)
+ assert.Equal(t, added[0].ID, "1")
+ assert.Len(t, updated, 1)
+ assert.Equal(t, updated[0].ID, "3")
+ assert.Equal(t, updated[0].ServerPort, int32(8080))
+ assert.Len(t, deleted, 1)
+ assert.Equal(t, deleted[0].ID, "2")
+}
+
func TestDiffUpstreams(t *testing.T) {
news := []*apisixv1.Upstream{
{
diff --git a/pkg/kube/translation/apisix_route.go b/pkg/kube/translation/apisix_route.go
index 69cbae4..bbf8705 100644
--- a/pkg/kube/translation/apisix_route.go
+++ b/pkg/kube/translation/apisix_route.go
@@ -294,5 +294,35 @@
}
func (t *translator) translateTCPRoute(ctx *TranslateContext, ar *configv2alpha1.ApisixRoute) error {
+ ruleNameMap := make(map[string]struct{})
+ for _, part := range ar.Spec.TCP {
+ if _, ok := ruleNameMap[part.Name]; ok {
+ return errors.New("duplicated route rule name")
+ }
+ ruleNameMap[part.Name] = struct{}{}
+ backend := &part.Backend
+ svcClusterIP, svcPort, err := t.getTCPServiceClusterIPAndPort(backend, ar)
+ if err != nil {
+ log.Errorw("failed to get service port in backend",
+ zap.Any("backend", backend),
+ zap.Any("apisix_route", ar),
+ zap.Error(err),
+ )
+ return err
+ }
+ sr := apisixv1.NewDefaultStreamRoute()
+ name := apisixv1.ComposeStreamRouteName(ar.Namespace, ar.Name, part.Name)
+ sr.ID = id.GenID(name)
+ sr.ServerPort = part.Match.IngressPort
+ // TODO use upstream id to refer the upstream object.
+ // Currently, APISIX doesn't use upstream_id field in
+ // APISIX, so we have to embed the entire upstream.
+ ups, err := t.translateUpstream(ar.Namespace, backend.ServiceName, backend.ResolveGranularity, svcClusterIP, svcPort)
+ if err != nil {
+ return err
+ }
+ sr.Upstream = ups
+ ctx.addStreamRoute(sr)
+ }
return nil
}
diff --git a/pkg/kube/translation/util.go b/pkg/kube/translation/util.go
index 76af4d3..16ced5a 100644
--- a/pkg/kube/translation/util.go
+++ b/pkg/kube/translation/util.go
@@ -70,6 +70,45 @@
return svc.Spec.ClusterIP, svcPort, nil
}
+func (t *translator) getTCPServiceClusterIPAndPort(backend *configv2alpha1.ApisixRouteTCPBackend, ar *configv2alpha1.ApisixRoute) (string, int32, error) {
+ svc, err := t.ServiceLister.Services(ar.Namespace).Get(backend.ServiceName)
+ if err != nil {
+ return "", 0, err
+ }
+ svcPort := int32(-1)
+loop:
+ for _, port := range svc.Spec.Ports {
+ switch backend.ServicePort.Type {
+ case intstr.Int:
+ if backend.ServicePort.IntVal == port.Port {
+ svcPort = port.Port
+ break loop
+ }
+ case intstr.String:
+ if backend.ServicePort.StrVal == port.Name {
+ svcPort = port.Port
+ break loop
+ }
+ }
+ }
+ if svcPort == -1 {
+ log.Errorw("ApisixRoute refers to non-existent Service port",
+ zap.Any("ApisixRoute", ar),
+ zap.String("port", backend.ServicePort.String()),
+ )
+ return "", 0, err
+ }
+
+ if backend.ResolveGranularity == "service" && svc.Spec.ClusterIP == "" {
+ log.Errorw("ApisixRoute refers to a headless service but want to use the service level resolve granularity",
+ zap.Any("ApisixRoute", ar),
+ zap.Any("service", svc),
+ )
+ return "", 0, errors.New("conflict headless service and backend resolve granularity")
+ }
+ return svc.Spec.ClusterIP, svcPort, nil
+}
+
func (t *translator) translateUpstream(namespace, svcName, svcResolveGranularity, svcClusterIP string, svcPort int32) (*apisixv1.Upstream, error) {
ups, err := t.TranslateUpstream(namespace, svcName, svcPort)
if err != nil {
diff --git a/pkg/types/apisix/v1/types.go b/pkg/types/apisix/v1/types.go
index 846182b..082beec 100644
--- a/pkg/types/apisix/v1/types.go
+++ b/pkg/types/apisix/v1/types.go
@@ -332,6 +332,7 @@
Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"`
ServerPort int32 `json:"server_port,omitempty" yaml:"server_port,omitempty"`
UpstreamId string `json:"upstream_id,omitempty" yaml:"upstream_id,omitempty"`
+ Upstream *Upstream `json:"upstream,omitempty" yaml:"upstream,omitempty"`
}
// GlobalRule represents the global_rule object in APISIX.
@@ -369,6 +370,16 @@
}
}
+// NewDefaultStreamRoute returns an empty StreamRoute with default values.
+func NewDefaultStreamRoute() *StreamRoute {
+ return &StreamRoute{
+ Desc: "Created by apisix-ingress-controller, DO NOT modify it manually",
+ Labels: map[string]string{
+ "managed-by": "apisix-ingress-controller",
+ },
+ }
+}
+
// ComposeUpstreamName uses namespace, name and port info to compose
// the upstream name.
func ComposeUpstreamName(namespace, name string, port int32) string {
@@ -403,3 +414,21 @@
return buf.String()
}
+
+// ComposeStreamRouteName uses namespace, name and rule name to compose
+// the stream_route name.
+func ComposeStreamRouteName(namespace, name string, rule string) string {
+ // FIXME Use sync.Pool to reuse this buffer if the upstream
+ // name composing code path is hot.
+ p := make([]byte, 0, len(namespace)+len(name)+len(rule)+6)
+ buf := bytes.NewBuffer(p)
+
+ buf.WriteString(namespace)
+ buf.WriteByte('_')
+ buf.WriteString(name)
+ buf.WriteByte('_')
+ buf.WriteString(rule)
+ buf.WriteString("_tcp")
+
+ return buf.String()
+}
diff --git a/samples/deploy/crd/v1beta1/ApisixRoute.yaml b/samples/deploy/crd/v1beta1/ApisixRoute.yaml
index d335379..89f4e1a 100644
--- a/samples/deploy/crd/v1beta1/ApisixRoute.yaml
+++ b/samples/deploy/crd/v1beta1/ApisixRoute.yaml
@@ -30,6 +30,12 @@
- JSONPath: .spec.http[].match.backends[].serviceName
name: Target Service
type: string
+ - JSONPath: .spec.tcp[].match.ingressPort
+ name: Ingress Server Port
+ type: integer
+ - JSONPath: .spec.tcp[].match.backend.serviceName
+ name: Target Service
+ type: string
- JSONPath: .metadata.creationTimestamp
name: Age
type: date
@@ -57,6 +63,9 @@
properties:
spec:
type: object
+ anyOf:
+ - required: ["http"]
+ - required: ["tcp"]
properties:
http:
type: array
@@ -197,3 +206,38 @@
required:
- name
- enable
+ tcp:
+ type: array
+ minItems: 1
+ items:
+ type: object
+ required: ["name", "match", "backend"]
+ properties:
+ name:
+ type: string
+ minLength: 1
+ match:
+ type: object
+ properties:
+ ingressPort:
+ type: integer
+ minimum: 1
+ maximum: 65535
+ required:
+ - ingressPort
+ backend:
+ type: object
+ properties:
+ serviceName:
+ type: string
+ minLength: 1
+ servicePort:
+ type: integer
+ minimum: 1
+ maximum: 65535
+ resolveGranualrity:
+ type: string
+ enum: ["endpoint", "service"]
+ required:
+ - serviceName
+ - servicePort
diff --git a/test/e2e/endpoints/endpoints.go b/test/e2e/endpoints/endpoints.go
index 0973a05..9148d02 100644
--- a/test/e2e/endpoints/endpoints.go
+++ b/test/e2e/endpoints/endpoints.go
@@ -26,25 +26,36 @@
)
var _ = ginkgo.Describe("endpoints", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("ignore applied only if there is an ApisixRoute referenced", func() {
time.Sleep(5 * time.Second)
assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(0), "checking number of upstreams")
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
ups := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.org
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /ip
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.org
+ paths:
+ - /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendSvcPort[0])
assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ups))
assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "checking number of upstreams")
@@ -54,19 +65,20 @@
ginkgo.Skip("now we don't handle endpoints delete event")
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
apisixRoute := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.com
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /ip
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.com
+ paths: /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendSvcPort[0])
assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
assert.Nil(ginkgo.GinkgoT(), s.EnsureNumApisixUpstreamsCreated(1), "checking number of upstreams")
@@ -80,30 +92,34 @@
})
var _ = ginkgo.Describe("port usage", func() {
- s := scaffold.NewScaffold(&scaffold.Options{
+ opts := &scaffold.Options{
Name: "endpoints-port",
Kubeconfig: scaffold.GetKubeconfig(),
APISIXConfigPath: "testdata/apisix-gw-config.yaml",
APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
IngressAPISIXReplicas: 1,
- HTTPBinServicePort: 8080, // use service port which is different from target port (80)
- })
+ HTTPBinServicePort: 8080,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("service port != target port", func() {
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
apisixRoute := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.com
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /ip
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.com
+ paths:
+ - /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendSvcPort[0])
assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
time.Sleep(5 * time.Second)
diff --git a/test/e2e/features/healthcheck.go b/test/e2e/features/healthcheck.go
index 75c11cd..bf4e7c9 100644
--- a/test/e2e/features/healthcheck.go
+++ b/test/e2e/features/healthcheck.go
@@ -25,7 +25,16 @@
)
var _ = ginkgo.Describe("health check", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("active check", func() {
backendSvc, backendPorts := s.DefaultHTTPBackend()
@@ -50,19 +59,21 @@
assert.Nil(ginkgo.GinkgoT(), err, "create ApisixUpstream")
ar := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.org
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /*
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.org
+ paths:
+ - /*
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendPorts[0])
err = s.CreateResourceFromString(ar)
assert.Nil(ginkgo.GinkgoT(), err)
@@ -110,19 +121,21 @@
assert.Nil(ginkgo.GinkgoT(), err, "create ApisixUpstream")
ar := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.org
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /*
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.org
+ paths:
+ - /*
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendPorts[0])
err = s.CreateResourceFromString(ar)
assert.Nil(ginkgo.GinkgoT(), err)
diff --git a/test/e2e/features/retries_timeout.go b/test/e2e/features/retries_timeout.go
index 9e3d757..64470ac 100644
--- a/test/e2e/features/retries_timeout.go
+++ b/test/e2e/features/retries_timeout.go
@@ -24,7 +24,16 @@
)
var _ = ginkgo.Describe("retries", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("active check", func() {
backendSvc, backendPorts := s.DefaultHTTPBackend()
@@ -41,19 +50,21 @@
time.Sleep(2 * time.Second)
ar := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
- name: httpbin-route
+ name: httpbin-route
spec:
- rules:
- - host: httpbin.org
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /*
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.org
+ paths:
+ - /*
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendPorts[0])
err = s.CreateResourceFromString(ar)
assert.Nil(ginkgo.GinkgoT(), err)
@@ -67,7 +78,16 @@
})
var _ = ginkgo.Describe("timeout", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("active check", func() {
backendSvc, backendPorts := s.DefaultHTTPBackend()
@@ -86,19 +106,21 @@
time.Sleep(2 * time.Second)
ar := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.org
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /*
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.org
+ paths:
+ - /*
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendPorts[0])
err = s.CreateResourceFromString(ar)
assert.Nil(ginkgo.GinkgoT(), err)
diff --git a/test/e2e/features/scheme.go b/test/e2e/features/scheme.go
index bdcd529..8f2334a 100644
--- a/test/e2e/features/scheme.go
+++ b/test/e2e/features/scheme.go
@@ -25,7 +25,16 @@
)
var _ = ginkgo.Describe("choose scheme", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("grpc", func() {
err := s.CreateResourceFromString(`
apiVersion: v1
@@ -64,19 +73,21 @@
scheme: grpc
`))
err = s.CreateResourceFromString(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: grpc-route
spec:
- rules:
- - host: grpc.local
- http:
- paths:
- - backend:
- serviceName: grpc-server-service
- servicePort: 50051
- path: /helloworld.Greeter/SayHello
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - grpc.local
+ paths:
+ - /helloworld.Greeter/SayHello
+ backend:
+ serviceName: grpc-server-service
+ servicePort: 50051
`)
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(2 * time.Second)
diff --git a/test/e2e/ingress/namespace.go b/test/e2e/ingress/namespace.go
index 9f4d1f9..57a211a 100644
--- a/test/e2e/ingress/namespace.go
+++ b/test/e2e/ingress/namespace.go
@@ -28,23 +28,34 @@
)
var _ = ginkgo.Describe("namespacing filtering", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("resources in other namespaces should be ignored", func() {
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
route := fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.com
- http:
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.com
paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /ip
+ - /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendSvcPort[0])
assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(route), "creating ApisixRoute")
@@ -62,19 +73,21 @@
// Now create another ApisixRoute in default namespace.
route = fmt.Sprintf(`
-apiVersion: apisix.apache.org/v1
+apiVersion: apisix.apache.org/v2alpha1
kind: ApisixRoute
metadata:
name: httpbin-route
spec:
- rules:
- - host: httpbin.com
- http:
- paths:
- - backend:
- serviceName: %s
- servicePort: %d
- path: /headers
+ http:
+ - name: rule1
+ match:
+ hosts:
+ - httpbin.com
+ paths:
+ - /headers
+ backend:
+ serviceName: %s
+ servicePort: %d
`, backendSvc, backendSvcPort[0])
assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromStringWithNamespace(route, "default"), "creating ApisixRoute")
diff --git a/test/e2e/ingress/sanity.go b/test/e2e/ingress/sanity.go
index 009efb3..7937a55 100644
--- a/test/e2e/ingress/sanity.go
+++ b/test/e2e/ingress/sanity.go
@@ -16,6 +16,7 @@
import (
"encoding/json"
+ "fmt"
"net/http"
"time"
@@ -30,25 +31,34 @@
}
var _ = ginkgo.Describe("single-route", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("/ip should return your ip", func() {
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
- s.CreateApisixRoute("httpbin-route", []scaffold.ApisixRouteRule{
- {
- Host: "httpbin.com",
- HTTP: scaffold.ApisixRouteRuleHTTP{
- Paths: []scaffold.ApisixRouteRuleHTTPPath{
- {
- Path: "/ip",
- Backend: scaffold.ApisixRouteRuleHTTPBackend{
- ServiceName: backendSvc,
- ServicePort: backendSvcPort[0],
- },
- },
- },
- },
- },
- })
+ ar := fmt.Sprintf(`
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixRoute
+metadata:
+ name: httpbin-route
+spec:
+ http:
+ - name: rule1
+ match:
+ paths:
+ - /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
+`, backendSvc, backendSvcPort[0])
+ assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar))
err := s.EnsureNumApisixRoutesCreated(1)
assert.Nil(ginkgo.GinkgoT(), err, "checking number of routes")
err = s.EnsureNumApisixUpstreamsCreated(1)
@@ -68,32 +78,43 @@
})
var _ = ginkgo.Describe("double-routes", func() {
- s := scaffold.NewDefaultScaffold()
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
ginkgo.It("double routes work independently", func() {
backendSvc, backendSvcPort := s.DefaultHTTPBackend()
- s.CreateApisixRoute("httpbin-route", []scaffold.ApisixRouteRule{
- {
- Host: "httpbin.com",
- HTTP: scaffold.ApisixRouteRuleHTTP{
- Paths: []scaffold.ApisixRouteRuleHTTPPath{
- {
- Path: "/ip",
- Backend: scaffold.ApisixRouteRuleHTTPBackend{
- ServiceName: backendSvc,
- ServicePort: backendSvcPort[0],
- },
- },
- {
- Path: "/json",
- Backend: scaffold.ApisixRouteRuleHTTPBackend{
- ServiceName: backendSvc,
- ServicePort: backendSvcPort[0],
- },
- },
- },
- },
- },
- })
+ ar := `
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixRoute
+metadata:
+ name: httpbin-route
+spec:
+ http:
+ - name: rule1
+ match:
+ paths:
+ - /ip
+ backend:
+ serviceName: %s
+ servicePort: %d
+ - name: rule2
+ match:
+ paths:
+ - /json
+ backend:
+ serviceName: %s
+ servicePort: %d
+`
+ ar = fmt.Sprintf(ar, backendSvc, backendSvcPort[0], backendSvc, backendSvcPort[0])
+ assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ar))
+
err := s.EnsureNumApisixRoutesCreated(2)
assert.Nil(ginkgo.GinkgoT(), err, "checking number of routes")
err = s.EnsureNumApisixUpstreamsCreated(1)
diff --git a/test/e2e/ingress/tcp.go b/test/e2e/ingress/tcp.go
new file mode 100644
index 0000000..b745527
--- /dev/null
+++ b/test/e2e/ingress/tcp.go
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package ingress
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/onsi/ginkgo"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
+)
+
+var _ = ginkgo.Describe("ApisixRoute Testing", func() {
+ opts := &scaffold.Options{
+ Name: "default",
+ Kubeconfig: scaffold.GetKubeconfig(),
+ APISIXConfigPath: "testdata/apisix-gw-config.yaml",
+ APISIXDefaultConfigPath: "testdata/apisix-gw-config-default.yaml",
+ IngressAPISIXReplicas: 1,
+ HTTPBinServicePort: 80,
+ APISIXRouteVersion: "apisix.apache.org/v2alpha1",
+ }
+ s := scaffold.NewScaffold(opts)
+ ginkgo.It("tcp proxy", func() {
+ backendSvc, backendSvcPort := s.DefaultHTTPBackend()
+ apisixRoute := fmt.Sprintf(`
+apiVersion: apisix.apache.org/v2alpha1
+kind: ApisixRoute
+metadata:
+ name: httpbin-tcp-route
+spec:
+ tcp:
+ - name: rule1
+ match:
+ ingressPort: 9100
+ backend:
+ serviceName: %s
+ servicePort: %d
+`, backendSvc, backendSvcPort[0])
+
+ assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute))
+ time.Sleep(3 * time.Second)
+
+ err := s.EnsureNumApisixStreamRoutesCreated(1)
+ assert.Nil(ginkgo.GinkgoT(), err, "Checking number of routes")
+
+ sr, err := s.ListApisixStreamRoutes()
+ assert.Nil(ginkgo.GinkgoT(), err)
+ assert.Len(ginkgo.GinkgoT(), sr, 1)
+ assert.Equal(ginkgo.GinkgoT(), sr[0].ServerPort, int32(9100))
+
+ resp := s.NewAPISIXClientWithTCPProxy().GET("/ip").Expect()
+ resp.Body().Contains("origin")
+
+ resp = s.NewAPISIXClientWithTCPProxy().GET("/get").WithHeader("x-my-header", "x-my-value").Expect()
+ resp.Body().Contains("x-my-value")
+ })
+})
diff --git a/test/e2e/scaffold/apisix.go b/test/e2e/scaffold/apisix.go
index 5719f45..84bed8e 100644
--- a/test/e2e/scaffold/apisix.go
+++ b/test/e2e/scaffold/apisix.go
@@ -120,6 +120,10 @@
port: 9443
protocol: TCP
targetPort: 9443
+ - name: tcp
+ port: 9100
+ protocol: TCP
+ targetPort: 9100
type: NodePort
`
)
diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go
index f165d8a..6fcc05b 100644
--- a/test/e2e/scaffold/k8s.go
+++ b/test/e2e/scaffold/k8s.go
@@ -162,6 +162,17 @@
return ensureNumApisixCRDsCreated(u.String(), desired)
}
+// EnsureNumApisixStreamRoutesCreated waits until desired number of Stream Routes are created in
+// APISIX cluster.
+func (s *Scaffold) EnsureNumApisixStreamRoutesCreated(desired int) error {
+ u := url.URL{
+ Scheme: "http",
+ Host: s.apisixAdminTunnel.Endpoint(),
+ Path: "/apisix/admin/stream_routes",
+ }
+ return ensureNumApisixCRDsCreated(u.String(), desired)
+}
+
// EnsureNumApisixUpstreamsCreated waits until desired number of Upstreams are created in
// APISIX cluster.
func (s *Scaffold) EnsureNumApisixUpstreamsCreated(desired int) error {
@@ -233,6 +244,26 @@
return cli.Cluster("").Route().List(context.TODO())
}
+// ListApisixStreamRoutes list all stream_routes from APISIX.
+func (s *Scaffold) ListApisixStreamRoutes() ([]*v1.StreamRoute, error) {
+ u := url.URL{
+ Scheme: "http",
+ Host: s.apisixAdminTunnel.Endpoint(),
+ Path: "/apisix/admin",
+ }
+ cli, err := apisix.NewClient()
+ if err != nil {
+ return nil, err
+ }
+ err = cli.AddCluster(&apisix.ClusterOptions{
+ BaseURL: u.String(),
+ })
+ if err != nil {
+ return nil, err
+ }
+ return cli.Cluster("").StreamRoute().List(context.TODO())
+}
+
// ListApisixTls list all ssl from APISIX
func (s *Scaffold) ListApisixTls() ([]*v1.Ssl, error) {
u := url.URL{
@@ -261,6 +292,8 @@
adminPort int
httpPort int
httpsPort int
+ tcpPort int
+ tcpNodePort int
)
for _, port := range s.apisixService.Spec.Ports {
if port.Name == "http" {
@@ -272,6 +305,9 @@
} else if port.Name == "http-admin" {
adminNodePort = int(port.NodePort)
adminPort = int(port.Port)
+ } else if port.Name == "tcp" {
+ tcpNodePort = int(port.NodePort)
+ tcpPort = int(port.Port)
}
}
@@ -281,6 +317,8 @@
httpNodePort, httpPort)
s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
httpsNodePort, httpsPort)
+ s.apisixTCPTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-service-e2e-test",
+ tcpNodePort, tcpPort)
if err := s.apisixAdminTunnel.ForwardPortE(s.t); err != nil {
return err
@@ -294,6 +332,10 @@
return err
}
s.addFinalizers(s.apisixHttpsTunnel.Close)
+ if err := s.apisixTCPTunnel.ForwardPortE(s.t); err != nil {
+ return err
+ }
+ s.addFinalizers(s.apisixTCPTunnel.Close)
return nil
}
diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go
index 8d4d505..ab63cd2 100644
--- a/test/e2e/scaffold/scaffold.go
+++ b/test/e2e/scaffold/scaffold.go
@@ -65,6 +65,7 @@
apisixAdminTunnel *k8s.Tunnel
apisixHttpTunnel *k8s.Tunnel
apisixHttpsTunnel *k8s.Tunnel
+ apisixTCPTunnel *k8s.Tunnel
// Used for template rendering.
EtcdServiceFQDN string
@@ -167,6 +168,26 @@
})
}
+// NewAPISIXClientWithTCPProxy creates the HTTP client but with the TCP proxy of APISIX.
+func (s *Scaffold) NewAPISIXClientWithTCPProxy() *httpexpect.Expect {
+ u := url.URL{
+ Scheme: "http",
+ Host: s.apisixTCPTunnel.Endpoint(),
+ }
+ return httpexpect.WithConfig(httpexpect.Config{
+ BaseURL: u.String(),
+ Client: &http.Client{
+ Transport: &http.Transport{},
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ },
+ },
+ Reporter: httpexpect.NewAssertReporter(
+ httpexpect.NewAssertReporter(ginkgo.GinkgoT()),
+ ),
+ })
+}
+
// NewAPISIXHttpsClient creates the default HTTPs client.
func (s *Scaffold) NewAPISIXHttpsClient(host string) *httpexpect.Expect {
u := url.URL{
diff --git a/test/e2e/testdata/apisix-gw-config.yaml b/test/e2e/testdata/apisix-gw-config.yaml
index a0ba901..4225a3a 100644
--- a/test/e2e/testdata/apisix-gw-config.yaml
+++ b/test/e2e/testdata/apisix-gw-config.yaml
@@ -53,10 +53,9 @@
http: 'radixtree_uri' # radixtree_uri: match route by uri(base on radixtree)
# radixtree_host_uri: match route by host + uri(base on radixtree)
ssl: 'radixtree_sni' # radixtree_sni: match route by SNI(base on radixtree)
- # stream_proxy: # TCP/UDP proxy
- # tcp: # TCP proxy port list
- # - 9100
- # - 9101
+ stream_proxy: # TCP/UDP proxy
+ tcp: # TCP proxy port list
+ - 9100
# udp: # UDP proxy port list
# - 9200
# - 9211
