KNOX-3214 - Add integration tests (#1116)
diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile
new file mode 100644
index 0000000..5d6056a
--- /dev/null
+++ b/.github/workflows/build/Dockerfile
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM eclipse-temurin:17-jre
+
+MAINTAINER moresandeep
+
+RUN useradd -ms /bin/bash gateway
+
+# Create directories
+RUN mkdir /knox-runtime
+RUN mkdir /knox-runtime/knoxshell
+
+# Copy artifacts
+COPY knox-temp-artifacts /knox-temp-artifacts-staging
+COPY knoxshell-temp-artifacts /knoxshell-temp-artifacts-staging
+
+# Move runtime to new location
+RUN mv /knox-temp-artifacts-staging/*/* /knox-runtime/ && \
+ mv /knoxshell-temp-artifacts-staging/*/* /knox-runtime/knoxshell/ && \
+ rm -rf /knox-temp-artifacts-staging /knoxshell-temp-artifacts-staging
+
+# Add configuration
+ADD master /knox-runtime/data/security/master
+ADD gateway-site.xml /knox-runtime/conf/gateway-site.xml
+ADD conf/topologies/knoxtoken.xml /knox-runtime/conf/topologies/knoxtoken.xml
+ADD conf/topologies/knoxldap.xml /knox-runtime/conf/topologies/knoxldap.xml
+ADD conf/topologies/health.xml /knox-runtime/conf/topologies/health.xml
+
+RUN chown -R gateway /knox-runtime/
+
+ADD ldap.sh /ldap.sh
+ADD gateway.sh /gateway.sh
+
+RUN chmod +x /ldap.sh /gateway.sh
diff --git a/.github/workflows/build/Dockerfile.local b/.github/workflows/build/Dockerfile.local
new file mode 100644
index 0000000..ba95ae1
--- /dev/null
+++ b/.github/workflows/build/Dockerfile.local
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM maven:3.9.9-eclipse-temurin-17
+
+MAINTAINER moresandeep
+
+# Install dependencies
+RUN apt-get update
+
+RUN apt-get install -y git
+
+RUN useradd -ms /bin/bash gateway
+
+# Clone our dev branch
+ARG knoxurl
+ARG branch
+
+RUN git clone -b $branch $knoxurl knox
+
+RUN mkdir /knox/knox-temp-artifacts
+RUN mkdir /knox/knoxshell-temp-artifacts
+
+# Update maven settings to ignore jcenter repo
+#ADD settings.xml /home/gateway/.m2/settings.xml
+#RUN mv /home/gateway/.m2/settings.xml ~/.m2/settings.xml
+ADD settings.xml /usr/share/maven/ref/
+
+# Skipping tests here for faster turnaround.
+RUN cd knox && mvn -settings /usr/share/maven/ref/settings.xml clean -Ppackage,release install -Dforbiddenapis.skip=true -Denforcer.skip -Dpmd.failOnViolation=false -DskipTests=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -Dpmd.skip=true -Drat.skip -DskipTests && tar -xvzf target/*/knox-*.tar.gz -C knox-temp-artifacts && tar -xvzf /knox/target/*/knoxshell-*.tar.gz -C knoxshell-temp-artifacts
+
+# move runtime to new location
+# See https://github.com/docker/compose/issues/4581#issuecomment-321386605
+# for KnoxShell dance
+RUN mkdir /knox-runtime
+RUN mkdir /knoxshell
+RUN mkdir /knox-runtime/knoxshell
+RUN mv /knox/knox-temp-artifacts/*/* /knox-runtime
+RUN mv /knox/knoxshell-temp-artifacts/*/* /knox-runtime/knoxshell
+
+# delete build artifacts
+RUN rm -rf /knox
+
+ADD master /knox-runtime/data/security/master
+# Enable websockets
+ADD gateway-site.xml /knox-runtime/conf/gateway-site.xml
+ADD conf/topologies/knoxtoken.xml /knox-runtime/conf/topologies/knoxtoken.xml
+ADD conf/topologies/health.xml /knox-runtime/conf/topologies/health.xml
+ADD conf/topologies/knoxldap.xml /knox-runtime/conf/topologies/knoxldap.xml
+
+RUN chown -R gateway /knox-runtime/
+
+# Cleanup
+RUN rm -rf /home/gateway/.m2/repository
+
+ADD ldap.sh /ldap.sh
+ADD gateway.sh /gateway.sh
+
+RUN chmod +x /ldap.sh
+RUN chmod +x /gateway.sh
+
diff --git a/.github/workflows/build/conf/topologies/health.xml b/.github/workflows/build/conf/topologies/health.xml
new file mode 100644
index 0000000..0117d9f
--- /dev/null
+++ b/.github/workflows/build/conf/topologies/health.xml
@@ -0,0 +1,29 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<topology>
+ <gateway>
+ <provider>
+ <role>authentication</role>
+ <name>Anonymous</name>
+ <enabled>true</enabled>
+ </provider>
+ </gateway>
+ <service>
+ <role>HEALTH</role>
+ </service>
+</topology>
\ No newline at end of file
diff --git a/.github/workflows/build/conf/topologies/knoxldap.xml b/.github/workflows/build/conf/topologies/knoxldap.xml
new file mode 100644
index 0000000..1027976
--- /dev/null
+++ b/.github/workflows/build/conf/topologies/knoxldap.xml
@@ -0,0 +1,82 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<topology>
+ <gateway>
+ <provider>
+ <role>authentication</role>
+ <name>ShiroProvider</name>
+ <enabled>true</enabled>
+ <param>
+ <name>sessionTimeout</name>
+ <value>30</value>
+ </param>
+ <param>
+ <name>main.ldapRealm</name>
+ <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm</value>
+ </param>
+ <param>
+ <name>main.ldapRealm.userDnTemplate</name>
+ <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>
+ </param>
+ <param>
+ <name>main.ldapRealm.contextFactory.url</name>
+ <value>ldap://ldap:33389</value>
+ </param>
+ <param>
+ <name>main.ldapRealm.contextFactory.authenticationMechanism</name>
+ <value>simple</value>
+ </param>
+ <param>
+ <name>urls./**</name>
+ <value>authcBasic</value>
+ </param>
+ </provider>
+ <provider>
+ <role>identity-assertion</role>
+ <name>Default</name>
+ <enabled>true</enabled>
+ <param>
+ <name>group.principal.mapping</name>
+ <value>admin=longGroupName1,longGroupName2,longGroupName3,longGroupName4</value>
+ </param>
+ </provider>
+ </gateway>
+ <service>
+ <role>KNOXTOKEN</role>
+ <param>
+ <name>knoxsso.token.ttl</name>
+ <value>86400000</value>
+ </param>
+ </service>
+ <service>
+ <role>KNOX-AUTH-SERVICE</role>
+ <param>
+ <name>preauth.auth.header.actor.id.name</name>
+ <value>x-knox-actor-username</value>
+ </param>
+ <param>
+ <name>preauth.auth.header.actor.groups.prefix</name>
+ <value>x-knox-actor-groups</value>
+ </param>
+ <param>
+ <name>preauth.group.filter.pattern</name>
+ <value>[^\s]+</value>
+ </param>
+ </service>
+</topology>
+
diff --git a/.github/workflows/build/conf/topologies/knoxtoken.xml b/.github/workflows/build/conf/topologies/knoxtoken.xml
new file mode 100644
index 0000000..d103603
--- /dev/null
+++ b/.github/workflows/build/conf/topologies/knoxtoken.xml
@@ -0,0 +1,56 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<topology>
+ <gateway>
+ <provider>
+ <role>federation</role>
+ <name>JWTProvider</name>
+ <enabled>true</enabled>
+ <param>
+ <name>knox.token.use.cookie</name>
+ <value>false</value>
+ </param>
+ <param>
+ <name>jwt.expected.sigalg</name>
+ <value>RS256</value>
+ </param>
+ </provider>
+ </gateway>
+ <service>
+ <role>KNOXTOKEN</role>
+ <param>
+ <name>knoxsso.token.ttl</name>
+ <value>86400000</value>
+ </param>
+ </service>
+ <service>
+ <role>KNOX-AUTH-SERVICE</role>
+ <param>
+ <name>preauth.auth.header.actor.id.name</name>
+ <value>x-knox-actor-username</value>
+ </param>
+ <param>
+ <name>preauth.auth.header.actor.groups.prefix</name>
+ <value>x-knox-actor-groups</value>
+ </param>
+ <param>
+ <name>preauth.group.filter.pattern</name>
+ <value>[^\s]+</value>
+ </param>
+ </service>
+</topology>
\ No newline at end of file
diff --git a/.github/workflows/build/gateway-site.xml b/.github/workflows/build/gateway-site.xml
new file mode 100644
index 0000000..add31b8
--- /dev/null
+++ b/.github/workflows/build/gateway-site.xml
@@ -0,0 +1,155 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration>
+
+ <property>
+ <name>gateway.service.alias.impl</name>
+ <value>org.apache.knox.gateway.services.security.impl.DefaultAliasService</value>
+ </property>
+ <property>
+ <name>gateway.port</name>
+ <value>8443</value>
+ <description>The HTTP port for the Gateway.</description>
+ </property>
+
+ <property>
+ <name>gateway.path</name>
+ <value>gateway</value>
+ <description>The default context path for the gateway.</description>
+ </property>
+
+ <property>
+ <name>gateway.gateway.conf.dir</name>
+ <value>deployments</value>
+ <description>The directory within GATEWAY_HOME that contains gateway topology files and deployments.</description>
+ </property>
+
+ <!-- @since 0.10 Websocket configs -->
+ <property>
+ <name>gateway.websocket.feature.enabled</name>
+ <value>true</value>
+ <description>Enable/Disable websocket feature.</description>
+ </property>
+
+ <property>
+ <name>gateway.scope.cookies.feature.enabled</name>
+ <value>false</value>
+ <description>Enable/Disable cookie scoping feature.</description>
+ </property>
+
+ <!-- @since 2.0.0 WebShell configs -->
+ <!-- must have websocket enabled to use webshell -->
+ <property>
+ <name>gateway.webshell.feature.enabled</name>
+ <value>true</value>
+ <description>Enable/Disable webshell feature.</description>
+ </property>
+ <property>
+ <name>gateway.webshell.max.concurrent.sessions</name>
+ <value>20</value>
+ <description>Maximum number of total concurrent webshell sessions</description>
+ </property>
+ <property>
+ <name>gateway.webshell.read.buffer.size</name>
+ <value>1024</value>
+ <description>Web Shell buffer size for reading</description>
+ </property>
+
+ <!-- @since 2.0.0 websocket JWT validation configs -->
+ <property>
+ <name>gateway.websocket.JWT.validation.feature.enabled</name>
+ <value>true</value>
+ <description>Enable/Disable websocket JWT validation at websocket layer.</description>
+ </property>
+
+ <!-- @since 1.5.0 homepage logout -->
+ <property>
+ <name>knox.homepage.logout.enabled</name>
+ <value>true</value>
+ <description>Enable/disable logout from the Knox Homepage.</description>
+ </property>
+
+ <!-- @since 1.6.0 token management related properties -->
+ <property>
+ <name>gateway.knox.token.eviction.grace.period</name>
+ <value>0</value>
+ <description>A duration (in seconds) beyond a token’s expiration to wait before evicting its state. This configuration only applies when server-managed token state is enabled either in gateway-site or at the topology level.</description>
+ </property>
+
+ <!-- Knox Admin related config -->
+ <property>
+ <name>gateway.knox.admin.groups</name>
+ <value>admin</value>
+ </property>
+
+ <!-- DEMO LDAP config for Hadoop Group Provider -->
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.LdapGroupsMapping</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.bind.user</name>
+ <value>uid=guest,ou=people,dc=hadoop,dc=apache,dc=org</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.bind.password</name>
+ <value>guest-password</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.url</name>
+ <value>ldap://localhost:33389</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.base</name>
+ <value></value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.user</name>
+ <value>(&(|(objectclass=person)(objectclass=applicationProcess))(cn={0}))</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.group</name>
+ <value>(objectclass=groupOfNames)</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.member</name>
+ <value>member</value>
+ </property>
+ <property>
+ <name>gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.group.name</name>
+ <value>cn</value>
+ </property>
+ <property>
+ <name>gateway.dispatch.whitelist.services</name>
+ <value>DATANODE,HBASEUI,HDFSUI,JOBHISTORYUI,NODEUI,YARNUI,knoxauth</value>
+ <description>The comma-delimited list of service roles for which the gateway.dispatch.whitelist should be applied.</description>
+ </property>
+ <property>
+ <name>gateway.dispatch.whitelist</name>
+ <value>^https?:\/\/(www\.local\.com|localhost|127\.0\.0\.1|0:0:0:0:0:0:0:1|::1):[0-9].*$</value>
+ <description>The whitelist to be applied for dispatches associated with the service roles specified by gateway.dispatch.whitelist.services.
+ If the value is DEFAULT, a domain-based whitelist will be derived from the Knox host.</description>
+ </property>
+ <property>
+ <name>gateway.xforwarded.header.context.append.servicename</name>
+ <value>LIVYSERVER</value>
+ <description>Add service name to x-forward-context header for the list of services defined above.</description>
+ </property>
+
+</configuration>
diff --git a/.github/workflows/build/gateway.sh b/.github/workflows/build/gateway.sh
new file mode 100755
index 0000000..f31dd44
--- /dev/null
+++ b/.github/workflows/build/gateway.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Start Knox
+java -jar /knox-runtime/bin/gateway.jar
\ No newline at end of file
diff --git a/.github/workflows/build/ldap.sh b/.github/workflows/build/ldap.sh
new file mode 100755
index 0000000..ad3d9c8
--- /dev/null
+++ b/.github/workflows/build/ldap.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+java -jar /knox-runtime/bin/ldap.jar /knox-runtime/conf
\ No newline at end of file
diff --git a/.github/workflows/build/master b/.github/workflows/build/master
new file mode 100644
index 0000000..3b3d9ed
--- /dev/null
+++ b/.github/workflows/build/master
@@ -0,0 +1,2 @@
+#1.0# Tue, Sep 23 2025 12:12:59.057
+TXpTZkVGM0VTYk09OjpGUncvV1JHOGIrVll5dGRWeXFKR1pRPT06Om9nV2x6UVdSM0UvbWhuQ0VnSlhuUWc9PQ==
diff --git a/.github/workflows/build/settings.xml b/.github/workflows/build/settings.xml
new file mode 100644
index 0000000..ea1f0de
--- /dev/null
+++ b/.github/workflows/build/settings.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+ https://maven.apache.org/xsd/settings-1.0.0.xsd">
+ <mirrors>
+ <mirror>
+ <id>replace-jcenter-with-central</id>
+ <mirrorOf>jcenter</mirrorOf>
+ <url>https://repo.maven.apache.org/maven2</url>
+ </mirror>
+ </mirrors>
+</settings>
diff --git a/.github/workflows/compose/docker-compose.yml b/.github/workflows/compose/docker-compose.yml
new file mode 100644
index 0000000..b114d3a
--- /dev/null
+++ b/.github/workflows/compose/docker-compose.yml
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# <p>
+# http://www.apache.org/licenses/LICENSE-2.0
+# <p>
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+services:
+ knox-dev:
+ build:
+ context: ../build
+ dockerfile: Dockerfile
+ image: apache/knox-dev:master
+
+ knox-dev-local:
+ build: &build_local
+ context: ../build
+ dockerfile: Dockerfile.local
+ args:
+ knoxurl: ${knoxurl:-https://github.com/apache/knox.git}
+ branch: ${branch:-master}
+ image: apache/knox-dev:local
+
+ ldap:
+ build: *build_local # Build LDAP service if not already built
+ image: apache/knox-dev:local
+ command: /ldap.sh
+
+ knox:
+ build: *build_local # Build Knox service if not already built
+ image: apache/knox-dev:local
+ command: /gateway.sh
+ volumes:
+# - ./topologies:/knox-runtime/conf/topologies
+ - ./logs:/knox-runtime/logs
+# - ./knoxshell:/knoxshell
+ depends_on:
+ - ldap
+
+ tests:
+ image: python:3.9-slim
+ working_dir: /tests
+ volumes:
+ - ../tests:/tests
+ environment:
+ - KNOX_GATEWAY_URL=https://knox:8443/
+ command: >
+ bash -c "pip install -r requirements.txt
+ && echo 'Waiting for knox...'
+ && sleep 30
+ && python -m unittest discover -p 'test_*.py'"
+ depends_on:
+ - knox
+
+
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..1e7fd42
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Apache Knox Docker Compose Tests
+
+on:
+ pull_request:
+ branches:
+ - '**' # triggers for all PRs
+ workflow_dispatch:
+
+jobs:
+ build-and-test:
+ if: github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'skip-tests')
+ runs-on: ubuntu-latest
+
+ env:
+ KNOX_URL: "https://github.com/apache/knox.git"
+ BRANCH: "${{ github.head_ref }}" # PR branch being tested
+
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v4
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: 'maven'
+
+ - name: Build with Maven
+ run: |
+ mvn clean -Ppackage,release install -T 1C \
+ -Dmaven.test.skip=true -Dpmd.skip=true -Dcpd.skip=true -Dcheckstyle.skip=true \
+ -Dspotbugs.skip=true -Drat.skip=true -Dforbiddenapis.skip=true -Denforcer.skip=true \
+ -Djacoco.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \
+ -Dshellcheck.skip=true -Dxml.skip=true \
+ -s .github/workflows/build/settings.xml
+
+ - name: Extract Artifacts
+ run: |
+ mkdir -p .github/workflows/build/knox-temp-artifacts .github/workflows/build/knoxshell-temp-artifacts
+ # Extract artifacts to the build directory where Dockerfile expects them
+ tar -xvzf target/*/knox-*.tar.gz -C .github/workflows/build/knox-temp-artifacts
+ tar -xvzf target/*/knoxshell-*.tar.gz -C .github/workflows/build/knoxshell-temp-artifacts
+
+ - name: Set up Docker Compose
+ run: docker compose version
+
+ - name: Build Docker Images
+ run: |
+ export knoxurl=${KNOX_URL}
+ export branch=${BRANCH:-master}
+ # Build only knox-dev which is the runtime image using artifacts
+ docker compose -f ./.github/workflows/compose/docker-compose.yml build knox-dev
+
+ - name: Start Knox and LDAP Services
+ run: docker compose -f ./.github/workflows/compose/docker-compose.yml up -d
+
+ - name: Wait for services to stabilize
+ run: sleep 30 # Adjust as needed for services startup time
+
+ - name: Run Knox Integration Tests
+ run: |
+ # Run the tests service defined in docker-compose.yml
+ docker compose -f ./.github/workflows/compose/docker-compose.yml up --exit-code-from tests tests
+
+ - name: Tear Down Docker Compose
+ if: always()
+ run: docker compose -f ./.github/workflows/compose/docker-compose.yml down --volumes
diff --git a/.github/workflows/tests/README.md b/.github/workflows/tests/README.md
new file mode 100644
index 0000000..c15672d
--- /dev/null
+++ b/.github/workflows/tests/README.md
@@ -0,0 +1,118 @@
+# Adding Test Cases to GitHub Workflow
+
+This directory contains Python integration tests that run as part of the GitHub workflow.
+
+## Directory Structure
+
+- `test_*.py`: Python test files containing test cases.
+- `requirements.txt`: Python dependencies required for running the tests.
+
+## How to Add a New Test Case
+
+1. **Create a New Test File**:
+ Create a new Python file in this directory (`.github/workflows/tests/`). The filename **must** start with `test_` (e.g., `test_auth.py`) to be automatically discovered by the test runner.
+
+2. **Implement Test Logic**:
+ Use the `unittest` framework to structure your tests. You can include multiple test methods in a single class, and multiple classes in a single file. Each method starting with `test_` will be executed as a separate test case.
+
+ ```python
+ # Licensed to the Apache Software Foundation (ASF) under one or more
+ # contributor license agreements. See the NOTICE file distributed with
+ # this work for additional information regarding copyright ownership.
+ # The ASF licenses this file to you under the Apache License, Version 2.0
+ # (the "License"); you may not use this file except in compliance with
+ # the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ import unittest
+ import requests
+ import os
+ import urllib3
+
+ # Suppress InsecureRequestWarning since we use verify=False for self-signed certs
+ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+ class TestMyFeature(unittest.TestCase):
+ def setUp(self):
+ # Get the Knox Gateway URL from environment variables
+ # Default to localhost for local debugging outside Docker
+ self.base_url = os.environ.get("KNOX_GATEWAY_URL", "https://localhost:8443/")
+
+ def test_my_endpoint(self):
+ """
+ Description of what this test checks.
+ """
+ url = f"{self.base_url}gateway/sandbox/webhdfs/v1/?op=LISTSTATUS"
+
+ print(f"Testing URL: {url}")
+
+ # Make the request
+ # verify=False is needed for the dev environment's self-signed certs
+ response = requests.get(url, verify=False)
+
+ # Assertions
+ self.assertEqual(response.status_code, 200)
+ # Add more assertions as needed
+
+ def test_another_endpoint(self):
+ """
+ Another test case in the same class.
+ """
+ # ... implementation ...
+ pass
+ ```
+
+3. **Add Dependencies**:
+ If your test requires additional Python libraries (other than `requests`), add them to `requirements.txt` in this directory.
+
+## Organizing Tests in Subdirectories
+
+You can organize tests into subdirectories (e.g., `tests/auth/`, `tests/proxy/`). For the test runner to discover them:
+
+1. The subdirectory **must** contain an `__init__.py` file (it can be empty).
+2. The test files inside must still match the `test_*.py` pattern.
+
+**Example structure:**
+
+```text
+tests/
+├── test_health.py
+├── auth/
+│ ├── __init__.py
+│ └── test_auth.py
+└── proxy/
+ ├── __init__.py
+ └── test_proxy.py
+```
+
+## How It Works
+
+The tests run in a dedicated Docker container defined in `../compose/docker-compose.yml`.
+
+1. The `tests` service mounts this directory (`.github/workflows/tests/`) to `/tests` inside the container.
+2. It installs dependencies from `requirements.txt`.
+3. It waits for the `knox` service to be ready.
+4. It runs `python -m unittest discover -p 'test_*.py'` to find and execute all test files.
+
+## Skipping Tests on Pull Requests
+
+If you want to skip the integration tests for a specific Pull Request (e.g., documentation changes), add the label **`skip-tests`** to the PR.
+
+## Running Tests Locally
+
+You can run these tests locally using Docker Compose from the `.github/workflows/compose` directory:
+
+```bash
+cd .github/workflows/compose
+docker-compose up --build --abort-on-container-exit
+```
+
+This will start the Knox environment and run the tests. The `tests` container will exit once tests are complete.
+
diff --git a/.github/workflows/tests/requirements.txt b/.github/workflows/tests/requirements.txt
new file mode 100644
index 0000000..3288e92
--- /dev/null
+++ b/.github/workflows/tests/requirements.txt
@@ -0,0 +1,2 @@
+requests
+
diff --git a/.github/workflows/tests/test_health.py b/.github/workflows/tests/test_health.py
new file mode 100644
index 0000000..421dc52
--- /dev/null
+++ b/.github/workflows/tests/test_health.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+import requests
+import urllib3
+
+# Suppress InsecureRequestWarning
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class TestKnoxHealth(unittest.TestCase):
+ def test_admin_api_health(self):
+ """
+ Basic health check to ensure Knox is up and running.
+ We expect a response 200 to indicate the server is up.
+ """
+ url = os.environ.get("KNOX_GATEWAY_URL", "https://localhost:8443/")
+ print(f"Checking connectivity to {url}...")
+ try:
+ response = requests.get(url + "health/v1/ping", verify=False, timeout=30)
+ print(f"Received status code: {response.status_code}")
+ self.assertEqual(response.status_code, 200)
+ except requests.exceptions.ConnectionError:
+ self.fail("Failed to connect to Knox on port 8443 - Connection refused")
+ except Exception as e:
+ self.fail(f"Health check failed with unexpected error: {e}")
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/.github/workflows/tests/test_knox_auth_service_and_LDAP.py b/.github/workflows/tests/test_knox_auth_service_and_LDAP.py
new file mode 100644
index 0000000..dd1b344
--- /dev/null
+++ b/.github/workflows/tests/test_knox_auth_service_and_LDAP.py
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import unittest
+import requests
+import urllib3
+from requests.auth import HTTPBasicAuth
+
+########################################################
+# This test is verifying the behavior of the Knox Auth Service + LDAP authentication.
+# It is using the 'auth/api/v1/pre' endpoint to get the actor ID and group headers.
+# It is using the 'guest' user to get the guest user headers.
+# It is using the 'admin' user to get the admin user headers.
+# It is verifying that the actor ID and group headers are correct.
+# It is verifying that the actor ID and group headers are not empty.
+# It is verifying that the actor ID and group headers are not None.
+########################################################
+
+# Suppress InsecureRequestWarning
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class TestKnoxAuthService(unittest.TestCase):
+ def setUp(self):
+ self.base_url = os.environ.get("KNOX_GATEWAY_URL", "https://localhost:8443/")
+ if not self.base_url.endswith("/"):
+ self.base_url += "/"
+ # The topology name is based on the filename knoxldap.xml
+ self.topology_url = self.base_url + "gateway/knoxldap/auth/api/v1/extauthz"
+
+ def test_auth_service_guest(self):
+ """
+ Verify that guest user gets the correct actor ID header.
+ """
+ print(f"\nTesting guest authentication against {self.topology_url}")
+ response = requests.get(
+ self.topology_url,
+ auth=HTTPBasicAuth('guest', 'guest-password'),
+ verify=False,
+ timeout=30
+ )
+
+ print(f"Status Code: {response.status_code}")
+ self.assertEqual(response.status_code, 200)
+
+ # Check for Actor ID header
+ # The config in knoxtoken.xml sets 'preauth.auth.header.actor.id.name' to 'x-knox-actor-username'
+ actor_id_header = 'x-knox-actor-username'
+ self.assertIn(actor_id_header, response.headers)
+ self.assertEqual(response.headers[actor_id_header], 'guest')
+ print(f"Verified {actor_id_header}: {response.headers[actor_id_header]}")
+
+ def test_auth_service_admin_groups(self):
+ """
+ Verify that admin user gets actor ID and group headers.
+ """
+ print(f"\nTesting admin authentication against {self.topology_url}")
+ response = requests.get(
+ self.topology_url,
+ auth=HTTPBasicAuth('admin', 'admin-password'),
+ verify=False,
+ timeout=30
+ )
+
+ print(f"Status Code: {response.status_code}")
+ self.assertEqual(response.status_code, 200)
+
+ # Check for Actor ID header
+ actor_id_header = 'x-knox-actor-username'
+ self.assertIn(actor_id_header, response.headers)
+ self.assertEqual(response.headers[actor_id_header], 'admin')
+ print(f"Verified {actor_id_header}: {response.headers[actor_id_header]}")
+
+ # Check for Group headers
+ # Config: 'preauth.auth.header.actor.groups.prefix' = 'x-knox-actor-groups'
+ # We mapped admin to 'longGroupName1,longGroupName2,longGroupName3,longGroupName4'
+
+ # We just verify that at least one header starting with the prefix exists
+ prefix = 'x-knox-actor-groups'
+ group_headers = [h for h in response.headers.keys() if h.lower().startswith(prefix.lower())]
+
+ self.assertTrue(len(group_headers) > 0, f"No headers found starting with {prefix}")
+
+ # Verify content of groups
+ all_groups = []
+ for h in group_headers:
+ all_groups.extend(response.headers[h].split(','))
+ print(f"Found group header {h}: {response.headers[h]}")
+
+ expected_groups = ['longGroupName1', 'longGroupName2', 'longGroupName3', 'longGroupName4']
+ for group in expected_groups:
+ self.assertIn(group, all_groups)
+
+if __name__ == '__main__':
+ unittest.main()
+
diff --git a/.gitignore b/.gitignore
index 11e463d..89404ec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@
*.patch
.checkstyle
**/package-lock.json
+.github/workflows/compose/logs/*
# other IDEs and editors
.vscode
diff --git a/pom.xml b/pom.xml
index 8551319..9f3bae9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -494,6 +494,7 @@
<exclude>.git/**</exclude>
<exclude>.pc/**</exclude>
<exclude>debian/**</exclude>
+ <exclude>.github/**</exclude>
<exclude>.gitignore/**</exclude>
<exclude>.svn/**</exclude>
<exclude>.idea/**</exclude>
