New implementation of VFSClassLoader (#3)
diff --git a/modules/classloaderfactory-substitute/.gitignore b/modules/classloaderfactory-substitute/.gitignore
deleted file mode 100644
index 3d5bdae..0000000
--- a/modules/classloaderfactory-substitute/.gitignore
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Maven ignores
-/target/
-
-# IDE ignores
-/.settings/
-/.project
-/.classpath
-/.pydevproject
-/.idea
-/*.iml
-/*.ipr
-/*.iws
-/nbproject/
-/nbactions.xml
-/nb-configuration.xml
-.vscode/
-.factorypath
diff --git a/modules/classloaderfactory-substitute/pom.xml b/modules/classloaderfactory-substitute/pom.xml
deleted file mode 100644
index d5475fc..0000000
--- a/modules/classloaderfactory-substitute/pom.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>classloader-extras</artifactId>
- <version>1.0.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <artifactId>classloaderfactory-substitute</artifactId>
- <name>ClassloaderFactory substitute</name>
- <description>This exists temporarily, until a version of Accumulo is published containing the required ClassloaderFactory interface</description>
- <properties>
- <eclipseFormatterStyle>../../contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
- <maven.deploy.skip>true</maven.deploy.skip>
- <!-- temporarily skip analyzing dependencies until things are fixed, in order to build -->
- <mdep.analyze.skip>true</mdep.analyze.skip>
- </properties>
-</project>
diff --git a/modules/classloaderfactory-substitute/src/main/java/org/apache/accumulo/core/spi/common/ClassLoaderFactory.java b/modules/classloaderfactory-substitute/src/main/java/org/apache/accumulo/core/spi/common/ClassLoaderFactory.java
deleted file mode 100644
index 7fb1d59..0000000
--- a/modules/classloaderfactory-substitute/src/main/java/org/apache/accumulo/core/spi/common/ClassLoaderFactory.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.spi.common;
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-/**
- * The ClassLoaderFactory is defined by the property general.context.factory. The factory
- * implementation is configured externally to Accumulo and will return a ClassLoader for a given
- * contextName.
- *
- */
-public interface ClassLoaderFactory {
-
- static class ClassLoaderFactoryConfiguration {
-
- public Iterator<Entry<String,String>> get() {
- return Collections.emptyIterator();
- }
- }
-
- /**
- * Initialize the ClassLoaderFactory. Implementations may need a reference to the configuration so
- * that it can clean up contexts that are no longer being used.
- *
- * @param conf
- * Accumulo configuration properties
- * @throws Exception
- * if error initializing ClassLoaderFactory
- */
- void initialize(ClassLoaderFactoryConfiguration conf) throws Exception;
-
- /**
- *
- * @param contextName
- * name of classloader context
- * @return classloader configured for the context
- * @throws IllegalArgumentException
- * if contextName is not supported
- */
- ClassLoader getClassLoader(String contextName) throws IllegalArgumentException;
-
-}
diff --git a/modules/example-iterators-a/.classpath b/modules/example-iterators-a/.classpath
new file mode 100644
index 0000000..0fb79cf
--- /dev/null
+++ b/modules/example-iterators-a/.classpath
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" output="target/classes" path="src/main/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
+ <attribute name="test" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ <attribute name="test" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="output" path="target/classes"/>
+</classpath>
diff --git a/modules/example-iterators-a/.gitignore b/modules/example-iterators-a/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/modules/example-iterators-a/.gitignore
@@ -0,0 +1 @@
+/target/
diff --git a/modules/example-iterators-a/.project b/modules/example-iterators-a/.project
new file mode 100644
index 0000000..8269290
--- /dev/null
+++ b/modules/example-iterators-a/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>example-iterators-a</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.eclipse.m2e.core.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/modules/example-iterators-a/.settings/org.eclipse.core.resources.prefs b/modules/example-iterators-a/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..29abf99
--- /dev/null
+++ b/modules/example-iterators-a/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,6 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
+encoding/<project>=UTF-8
diff --git a/modules/example-iterators-a/.settings/org.eclipse.jdt.core.prefs b/modules/example-iterators-a/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..b5490a0
--- /dev/null
+++ b/modules/example-iterators-a/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,8 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
+org.eclipse.jdt.core.compiler.compliance=11
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=enabled
+org.eclipse.jdt.core.compiler.source=11
diff --git a/modules/example-iterators-a/.settings/org.eclipse.m2e.core.prefs b/modules/example-iterators-a/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 0000000..f897a7f
--- /dev/null
+++ b/modules/example-iterators-a/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/modules/example-iterators-a/pom.xml b/modules/example-iterators-a/pom.xml
new file mode 100644
index 0000000..3d5d8b5
--- /dev/null
+++ b/modules/example-iterators-a/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>classloader-extras</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <artifactId>example-iterators-a</artifactId>
+ <properties>
+ <eclipseFormatterStyle>../../contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/modules/example-iterators-a/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java b/modules/example-iterators-a/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java
new file mode 100644
index 0000000..9082fac
--- /dev/null
+++ b/modules/example-iterators-a/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java
@@ -0,0 +1,87 @@
+package org.apache.accumulo.classloader.vfs.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * Iterator that only returns 'foo' for the value.
+ */
+public class ExampleIterator implements SortedKeyValueIterator<Key,Value> {
+
+ protected SortedKeyValueIterator<Key,Value> source;
+
+ public ExampleIterator deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ public Key getTopKey() {
+ return source.getTopKey();
+ }
+
+ public Value getTopValue() {
+ return new Value("foo".getBytes(StandardCharsets.UTF_8));
+ }
+
+ public boolean hasTop() {
+ return source.hasTop();
+ }
+
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
+ IteratorEnvironment env) {
+ this.source = source;
+ }
+
+ public void next() throws IOException {
+ source.next();
+ }
+
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
+ throws IOException {
+ source.seek(range, columnFamilies, inclusive);
+ }
+}
diff --git a/modules/example-iterators-b/.classpath b/modules/example-iterators-b/.classpath
new file mode 100644
index 0000000..0fb79cf
--- /dev/null
+++ b/modules/example-iterators-b/.classpath
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" output="target/classes" path="src/main/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="src" output="target/test-classes" path="src/test/java">
+ <attributes>
+ <attribute name="optional" value="true"/>
+ <attribute name="maven.pomderived" value="true"/>
+ <attribute name="test" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ <attribute name="test" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+ <attributes>
+ <attribute name="maven.pomderived" value="true"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="output" path="target/classes"/>
+</classpath>
diff --git a/modules/example-iterators-b/.gitignore b/modules/example-iterators-b/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/modules/example-iterators-b/.gitignore
@@ -0,0 +1 @@
+/target/
diff --git a/modules/example-iterators-b/.project b/modules/example-iterators-b/.project
new file mode 100644
index 0000000..8269290
--- /dev/null
+++ b/modules/example-iterators-b/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>example-iterators-a</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ <buildCommand>
+ <name>org.eclipse.m2e.core.maven2Builder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.eclipse.m2e.core.maven2Nature</nature>
+ </natures>
+</projectDescription>
diff --git a/modules/example-iterators-b/.settings/org.eclipse.core.resources.prefs b/modules/example-iterators-b/.settings/org.eclipse.core.resources.prefs
new file mode 100644
index 0000000..29abf99
--- /dev/null
+++ b/modules/example-iterators-b/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,6 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
+encoding/<project>=UTF-8
diff --git a/modules/example-iterators-b/.settings/org.eclipse.jdt.core.prefs b/modules/example-iterators-b/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..b5490a0
--- /dev/null
+++ b/modules/example-iterators-b/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,8 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
+org.eclipse.jdt.core.compiler.compliance=11
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=enabled
+org.eclipse.jdt.core.compiler.source=11
diff --git a/modules/example-iterators-b/.settings/org.eclipse.m2e.core.prefs b/modules/example-iterators-b/.settings/org.eclipse.m2e.core.prefs
new file mode 100644
index 0000000..f897a7f
--- /dev/null
+++ b/modules/example-iterators-b/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1
diff --git a/modules/example-iterators-b/pom.xml b/modules/example-iterators-b/pom.xml
new file mode 100644
index 0000000..ce6b7c5
--- /dev/null
+++ b/modules/example-iterators-b/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>classloader-extras</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <artifactId>example-iterators-b</artifactId>
+ <properties>
+ <eclipseFormatterStyle>../../contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/modules/example-iterators-b/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java b/modules/example-iterators-b/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java
new file mode 100644
index 0000000..4c2617b
--- /dev/null
+++ b/modules/example-iterators-b/src/main/java/org/apache/accumulo/classloader/vfs/examples/ExampleIterator.java
@@ -0,0 +1,87 @@
+package org.apache.accumulo.classloader.vfs.examples;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * Iterator that only returns 'bar' for the value.
+ */
+public class ExampleIterator implements SortedKeyValueIterator<Key,Value> {
+
+ protected SortedKeyValueIterator<Key,Value> source;
+
+ public ExampleIterator deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+
+ public Key getTopKey() {
+ return source.getTopKey();
+ }
+
+ public Value getTopValue() {
+ return new Value("bar".getBytes(StandardCharsets.UTF_8));
+ }
+
+ public boolean hasTop() {
+ return source.hasTop();
+ }
+
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
+ IteratorEnvironment env) {
+ this.source = source;
+ }
+
+ public void next() throws IOException {
+ source.next();
+ }
+
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
+ throws IOException {
+ source.seek(range, columnFamilies, inclusive);
+ }
+}
diff --git a/modules/vfs-class-loader/README.md b/modules/vfs-class-loader/README.md
index 8b53cdb..60a9762 100644
--- a/modules/vfs-class-loader/README.md
+++ b/modules/vfs-class-loader/README.md
@@ -15,13 +15,17 @@
limitations under the License.
-->
-# VFS Reloading ClassLoader
+# Accumulo VFS ClassLoader
-This module contains a [ClassLoader](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/ClassLoader.html) implementation that can be used as the [System](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/ClassLoader.html#getSystemClassLoader()) ClassLoader.
+This module contains a [ClassLoader](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/ClassLoader.html) implementation that can be used as the JVM [System](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/ClassLoader.html#getSystemClassLoader()) ClassLoader or a ClassLoader for Accumulo table contexts.
-## Configuration
+## System Class Loader
-To use this ClassLoader as the System ClassLoader you must set the JVM system property **java.system.class.loader** to the fully qualified class name (org.apache.accumulo.classloader.vfs.ReloadingVFSClassLoader). This jar and it's dependent jars must be on the **java.class.path**.
+To use this ClassLoader as the System ClassLoader you must set the JVM system property **java.system.class.loader** to the fully qualified class name (org.apache.accumulo.classloader.vfs.AccumuloVFSClassLoader). This jar and it's dependent jars must be on the **java.class.path**.
+
+NOTE: When used in this manner the reloading feature is disabled because the JVM caches classes created from the standard JVM hierarchy and thus reloading has no effect.
+
+### Configuration
To set the classpath for this ClassLoader you must define the system property **vfs.class.loader.classpath** and set it to locations that are supported by [Apache Commons VFS](http://commons.apache.org/proper/commons-vfs/filesystems.html).
@@ -29,6 +33,39 @@
This ClassLoader follows the normal parent delegation model but can be set to load classes and resources first, before checking if the parent classloader can, by setting the system property **vfs.class.loader.delegation** to "post".
+## Accumulo Table Context ClassLoader
+
+To use this ClassLoader as the ContextClassLoader you must set the JVM system property **vfs.context.class.loader.config** to a valid JSON configuration file and set the Accumulo configuration property **general.context.class.loader.factory** to the fully qualified class name of the ContextClassLoaderFactory implementation (org.apache.accumulo.classloader.vfs.context.ReloadingVFSContextClassLoaderFactory). This jar and it's dependent jars must be on the **java.class.path**.
+
+### Configuration
+
+You will need to define the supported contexts and their configuration in a JSON formatted file. For example:
+
+```
+{
+ "contexts": [
+ {
+ "name": "cx1",
+ "config": {
+ "classPath": "file:///tmp/foo",
+ "postDelegate": true,
+ "monitorIntervalMs": 30000
+ }
+ },
+ {
+ "name": "cx2",
+ "config": {
+ "classPath": "file:///tmp/bar",
+ "postDelegate": false,
+ "monitorIntervalMs": 30000
+ }
+ }
+ ]
+}
+```
+
+## Additional Configuration
+
Finally, this ClassLoader keeps a local cache of objects pulled from remote systems (via http, etc.). The default location for this cache directory is the value of the system property **java.io.tmpdir**. To change this location set the system property **vfs.cache.dir** to an existing directory.
## Implementation
diff --git a/modules/vfs-class-loader/TESTING.md b/modules/vfs-class-loader/TESTING.md
new file mode 100644
index 0000000..c15998f
--- /dev/null
+++ b/modules/vfs-class-loader/TESTING.md
@@ -0,0 +1,251 @@
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Setup
+
+After running `mvn clean package` add the built jars to HDFS:
+
+```
+hadoop fs -mkdir -p /iterators/legacy/foo
+hadoop fs -mkdir -p /iterators/legacy/bar
+hadoop fs -mkdir -p /iterators/new/foo
+hadoop fs -mkdir -p /iterators/new/bar
+hadoop fs -mkdir -p /iterators/system
+hadoop fs -put -f modules/example-iterators-a/target/example-iterators-a-1.0.0-SNAPSHOT.jar /iterators/legacy/foo/examples.jar
+hadoop fs -put -f modules/example-iterators-a/target/example-iterators-a-1.0.0-SNAPSHOT.jar /iterators/new/foo/examples.jar
+hadoop fs -put -f modules/example-iterators-b/target/example-iterators-b-1.0.0-SNAPSHOT.jar /iterators/legacy/bar/examples.jar
+hadoop fs -put -f modules/example-iterators-b/target/example-iterators-b-1.0.0-SNAPSHOT.jar /iterators/new/bar/examples.jar
+hadoop fs -cp -f /iterators/legacy/foo/examples.jar /iterators/system/examples.jar
+```
+
+Copy the new class loader jar to /tmp:
+
+```
+cp ./modules/vfs-class-loader/target/vfs-reloading-classloader-1.0.0-SNAPSHOT.jar /tmp/.
+```
+
+## Running Accumulo with new VFS ClassLoader as System ClassLoader
+
+### Configure Accumulo to use new classloader
+
+Stop Accumulo if it's running and add the following to the accumulo-env.sh:
+
+```
+a. Add vfs-reloading-classloader-1.0.0-SNAPSHOT.jar to CLASSPATH
+b. Add "-Djava.system.class.loader=org.apache.accumulo.classloader.vfs.AccumuloVFSClassLoader" to JAVA_OPTS
+c. Add "-Dvfs.class.loader.classpath=hdfs://localhost:9000/iterators/system/.*" to JAVA_OPTS
+d. Add "-Dvfs.classpath.monitor.seconds=10" to JAVA_OPTS
+e. (optional) Add "-Dvfs.class.loader.debug=true" to JAVA_OPTS
+```
+
+### Test setting iterator retrieved from jar in HDFS with System ClassLoader
+
+The goal of this test is to create a table, insert some data, and change the value of the data using an iterator that is loaded from HDFS via the AccumuloVFSClassLoader set up as the System ClassLoader. After setting the iterator the value should be `foo` in subsequent scans.
+
+```
+createtable test
+insert a b c this_is_a_test
+scan
+setiter -class org.apache.accumulo.classloader.vfs.examples.ExampleIterator -scan -t test -name example -p 100
+scan
+```
+
+## Setting scan context on table (Legacy)
+
+### Define a Table Context and load the iterator class with the same name, but different behavior
+
+In this test we will define a context name with an associated classpath. Then we will set that context on the table. Note that
+we did not change the iterator class name, the context classloader will load a new class with the same name. Scans performed after the context is set on the table should return the value `bar`.
+
+a. Set Accumulo Classpath Context property:
+
+```
+config -s general.vfs.context.classpath.cx1=hdfs://localhost:9000/iterators/legacy/bar/.*
+config -s general.vfs.context.classpath.cx1.delegation=post
+```
+
+b. Set Accumulo Table Context property:
+
+```
+config -t test -s table.classpath.context=cx1
+```
+
+c. Test context classpath iterator setting:
+
+```
+scan
+```
+
+### Testing Reloading
+
+This test will continue from the previous test and we will copy a jar over the jar referenced in the cx1 context classpath. The legacy AccumuloReloadingVFSClassLoader has a hard-coded filesystem monitor time of 5 minutes, so we will need to wait some number of minutes after overwriting the jar before the scans will return the new value of `foo`.
+
+a. Copy the example-a.jar over the context-examples.jar to force a reload. The value in the scan result should change from `bar` back to `foo`.
+
+```
+hadoop fs -rm /iterators/legacy/bar/examples.jar
+hadoop fs -cp -f /iterators/legacy/foo/examples.jar /iterators/legacy/bar/examples2.jar
+```
+
+b. Wait 10 minutes for a reload
+
+c. Test that class loader has been updated and is returning the value 'foo'
+
+```
+scan
+```
+
+### Change the context on the table
+
+In this test we will unset the properties that we set in the previous tests. Instead of testing reloading we are testing that changing the context will have the same effect on the iterator class.
+
+a. Unset prior properties and define two contexts:
+
+```
+deleteiter -n example -t test -scan
+config -t test -d table.classpath.context
+config -d general.vfs.context.classpath.cx1
+config -d general.vfs.context.classpath.cx1.delegation
+config -s general.vfs.context.classpath.cx1=hdfs://localhost:9000/iterators/new/foo/examples.jar
+config -s general.vfs.context.classpath.cx1.delegation=post
+config -s general.vfs.context.classpath.cx2=hdfs://localhost:9000/iterators/new/bar/examples.jar
+config -s general.vfs.context.classpath.cx2.delegation=post
+```
+
+b. Set Accumulo Table Context property:
+
+```
+config -t test -s table.classpath.context=cx1
+```
+
+c. Test context classpath iterator setting:
+
+The initial scan command should return the value `this_is_a_test`. After setting the iterator, the scan should return `foo`.
+
+```
+scan
+setiter -class org.apache.accumulo.classloader.vfs.examples.ExampleIterator -scan -t test -name example -p 100
+scan
+```
+
+d. Change the context Table Context property:
+
+```
+config -t test -s table.classpath.context=cx2
+```
+
+e. Test Context change
+
+After the context change, the scan should return `bar`.
+
+```
+scan
+```
+
+## Setting scan context on table (New)
+
+For this test we will use the new ReloadingVFSContextClassLoaderFactory for the table context classloaders.
+
+a. First, let's clean up from the prior tests
+
+```
+droptable -f test
+config -d general.vfs.context.classpath.cx1
+config -d general.vfs.context.classpath.cx1.delegation
+config -d general.vfs.context.classpath.cx2
+config -d general.vfs.context.classpath.cx2.delegation
+```
+
+b. Then, create a file on the local filesystem for the context configuration.
+
+```
+{
+ "contexts": [
+ {
+ "name": "cxA",
+ "config": {
+ "classPath": "hdfs://localhost:9000/iterators/new/foo/.*",
+ "postDelegate": true,
+ "monitorIntervalMs": 10000
+ }
+ },
+ {
+ "name": "cxB",
+ "config": {
+ "classPath": "hdfs://localhost:9000/iterators/new/bar/.*",
+ "postDelegate": true,
+ "monitorIntervalMs": 10000
+ }
+ }
+ ]
+}
+```
+
+c. Next, shutdown Accumulo and make the following changes in the accumulo configuration. Then re-start Accumulo.
+
+```
+a. Add "general.context.class.loader.factory=org.apache.accumulo.classloader.vfs.context.ReloadingVFSContextClassLoaderFactory" to accumulo.properties
+b. Add "-Dvfs.context.class.loader.config=file:///path/to/config/file.json" to JAVA_OPTS
+```
+
+### Test setting iterator retrieved from jar in HDFS with System ClassLoader
+
+If you did not remove the configuration for the new System ClassLoader, then the following test should work as it should load the ExampleIterator class from the System ClassLoader.
+
+a. Create the table as we did in the last test. Create a table, insert some data, and change the value of the data using an iterator that is loaded from HDFS via the VFSClassLoader set up as the System ClassLoader. After setting the iterator the value should be `foo` in subsequent scans.
+
+```
+createtable test
+insert a b c this_is_a_test
+scan
+setiter -class org.apache.accumulo.classloader.vfs.examples.ExampleIterator -scan -t test -name example -p 100
+scan
+```
+
+### Change the context on the table
+
+Change the contexts on the table to test the classes being loaded from the different jars.
+
+a. Set the table context to cxA. The scan on the table should return the value `foo`.
+
+```
+config -t test -s table.classpath.context=cxA
+scan
+```
+
+b. Set the table context to cxB. The scan on the table should return the value `bar`.
+
+```
+config -t test -s table.classpath.context=cxB
+scan
+```
+
+### Testing Reloading
+
+Now we are going to remove the jar from the `cxB` context directory and replace it with the jar from the `cxA` context directory.
+
+a. Test the reloading by removing the existing jar from HDFS, copy /iterators/example-a/examples.jar to /iterators/example-b/examples2.jar and rescan. The value from the scan should be `foo`. Note that the context set on the table is `cxB` which returned `bar` in the previous test.
+
+NOTE: Overwriting the example-b/examples.jar does not work, it does not appear that VFS pulls down the jar from HDFS into the local cache directory and it continues to serve up the old class.
+
+```
+hadoop fs -rm /iterators/new/bar/examples.jar
+hadoop fs -cp -f /iterators/new/foo/examples.jar /iterators/new/bar/examples2.jar
+scan
+```
diff --git a/modules/vfs-class-loader/WIP.txt b/modules/vfs-class-loader/WIP.txt
deleted file mode 100644
index 09edcb9..0000000
--- a/modules/vfs-class-loader/WIP.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-
-1. Running Accumulo with new VFS ClassLoader as SystemClassLoader
-
- a. Add accumulo/test/target/TestJar-Iterators.jar to HDFS directory
- hadoop fs -mkdir /iterators
- hadoop fs -put TestJar-Iterators.jar /iterators/.
- b. Add vfs-reloading-classloader-0.0.1-SNAPSHOT.jar to CLASSPATH
- c. Add "-Djava.system.class.loader=org.apache.accumulo.classloader.vfs.ReloadingVFSClassLoader" to JAVA_OPTS
- d. Add "-Dvfs.class.loader.classpath=hdfs://localhost:9000/iterators/TestJar-Iterators.jar" to JAVA_OPTS
- e. Add "-Dvfs.class.loader.debug=true" to JAVA_OPTS
- f. Create table , insert some test data
- createtable test
- insert a b c this_is_a_test
- scan
- setiter -class org.apache.accumulo.test.functional.ValueReversingIterator -scan -t test -name reverse -p 100
-
diff --git a/modules/vfs-class-loader/pom.xml b/modules/vfs-class-loader/pom.xml
index 981c258..ee98b9d 100644
--- a/modules/vfs-class-loader/pom.xml
+++ b/modules/vfs-class-loader/pom.xml
@@ -28,24 +28,19 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>vfs-reloading-classloader</artifactId>
- <name>VFS Reloading ClassLoader</name>
+ <name>classloader-extras-vfs-reloading</name>
<properties>
<eclipseFormatterStyle>../../contrib/Eclipse-Accumulo-Codestyle.xml</eclipseFormatterStyle>
- <!-- temporarily skip analyzing dependencies until things are fixed, in order to build -->
- <mdep.analyze.skip>true</mdep.analyze.skip>
- <!-- temporarily skip spotbugs until things are fixed, in order to build -->
- <spotbugs.skip>true</spotbugs.skip>
+ <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
</properties>
<dependencies>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>3.7.1</version>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-vfs2</artifactId>
- <version>2.6.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -54,74 +49,53 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <version>2.13.1</version>
- </dependency>
- <dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>2.8.6</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.7</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <version>1.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
- <artifactId>classloaderfactory-substitute</artifactId>
- <version>${project.version}</version>
+ <artifactId>accumulo-core</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
- <version>3.2.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.30</version>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>28.2-jre</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.13</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-start</artifactId>
- <version>2.1.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-minicluster</artifactId>
- <version>3.2.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- <version>2.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -140,7 +114,7 @@
<pluginExecutionFilter>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <versionRange>[3.0.0,)</versionRange>
+ <versionRange>[1.6.0,)</versionRange>
<goals>
<goal>exec</goal>
</goals>
@@ -167,7 +141,6 @@
<exclude>**/LICENSE</exclude>
<exclude>**/NOTICE</exclude>
<exclude>**/target/**</exclude>
- <exclude>contrib/javadoc11.patch</exclude>
</excludes>
<mapping combine.children="append">
<!-- general mappings; module-specific mappings appear in their respective pom -->
diff --git a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoader.java b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoader.java
similarity index 65%
rename from modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoader.java
rename to modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoader.java
index 755f0d8..ea5a303 100644
--- a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoader.java
+++ b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoader.java
@@ -18,8 +18,6 @@
*/
package org.apache.accumulo.classloader.vfs;
-import static java.util.concurrent.TimeUnit.SECONDS;
-
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
@@ -27,6 +25,7 @@
import java.util.Arrays;
import java.util.Enumeration;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.RejectedExecutionException;
@@ -37,6 +36,7 @@
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.vfs2.FileChangeEvent;
@@ -45,11 +45,12 @@
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.impl.DefaultFileMonitor;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.provider.hdfs.HdfsFileObject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
/**
* <p>
* A {@code ClassLoader} implementation that watches for changes in any of the files/directories in
@@ -74,7 +75,7 @@
* This class will attempt to perform substitution on any environment variables found in the values.
* For example, the system property <b>vfs.cache.dir</b> can be set to <b>$HOME/cache</b>.
*/
-public class ReloadingVFSClassLoader extends ClassLoader implements Closeable, FileListener {
+public class AccumuloVFSClassLoader extends ClassLoader implements Closeable, FileListener {
public static final String VFS_CLASSPATH_MONITOR_INTERVAL = "vfs.classpath.monitor.seconds";
public static final String VFS_CACHE_DIR_PROPERTY = "vfs.cache.dir";
@@ -84,13 +85,13 @@
private static final String VFS_CACHE_DIR_DEFAULT = "java.io.tmpdir";
- // set to 5 mins. The rationale behind this large time is to avoid a gazillion tservers all asking
+ // set to 5 mins. The rationale behind this large time is to avoid a tservers all asking
// the name node for info too frequently.
private static final long DEFAULT_TIMEOUT = TimeUnit.MINUTES.toMillis(5);
private static boolean DEBUG = false;
private static String CLASSPATH = null;
- private static Boolean PRE_DELEGATION = null;
+ private static Boolean POST_DELEGATION = null;
private static Long MONITOR_INTERVAL = null;
private static boolean VM_INITIALIZED = false;
@@ -98,36 +99,40 @@
private volatile long maxRetries = -1;
private volatile long sleepInterval = 1000;
private volatile boolean vfsInitializing = false;
+ private volatile boolean vfsInitialized = false;
- private final ThreadPoolExecutor executor;
private final ClassLoader parent;
private final ReentrantReadWriteLock updateLock = new ReentrantReadWriteLock(true);
private final String name;
private final String classpath;
- private final Boolean preDelegation;
+ private final Boolean postDelegation;
private final long monitorInterval;
- private DefaultFileMonitor monitor;
+ private Optional<Monitor> fileMonitor = Optional.empty();
private FileObject[] files;
private VFSClassLoaderWrapper cl = null;
- private DefaultFileSystemManager vfs = null;
static {
DEBUG = Boolean.parseBoolean(System.getProperty(VFS_CLASSLOADER_DEBUG, "false"));
CLASSPATH = getClassPathProperty();
- PRE_DELEGATION = getPreDelegationModelProperty();
+ POST_DELEGATION = getDelegationModelProperty();
MONITOR_INTERVAL = getMonitorIntervalProperty();
}
private static void printDebug(String msg) {
if (!DEBUG)
return;
- System.out
- .println(String.format("%d ReloadingVFSClassLoader: %s", System.currentTimeMillis(), msg));
+ System.out.println(
+ String.format("DEBUG: %d AccumuloVFSClassLoader: %s", System.currentTimeMillis(), msg));
}
private static void printError(String msg) {
- System.err
- .println(String.format("%d ReloadingVFSClassLoader: %s", System.currentTimeMillis(), msg));
+ System.err.println(
+ String.format("ERROR: %d AccumuloVFSClassLoader: %s", System.currentTimeMillis(), msg));
+ }
+
+ private static void printWarn(String msg) {
+ System.err.println(
+ String.format("WARN: %d AccumuloVFSClassLoader: %s", System.currentTimeMillis(), msg));
}
/**
@@ -138,7 +143,7 @@
private static String getClassPathProperty() {
String cp = System.getProperty(VFS_CLASSLOADER_CLASSPATH);
if (null == cp || cp.isBlank()) {
- printError(VFS_CLASSLOADER_CLASSPATH + " system property not set, using default of \"\"");
+ printWarn(VFS_CLASSLOADER_CLASSPATH + " system property not set, using default of \"\"");
cp = "";
}
String result = replaceEnvVars(cp, System.getenv());
@@ -151,14 +156,14 @@
*
* @return true if pre delegaion, false if post delegation
*/
- private static boolean getPreDelegationModelProperty() {
- String delegation = System.getProperty(VFS_CLASSLOADER_DELEGATION);
- boolean preDelegation = true;
- if (null != delegation && delegation.equalsIgnoreCase("post")) {
- preDelegation = false;
+ private static boolean getDelegationModelProperty() {
+ String property = System.getProperty(VFS_CLASSLOADER_DELEGATION);
+ boolean postDelegation = false;
+ if (null != property && property.equalsIgnoreCase("post")) {
+ postDelegation = true;
}
- printDebug("ClassLoader configured for pre-delegation: " + preDelegation);
- return preDelegation;
+ printDebug("ClassLoader configured for pre-delegation: " + postDelegation);
+ return postDelegation;
}
/**
@@ -170,7 +175,7 @@
// Get configuration properties from the environment variables
String vfsCacheDir = System.getProperty(VFS_CACHE_DIR_PROPERTY);
if (null == vfsCacheDir || vfsCacheDir.isBlank()) {
- printError(VFS_CACHE_DIR_PROPERTY + " system property not set, using default of "
+ printWarn(VFS_CACHE_DIR_PROPERTY + " system property not set, using default of "
+ VFS_CACHE_DIR_DEFAULT);
vfsCacheDir = System.getProperty(VFS_CACHE_DIR_DEFAULT);
}
@@ -210,7 +215,7 @@
try {
return TimeUnit.SECONDS.toMillis(Long.parseLong(interval));
} catch (NumberFormatException e) {
- printError(VFS_CLASSPATH_MONITOR_INTERVAL + " system property not set, using default of "
+ printWarn(VFS_CLASSPATH_MONITOR_INTERVAL + " system property not set, using default of "
+ DEFAULT_TIMEOUT);
return DEFAULT_TIMEOUT;
}
@@ -218,117 +223,144 @@
return DEFAULT_TIMEOUT;
}
- /**
- * This task replaces the delegate classloader with a new instance when the filesystem has
- * changed. This will orphan the old classloader and the only references to the old classloader
- * are from the objects that it loaded.
- */
- private final Runnable refresher = new Runnable() {
- @Override
- public void run() {
- while (!executor.isTerminating()) {
- try {
- printDebug("Recreating delegate classloader due to filesystem change event");
- updateDelegateClassloader();
- return;
- } catch (Exception e) {
- e.printStackTrace();
+ private class Monitor {
+
+ /**
+ * This task replaces the delegate classloader with a new instance when the filesystem has
+ * changed. This will orphan the old classloader and the only references to the old classloader
+ * are from the objects that it loaded.
+ */
+ private final Runnable refresher = new Runnable() {
+ @Override
+ public void run() {
+ while (!executor.isTerminating()) {
try {
- Thread.sleep(getMonitorInterval());
- } catch (InterruptedException ie) {
- ie.printStackTrace();
+ printDebug("Recreating delegate classloader due to filesystem change event");
+ updateDelegateClassloader();
+ return;
+ } catch (Exception e) {
+ e.printStackTrace();
+ try {
+ Thread.sleep(getMonitorInterval());
+ } catch (InterruptedException ie) {
+ ie.printStackTrace();
+ }
}
}
}
- }
- };
-
- public ReloadingVFSClassLoader(ClassLoader parent) {
- super(ReloadingVFSClassLoader.class.getSimpleName(), parent);
- printDebug("Parent ClassLoader: " + parent.getClass().getName());
- this.name = ReloadingVFSClassLoader.class.getSimpleName();
- this.parent = parent;
- this.classpath = CLASSPATH;
- this.preDelegation = PRE_DELEGATION;
- this.monitorInterval = MONITOR_INTERVAL;
-
- BlockingQueue<Runnable> queue = new ArrayBlockingQueue<>(2);
- ThreadFactory factory = r -> {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
};
- executor = new ThreadPoolExecutor(1, 1, 1, SECONDS, queue, factory);
+
+ private final ThreadPoolExecutor executor;
+ private final DefaultFileMonitor monitor;
+
+ private Monitor(AccumuloVFSClassLoader fileMonitor) {
+ BlockingQueue<Runnable> queue = new ArrayBlockingQueue<>(2);
+ ThreadFactory factory = r -> {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ };
+ this.executor = new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, queue, factory);
+ this.monitor = new DefaultFileMonitor(fileMonitor);
+
+ monitor.setDelay(getMonitorInterval());
+ monitor.setRecursive(false);
+ monitor.start();
+ printDebug("Monitor started with interval set to: " + monitor.getDelay());
+ }
+
+ private FileMonitor getMonitor() {
+ return this.monitor;
+ }
+
+ private void scheduleRefresh() {
+ try {
+ this.executor.execute(refresher);
+ } catch (RejectedExecutionException e) {
+ printDebug("Ignoring refresh request (already refreshing)");
+ }
+ }
+
+ private void shutdown() {
+ this.executor.shutdownNow();
+ this.monitor.stop();
+ }
+
}
- protected DefaultFileSystemManager getFileSystem() {
- if (null == this.vfs) {
+ public AccumuloVFSClassLoader(ClassLoader parent) {
+ super(AccumuloVFSClassLoader.class.getSimpleName(), parent);
+ printDebug("Parent ClassLoader: " + parent.getClass().getName());
+ this.name = AccumuloVFSClassLoader.class.getSimpleName();
+ this.parent = parent;
+ this.classpath = CLASSPATH;
+ this.postDelegation = POST_DELEGATION;
+ this.monitorInterval = MONITOR_INTERVAL;
+ }
+
+ private void initializeFileSystem() {
+ if (!this.vfsInitialized) {
if (DEBUG) {
VFSManager.enableDebug();
}
try {
- this.vfs = VFSManager.generateVfs();
+ if (DEBUG) {
+ printDebug("Creating new VFS File System");
+ }
+ VFSManager.initialize();
} catch (FileSystemException e) {
printError("Error creating FileSystem: " + e.getMessage());
- e.printStackTrace();
+ throw new RuntimeException("Problem creating VFS file system", e);
}
printDebug("VFS File System created.");
}
- return this.vfs;
}
protected String getClassPath() {
return this.classpath;
}
- protected boolean isPreDelegationModel() {
- return this.preDelegation;
+ protected boolean isPostDelegationModel() {
+ printDebug("isPostDelegationModel called, returning " + this.postDelegation);
+ return this.postDelegation;
}
protected long getMonitorInterval() {
return this.monitorInterval;
}
- private synchronized FileMonitor getFileMonitor() {
- if (null == this.monitor) {
- this.monitor = new DefaultFileMonitor(this);
- monitor.setDelay(getMonitorInterval());
- monitor.setRecursive(false);
- monitor.start();
- printDebug("Monitor started with interval set to: " + monitor.getDelay());
- }
- return this.monitor;
- }
-
private void addFileToMonitor(FileObject file) throws RuntimeException {
try {
- getFileMonitor().addFile(file);
+ fileMonitor.ifPresent(u -> {
+ u.getMonitor().addFile(file);
+ });
} catch (RuntimeException re) {
- if (re.getMessage().contains("files-cache"))
+ if (re.getMessage().contains("files-cache")) {
printDebug("files-cache error adding " + file.toString() + " to VFS monitor. "
+ "There is no implementation for files-cache in VFS2");
- else
+ } else {
printDebug("Runtime error adding " + file.toString() + " to VFS monitor");
-
+ }
re.printStackTrace();
throw re;
}
}
+ @SuppressFBWarnings(value = "SWL_SLEEP_WITH_LOCK_HELD")
private synchronized void updateDelegateClassloader() throws Exception {
try {
- updateLock.writeLock().lock();
// Re-resolve the files on the classpath, things may have changed.
long retries = 0;
long currentSleepMillis = sleepInterval;
- FileObject[] classpathFiles = VFSManager.resolve(getFileSystem(), this.getClassPath());
+ printDebug("Looking for files on classpath: " + this.getClassPath());
+ FileObject[] classpathFiles = VFSManager.resolve(this.getClassPath());
if (classpathFiles.length == 0) {
while (classpathFiles.length == 0 && retryPermitted(retries)) {
try {
- printDebug("VFS path was empty. Waiting " + currentSleepMillis + " ms to retry");
+ printWarn("VFS path was empty. Waiting " + currentSleepMillis + " ms to retry");
Thread.sleep(currentSleepMillis);
- classpathFiles = VFSManager.resolve(getFileSystem(), this.getClassPath());
+ classpathFiles = VFSManager.resolve(this.getClassPath());
retries++;
currentSleepMillis = Math.min(maxWaitInterval, currentSleepMillis + sleepInterval);
} catch (InterruptedException e) {
@@ -339,9 +371,17 @@
}
}
if (classpathFiles.length == 0) {
- printError("ReloadingVFSClassLoader has no resources on classpath");
+ printError("AccumuloVFSClassLoader has no resources on classpath");
}
this.files = classpathFiles;
+ // Remove old files from monitor
+ VFSClassLoaderWrapper currentDelegate = this.cl;
+ if (null != currentDelegate) {
+ forEachCatchRTEs(Arrays.stream(currentDelegate.getFileObjects()), f -> {
+ removeFile(f);
+ printDebug("removed from monitor: " + f.toString());
+ });
+ }
// There is a chance that the listener was removed from the top level directory or
// its children if they were deleted within some time window. Re-add files to be
// monitored. The Monitor will ignore files that are already/still being monitored.
@@ -349,58 +389,54 @@
// and can collect them to list or reduce into one exception
forEachCatchRTEs(Arrays.stream(this.files), f -> {
addFileToMonitor(f);
- printDebug("monitoring: " + f.toString());
+ printDebug("now monitoring: " + f.toString());
});
// Create the new classloader delegate
- printDebug("Rebuilding dynamic classloader using files: " + stringify(this.files));
- VFSClassLoaderWrapper cl;
- if (this.isPreDelegationModel()) {
+ if (DEBUG) {
+ printDebug("Rebuilding dynamic classloader using files: "
+ + Arrays.stream(this.files).map(Object::toString).collect(Collectors.joining(",")));
+ }
+ VFSClassLoaderWrapper newDelegate;
+ if (!this.isPostDelegationModel()) {
// This is the normal classloader parent delegation model
- cl = new VFSClassLoaderWrapper(this.files, getFileSystem(), parent);
+ printDebug("Creating new pre-delegating VFSClassLoaderWrapper");
+ newDelegate = new VFSClassLoaderWrapper(this.files, VFSManager.get(), parent);
} else {
// This delegates to the parent after we lookup locally first.
- cl = new VFSClassLoaderWrapper(this.files, getFileSystem()) {
+ printDebug("Creating new post-delegating VFSClassLoaderWrapper");
+ newDelegate = new VFSClassLoaderWrapper(this.files, VFSManager.get(), parent) {
@Override
public synchronized Class<?> loadClass(String name, boolean resolve)
throws ClassNotFoundException {
- Class<?> c = findLoadedClass(name);
- if (c != null)
+ // Check to see if this ClassLoader has already loaded the class
+ Class<?> c = this.findLoadedClass(name);
+ if (c != null) {
+ if (DEBUG) {
+ printDebug("Returning already loaded class: " + name + "@" + c.hashCode()
+ + " from classloader: " + c.getClassLoader().hashCode());
+ }
return c;
+ }
try {
// try finding this class here instead of parent
- return findClass(name);
+ Class<?> clazz = super.findClass(name);
+ if (DEBUG) {
+ printDebug("Returning newly loaded class: " + name + "@" + clazz.hashCode()
+ + " from classloader: " + clazz.getClassLoader().hashCode());
+ }
+ return clazz;
} catch (ClassNotFoundException e) {
-
+ printDebug("Class " + name + " not found in classloader: " + this.hashCode()
+ + ", delegating to parent.");
}
+ printDebug("Loading class " + name + " from parent classloader");
return super.loadClass(name, resolve);
}
};
}
- // An HDFS FileSystem and Configuration object were created for each unique HDFS namespace
- // in the call to resolve above. The HDFS Client did us a favor and cached these objects
- // so that the next time someone calls FileSystem.get(uri), they get the cached object.
- // However, these objects were created not with the VFS classloader, but the
- // classloader above it. We need to override the classloader on the Configuration objects.
- // Ran into an issue were log recovery was being attempted and SequenceFile$Reader was
- // trying to instantiate the key class via WritableName.getClass(String, Configuration)
- printDebug("Setting ClassLoader on HDFS FileSystem objects");
- for (FileObject fo : this.files) {
- if (fo instanceof HdfsFileObject) {
- String uri = fo.getName().getRootURI();
- Configuration c = new Configuration(true);
- c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
- try {
- FileSystem fs = FileSystem.get(c);
- fs.getConf().setClassLoader(cl);
- } catch (IOException e) {
- throw new RuntimeException("Error setting classloader on HDFS FileSystem object", e);
- }
- }
- }
-
- // Update the delegate reference to the new classloader
- this.cl = cl;
- printDebug("ReloadingVFSClassLoader set.");
+ updateLock.writeLock().lock();
+ this.cl = newDelegate;
+ printDebug("AccumuloVFSClassLoader set, hash=" + this.cl.hashCode());
} finally {
updateLock.writeLock().unlock();
}
@@ -416,7 +452,11 @@
*/
private void removeFile(FileObject file) throws RuntimeException {
try {
- getFileMonitor().removeFile(file);
+ fileMonitor.ifPresent(u -> {
+ u.getMonitor().removeFile(file);
+ // VFS DefaultFileMonitor does not remove listener from the file on remove
+ file.getFileSystem().removeListener(file, this);
+ });
} catch (RuntimeException re) {
printError("Error removing file from VFS cache: " + file.toString());
re.printStackTrace();
@@ -427,42 +467,36 @@
@Override
public void fileCreated(FileChangeEvent event) throws Exception {
printDebug(event.getFileObject().getURL().toString() + " created, recreating classloader");
- scheduleRefresh();
+ fileMonitor.ifPresent(u -> u.scheduleRefresh());
}
@Override
public void fileDeleted(FileChangeEvent event) throws Exception {
printDebug(event.getFileObject().getURL().toString() + " deleted, recreating classloader");
- scheduleRefresh();
+ fileMonitor.ifPresent(u -> u.scheduleRefresh());
}
@Override
public void fileChanged(FileChangeEvent event) throws Exception {
printDebug(event.getFileObject().getURL().toString() + " changed, recreating classloader");
- scheduleRefresh();
- }
-
- private void scheduleRefresh() {
- try {
- executor.execute(refresher);
- } catch (RejectedExecutionException e) {
- printDebug("Ignoring refresh request (already refreshing)");
- }
+ fileMonitor.ifPresent(u -> u.scheduleRefresh());
}
@Override
public void close() {
- forEachCatchRTEs(Stream.of(this.files), f -> {
- removeFile(f);
- printDebug("Closing, removing file from monitoring: " + f.toString());
- });
+ if (null != this.files) {
+ forEachCatchRTEs(Stream.of(this.files), f -> {
+ // remove file from monitor
+ removeFile(f);
+ printDebug("Closing, removed file from monitoring: " + f.toString());
+ });
+ }
+ fileMonitor.ifPresent(u -> u.shutdown());
+ fileMonitor = Optional.empty();
- this.executor.shutdownNow();
- this.monitor.stop();
- if (null != this.vfs)
- VFSManager.returnVfs(this.vfs);
- vfs = null;
+ this.cl = null;
+
}
public static <T> void forEachCatchRTEs(Stream<T> stream, Consumer<T> consumer) {
@@ -511,20 +545,47 @@
// the VM is fully initialized.
if (!isVMInitialized() || vfsInitializing) {
return this.parent;
- } else if (null == this.vfs) {
+ } else if (!this.vfsInitialized) {
this.vfsInitializing = true;
printDebug("getDelegateClassLoader() initializing VFS.");
- getFileSystem();
- if (null == getFileSystem()) {
- // Some error happened
- throw new RuntimeException("Problem creating VFS file system");
- }
+ initializeFileSystem();
+ this.vfsInitialized = true;
printDebug("getDelegateClassLoader() VFS initialized.");
}
if (null == this.cl) {
try {
+ if (!isSystemClassLoader()) {
+ printDebug("Reloading enabled, creating monitor");
+ fileMonitor = Optional.of(new Monitor(this));
+ } else {
+ printDebug("Reloading disabled as this is the java.system.class.loader");
+ }
printDebug("Creating initial delegate class loader");
updateDelegateClassloader();
+ if (isSystemClassLoader()) {
+ // An HDFS FileSystem and Configuration object were created for each unique HDFS namespace
+ // in the call to resolve above. The HDFS Client did us a favor and cached these objects
+ // so that the next time someone calls FileSystem.get(uri), they get the cached object.
+ // However, these objects were created not with the VFS classloader, but the
+ // classloader above it. We need to override the classloader on the Configuration objects.
+ // Ran into an issue were log recovery was being attempted and SequenceFile$Reader was
+ // trying to instantiate the key class via WritableName.getClass(String, Configuration)
+ printDebug("Setting ClassLoader on HDFS FileSystem objects");
+ for (FileObject fo : this.files) {
+ if (fo instanceof HdfsFileObject) {
+ String uri = fo.getName().getRootURI();
+ Configuration c = new Configuration(true);
+ c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
+ try {
+ FileSystem fs = FileSystem.get(c);
+ fs.getConf().setClassLoader(cl);
+ } catch (IOException e) {
+ throw new RuntimeException("Error setting classloader on HDFS FileSystem object",
+ e);
+ }
+ }
+ }
+ }
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Error creating initial delegate classloader", e);
@@ -587,7 +648,11 @@
return name;
}
- private boolean isVMInitialized() {
+ private boolean isSystemClassLoader() {
+ return ClassLoader.getSystemClassLoader().equals(this);
+ }
+
+ protected boolean isVMInitialized() {
if (VM_INITIALIZED) {
return VM_INITIALIZED;
} else {
@@ -597,7 +662,7 @@
try {
printDebug(
"System ClassLoader: " + ClassLoader.getSystemClassLoader().getClass().getName());
- VM_INITIALIZED = ClassLoader.getSystemClassLoader().equals(this);
+ VM_INITIALIZED = isSystemClassLoader();
} catch (IllegalStateException e) {
// VM is still initializing
VM_INITIALIZED = false;
@@ -652,12 +717,18 @@
getDelegateClassLoader().clearAssertionStatus();
}
+ public ClassLoader unwrap() {
+ return getDelegateClassLoader();
+ }
+
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((parent.getName() == null) ? 0 : parent.getName().hashCode());
+ if (null != parent) {
+ result = prime * result + ((parent.getName() == null) ? 0 : parent.getName().hashCode());
+ }
return result;
}
@@ -669,7 +740,7 @@
return false;
if (getClass() != obj.getClass())
return false;
- ReloadingVFSClassLoader other = (ReloadingVFSClassLoader) obj;
+ AccumuloVFSClassLoader other = (AccumuloVFSClassLoader) obj;
if (name == null) {
if (other.name != null)
return false;
@@ -702,23 +773,21 @@
// VisibleForTesting intentionally not using annotation from Guava
// because it adds unwanted dependency
- void setMaxRetries(long maxRetries) {
+ public void setMaxRetries(long maxRetries) {
this.maxRetries = maxRetries;
}
// VisibleForTesting intentionally not using annotation from Guava
// because it adds unwanted dependency
- void setVMInitializedForTests() {
+ @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD",
+ justification = "used for tests")
+ public void setVMInitializedForTests() {
VM_INITIALIZED = true;
}
- // VisibleForTesting intentionally not using annotation from Guava
- // because it adds unwanted dependency
- void setVFSForTests(DefaultFileSystemManager vfs) {
- this.vfs = vfs;
- }
-
- void enableDebugForTests() {
+ @SuppressFBWarnings(value = "ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD",
+ justification = "used for tests")
+ public void enableDebugForTests() {
DEBUG = true;
}
}
diff --git a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ClassPathPrinter.java b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ClassPathPrinter.java
index 7112d25..dc5e394 100644
--- a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ClassPathPrinter.java
+++ b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/ClassPathPrinter.java
@@ -93,12 +93,12 @@
printJar(out, u.getFile(), debug, sawFirst);
sawFirst = true;
}
- } else if (classLoader instanceof ReloadingVFSClassLoader) {
+ } else if (classLoader instanceof AccumuloVFSClassLoader) {
if (debug) {
out.print("Level " + level + ": ReloadingVFSClassLoader, classpath items are:\n");
}
@SuppressWarnings("resource")
- ReloadingVFSClassLoader vcl = (ReloadingVFSClassLoader) classLoader;
+ AccumuloVFSClassLoader vcl = (AccumuloVFSClassLoader) classLoader;
ClassLoader delegate = vcl.getDelegateClassLoader();
if (delegate instanceof VFSClassLoaderWrapper) {
VFSClassLoaderWrapper wrapper = (VFSClassLoaderWrapper) delegate;
diff --git a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSClassLoaderWrapper.java b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSClassLoaderWrapper.java
index 7f17684..d0c7c89 100644
--- a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSClassLoaderWrapper.java
+++ b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSClassLoaderWrapper.java
@@ -80,4 +80,8 @@
return super.loadClass(name, resolve);
}
+ @Override
+ public String toString() {
+ return "VFSClassLoaderWrapper (hashCode: " + this.hashCode() + ")";
+ }
}
diff --git a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSManager.java b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSManager.java
index ba153d0..04de7f4 100644
--- a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSManager.java
+++ b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/VFSManager.java
@@ -21,10 +21,7 @@
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.ref.WeakReference;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.CacheStrategy;
@@ -54,22 +51,30 @@
}
}
- private static List<WeakReference<DefaultFileSystemManager>> vfsInstances =
- Collections.synchronizedList(new ArrayList<>());
+ private static DefaultFileSystemManager VFS = null;
private static volatile boolean DEBUG = false;
- static void enableDebug() {
- DEBUG = true;
- }
-
static {
// Register the shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(new AccumuloVFSManagerShutdownThread()));
}
- public static FileObject[] resolve(FileSystemManager vfs, String uris)
- throws FileSystemException {
- return resolve(vfs, uris, new ArrayList<>());
+ public static void enableDebug() {
+ DEBUG = true;
+ }
+
+ private static void printDebug(String msg) {
+ if (!DEBUG)
+ return;
+ System.out.println(String.format("DEBUG: %d VFSManager: %s", System.currentTimeMillis(), msg));
+ }
+
+ private static void printError(String msg) {
+ System.err.println(String.format("ERROR: %d VFSManager: %s", System.currentTimeMillis(), msg));
+ }
+
+ public static FileObject[] resolve(String uris) throws FileSystemException {
+ return resolve(VFS, uris, new ArrayList<>());
}
static FileObject[] resolve(FileSystemManager vfs, String uris,
@@ -90,8 +95,9 @@
continue;
}
- path = ReloadingVFSClassLoader.replaceEnvVars(path, System.getenv());
+ path = AccumuloVFSClassLoader.replaceEnvVars(path, System.getenv());
+ printDebug("Resolving path element: " + path);
FileObject fo = vfs.resolveFile(path);
switch (fo.getType()) {
@@ -115,17 +121,21 @@
}
}
} else {
- if (DEBUG)
- System.out.println("classpath entry " + fo.getParent().toString() + " is "
+ if (DEBUG) {
+ printDebug("classpath entry " + fo.getParent().toString() + " is "
+ fo.getParent().getType().toString());
+ }
}
} else {
- if (DEBUG)
- System.out.println("ignoring classpath entry: " + fo.toString());
+ if (DEBUG) {
+ printDebug("ignoring classpath entry: " + fo.toString());
+ }
}
break;
default:
- System.out.println("ignoring classpath entry: " + fo.toString());
+ if (DEBUG) {
+ printDebug("ignoring classpath entry: " + fo.toString());
+ }
break;
}
@@ -134,85 +144,79 @@
return classpath.toArray(new FileObject[classpath.size()]);
}
- public static DefaultFileSystemManager generateVfs() throws FileSystemException {
- DefaultFileSystemManager vfs = new DefaultFileSystemManager();
- vfs.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
- vfs.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
- vfs.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
- vfs.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
- vfs.addProvider("file", new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
- vfs.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
- vfs.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
- vfs.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
- vfs.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
- vfs.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
- vfs.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
- vfs.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
- vfs.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
- vfs.addProvider("hdfs", new HdfsFileProvider());
- vfs.addExtensionMap("jar", "jar");
- vfs.addExtensionMap("zip", "zip");
- vfs.addExtensionMap("gz", "gz");
- vfs.addExtensionMap("tar", "tar");
- vfs.addExtensionMap("tbz2", "tar");
- vfs.addExtensionMap("tgz", "tar");
- vfs.addExtensionMap("bz2", "bz2");
- vfs.addMimeTypeMap("application/x-tar", "tar");
- vfs.addMimeTypeMap("application/x-gzip", "gz");
- vfs.addMimeTypeMap("application/zip", "zip");
- vfs.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
- vfs.setFilesCache(new SoftRefFilesCache());
- File cacheDir = computeTopCacheDir();
- vfs.setReplicator(new UniqueFileReplicator(cacheDir));
- vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE);
- vfs.init();
- vfsInstances.add(new WeakReference<>(vfs));
- return vfs;
+ public static void initialize() throws FileSystemException {
+ if (null == VFS) {
+ VFS = new DefaultFileSystemManager();
+ VFS.addProvider("res", new org.apache.commons.vfs2.provider.res.ResourceFileProvider());
+ VFS.addProvider("zip", new org.apache.commons.vfs2.provider.zip.ZipFileProvider());
+ VFS.addProvider("gz", new org.apache.commons.vfs2.provider.gzip.GzipFileProvider());
+ VFS.addProvider("ram", new org.apache.commons.vfs2.provider.ram.RamFileProvider());
+ VFS.addProvider("file",
+ new org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider());
+ VFS.addProvider("jar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("http", new org.apache.commons.vfs2.provider.http.HttpFileProvider());
+ VFS.addProvider("https", new org.apache.commons.vfs2.provider.https.HttpsFileProvider());
+ VFS.addProvider("ftp", new org.apache.commons.vfs2.provider.ftp.FtpFileProvider());
+ VFS.addProvider("ftps", new org.apache.commons.vfs2.provider.ftps.FtpsFileProvider());
+ VFS.addProvider("war", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("par", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("ear", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("sar", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("ejb3", new org.apache.commons.vfs2.provider.jar.JarFileProvider());
+ VFS.addProvider("tmp", new org.apache.commons.vfs2.provider.temp.TemporaryFileProvider());
+ VFS.addProvider("tar", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ VFS.addProvider("tbz2", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ VFS.addProvider("tgz", new org.apache.commons.vfs2.provider.tar.TarFileProvider());
+ VFS.addProvider("bz2", new org.apache.commons.vfs2.provider.bzip2.Bzip2FileProvider());
+ VFS.addProvider("hdfs", new HdfsFileProvider());
+ VFS.addExtensionMap("jar", "jar");
+ VFS.addExtensionMap("zip", "zip");
+ VFS.addExtensionMap("gz", "gz");
+ VFS.addExtensionMap("tar", "tar");
+ VFS.addExtensionMap("tbz2", "tar");
+ VFS.addExtensionMap("tgz", "tar");
+ VFS.addExtensionMap("bz2", "bz2");
+ VFS.addMimeTypeMap("application/x-tar", "tar");
+ VFS.addMimeTypeMap("application/x-gzip", "gz");
+ VFS.addMimeTypeMap("application/zip", "zip");
+ VFS.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
+ VFS.setFilesCache(new SoftRefFilesCache());
+ File cacheDir = computeTopCacheDir();
+ VFS.setReplicator(new UniqueFileReplicator(cacheDir));
+ VFS.setCacheStrategy(CacheStrategy.ON_RESOLVE);
+ VFS.init();
+ }
+ }
+
+ public static FileSystemManager get() {
+ return VFS;
}
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN",
justification = "tmpdir is controlled by admin, not unchecked user input")
private static File computeTopCacheDir() {
- String cacheDirPath = ReloadingVFSClassLoader.getVFSCacheDir();
+ String cacheDirPath = AccumuloVFSClassLoader.getVFSCacheDir();
String procName = ManagementFactory.getRuntimeMXBean().getName();
return new File(cacheDirPath,
"accumulo-vfs-manager-cache-" + procName + "-" + System.getProperty("user.name", "nouser"));
}
- public static void returnVfs(DefaultFileSystemManager vfs) {
- if (DEBUG) {
- System.out.println("Closing VFS instance.");
- }
+ private static void close() {
+ printDebug("Closing VFS instance.");
FileReplicator replicator;
try {
- replicator = vfs.getReplicator();
+ replicator = VFS.getReplicator();
if (replicator instanceof UniqueFileReplicator) {
((UniqueFileReplicator) replicator).close();
}
} catch (FileSystemException e) {
- System.err.println("Error occurred closing VFS instance: " + e.getMessage());
+ printError("Error occurred closing VFS instance: " + e.getMessage());
}
- vfs.close();
- }
-
- public static void close() {
- for (WeakReference<DefaultFileSystemManager> vfsInstance : vfsInstances) {
- DefaultFileSystemManager ref = vfsInstance.get();
- if (ref != null) {
- returnVfs(ref);
- }
- }
+ VFS.close();
try {
FileUtils.deleteDirectory(computeTopCacheDir());
} catch (IOException e) {
- System.err.println("IOException deleting cache directory");
+ printError("IOException deleting cache directory");
e.printStackTrace();
}
}
diff --git a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactory.java b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactory.java
index a4f93f9..9f73078 100644
--- a/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactory.java
+++ b/modules/vfs-class-loader/src/main/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactory.java
@@ -19,13 +19,20 @@
package org.apache.accumulo.classloader.vfs.context;
import java.io.File;
+import java.net.URI;
import java.nio.file.Files;
-import java.util.HashMap;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
-import org.apache.accumulo.classloader.vfs.ReloadingVFSClassLoader;
-import org.apache.accumulo.core.spi.common.ClassLoaderFactory;
+import org.apache.accumulo.classloader.vfs.AccumuloVFSClassLoader;
+import org.apache.accumulo.core.client.PluginEnvironment.Configuration;
+import org.apache.accumulo.core.spi.common.ContextClassLoaderFactory;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
@@ -33,10 +40,10 @@
* A ClassLoaderFactory implementation that uses a ReloadingVFSClassLoader per defined context.
* Configuration of this class is done with a JSON file whose location is defined by the system
* property <b>vfs.context.class.loader.config</b>. To use this ClassLoaderFactory you need to set
- * the Accumulo configuration property <b>general.context.factory</b> to the fully qualified name of
- * this class, create a configuration file that defines the supported contexts and their
- * configuration, and set <b>vfs.context.class.loader.config</b> to the location of the
- * configuration file.
+ * the Accumulo configuration property <b>general.context.class.loader.factory</b> to the fully
+ * qualified name of this class, create a configuration file that defines the supported contexts and
+ * their configuration, and set the system property <b>vfs.context.class.loader.config</b> to the
+ * location of the configuration file.
*
* <p>
* Example configuration file:
@@ -64,7 +71,7 @@
* }
* </pre>
*/
-public class ReloadingVFSContextClassLoaderFactory implements ClassLoaderFactory {
+public class ReloadingVFSContextClassLoaderFactory implements ContextClassLoaderFactory {
public static class Contexts {
List<Context> contexts;
@@ -78,6 +85,11 @@
}
@Override
+ public String toString() {
+ return "Contexts [contexts=" + contexts + "]";
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
@@ -124,6 +136,11 @@
}
@Override
+ public String toString() {
+ return "Context [name=" + name + ", config=" + config + "]";
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
@@ -165,7 +182,7 @@
}
public void setClassPath(String classPath) {
- this.classPath = ReloadingVFSClassLoader.replaceEnvVars(classPath, System.getenv());
+ this.classPath = AccumuloVFSClassLoader.replaceEnvVars(classPath, System.getenv());
}
public boolean getPostDelegate() {
@@ -185,6 +202,12 @@
}
@Override
+ public String toString() {
+ return "ContextConfig [classPath=" + classPath + ", postDelegate=" + postDelegate
+ + ", monitorIntervalMs=" + monitorIntervalMs + "]";
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
@@ -216,8 +239,11 @@
}
}
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReloadingVFSContextClassLoaderFactory.class);
public static final String CONFIG_LOCATION = "vfs.context.class.loader.config";
- private static final Map<String,ReloadingVFSClassLoader> CONTEXTS = new HashMap<>();
+ private static final Map<String,AccumuloVFSClassLoader> CONTEXTS = new ConcurrentHashMap<>();
+ private Contexts contextDefinitions = null;
protected String getConfigFileLocation() {
String loc = System.getProperty(CONFIG_LOCATION);
@@ -229,43 +255,75 @@
}
@Override
- public void initialize(ClassLoaderFactoryConfiguration conf) throws Exception {
+ public void initialize(Configuration contextProperties) throws Exception {
// Properties
- File f = new File(getConfigFileLocation());
+ String conf = getConfigFileLocation();
+ File f = new File(new URI(conf));
if (!f.canRead()) {
- throw new RuntimeException("Unable to read configuration file: " + f.getAbsolutePath());
+ throw new RuntimeException("Unable to read configuration file: " + conf);
}
Gson g = new Gson();
- Contexts con = g.fromJson(Files.newBufferedReader(f.toPath()), Contexts.class);
+ LOG.debug("Context configuration: {}", FileUtils.readFileToString(f, "UTF-8"));
+ contextDefinitions = g.fromJson(Files.newBufferedReader(f.toPath()), Contexts.class);
+ LOG.debug("Deserialized JSON: {}", contextDefinitions);
+ contextDefinitions.getContexts().forEach(c -> {
+ CONTEXTS.put(c.getName(), create(c));
+ });
+ }
- con.getContexts().forEach(c -> {
- CONTEXTS.put(c.getName(), new ReloadingVFSClassLoader(
- ReloadingVFSContextClassLoaderFactory.class.getClassLoader()) {
- @Override
- protected String getClassPath() {
- return c.getConfig().getClassPath();
- }
+ protected AccumuloVFSClassLoader create(Context c) {
+ LOG.debug("Creating ReloadingVFSClassLoader for context: {})", c.getName());
+ return AccessController.doPrivileged(new PrivilegedAction<AccumuloVFSClassLoader>() {
+ @Override
+ public AccumuloVFSClassLoader run() {
+ return new AccumuloVFSClassLoader(
+ ReloadingVFSContextClassLoaderFactory.class.getClassLoader()) {
+ @Override
+ protected String getClassPath() {
+ return c.getConfig().getClassPath();
+ }
- @Override
- protected boolean isPreDelegationModel() {
- return !(c.getConfig().getPostDelegate());
- }
+ @Override
+ protected boolean isPostDelegationModel() {
+ LOG.debug("isPostDelegationModel called, returning {}",
+ c.getConfig().getPostDelegate());
+ return c.getConfig().getPostDelegate();
+ }
- @Override
- protected long getMonitorInterval() {
- return c.getConfig().getMonitorIntervalMs();
- }
- });
+ @Override
+ protected long getMonitorInterval() {
+ return c.getConfig().getMonitorIntervalMs();
+ }
+
+ @Override
+ protected boolean isVMInitialized() {
+ // The classloader is not being set using
+ // `java.system.class.loader`, so the VM is initialized.
+ return true;
+ }
+ };
+ }
});
}
@Override
- public ClassLoader getClassLoader(String contextName) throws IllegalArgumentException {
+ public synchronized ClassLoader getClassLoader(String contextName)
+ throws IllegalArgumentException {
if (!CONTEXTS.containsKey(contextName)) {
throw new IllegalArgumentException(
"ReloadingVFSContextClassLoaderFactory not configured for context: " + contextName);
}
- return CONTEXTS.get(contextName);
+ // The JVM maintains a cache of loaded classes where the key is the ClassLoader
+ // instance and the loaded Class name (see ClassLoader.findLoadedClass()). So
+ // that we can return new implementations of the same class when the context
+ // changes we need to load the class from the delegate classloader directly
+ // since the delegate instance will be recreated when the context changes.
+ return CONTEXTS.get(contextName).unwrap();
}
+ public void closeForTests() {
+ CONTEXTS.forEach((k, v) -> {
+ v.close();
+ });
+ }
}
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloDFSBase.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloDFSBase.java
index 4c6c27d..d3a20f6 100644
--- a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloDFSBase.java
+++ b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloDFSBase.java
@@ -18,11 +18,13 @@
*/
package org.apache.accumulo.classloader.vfs;
+import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
-import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.cache.DefaultFilesCache;
@@ -42,9 +44,9 @@
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class AccumuloDFSBase {
- protected static Configuration conf = null;
- protected static DefaultFileSystemManager vfs = null;
- protected static MiniDFSCluster cluster = null;
+ private static Configuration conf = null;
+ private static DefaultFileSystemManager vfs = null;
+ private static MiniDFSCluster cluster = null;
private static URI HDFS_URI;
@@ -52,6 +54,33 @@
return HDFS_URI;
}
+ public static String computeDatanodeDirectoryPermission() {
+ // MiniDFSCluster will check the permissions on the data directories, but does not
+ // do a good job of setting them properly. We need to get the users umask and set
+ // the appropriate Hadoop property so that the data directories will be created
+ // with the correct permissions.
+ try {
+ Process p = Runtime.getRuntime().exec("/bin/sh -c umask");
+ try (BufferedReader bri =
+ new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) {
+ String line = bri.readLine();
+ p.waitFor();
+
+ if (line == null) {
+ throw new IOException("umask input stream closed prematurely");
+ }
+ short umask = Short.parseShort(line.trim(), 8);
+ // Need to set permission to 777 xor umask
+ // leading zero makes java interpret as base 8
+ int newPermission = 0777 ^ umask;
+
+ return String.format("%03o", newPermission);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error getting umask from O/S", e);
+ }
+ }
+
@BeforeClass
public static void miniDfsClusterSetup() {
System.setProperty("java.io.tmpdir", System.getProperty("user.dir") + "/target");
@@ -63,7 +92,7 @@
conf = new Configuration();
conf.set("hadoop.security.token.service.use_ip", "true");
- conf.set("dfs.datanode.data.dir.perm", MiniDFSUtil.computeDatanodeDirectoryPermission());
+ conf.set("dfs.datanode.data.dir.perm", computeDatanodeDirectoryPermission());
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize
try {
@@ -124,6 +153,18 @@
}
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ public MiniDFSCluster getCluster() {
+ return cluster;
+ }
+
+ public DefaultFileSystemManager getDefaultFileSystemManager() {
+ return vfs;
+ }
+
@AfterClass
public static void tearDownMiniDfsCluster() {
if (null != cluster) {
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoaderTest.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoaderTest.java
new file mode 100644
index 0000000..e86b296
--- /dev/null
+++ b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/AccumuloVFSClassLoaderTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.vfs;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
+public class AccumuloVFSClassLoaderTest {
+
+ @Rule
+ public TemporaryFolder folder1 =
+ new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
+ String folderPath;
+
+ @Before
+ public void setup() throws Exception {
+ System.setProperty(AccumuloVFSClassLoader.VFS_CLASSPATH_MONITOR_INTERVAL, "1");
+ VFSManager.initialize();
+
+ folderPath = folder1.getRoot().toURI() + ".*";
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ folder1.newFile("HelloWorld.jar"));
+ }
+
+ FileObject[] createFileSystems(FileObject[] fos) throws FileSystemException {
+ FileObject[] rfos = new FileObject[fos.length];
+ for (int i = 0; i < fos.length; i++) {
+ if (VFSManager.get().canCreateFileSystem(fos[i])) {
+ rfos[i] = VFSManager.get().createFileSystem(fos[i]);
+ } else {
+ rfos[i] = fos[i];
+ }
+ }
+
+ return rfos;
+ }
+
+ @Test
+ public void testConstructor() throws Exception {
+ FileObject testDir = VFSManager.get().resolveFile(folder1.getRoot().toURI().toString());
+ FileObject[] dirContents = testDir.getChildren();
+
+ AccumuloVFSClassLoader arvcl = new AccumuloVFSClassLoader(ClassLoader.getSystemClassLoader()) {
+ @Override
+ protected String getClassPath() {
+ return folderPath;
+ }
+ };
+ arvcl.setVMInitializedForTests();
+
+ FileObject[] files = ((VFSClassLoaderWrapper) arvcl.getDelegateClassLoader()).getFileObjects();
+ assertArrayEquals(createFileSystems(dirContents), files);
+
+ arvcl.close();
+ }
+
+}
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ClassPathPrinterTest.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ClassPathPrinterTest.java
index 6845b2a..aa063bc 100644
--- a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ClassPathPrinterTest.java
+++ b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ClassPathPrinterTest.java
@@ -24,7 +24,6 @@
import java.io.File;
import java.net.MalformedURLException;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -53,9 +52,9 @@
@Test
public void testPrintClassPath() throws Exception {
File conf = folder1.newFile("accumulo.properties");
- DefaultFileSystemManager vfs = VFSManager.generateVfs();
+ VFSManager.initialize();
- ReloadingVFSClassLoader cl = new ReloadingVFSClassLoader(parent) {
+ AccumuloVFSClassLoader cl = new AccumuloVFSClassLoader(parent) {
@Override
protected String getClassPath() {
try {
@@ -65,13 +64,8 @@
}
}
- @Override
- protected DefaultFileSystemManager getFileSystem() {
- return vfs;
- }
};
cl.setVMInitializedForTests();
- cl.setVFSForTests(vfs);
assertPattern(ClassPathPrinter.getClassPath(cl, true), "(?s).*\\s+.*\\n$", true);
assertTrue(ClassPathPrinter.getClassPath(cl, true)
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoaderTest.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoaderTest.java
deleted file mode 100644
index 739b5ac..0000000
--- a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/ReloadingVFSClassLoaderTest.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.classloader.vfs;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.vfs2.FileObject;
-import org.apache.commons.vfs2.FileSystemException;
-import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-
-@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
-public class ReloadingVFSClassLoaderTest {
-
- @Rule
- public TemporaryFolder folder1 =
- new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
- String folderPath;
- private DefaultFileSystemManager vfs;
-
- @Before
- public void setup() throws Exception {
- System.setProperty(ReloadingVFSClassLoader.VFS_CLASSPATH_MONITOR_INTERVAL, "1");
- vfs = VFSManager.generateVfs();
-
- folderPath = folder1.getRoot().toURI() + ".*";
-
- FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
- folder1.newFile("HelloWorld.jar"));
- }
-
- FileObject[] createFileSystems(FileObject[] fos) throws FileSystemException {
- FileObject[] rfos = new FileObject[fos.length];
- for (int i = 0; i < fos.length; i++) {
- if (vfs.canCreateFileSystem(fos[i])) {
- rfos[i] = vfs.createFileSystem(fos[i]);
- } else {
- rfos[i] = fos[i];
- }
- }
-
- return rfos;
- }
-
- @Test
- public void testConstructor() throws Exception {
- FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
- FileObject[] dirContents = testDir.getChildren();
-
- ReloadingVFSClassLoader arvcl =
- new ReloadingVFSClassLoader(ClassLoader.getSystemClassLoader()) {
- @Override
- protected String getClassPath() {
- return folderPath;
- }
-
- @Override
- protected DefaultFileSystemManager getFileSystem() {
- return vfs;
- }
- };
- arvcl.setVMInitializedForTests();
- arvcl.setVFSForTests(vfs);
-
- FileObject[] files = ((VFSClassLoaderWrapper) arvcl.getDelegateClassLoader()).getFileObjects();
- assertArrayEquals(createFileSystems(dirContents), files);
-
- arvcl.close();
- }
-
- @Test
- public void testReloading() throws Exception {
- FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
- FileObject[] dirContents = testDir.getChildren();
-
- ReloadingVFSClassLoader arvcl =
- new ReloadingVFSClassLoader(ClassLoader.getSystemClassLoader()) {
- @Override
- protected String getClassPath() {
- return folderPath;
- }
-
- @Override
- protected long getMonitorInterval() {
- return 500l;
- }
-
- @Override
- protected DefaultFileSystemManager getFileSystem() {
- return vfs;
- }
- };
- arvcl.setVMInitializedForTests();
- arvcl.setVFSForTests(vfs);
-
- FileObject[] files = ((VFSClassLoaderWrapper) arvcl.getDelegateClassLoader()).getFileObjects();
- assertArrayEquals(createFileSystems(dirContents), files);
-
- // set retry settings sufficiently low that not everything is reloaded in the first round
- arvcl.setMaxRetries(1);
-
- Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
- Object o1 = clazz1.getDeclaredConstructor().newInstance();
- assertEquals("Hello World!", o1.toString());
-
- // Check that the class is the same before the update
- Class<?> clazz1_5 = arvcl.loadClass("test.HelloWorld");
- assertEquals(clazz1, clazz1_5);
-
- assertTrue(new File(folder1.getRoot(), "HelloWorld.jar").delete());
-
- Thread.sleep(1000);
-
- // Update the class
- FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
- folder1.newFile("HelloWorld2.jar"));
-
- // Wait for the monitor to notice
- Thread.sleep(1000);
-
- Class<?> clazz2 = arvcl.loadClass("test.HelloWorld");
- Object o2 = clazz2.getDeclaredConstructor().newInstance();
- assertEquals("Hello World!", o2.toString());
-
- // This is false because they are loaded by a different classloader
- assertNotEquals(clazz1, clazz2);
- assertNotEquals(o1, o2);
-
- arvcl.close();
- }
-
- @Test
- public void testReloadingWithLongerTimeout() throws Exception {
- FileObject testDir = vfs.resolveFile(folder1.getRoot().toURI().toString());
- FileObject[] dirContents = testDir.getChildren();
-
- ReloadingVFSClassLoader arvcl =
- new ReloadingVFSClassLoader(ClassLoader.getSystemClassLoader()) {
- @Override
- protected String getClassPath() {
- return folderPath;
- }
-
- @Override
- protected long getMonitorInterval() {
- return 1000l;
- }
-
- @Override
- protected DefaultFileSystemManager getFileSystem() {
- return vfs;
- }
- };
- arvcl.setVMInitializedForTests();
- arvcl.setVFSForTests(vfs);
-
- FileObject[] files = ((VFSClassLoaderWrapper) arvcl.getDelegateClassLoader()).getFileObjects();
- assertArrayEquals(createFileSystems(dirContents), files);
-
- // set retry settings sufficiently high such that reloading happens in the first rounds
- arvcl.setMaxRetries(3);
-
- Class<?> clazz1 = arvcl.loadClass("test.HelloWorld");
- Object o1 = clazz1.getDeclaredConstructor().newInstance();
- assertEquals("Hello World!", o1.toString());
-
- // Check that the class is the same before the update
- Class<?> clazz1_5 = arvcl.loadClass("test.HelloWorld");
- assertEquals(clazz1, clazz1_5);
-
- assertTrue(new File(folder1.getRoot(), "HelloWorld.jar").delete());
-
- Thread.sleep(3000);
-
- // Update the class
- FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
- folder1.newFile("HelloWorld2.jar"));
-
- // Wait for the monitor to notice
- Thread.sleep(3000);
-
- Class<?> clazz2 = arvcl.loadClass("test.HelloWorld");
- Object o2 = clazz2.getDeclaredConstructor().newInstance();
- assertEquals("Hello World!", o2.toString());
-
- // This is false because even though it's the same class, it's loaded from a different jar
- // this is a change in behavior from previous versions of vfs2 where it would load the same
- // class from different jars as though it was from the first jar
- assertNotEquals(clazz1, clazz2);
- assertNotSame(o1, o2);
- assertEquals(clazz1.getName(), clazz2.getName());
- assertEquals(o1.toString(), o2.toString());
-
- arvcl.close();
- }
-
-}
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/VfsClassLoaderTest.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/VfsClassLoaderTest.java
index 4778872..0d9a1e8 100644
--- a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/VfsClassLoaderTest.java
+++ b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/VfsClassLoaderTest.java
@@ -22,11 +22,15 @@
import static org.junit.Assert.assertTrue;
import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import org.apache.commons.vfs2.FileChangeEvent;
import org.apache.commons.vfs2.FileListener;
import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
import org.apache.commons.vfs2.impl.DefaultFileMonitor;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -40,11 +44,12 @@
private FileSystem hdfs = null;
private VFSClassLoader cl = null;
+ private DefaultFileSystemManager vfs = null;
@Before
public void setup() throws Exception {
- this.hdfs = cluster.getFileSystem();
+ this.hdfs = this.getCluster().getFileSystem();
this.hdfs.mkdirs(TEST_DIR);
// Copy jar file to TEST_DIR
@@ -53,11 +58,21 @@
Path dst = new Path(TEST_DIR, src.getName());
this.hdfs.copyFromLocalFile(src, dst);
+ vfs = this.getDefaultFileSystemManager();
FileObject testDir = vfs.resolveFile(TEST_DIR.toUri().toString());
FileObject[] dirContents = testDir.getChildren();
- // Point the VFSClassLoader to all of the objects in TEST_DIR
- this.cl = new VFSClassLoader(dirContents, vfs);
+ this.cl = AccessController.doPrivileged(new PrivilegedAction<VFSClassLoader>() {
+ @Override
+ public VFSClassLoader run() {
+ // Point the VFSClassLoader to all of the objects in TEST_DIR
+ try {
+ return new VFSClassLoader(dirContents, vfs);
+ } catch (FileSystemException e) {
+ throw new RuntimeException("Error creating VFSClassLoader", e);
+ }
+ }
+ });
}
@Test
diff --git a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactoryTest.java b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactoryTest.java
index 4be5c46..302b9b6 100644
--- a/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactoryTest.java
+++ b/modules/vfs-class-loader/src/test/java/org/apache/accumulo/classloader/vfs/context/ReloadingVFSContextClassLoaderFactoryTest.java
@@ -21,45 +21,117 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.BufferedWriter;
import java.io.File;
import java.nio.file.Files;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.List;
+import org.apache.accumulo.classloader.vfs.AccumuloVFSClassLoader;
import org.apache.accumulo.classloader.vfs.context.ReloadingVFSContextClassLoaderFactory.Context;
import org.apache.accumulo.classloader.vfs.context.ReloadingVFSContextClassLoaderFactory.ContextConfig;
import org.apache.accumulo.classloader.vfs.context.ReloadingVFSContextClassLoaderFactory.Contexts;
+import org.apache.commons.io.FileUtils;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not set by user input")
public class ReloadingVFSContextClassLoaderFactoryTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReloadingVFSContextClassLoaderFactoryTest.class);
+
+ private static class TestReloadingVFSContextClassLoaderFactory
+ extends ReloadingVFSContextClassLoaderFactory {
+
+ private final String dir;
+
+ public TestReloadingVFSContextClassLoaderFactory(String dir) {
+ this.dir = dir;
+ }
+
+ @Override
+ protected AccumuloVFSClassLoader create(Context c) {
+ AccumuloVFSClassLoader acl =
+ AccessController.doPrivileged(new PrivilegedAction<AccumuloVFSClassLoader>() {
+ @Override
+ public AccumuloVFSClassLoader run() {
+ AccumuloVFSClassLoader cl = new AccumuloVFSClassLoader(
+ ReloadingVFSContextClassLoaderFactory.class.getClassLoader()) {
+ @Override
+ protected String getClassPath() {
+ return dir;
+ }
+
+ @Override
+ protected boolean isPostDelegationModel() {
+ LOG.debug("isPostDelegationModel called, returning {}",
+ c.getConfig().getPostDelegate());
+ return c.getConfig().getPostDelegate();
+ }
+
+ @Override
+ protected long getMonitorInterval() {
+ return 500l;
+ }
+
+ @Override
+ protected boolean isVMInitialized() {
+ return true;
+ }
+ };
+ cl.setVMInitializedForTests();
+ cl.setMaxRetries(2);
+ return cl;
+ }
+
+ });
+ return acl;
+ }
+ }
+
@Rule
- public TemporaryFolder temp = new TemporaryFolder();
+ public TemporaryFolder TEMP =
+ new TemporaryFolder(new File(System.getProperty("user.dir") + "/target"));
private static final Contexts c = new Contexts();
+ private static File foo = new File(System.getProperty("user.dir") + "/target/foo");
+ private static File bar = new File(System.getProperty("user.dir") + "/target/bar");
+
@BeforeClass
public static void setup() throws Exception {
+
+ assertTrue(foo.mkdir());
+ assertTrue(bar.mkdir());
+
+ System.setProperty(AccumuloVFSClassLoader.VFS_CLASSLOADER_DEBUG, "true");
ContextConfig cc1 = new ContextConfig();
- cc1.setClassPath("file:///tmp/foo");
+ cc1.setClassPath(foo.toURI() + ".*");
cc1.setPostDelegate(true);
- cc1.setMonitorIntervalMs(30000);
+ cc1.setMonitorIntervalMs(1000);
Context c1 = new Context();
c1.setName("cx1");
c1.setConfig(cc1);
ContextConfig cc2 = new ContextConfig();
- cc2.setClassPath("file:///tmp/bar");
+ cc2.setClassPath(bar.toURI() + ".*");
cc2.setPostDelegate(false);
- cc2.setMonitorIntervalMs(30000);
+ cc2.setMonitorIntervalMs(1000);
Context c2 = new Context();
c2.setName("cx2");
c2.setConfig(cc2);
@@ -85,7 +157,13 @@
@Test
public void testCreation() throws Exception {
- File f = temp.newFile();
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ new File(foo, "HelloWorld.jar"));
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ new File(bar, "HelloWorld2.jar"));
+
+ File f = TEMP.newFile();
f.deleteOnExit();
Gson g = new Gson();
String contexts = g.toJson(c);
@@ -95,7 +173,7 @@
ReloadingVFSContextClassLoaderFactory cl = new ReloadingVFSContextClassLoaderFactory() {
@Override
protected String getConfigFileLocation() {
- return f.getAbsolutePath();
+ return f.toURI().toString();
}
};
cl.initialize(null);
@@ -107,7 +185,68 @@
}
cl.getClassLoader("cx1");
cl.getClassLoader("cx2");
+ cl.closeForTests();
+ }
+ @Test
+ public void testReloading() throws Exception {
+
+ System.setProperty(AccumuloVFSClassLoader.VFS_CLASSPATH_MONITOR_INTERVAL, "1");
+
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ new File(foo, "HelloWorld.jar"));
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ new File(bar, "HelloWorld2.jar"));
+
+ File f = TEMP.newFile();
+ f.deleteOnExit();
+ Gson g = new Gson();
+ String contexts = g.toJson(c);
+ try (BufferedWriter writer = Files.newBufferedWriter(f.toPath(), UTF_8, WRITE)) {
+ writer.write(contexts);
+ }
+
+ TestReloadingVFSContextClassLoaderFactory factory =
+ new TestReloadingVFSContextClassLoaderFactory(foo.toURI() + ".*") {
+ @Override
+ protected String getConfigFileLocation() {
+ return f.toURI().toString();
+ }
+ };
+ factory.initialize(null);
+
+ ClassLoader cl1 = factory.getClassLoader("cx1");
+ Class<?> clazz1 = cl1.loadClass("test.HelloWorld");
+ Object o1 = clazz1.getDeclaredConstructor().newInstance();
+ assertEquals("Hello World!", o1.toString());
+
+ // Check that the class is the same before the update
+ Class<?> clazz1_5 = cl1.loadClass("test.HelloWorld");
+ assertEquals(clazz1, clazz1_5);
+
+ assertTrue(new File(foo, "HelloWorld.jar").delete());
+
+ Thread.sleep(1000);
+
+ // Update the class
+ FileUtils.copyURLToFile(this.getClass().getResource("/HelloWorld.jar"),
+ new File(foo, "HelloWorld2.jar"));
+
+ // Wait for the monitor to notice
+ Thread.sleep(1000);
+
+ ClassLoader cl2 = factory.getClassLoader("cx1");
+ assertNotEquals(cl1, cl2);
+
+ Class<?> clazz2 = factory.getClassLoader("cx1").loadClass("test.HelloWorld");
+ Object o2 = clazz2.getDeclaredConstructor().newInstance();
+ assertEquals("Hello World!", o2.toString());
+
+ // This is false because they are loaded by a different classloader
+ assertNotEquals(clazz1, clazz2);
+ assertNotEquals(o1, o2);
+
+ factory.closeForTests();
}
}
diff --git a/modules/vfs-class-loader/src/test/resources/log4j2-test.properties b/modules/vfs-class-loader/src/test/resources/log4j2-test.properties
index 6dcf0c5..efe0a38 100644
--- a/modules/vfs-class-loader/src/test/resources/log4j2-test.properties
+++ b/modules/vfs-class-loader/src/test/resources/log4j2-test.properties
@@ -27,9 +27,6 @@
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{ISO8601} [%-8c{2}] %-5p: %m%n
-logger.01.name = org.apache.accumulo.server.util.TabletIterator
-logger.01.level = error
-
rootLogger.level = info
rootLogger.appenderRef.console.ref = STDOUT
diff --git a/modules/vfs-class-loader/test-configurations/new-context-configuration.json b/modules/vfs-class-loader/test-configurations/new-context-configuration.json
new file mode 100644
index 0000000..eacd78f
--- /dev/null
+++ b/modules/vfs-class-loader/test-configurations/new-context-configuration.json
@@ -0,0 +1,20 @@
+{
+ "contexts": [
+ {
+ "name": "cxA",
+ "config": {
+ "classPath": "hdfs://localhost:9000/iterators/example-a/examples.jar",
+ "postDelegate": true,
+ "monitorIntervalMs": 10000
+ }
+ },
+ {
+ "name": "cxB",
+ "config": {
+ "classPath": "hdfs://localhost:9000/iterators/example-b/examples.jar",
+ "postDelegate": true,
+ "monitorIntervalMs": 10000
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index f91a08b..6690863 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,8 @@
</mailingList>
</mailingLists>
<modules>
- <module>modules/classloaderfactory-substitute</module>
+ <module>modules/example-iterators-a</module>
+ <module>modules/example-iterators-b</module>
<module>modules/vfs-class-loader</module>
</modules>
<scm>
@@ -97,14 +98,120 @@
<project.build.outputTimestamp>2020-08-27T15:56:15Z</project.build.outputTimestamp>
<sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor>
</properties>
- <dependencies>
- <!-- spotbugs-annotations provides SuppressFBWarnings annotation -->
- <dependency>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-annotations</artifactId>
- <version>4.1.2</version>
- </dependency>
- </dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <!-- spotbugs-annotations provides SuppressFBWarnings annotation -->
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-annotations</artifactId>
+ <version>4.1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>3.7.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-vfs2</artifactId>
+ <version>2.6.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.6</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.7</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>1.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-core</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ <version>3.2.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>2.13.1</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.30</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-start</artifactId>
+ <version>2.1.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ <version>3.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ <version>3.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>3.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>2.13.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<build>
<pluginManagement>
<plugins>
@@ -121,6 +228,7 @@
<configuration>
<excludes combine.children="append">
<exclude>.github/**</exclude>
+ <exclude>**/*.json</exclude>
</excludes>
</configuration>
</plugin>
