| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.hadoop.conf; |
| |
| import java.io.BufferedInputStream; |
| import java.io.BufferedWriter; |
| import java.io.ByteArrayInputStream; |
| import java.io.ByteArrayOutputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStreamWriter; |
| import java.io.StringWriter; |
| import java.net.InetAddress; |
| import java.net.InetSocketAddress; |
| import java.net.URI; |
| import java.nio.charset.StandardCharsets; |
| import java.nio.file.Files; |
| import java.nio.file.Paths; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.ConcurrentModificationException; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Properties; |
| import java.util.Random; |
| import java.util.Set; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| import java.util.regex.Pattern; |
| import static java.util.concurrent.TimeUnit.*; |
| |
| import com.fasterxml.jackson.databind.ObjectMapper; |
| import org.apache.hadoop.fs.CommonConfigurationKeysPublic; |
| import org.assertj.core.api.Assertions; |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| import static org.apache.hadoop.conf.StorageUnit.BYTES; |
| import static org.apache.hadoop.conf.StorageUnit.GB; |
| import static org.apache.hadoop.conf.StorageUnit.KB; |
| import static org.apache.hadoop.conf.StorageUnit.MB; |
| import static org.apache.hadoop.conf.StorageUnit.TB; |
| import static org.junit.Assert.*; |
| |
| import org.apache.commons.lang3.StringUtils; |
| import org.apache.hadoop.conf.Configuration.IntegerRanges; |
| import org.apache.hadoop.fs.FileUtil; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.logging.LogCapturer; |
| import org.apache.hadoop.net.NetUtils; |
| import org.apache.hadoop.security.alias.CredentialProvider; |
| import org.apache.hadoop.security.alias.CredentialProviderFactory; |
| import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider; |
| import org.apache.hadoop.test.GenericTestUtils; |
| |
| import static org.apache.hadoop.util.PlatformName.IBM_JAVA; |
| |
| import org.mockito.Mockito; |
| import org.slf4j.LoggerFactory; |
| |
| public class TestConfiguration { |
| |
| private static final double DOUBLE_DELTA = 0.000000001f; |
| private Configuration conf; |
| final static String CONFIG = new File("./test-config-TestConfiguration.xml").getAbsolutePath(); |
| final static String CONFIG2 = new File("./test-config2-TestConfiguration.xml").getAbsolutePath(); |
| final static String CONFIG_CORE = new File("./core-site.xml") |
| .getAbsolutePath(); |
| final static String CONFIG_FOR_ENUM = new File("./test-config-enum-TestConfiguration.xml").getAbsolutePath(); |
| final static String CONFIG_FOR_URI = new File( |
| "./test-config-uri-TestConfiguration.xml").toURI().toString(); |
| |
| private static final String CONFIG_MULTI_BYTE = new File( |
| "./test-config-multi-byte-TestConfiguration.xml").getAbsolutePath(); |
| private static final String CONFIG_MULTI_BYTE_SAVED = new File( |
| "./test-config-multi-byte-saved-TestConfiguration.xml").getAbsolutePath(); |
| final static Random RAN = new Random(); |
| final static String XMLHEADER = |
| IBM_JAVA?"<?xml version=\"1.0\" encoding=\"UTF-8\"?><configuration>": |
| "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><configuration>"; |
| |
| /** Four apostrophes. */ |
| public static final String ESCAPED = "''''"; |
| |
| private static final String SENSITIVE_CONFIG_KEYS = |
| CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; |
| |
| private BufferedWriter out; |
| |
| @Before |
| public void setUp() throws Exception { |
| conf = new Configuration(); |
| } |
| |
| @After |
| public void tearDown() throws Exception { |
| if(out != null) { |
| out.close(); |
| } |
| new File(CONFIG).delete(); |
| new File(CONFIG2).delete(); |
| new File(CONFIG_FOR_ENUM).delete(); |
| new File(new URI(CONFIG_FOR_URI)).delete(); |
| new File(CONFIG_MULTI_BYTE).delete(); |
| new File(CONFIG_MULTI_BYTE_SAVED).delete(); |
| new File(CONFIG_CORE).delete(); |
| } |
| |
| private void startConfig() throws IOException{ |
| out.write("<?xml version=\"1.0\"?>\n"); |
| out.write("<configuration>\n"); |
| } |
| |
| private void writeHeader() throws IOException{ |
| out.write("<?xml version=\"1.0\"?>\n"); |
| } |
| |
| private void writeHeader(String encoding) throws IOException{ |
| out.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); |
| } |
| |
| private void writeConfiguration() throws IOException{ |
| out.write("<configuration>\n"); |
| } |
| |
| private void endConfig() throws IOException{ |
| out.write("</configuration>\n"); |
| out.flush(); |
| out.close(); |
| } |
| |
| private void startInclude(String filename) throws IOException { |
| out.write("<xi:include href=\"" + filename + "\" xmlns:xi=\"http://www.w3.org/2001/XInclude\" >\n "); |
| } |
| |
| private void endInclude() throws IOException{ |
| out.write("</xi:include>\n "); |
| } |
| |
| private void startFallback() throws IOException { |
| out.write("<xi:fallback>\n "); |
| } |
| |
| private void endFallback() throws IOException { |
| out.write("</xi:fallback>\n "); |
| } |
| |
| private void declareEntity(String root, String entity, String value) |
| throws IOException { |
| out.write("<!DOCTYPE " + root |
| + " [\n<!ENTITY " + entity + " \"" + value + "\">\n]>"); |
| } |
| |
| private void declareSystemEntity(String root, String entity, String value) |
| throws IOException { |
| out.write("<!DOCTYPE " + root |
| + " [\n<!ENTITY " + entity + " SYSTEM \"" + value + "\">\n]>"); |
| } |
| |
| @Test |
| public void testInputStreamResource() throws Exception { |
| StringWriter writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "A", "A"); |
| endConfig(); |
| |
| InputStream in1 = Mockito.spy(new ByteArrayInputStream( |
| writer.toString().getBytes())); |
| Configuration conf = new Configuration(false); |
| conf.addResource(in1); |
| assertEquals("A", conf.get("prop")); |
| Mockito.verify(in1, Mockito.times(1)).close(); |
| InputStream in2 = new ByteArrayInputStream(writer.toString().getBytes()); |
| conf.addResource(in2); |
| assertEquals("A", conf.get("prop")); |
| } |
| |
| @Test |
| public void testFinalWarnings() throws Exception { |
| // Make a configuration file with a final property |
| StringWriter writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "A", "A", true); |
| endConfig(); |
| byte[] bytes = writer.toString().getBytes(); |
| InputStream in1 = new ByteArrayInputStream(bytes); |
| |
| // Make a second config file with a final property with a different value |
| writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "BB", "BB", true); |
| endConfig(); |
| byte[] bytes2 = writer.toString().getBytes(); |
| InputStream in2 = new ByteArrayInputStream(bytes2); |
| |
| // Attach our own log appender so we can verify output |
| LogCapturer logCapturer = LogCapturer.captureLogs(LoggerFactory.getLogger("root")); |
| |
| try { |
| // Add the 2 different resources - this should generate a warning |
| conf.addResource(in1); |
| conf.addResource(in2); |
| assertEquals("should see the first value", "A", conf.get("prop")); |
| |
| String renderedMessage = logCapturer.getOutput(); |
| assertTrue("did not see expected string inside message " + renderedMessage, |
| renderedMessage.contains( |
| "an attempt to override final parameter: " + "prop; Ignoring.")); |
| } finally { |
| // Make sure the appender is removed |
| logCapturer.stopCapturing(); |
| } |
| } |
| |
| @Test |
| public void testNoFinalWarnings() throws Exception { |
| // Make a configuration file with a final property |
| StringWriter writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "A", "A", true); |
| endConfig(); |
| byte[] bytes = writer.toString().getBytes(); |
| // The 2 input streams both have the same config file |
| InputStream in1 = new ByteArrayInputStream(bytes); |
| InputStream in2 = new ByteArrayInputStream(bytes); |
| |
| // Attach our own log appender so we can verify output |
| LogCapturer logCapturer = LogCapturer.captureLogs(LoggerFactory.getLogger("root")); |
| |
| try { |
| // Add the resource twice from a stream - should not generate warnings |
| conf.addResource(in1); |
| conf.addResource(in2); |
| assertEquals("A", conf.get("prop")); |
| |
| String appenderOutput = logCapturer.getOutput(); |
| assertTrue("adding same resource twice should not cause logging", |
| appenderOutput.isEmpty()); |
| } finally { |
| // Make sure the appender is removed |
| logCapturer.stopCapturing(); |
| } |
| } |
| |
| @Test |
| public void testFinalWarningsMultiple() throws Exception { |
| // Make a configuration file with a repeated final property |
| StringWriter writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "A", "A", true); |
| declareProperty("prop", "A", "A", true); |
| endConfig(); |
| byte[] bytes = writer.toString().getBytes(); |
| InputStream in1 = new ByteArrayInputStream(bytes); |
| |
| // Attach our own log appender so we can verify output |
| LogCapturer logCapturer = LogCapturer.captureLogs(LoggerFactory.getLogger("root")); |
| |
| try { |
| // Add the resource - this should not produce a warning |
| conf.addResource(in1); |
| assertEquals("should see the value", "A", conf.get("prop")); |
| |
| String appenderOutput = logCapturer.getOutput(); |
| assertTrue("adding same resource twice should not cause logging", |
| appenderOutput.isEmpty()); |
| } finally { |
| // Make sure the appender is removed |
| logCapturer.stopCapturing(); |
| } |
| } |
| |
| @Test |
| public void testFinalWarningsMultipleOverride() throws Exception { |
| // Make a configuration file with 2 final properties with different values |
| StringWriter writer = new StringWriter(); |
| out = new BufferedWriter(writer); |
| startConfig(); |
| declareProperty("prop", "A", "A", true); |
| declareProperty("prop", "BB", "BB", true); |
| endConfig(); |
| byte[] bytes = writer.toString().getBytes(); |
| InputStream in1 = new ByteArrayInputStream(bytes); |
| |
| // Attach our own log appender so we can verify output |
| LogCapturer logCapturer = LogCapturer.captureLogs(LoggerFactory.getLogger("root")); |
| |
| try { |
| // Add the resource - this should produce a warning |
| conf.addResource(in1); |
| assertEquals("should see the value", "A", conf.get("prop")); |
| |
| String renderedMessage = logCapturer.getOutput(); |
| assertTrue("did not see expected string inside message " + renderedMessage, |
| renderedMessage.contains( |
| "an attempt to override final parameter: " + "prop; Ignoring.")); |
| } finally { |
| // Make sure the appender is removed |
| logCapturer.stopCapturing(); |
| } |
| } |
| |
| /** |
| * Tests use of multi-byte characters in property names and values. This test |
| * round-trips multi-byte string literals through saving and loading of config |
| * and asserts that the same values were read. |
| */ |
| @Test |
| public void testMultiByteCharacters() throws IOException { |
| String priorDefaultEncoding = System.getProperty("file.encoding"); |
| try { |
| System.setProperty("file.encoding", "US-ASCII"); |
| String name = "multi_byte_\u611b_name"; |
| String value = "multi_byte_\u0641_value"; |
| out = new BufferedWriter(new OutputStreamWriter( |
| new FileOutputStream(CONFIG_MULTI_BYTE), "UTF-8")); |
| startConfig(); |
| declareProperty(name, value, value); |
| endConfig(); |
| |
| Configuration conf = new Configuration(false); |
| conf.addResource(new Path(CONFIG_MULTI_BYTE)); |
| assertEquals(value, conf.get(name)); |
| try (FileOutputStream fos = |
| new FileOutputStream(CONFIG_MULTI_BYTE_SAVED)) { |
| conf.writeXml(fos); |
| } |
| |
| conf = new Configuration(false); |
| conf.addResource(new Path(CONFIG_MULTI_BYTE_SAVED)); |
| assertEquals(value, conf.get(name)); |
| } finally { |
| System.setProperty("file.encoding", priorDefaultEncoding); |
| } |
| } |
| |
| @Test |
| public void testVariableSubstitution() throws IOException { |
| // stubbing only environment dependent functions |
| Configuration mock = Mockito.spy(conf); |
| Mockito.when(mock.getProperty("user.name")).thenReturn("hadoop_user"); |
| Mockito.when(mock.getenv("FILE_NAME")).thenReturn("hello"); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| declareProperty("my.int", "${intvar}", "42"); |
| declareProperty("intvar", "42", "42"); |
| declareProperty("my.base", "/tmp/${user.name}", "/tmp/hadoop_user"); |
| declareProperty("my.file", "${env.FILE_NAME}", "hello"); |
| declareProperty("my.suffix", ".txt", ".txt"); |
| declareProperty("my.relfile", "${my.file}${my.suffix}", "hello.txt"); |
| declareProperty("my.fullfile", "${my.base}/${my.file}${my.suffix}", "/tmp/hadoop_user/hello.txt"); |
| // check that undefined variables are returned as-is |
| declareProperty("my.failsexpand", "a${my.undefvar}b", "a${my.undefvar}b"); |
| // check that multiple variable references are resolved |
| declareProperty("my.user.group", "${user.name} ${user.name}", |
| "hadoop_user hadoop_user"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| mock.addResource(fileResource); |
| |
| for (Prop p : props) { |
| System.out.println("p=" + p.name); |
| String gotVal = mock.get(p.name); |
| String gotRawVal = mock.getRaw(p.name); |
| assertEq(p.val, gotRawVal); |
| assertEq(p.expectEval, gotVal); |
| } |
| |
| // check that expansion also occurs for getInt() |
| assertTrue(mock.getInt("intvar", -1) == 42); |
| assertTrue(mock.getInt("my.int", -1) == 42); |
| } |
| |
| /** |
| * Checks if variable substitution is accessible via a public API. |
| */ |
| @Test |
| public void testCommonVariableSubstitution() { |
| conf.set("intvar", String.valueOf(42)); |
| String intVar = conf.substituteCommonVariables("${intvar}"); |
| |
| assertEquals("42", intVar); |
| } |
| |
| @Test |
| public void testEnvDefault() throws IOException { |
| Configuration mock = Mockito.spy(conf); |
| Mockito.when(mock.getenv("NULL_VALUE")).thenReturn(null); |
| Mockito.when(mock.getenv("EMPTY_VALUE")).thenReturn(""); |
| Mockito.when(mock.getenv("SOME_VALUE")).thenReturn("some value"); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| |
| // if var is unbound, literal ${var} is returned |
| declareProperty("null1", "${env.NULL_VALUE}", "${env.NULL_VALUE}"); |
| declareProperty("null2", "${env.NULL_VALUE-a}", "a"); |
| declareProperty("null3", "${env.NULL_VALUE:-b}", "b"); |
| declareProperty("empty1", "${env.EMPTY_VALUE}", ""); |
| declareProperty("empty2", "${env.EMPTY_VALUE-c}", ""); |
| declareProperty("empty3", "${env.EMPTY_VALUE:-d}", "d"); |
| declareProperty("some1", "${env.SOME_VALUE}", "some value"); |
| declareProperty("some2", "${env.SOME_VALUE-e}", "some value"); |
| declareProperty("some3", "${env.SOME_VALUE:-f}", "some value"); |
| |
| // some edge cases |
| declareProperty("edge1", "${env.NULL_VALUE-g-h}", "g-h"); |
| declareProperty("edge2", "${env.NULL_VALUE:-i:-j}", "i:-j"); |
| declareProperty("edge3", "${env.NULL_VALUE-}", ""); |
| declareProperty("edge4", "${env.NULL_VALUE:-}", ""); |
| declareProperty("edge5", "${env.NULL_VALUE:}", "${env.NULL_VALUE:}"); |
| |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| mock.addResource(fileResource); |
| |
| for (Prop p : props) { |
| System.out.println("p=" + p.name); |
| String gotVal = mock.get(p.name); |
| String gotRawVal = mock.getRaw(p.name); |
| assertEq(p.val, gotRawVal); |
| assertEq(p.expectEval, gotVal); |
| } |
| } |
| |
| /** |
| * Verify that when a configuration is restricted, environment |
| * variables and system properties will be unresolved. |
| * The fallback patterns for the variables are still parsed. |
| */ |
| @Test |
| public void testRestrictedEnv() throws IOException { |
| // this test relies on env.PATH being set on all platforms a |
| // test run will take place on, and the java.version sysprop |
| // set in all JVMs. |
| // Restricted configurations will not get access to these values, so |
| // will either be unresolved or, for env vars with fallbacks: the fallback |
| // values. |
| |
| conf.setRestrictSystemProperties(true); |
| |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| // a simple property to reference |
| declareProperty("d", "D", "D"); |
| |
| // system property evaluation stops working completely |
| declareProperty("system1", "${java.version}", "${java.version}"); |
| |
| // the env variable does not resolve |
| declareProperty("secret1", "${env.PATH}", "${env.PATH}"); |
| |
| // but all the fallback options do work |
| declareProperty("secret2", "${env.PATH-a}", "a"); |
| declareProperty("secret3", "${env.PATH:-b}", "b"); |
| declareProperty("secret4", "${env.PATH:-}", ""); |
| declareProperty("secret5", "${env.PATH-}", ""); |
| // special case |
| declareProperty("secret6", "${env.PATH:}", "${env.PATH:}"); |
| // safety check |
| declareProperty("secret7", "${env.PATH:--}", "-"); |
| |
| // recursive eval of the fallback |
| declareProperty("secret8", "${env.PATH:-${d}}", "D"); |
| |
| // if the fallback doesn't resolve, the result is the whole variable raw. |
| declareProperty("secret9", "${env.PATH:-$d}}", "${env.PATH:-$d}}"); |
| |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| for (Prop p : props) { |
| System.out.println("p=" + p.name); |
| String gotVal = conf.get(p.name); |
| String gotRawVal = conf.getRaw(p.name); |
| assertEq(p.val, gotRawVal); |
| assertEq(p.expectEval, gotVal); |
| } |
| } |
| |
| @Test |
| public void testFinalParam() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| declareProperty("my.var", "", "", true); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| Configuration conf1 = new Configuration(); |
| conf1.addResource(fileResource); |
| assertNull("my var is not null", conf1.get("my.var")); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| declareProperty("my.var", "myval", "myval", false); |
| endConfig(); |
| fileResource = new Path(CONFIG2); |
| |
| Configuration conf2 = new Configuration(conf1); |
| conf2.addResource(fileResource); |
| assertNull("my var is not final", conf2.get("my.var")); |
| } |
| |
| @Test |
| public void testCompactFormat() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendCompactFormatProperty("a", "b"); |
| appendCompactFormatProperty("c", "d", true); |
| appendCompactFormatProperty("e", "f", false, "g"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| Configuration conf = new Configuration(false); |
| conf.addResource(fileResource); |
| |
| assertEquals("b", conf.get("a")); |
| |
| assertEquals("d", conf.get("c")); |
| Set<String> s = conf.getFinalParameters(); |
| assertEquals(1, s.size()); |
| assertTrue(s.contains("c")); |
| |
| assertEquals("f", conf.get("e")); |
| String[] sources = conf.getPropertySources("e"); |
| assertEquals(2, sources.length); |
| assertEquals("g", sources[0]); |
| assertEquals(fileResource.toString(), sources[1]); |
| } |
| |
| public static void assertEq(Object a, Object b) { |
| System.out.println("assertEq: " + a + ", " + b); |
| assertEquals(a, b); |
| } |
| |
| static class Prop { |
| String name; |
| String val; |
| String expectEval; |
| } |
| |
| ArrayList<Prop> props = new ArrayList<Prop>(); |
| |
| void declareProperty(String name, String val, String expectEval) |
| throws IOException { |
| declareProperty(name, val, expectEval, false); |
| } |
| |
| void declareProperty(String name, String val, String expectEval, |
| boolean isFinal) |
| throws IOException { |
| appendProperty(name, val, isFinal); |
| Prop p = new Prop(); |
| p.name = name; |
| p.val = val; |
| p.expectEval = expectEval; |
| props.add(p); |
| } |
| |
| void appendProperty(String name, String val) throws IOException { |
| appendProperty(name, val, false); |
| } |
| |
| void appendProperty(String name, String val, boolean isFinal, |
| String ... sources) |
| throws IOException { |
| out.write("<property>"); |
| out.write("<name>"); |
| out.write(name); |
| out.write("</name>"); |
| out.write("<value>"); |
| out.write(val); |
| out.write("</value>"); |
| if (isFinal) { |
| out.write("<final>true</final>"); |
| } |
| for(String s : sources) { |
| out.write("<source>"); |
| out.write(s); |
| out.write("</source>"); |
| } |
| out.write("</property>\n"); |
| } |
| |
| private void appendPropertyByTag(String name, String val, String tags, |
| String... sources) throws IOException { |
| appendPropertyByTag(name, val, false, tags, sources); |
| } |
| |
| private void appendPropertyByTag(String name, String val, boolean isFinal, |
| String tag, String... sources) throws IOException { |
| out.write("<property>"); |
| out.write("<name>"); |
| out.write(name); |
| out.write("</name>"); |
| out.write("<value>"); |
| out.write(val); |
| out.write("</value>"); |
| if (isFinal) { |
| out.write("<final>true</final>"); |
| } |
| for (String s : sources) { |
| out.write("<source>"); |
| out.write(s); |
| out.write("</source>"); |
| } |
| out.write("<tag>"); |
| out.write(tag); |
| out.write("</tag>"); |
| out.write("</property>\n"); |
| } |
| |
| void appendCompactFormatProperty(String name, String val) throws IOException { |
| appendCompactFormatProperty(name, val, false); |
| } |
| |
| void appendCompactFormatProperty(String name, String val, boolean isFinal) |
| throws IOException { |
| appendCompactFormatProperty(name, val, isFinal, null); |
| } |
| |
| void appendCompactFormatProperty(String name, String val, boolean isFinal, |
| String source) |
| throws IOException { |
| out.write("<property "); |
| out.write("name=\""); |
| out.write(name); |
| out.write("\" "); |
| out.write("value=\""); |
| out.write(val); |
| out.write("\" "); |
| if (isFinal) { |
| out.write("final=\"true\" "); |
| } |
| if (source != null) { |
| out.write("source=\""); |
| out.write(source); |
| out.write("\" "); |
| } |
| out.write("/>\n"); |
| } |
| |
| @Test |
| public void testOverlay() throws IOException{ |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("a","b"); |
| appendProperty("b","c"); |
| appendProperty("d","e"); |
| appendProperty("e","f", true); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("a","b"); |
| appendProperty("b","d"); |
| appendProperty("e","e"); |
| endConfig(); |
| |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| //set dynamically something |
| conf.set("c","d"); |
| conf.set("a","d"); |
| |
| Configuration clone=new Configuration(conf); |
| clone.addResource(new Path(CONFIG2)); |
| |
| assertEquals(clone.get("a"), "d"); |
| assertEquals(clone.get("b"), "d"); |
| assertEquals(clone.get("c"), "d"); |
| assertEquals(clone.get("d"), "e"); |
| assertEquals(clone.get("e"), "f"); |
| |
| } |
| |
| @Test |
| public void testCommentsInValue() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("my.comment", "this <!--comment here--> contains a comment"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| //two spaces one after "this", one before "contains" |
| assertEquals("this contains a comment", conf.get("my.comment")); |
| } |
| |
| @Test |
| public void testEscapedCharactersInValue() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("my.comment", ESCAPED); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| //two spaces one after "this", one before "contains" |
| assertEquals("''''", conf.get("my.comment")); |
| } |
| |
| @Test |
| public void testTrim() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| String[] whitespaces = {"", " ", "\n", "\t"}; |
| String[] name = new String[100]; |
| for(int i = 0; i < name.length; i++) { |
| name[i] = "foo" + i; |
| StringBuilder prefix = new StringBuilder(); |
| StringBuilder postfix = new StringBuilder(); |
| for(int j = 0; j < 3; j++) { |
| prefix.append(whitespaces[RAN.nextInt(whitespaces.length)]); |
| postfix.append(whitespaces[RAN.nextInt(whitespaces.length)]); |
| } |
| |
| appendProperty(prefix + name[i] + postfix, name[i] + ".value"); |
| } |
| endConfig(); |
| |
| conf.addResource(new Path(CONFIG)); |
| for(String n : name) { |
| assertEquals(n + ".value", conf.get(n)); |
| } |
| } |
| |
| @Test |
| public void testGetLocalPath() throws IOException { |
| Configuration conf = new Configuration(); |
| String[] dirs = new String[]{"a", "b", "c"}; |
| for (int i = 0; i < dirs.length; i++) { |
| dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString(); |
| } |
| conf.set("dirs", StringUtils.join(dirs, ",")); |
| for (int i = 0; i < 1000; i++) { |
| String localPath = conf.getLocalPath("dirs", "dir" + i).toString(); |
| assertTrue("Path doesn't end in specified dir: " + localPath, |
| localPath.endsWith("dir" + i)); |
| assertFalse("Path has internal whitespace: " + localPath, |
| localPath.contains(" ")); |
| } |
| } |
| |
| @Test |
| public void testGetFile() throws IOException { |
| Configuration conf = new Configuration(); |
| String[] dirs = new String[]{"a", "b", "c"}; |
| for (int i = 0; i < dirs.length; i++) { |
| dirs[i] = new Path(GenericTestUtils.getTempPath(dirs[i])).toString(); |
| } |
| conf.set("dirs", StringUtils.join(dirs, ",")); |
| for (int i = 0; i < 1000; i++) { |
| String localPath = conf.getFile("dirs", "dir" + i).toString(); |
| assertTrue("Path doesn't end in specified dir: " + localPath, |
| localPath.endsWith("dir" + i)); |
| assertFalse("Path has internal whitespace: " + localPath, |
| localPath.contains(" ")); |
| } |
| } |
| |
| @Test |
| public void testToString() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| String expectedOutput = |
| "Configuration: core-default.xml, core-site.xml, " + |
| fileResource.toString(); |
| assertEquals(expectedOutput, conf.toString()); |
| } |
| |
| @Test |
| public void testWriteXml() throws IOException { |
| Configuration conf = new Configuration(); |
| ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
| conf.writeXml(baos); |
| String result = baos.toString(); |
| assertTrue("Result has proper header", result.startsWith(XMLHEADER)); |
| |
| assertTrue("Result has proper footer", result.endsWith("</configuration>")); |
| } |
| |
| @Test |
| public void testIncludes() throws Exception { |
| tearDown(); |
| System.out.println("XXX testIncludes"); |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("a","b"); |
| appendProperty("c","d"); |
| endConfig(); |
| File fileUri = new File(new URI(CONFIG_FOR_URI)); |
| out=new BufferedWriter(new FileWriter(fileUri)); |
| startConfig(); |
| appendProperty("e", "f"); |
| appendProperty("g", "h"); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| startInclude(CONFIG2); |
| endInclude(); |
| startInclude(CONFIG_FOR_URI); |
| endInclude(); |
| appendProperty("i", "j"); |
| appendProperty("k", "l"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(conf.get("a"), "b"); |
| assertEquals(conf.get("c"), "d"); |
| assertEquals(conf.get("e"), "f"); |
| assertEquals(conf.get("g"), "h"); |
| assertEquals(conf.get("i"), "j"); |
| assertEquals(conf.get("k"), "l"); |
| tearDown(); |
| } |
| |
| @Test |
| public void testCharsetInDocumentEncoding() throws Exception { |
| tearDown(); |
| out=new BufferedWriter(new OutputStreamWriter(new FileOutputStream(CONFIG), |
| StandardCharsets.ISO_8859_1)); |
| writeHeader(StandardCharsets.ISO_8859_1.displayName()); |
| writeConfiguration(); |
| appendProperty("a", "b"); |
| appendProperty("c", "Müller"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(conf.get("a"), "b"); |
| assertEquals(conf.get("c"), "Müller"); |
| tearDown(); |
| } |
| |
| @Test |
| public void testEntityReference() throws Exception { |
| tearDown(); |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| writeHeader(); |
| declareEntity("configuration", "d", "d"); |
| writeConfiguration(); |
| appendProperty("a", "b"); |
| appendProperty("c", "&d;"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(conf.get("a"), "b"); |
| assertEquals(conf.get("c"), "d"); |
| tearDown(); |
| } |
| |
| @Test |
| public void testSystemEntityReference() throws Exception { |
| tearDown(); |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| out.write("d"); |
| out.close(); |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| writeHeader(); |
| declareSystemEntity("configuration", "d", |
| new Path(CONFIG2).toUri().toString()); |
| writeConfiguration(); |
| appendProperty("a", "b"); |
| appendProperty("c", "&d;"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(conf.get("a"), "b"); |
| assertEquals(conf.get("c"), "d"); |
| tearDown(); |
| } |
| |
| @Test |
| public void testIncludesWithFallback() throws Exception { |
| tearDown(); |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("a","b"); |
| appendProperty("c","d"); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| startInclude(CONFIG2); |
| startFallback(); |
| appendProperty("a", "b.fallback"); |
| appendProperty("c", "d.fallback", true); |
| endFallback(); |
| endInclude(); |
| appendProperty("e","f"); |
| appendProperty("g","h"); |
| startInclude("MissingConfig.xml"); |
| startFallback(); |
| appendProperty("i", "j.fallback"); |
| appendProperty("k", "l.fallback", true); |
| endFallback(); |
| endInclude(); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals("b", conf.get("a")); |
| assertEquals("d", conf.get("c")); |
| assertEquals("f", conf.get("e")); |
| assertEquals("h", conf.get("g")); |
| assertEquals("j.fallback", conf.get("i")); |
| assertEquals("l.fallback", conf.get("k")); |
| tearDown(); |
| } |
| |
| // When a resource is parsed as an input stream the first time, included |
| // properties are saved within the config. However, the included properties |
| // are not cached in the resource object. So, if an additional resource is |
| // added after the config is parsed the first time, the config loses the |
| // prperties that were included from the first resource. |
| @Test |
| public void testIncludesFromInputStreamWhenResourceAdded() throws Exception { |
| tearDown(); |
| |
| // CONFIG includes CONFIG2. CONFIG2 includes CONFIG_FOR_ENUM |
| out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM)); |
| startConfig(); |
| appendProperty("e", "SecondLevelInclude"); |
| appendProperty("f", "SecondLevelInclude"); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| startInclude(CONFIG_FOR_ENUM); |
| endInclude(); |
| appendProperty("c","FirstLevelInclude"); |
| appendProperty("d","FirstLevelInclude"); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| startInclude(CONFIG2); |
| endInclude(); |
| appendProperty("a", "1"); |
| appendProperty("b", "2"); |
| endConfig(); |
| |
| // Add CONFIG as an InputStream resource. |
| File file = new File(CONFIG); |
| BufferedInputStream bis = |
| new BufferedInputStream(new FileInputStream(file)); |
| conf.addResource(bis); |
| |
| // The first time the conf is parsed, verify that all properties were read |
| // from all levels of includes. |
| assertEquals("1", conf.get("a")); |
| assertEquals("2", conf.get("b")); |
| assertEquals("FirstLevelInclude", conf.get("c")); |
| assertEquals("FirstLevelInclude", conf.get("d")); |
| assertEquals("SecondLevelInclude", conf.get("e")); |
| assertEquals("SecondLevelInclude", conf.get("f")); |
| |
| // Add another resource to the conf. |
| out=new BufferedWriter(new FileWriter(CONFIG_MULTI_BYTE)); |
| startConfig(); |
| appendProperty("g", "3"); |
| appendProperty("h", "4"); |
| endConfig(); |
| |
| Path fileResource = new Path(CONFIG_MULTI_BYTE); |
| conf.addResource(fileResource); |
| |
| // Verify that all properties were read from all levels of includes the |
| // second time the conf is parsed. |
| assertEquals("1", conf.get("a")); |
| assertEquals("2", conf.get("b")); |
| assertEquals("FirstLevelInclude", conf.get("c")); |
| assertEquals("FirstLevelInclude", conf.get("d")); |
| assertEquals("SecondLevelInclude", conf.get("e")); |
| assertEquals("SecondLevelInclude", conf.get("f")); |
| assertEquals("3", conf.get("g")); |
| assertEquals("4", conf.get("h")); |
| |
| tearDown(); |
| } |
| |
| @Test |
| public void testOrderOfDuplicatePropertiesWithInclude() throws Exception { |
| tearDown(); |
| |
| // Property "a" is set to different values inside and outside of includes. |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("a", "a-InsideInclude"); |
| appendProperty("b", "b-InsideInclude"); |
| endConfig(); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("a","a-OutsideInclude"); |
| startInclude(CONFIG2); |
| endInclude(); |
| appendProperty("b","b-OutsideInclude"); |
| endConfig(); |
| |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| assertEquals("a-InsideInclude", conf.get("a")); |
| assertEquals("b-OutsideInclude", conf.get("b")); |
| |
| tearDown(); |
| } |
| |
| @Test |
| public void testRelativeIncludes() throws Exception { |
| tearDown(); |
| String relConfig = new File("./tmp/test-config.xml").getAbsolutePath(); |
| String relConfig2 = new File("./tmp/test-config2.xml").getAbsolutePath(); |
| |
| new File(new File(relConfig).getParent()).mkdirs(); |
| out = new BufferedWriter(new FileWriter(relConfig2)); |
| startConfig(); |
| appendProperty("a", "b"); |
| endConfig(); |
| |
| out = new BufferedWriter(new FileWriter(relConfig)); |
| startConfig(); |
| // Add the relative path instead of the absolute one. |
| startInclude(new File(relConfig2).getName()); |
| endInclude(); |
| appendProperty("c", "d"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(relConfig); |
| conf.addResource(fileResource); |
| assertEquals(conf.get("a"), "b"); |
| assertEquals(conf.get("c"), "d"); |
| |
| // Cleanup |
| new File(relConfig).delete(); |
| new File(relConfig2).delete(); |
| new File(new File(relConfig).getParent()).delete(); |
| } |
| |
| @Test |
| public void testRelativeIncludesWithLoadingViaUri() throws Exception { |
| tearDown(); |
| File configFile = new File("./tmp/test-config.xml"); |
| File configFile2 = new File("./tmp/test-config2.xml"); |
| |
| new File(configFile.getParent()).mkdirs(); |
| out = new BufferedWriter(new FileWriter(configFile2)); |
| startConfig(); |
| appendProperty("a", "b"); |
| endConfig(); |
| |
| out = new BufferedWriter(new FileWriter(configFile)); |
| startConfig(); |
| // Add the relative path instead of the absolute one. |
| startInclude(configFile2.getName()); |
| endInclude(); |
| appendProperty("c", "d"); |
| endConfig(); |
| |
| // verify that the includes file contains all properties |
| Path fileResource = new Path(configFile.toURI()); |
| conf.addResource(fileResource); |
| assertEquals("b", conf.get("a")); |
| assertEquals("d", conf.get("c")); |
| |
| // Cleanup |
| configFile.delete(); |
| configFile2.delete(); |
| new File(configFile.getParent()).delete(); |
| } |
| |
| @Test |
| public void testIntegerRanges() { |
| Configuration conf = new Configuration(); |
| conf.set("first", "-100"); |
| conf.set("second", "4-6,9-10,27"); |
| conf.set("third", "34-"); |
| Configuration.IntegerRanges range = conf.getRange("first", null); |
| System.out.println("first = " + range); |
| assertEquals(true, range.isIncluded(0)); |
| assertEquals(true, range.isIncluded(1)); |
| assertEquals(true, range.isIncluded(100)); |
| assertEquals(false, range.isIncluded(101)); |
| range = conf.getRange("second", null); |
| System.out.println("second = " + range); |
| assertEquals(false, range.isIncluded(3)); |
| assertEquals(true, range.isIncluded(4)); |
| assertEquals(true, range.isIncluded(6)); |
| assertEquals(false, range.isIncluded(7)); |
| assertEquals(false, range.isIncluded(8)); |
| assertEquals(true, range.isIncluded(9)); |
| assertEquals(true, range.isIncluded(10)); |
| assertEquals(false, range.isIncluded(11)); |
| assertEquals(false, range.isIncluded(26)); |
| assertEquals(true, range.isIncluded(27)); |
| assertEquals(false, range.isIncluded(28)); |
| range = conf.getRange("third", null); |
| System.out.println("third = " + range); |
| assertEquals(false, range.isIncluded(33)); |
| assertEquals(true, range.isIncluded(34)); |
| assertEquals(true, range.isIncluded(100000000)); |
| } |
| |
| @Test |
| public void testGetRangeIterator() throws Exception { |
| Configuration config = new Configuration(false); |
| IntegerRanges ranges = config.getRange("Test", ""); |
| assertFalse("Empty range has values", ranges.iterator().hasNext()); |
| ranges = config.getRange("Test", "5"); |
| Set<Integer> expected = new HashSet<Integer>(Arrays.asList(5)); |
| Set<Integer> found = new HashSet<Integer>(); |
| for(Integer i: ranges) { |
| found.add(i); |
| } |
| assertEquals(expected, found); |
| |
| ranges = config.getRange("Test", "5-10,13-14"); |
| expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,13,14)); |
| found = new HashSet<Integer>(); |
| for(Integer i: ranges) { |
| found.add(i); |
| } |
| assertEquals(expected, found); |
| |
| ranges = config.getRange("Test", "8-12, 5- 7"); |
| expected = new HashSet<Integer>(Arrays.asList(5,6,7,8,9,10,11,12)); |
| found = new HashSet<Integer>(); |
| for(Integer i: ranges) { |
| found.add(i); |
| } |
| assertEquals(expected, found); |
| } |
| |
| @Test |
| public void testHexValues() throws IOException{ |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.hex1", "0x10"); |
| appendProperty("test.hex2", "0xF"); |
| appendProperty("test.hex3", "-0x10"); |
| // Invalid? |
| appendProperty("test.hex4", "-0x10xyz"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(16, conf.getInt("test.hex1", 0)); |
| assertEquals(16, conf.getLong("test.hex1", 0)); |
| assertEquals(15, conf.getInt("test.hex2", 0)); |
| assertEquals(15, conf.getLong("test.hex2", 0)); |
| assertEquals(-16, conf.getInt("test.hex3", 0)); |
| assertEquals(-16, conf.getLong("test.hex3", 0)); |
| try { |
| conf.getLong("test.hex4", 0); |
| fail("Property had invalid long value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| try { |
| conf.getInt("test.hex4", 0); |
| fail("Property had invalid int value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| } |
| |
| @Test |
| public void testIntegerValues() throws IOException{ |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.int1", "20"); |
| appendProperty("test.int2", "020"); |
| appendProperty("test.int3", "-20"); |
| appendProperty("test.int4", " -20 "); |
| appendProperty("test.int5", " -20xyz "); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(20, conf.getInt("test.int1", 0)); |
| assertEquals(20, conf.getLong("test.int1", 0)); |
| assertEquals(20, conf.getLongBytes("test.int1", 0)); |
| assertEquals(20, conf.getInt("test.int2", 0)); |
| assertEquals(20, conf.getLong("test.int2", 0)); |
| assertEquals(20, conf.getLongBytes("test.int2", 0)); |
| assertEquals(-20, conf.getInt("test.int3", 0)); |
| assertEquals(-20, conf.getLong("test.int3", 0)); |
| assertEquals(-20, conf.getLongBytes("test.int3", 0)); |
| assertEquals(-20, conf.getInt("test.int4", 0)); |
| assertEquals(-20, conf.getLong("test.int4", 0)); |
| assertEquals(-20, conf.getLongBytes("test.int4", 0)); |
| try { |
| conf.getInt("test.int5", 0); |
| fail("Property had invalid int value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| } |
| |
| @Test |
| public void testHumanReadableValues() throws IOException { |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.humanReadableValue1", "1m"); |
| appendProperty("test.humanReadableValue2", "1M"); |
| appendProperty("test.humanReadableValue5", "1MBCDE"); |
| |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(1048576, conf.getLongBytes("test.humanReadableValue1", 0)); |
| assertEquals(1048576, conf.getLongBytes("test.humanReadableValue2", 0)); |
| try { |
| conf.getLongBytes("test.humanReadableValue5", 0); |
| fail("Property had invalid human readable value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| } |
| |
| @Test |
| public void testBooleanValues() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.bool1", "true"); |
| appendProperty("test.bool2", "false"); |
| appendProperty("test.bool3", " true "); |
| appendProperty("test.bool4", " false "); |
| appendProperty("test.bool5", "foo"); |
| appendProperty("test.bool6", "TRUE"); |
| appendProperty("test.bool7", "FALSE"); |
| appendProperty("test.bool8", ""); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(true, conf.getBoolean("test.bool1", false)); |
| assertEquals(false, conf.getBoolean("test.bool2", true)); |
| assertEquals(true, conf.getBoolean("test.bool3", false)); |
| assertEquals(false, conf.getBoolean("test.bool4", true)); |
| assertEquals(true, conf.getBoolean("test.bool5", true)); |
| assertEquals(true, conf.getBoolean("test.bool6", false)); |
| assertEquals(false, conf.getBoolean("test.bool7", true)); |
| assertEquals(false, conf.getBoolean("test.bool8", false)); |
| } |
| |
| @Test |
| public void testFloatValues() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.float1", "3.1415"); |
| appendProperty("test.float2", "003.1415"); |
| appendProperty("test.float3", "-3.1415"); |
| appendProperty("test.float4", " -3.1415 "); |
| appendProperty("test.float5", "xyz-3.1415xyz"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(3.1415f, conf.getFloat("test.float1", 0.0f), DOUBLE_DELTA); |
| assertEquals(3.1415f, conf.getFloat("test.float2", 0.0f), DOUBLE_DELTA); |
| assertEquals(-3.1415f, conf.getFloat("test.float3", 0.0f), DOUBLE_DELTA); |
| assertEquals(-3.1415f, conf.getFloat("test.float4", 0.0f), DOUBLE_DELTA); |
| try { |
| conf.getFloat("test.float5", 0.0f); |
| fail("Property had invalid float value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| } |
| |
| @Test |
| public void testDoubleValues() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.double1", "3.1415"); |
| appendProperty("test.double2", "003.1415"); |
| appendProperty("test.double3", "-3.1415"); |
| appendProperty("test.double4", " -3.1415 "); |
| appendProperty("test.double5", "xyz-3.1415xyz"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals(3.1415, conf.getDouble("test.double1", 0.0), DOUBLE_DELTA); |
| assertEquals(3.1415, conf.getDouble("test.double2", 0.0), DOUBLE_DELTA); |
| assertEquals(-3.1415, conf.getDouble("test.double3", 0.0), DOUBLE_DELTA); |
| assertEquals(-3.1415, conf.getDouble("test.double4", 0.0), DOUBLE_DELTA); |
| try { |
| conf.getDouble("test.double5", 0.0); |
| fail("Property had invalid double value, but was read successfully."); |
| } catch (NumberFormatException e) { |
| // pass |
| } |
| } |
| |
| @Test |
| public void testGetClass() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.class1", "java.lang.Integer"); |
| appendProperty("test.class2", " java.lang.Integer "); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals("java.lang.Integer", |
| conf.getClass("test.class1", null).getCanonicalName()); |
| assertEquals("java.lang.Integer", |
| conf.getClass("test.class2", null).getCanonicalName()); |
| } |
| |
| @Test |
| public void testGetClasses() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.classes1", "java.lang.Integer,java.lang.String"); |
| appendProperty("test.classes2", " java.lang.Integer , java.lang.String "); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| String[] expectedNames = {"java.lang.Integer", "java.lang.String"}; |
| Class<?>[] defaultClasses = {}; |
| Class<?>[] classes1 = conf.getClasses("test.classes1", defaultClasses); |
| Class<?>[] classes2 = conf.getClasses("test.classes2", defaultClasses); |
| assertArrayEquals(expectedNames, extractClassNames(classes1)); |
| assertArrayEquals(expectedNames, extractClassNames(classes2)); |
| } |
| |
| @Test |
| public void testGetStringCollection() { |
| Configuration c = new Configuration(); |
| c.set("x", " a, b\n,\nc "); |
| Collection<String> strs = c.getTrimmedStringCollection("x"); |
| assertEquals(3, strs.size()); |
| assertArrayEquals(new String[]{ "a", "b", "c" }, |
| strs.toArray(new String[0])); |
| |
| // Check that the result is mutable |
| strs.add("z"); |
| |
| // Make sure same is true for missing config |
| strs = c.getStringCollection("does-not-exist"); |
| assertEquals(0, strs.size()); |
| strs.add("z"); |
| } |
| |
| @Test |
| public void testGetTrimmedStringCollection() { |
| Configuration c = new Configuration(); |
| c.set("x", "a, b, c"); |
| Collection<String> strs = c.getStringCollection("x"); |
| assertEquals(3, strs.size()); |
| assertArrayEquals(new String[]{ "a", " b", " c" }, |
| strs.toArray(new String[0])); |
| |
| // Check that the result is mutable |
| strs.add("z"); |
| |
| // Make sure same is true for missing config |
| strs = c.getStringCollection("does-not-exist"); |
| assertEquals(0, strs.size()); |
| strs.add("z"); |
| } |
| |
| private static String[] extractClassNames(Class<?>[] classes) { |
| String[] classNames = new String[classes.length]; |
| for (int i = 0; i < classNames.length; i++) { |
| classNames[i] = classes[i].getCanonicalName(); |
| } |
| return classNames; |
| } |
| |
| enum Dingo { FOO, BAR }; |
| enum Yak { RAB, FOO }; |
| @Test |
| public void testEnum() { |
| Configuration conf = new Configuration(); |
| conf.setEnum("test.enum", Dingo.FOO); |
| assertSame(Dingo.FOO, conf.getEnum("test.enum", Dingo.BAR)); |
| assertSame(Yak.FOO, conf.getEnum("test.enum", Yak.RAB)); |
| conf.setEnum("test.enum", Dingo.FOO); |
| boolean fail = false; |
| try { |
| conf.setEnum("test.enum", Dingo.BAR); |
| conf.getEnum("test.enum", Yak.FOO); |
| } catch (IllegalArgumentException e) { |
| fail = true; |
| } |
| assertTrue(fail); |
| } |
| |
| @Test |
| public void testEnumFromXml() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG_FOR_ENUM)); |
| startConfig(); |
| appendProperty("test.enum"," \t \n FOO \t \n"); |
| appendProperty("test.enum2"," \t \n Yak.FOO \t \n"); |
| endConfig(); |
| |
| Configuration conf = new Configuration(); |
| Path fileResource = new Path(CONFIG_FOR_ENUM); |
| conf.addResource(fileResource); |
| assertSame(Yak.FOO, conf.getEnum("test.enum", Yak.FOO)); |
| boolean fail = false; |
| try { |
| conf.getEnum("test.enum2", Yak.FOO); |
| } catch (IllegalArgumentException e) { |
| fail = true; |
| } |
| assertTrue(fail); |
| } |
| |
| @Test |
| public void testTimeDuration() { |
| Configuration conf = new Configuration(false); |
| |
| assertEquals(7000L, |
| conf.getTimeDuration("test.time.a", 7L, SECONDS, MILLISECONDS)); |
| |
| conf.setTimeDuration("test.time.a", 7L, SECONDS); |
| assertEquals("7s", conf.get("test.time.a")); |
| assertEquals(0L, conf.getTimeDuration("test.time.a", 30, MINUTES)); |
| assertEquals(0L, conf.getTimeDuration("test.time.a", 30, SECONDS, MINUTES)); |
| assertEquals(7L, conf.getTimeDuration("test.time.a", 30, SECONDS)); |
| assertEquals(7L, |
| conf.getTimeDuration("test.time.a", 30, MILLISECONDS, SECONDS)); |
| assertEquals(7000L, conf.getTimeDuration("test.time.a", 30, MILLISECONDS)); |
| assertEquals(7000000L, |
| conf.getTimeDuration("test.time.a", 30, MICROSECONDS)); |
| assertEquals(7000000000L, |
| conf.getTimeDuration("test.time.a", 30, NANOSECONDS)); |
| conf.setTimeDuration("test.time.b", 1, DAYS); |
| assertEquals("1d", conf.get("test.time.b")); |
| assertEquals(1, conf.getTimeDuration("test.time.b", 1, DAYS)); |
| assertEquals(24, conf.getTimeDuration("test.time.b", 1, HOURS)); |
| assertEquals(MINUTES.convert(1, DAYS), |
| conf.getTimeDuration("test.time.b", 1, MINUTES)); |
| |
| // check default |
| assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS)); |
| conf.set("test.time.X", "30"); |
| assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS)); |
| assertEquals(30000L, |
| conf.getTimeDuration("test.time.X", 40, SECONDS, MILLISECONDS)); |
| assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS)); |
| assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS)); |
| assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS)); |
| conf.set("test.time.c", "30"); |
| assertEquals(30L, conf.getTimeDuration("test.time.c", "40s", SECONDS)); |
| |
| // check suffix insensitive |
| conf.set("test.time.d", "30S"); |
| assertEquals(30L, conf.getTimeDuration("test.time.d", 40, SECONDS)); |
| |
| for (Configuration.ParsedTimeDuration ptd : |
| Configuration.ParsedTimeDuration.values()) { |
| conf.setTimeDuration("test.time.unit", 1, ptd.unit()); |
| assertEquals(1 + ptd.suffix(), conf.get("test.time.unit")); |
| assertEquals(1, conf.getTimeDuration("test.time.unit", 2, ptd.unit())); |
| } |
| } |
| |
| @Test |
| public void testStorageUnit() { |
| final String key = "valid.key"; |
| final String nonKey = "not.a.key"; |
| Configuration conf = new Configuration(false); |
| |
| conf.setStorageSize(key, 10, MB); |
| // This call returns the value specified in the Key as a double in MBs. |
| Assertions.assertThat(conf.getStorageSize(key, "1GB", MB)) |
| .isEqualTo(10.0); |
| |
| // Since this key is missing, This call converts the default value of 1GB |
| // to MBs are returns that value. |
| Assertions.assertThat(conf.getStorageSize(nonKey, "1GB", MB)) |
| .isEqualTo(1024.0); |
| |
| |
| conf.setStorageSize(key, 1024, BYTES); |
| Assertions.assertThat(conf.getStorageSize(key, 100, KB)).isEqualTo(1.0); |
| |
| Assertions.assertThat(conf.getStorageSize(nonKey, 100.0, KB)) |
| .isEqualTo(100.0); |
| |
| // We try out different kind of String formats to see if they work and |
| // during read, we also try to read using a different Storage Units. |
| conf.setStrings(key, "1TB"); |
| Assertions.assertThat(conf.getStorageSize(key, "1PB", GB)) |
| .isEqualTo(1024.0); |
| |
| conf.setStrings(key, "1bytes"); |
| Assertions.assertThat(conf.getStorageSize(key, "1PB", KB)) |
| .isEqualTo(0.001); |
| |
| conf.setStrings(key, "2048b"); |
| Assertions.assertThat(conf.getStorageSize(key, "1PB", KB)).isEqualTo(2.0); |
| |
| conf.setStrings(key, "64 GB"); |
| Assertions.assertThat(conf.getStorageSize(key, "1PB", GB)).isEqualTo(64.0); |
| |
| // Match the parsing patterns of getLongBytes, which takes single char |
| // suffix. |
| conf.setStrings(key, "1T"); |
| Assertions.assertThat(conf.getStorageSize(key, "1GB", TB)).isEqualTo(1.0); |
| |
| conf.setStrings(key, "1k"); |
| Assertions.assertThat(conf.getStorageSize(key, "1GB", KB)).isEqualTo(1.0); |
| |
| conf.setStrings(key, "10m"); |
| Assertions.assertThat(conf.getStorageSize(key, "1GB", MB)).isEqualTo(10.0); |
| |
| |
| |
| // Missing format specification, this should throw. |
| conf.setStrings(key, "100"); |
| assertThrows(IllegalArgumentException.class, |
| () -> conf.getStorageSize(key, "1PB", GB)); |
| |
| // illegal format specification, this should throw. |
| conf.setStrings(key, "1HB"); |
| assertThrows(IllegalArgumentException.class, |
| () -> conf.getStorageSize(key, "1PB", GB)); |
| |
| // Illegal number specification, this should throw. |
| conf.setStrings(key, "HadoopGB"); |
| assertThrows(IllegalArgumentException.class, |
| () -> conf.getStorageSize(key, "1PB", GB)); |
| } |
| |
| @Test |
| public void testTimeDurationWarning() { |
| // check warn for possible loss of precision |
| final String warnFormat = "Possible loss of precision converting %s" + |
| " to %s for test.time.warn"; |
| final ArrayList<String> warnchk = new ArrayList<>(); |
| Configuration wconf = new Configuration(false) { |
| @Override |
| void logDeprecation(String message) { |
| warnchk.add(message); |
| } |
| }; |
| String[] convDAYS = new String[]{"23h", "30m", "40s", "10us", "40000ms"}; |
| for (String s : convDAYS) { |
| wconf.set("test.time.warn", s); |
| assertEquals(0, wconf.getTimeDuration("test.time.warn", 1, DAYS)); |
| } |
| for (int i = 0; i < convDAYS.length; ++i) { |
| String wchk = String.format(warnFormat, convDAYS[i], "DAYS"); |
| assertEquals(wchk, warnchk.get(i)); |
| } |
| |
| warnchk.clear(); |
| wconf.setTimeDuration("test.time.warn", 1441, MINUTES); |
| assertEquals(1, wconf.getTimeDuration("test.time.warn", 0, DAYS)); |
| assertEquals(24, wconf.getTimeDuration("test.time.warn", 0, HOURS)); |
| String dchk = String.format(warnFormat, "1441m", "DAYS"); |
| assertEquals(dchk, warnchk.get(0)); |
| String hchk = String.format(warnFormat, "1441m", "HOURS"); |
| assertEquals(hchk, warnchk.get(1)); |
| assertEquals(1441, wconf.getTimeDuration("test.time.warn", 0, MINUTES)); |
| // no warning |
| assertEquals(2, warnchk.size()); |
| assertEquals(86460, wconf.getTimeDuration("test.time.warn", 0, SECONDS)); |
| // no warning |
| assertEquals(2, warnchk.size()); |
| } |
| |
| @Test |
| public void testPattern() throws IOException { |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.pattern1", ""); |
| appendProperty("test.pattern2", "("); |
| appendProperty("test.pattern3", "a+b"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| Pattern defaultPattern = Pattern.compile("x+"); |
| // Return default if missing |
| assertEquals(defaultPattern.pattern(), |
| conf.getPattern("xxxxx", defaultPattern).pattern()); |
| // Return null if empty and default is null |
| assertNull(conf.getPattern("test.pattern1", null)); |
| // Return default for empty |
| assertEquals(defaultPattern.pattern(), |
| conf.getPattern("test.pattern1", defaultPattern).pattern()); |
| // Return default for malformed |
| assertEquals(defaultPattern.pattern(), |
| conf.getPattern("test.pattern2", defaultPattern).pattern()); |
| // Works for correct patterns |
| assertEquals("a+b", |
| conf.getPattern("test.pattern3", defaultPattern).pattern()); |
| } |
| |
| @Test |
| public void testPropertySource() throws IOException { |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.foo", "bar"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| conf.set("fs.defaultFS", "value"); |
| String [] sources = conf.getPropertySources("test.foo"); |
| assertEquals(1, sources.length); |
| assertEquals( |
| "Resource string returned for a file-loaded property" + |
| " must be a proper absolute path", |
| fileResource, |
| new Path(sources[0])); |
| assertArrayEquals("Resource string returned for a set() property must be " + |
| "\"programmatically\"", |
| new String[]{"programmatically"}, |
| conf.getPropertySources("fs.defaultFS")); |
| assertArrayEquals("Resource string returned for an unset property must " |
| + "be null", |
| null, conf.getPropertySources("fs.defaultFoo")); |
| } |
| |
| @Test |
| public void testMultiplePropertySource() throws IOException { |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.foo", "bar", false, "a", "b", "c"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| String [] sources = conf.getPropertySources("test.foo"); |
| assertEquals(4, sources.length); |
| assertEquals("a", sources[0]); |
| assertEquals("b", sources[1]); |
| assertEquals("c", sources[2]); |
| assertEquals( |
| "Resource string returned for a file-loaded property" + |
| " must be a proper absolute path", |
| fileResource, |
| new Path(sources[3])); |
| } |
| |
| @Test |
| public void testSocketAddress() { |
| Configuration conf = new Configuration(); |
| final String defaultAddr = "host:1"; |
| final int defaultPort = 2; |
| InetSocketAddress addr = null; |
| |
| addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort); |
| assertEquals(defaultAddr, NetUtils.getHostPortString(addr)); |
| |
| conf.set("myAddress", "host2"); |
| addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort); |
| assertEquals("host2:"+defaultPort, NetUtils.getHostPortString(addr)); |
| |
| conf.set("myAddress", "host2:3"); |
| addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort); |
| assertEquals("host2:3", NetUtils.getHostPortString(addr)); |
| |
| conf.set("myAddress", " \n \t host4:5 \t \n "); |
| addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort); |
| assertEquals("host4:5", NetUtils.getHostPortString(addr)); |
| |
| boolean threwException = false; |
| conf.set("myAddress", "bad:-port"); |
| try { |
| addr = conf.getSocketAddr("myAddress", defaultAddr, defaultPort); |
| } catch (IllegalArgumentException iae) { |
| threwException = true; |
| assertEquals("Does not contain a valid host:port authority: " + |
| "bad:-port (configuration property 'myAddress')", |
| iae.getMessage()); |
| |
| } finally { |
| assertTrue(threwException); |
| } |
| } |
| |
| @Test |
| public void testSetSocketAddress() { |
| Configuration conf = new Configuration(); |
| NetUtils.addStaticResolution("host", "127.0.0.1"); |
| final String defaultAddr = "host:1"; |
| |
| InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr); |
| conf.setSocketAddr("myAddress", addr); |
| assertEquals(defaultAddr, NetUtils.getHostPortString(addr)); |
| } |
| |
| @Test |
| public void testUpdateSocketAddress() throws IOException { |
| InetSocketAddress addr = NetUtils.createSocketAddrForHost("host", 1); |
| InetSocketAddress connectAddr = conf.updateConnectAddr("myAddress", addr); |
| assertEquals(connectAddr.getHostName(), addr.getHostName()); |
| |
| addr = new InetSocketAddress(1); |
| connectAddr = conf.updateConnectAddr("myAddress", addr); |
| assertEquals(connectAddr.getHostName(), |
| InetAddress.getLocalHost().getHostName()); |
| } |
| |
| @Test |
| public void testReload() throws IOException { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.key1", "final-value1", true); |
| appendProperty("test.key2", "value2"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("test.key1", "value1"); |
| appendProperty("test.key3", "value3"); |
| endConfig(); |
| Path fileResource1 = new Path(CONFIG2); |
| conf.addResource(fileResource1); |
| |
| // add a few values via set. |
| conf.set("test.key3", "value4"); |
| conf.set("test.key4", "value5"); |
| |
| assertEquals("final-value1", conf.get("test.key1")); |
| assertEquals("value2", conf.get("test.key2")); |
| assertEquals("value4", conf.get("test.key3")); |
| assertEquals("value5", conf.get("test.key4")); |
| |
| // change values in the test file... |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.key1", "final-value1"); |
| appendProperty("test.key3", "final-value3", true); |
| endConfig(); |
| |
| conf.reloadConfiguration(); |
| assertEquals("value1", conf.get("test.key1")); |
| // overlayed property overrides. |
| assertEquals("value4", conf.get("test.key3")); |
| assertEquals(null, conf.get("test.key2")); |
| assertEquals("value5", conf.get("test.key4")); |
| } |
| |
| @Test |
| public void testSize() { |
| Configuration conf = new Configuration(false); |
| conf.set("a", "A"); |
| conf.set("b", "B"); |
| assertEquals(2, conf.size()); |
| } |
| |
| @Test |
| public void testClear() { |
| Configuration conf = new Configuration(false); |
| conf.set("a", "A"); |
| conf.set("b", "B"); |
| conf.clear(); |
| assertEquals(0, conf.size()); |
| assertFalse(conf.iterator().hasNext()); |
| } |
| |
| public static class Fake_ClassLoader extends ClassLoader { |
| } |
| |
| @Test |
| public void testClassLoader() { |
| Configuration conf = new Configuration(false); |
| conf.setQuietMode(false); |
| conf.setClassLoader(new Fake_ClassLoader()); |
| Configuration other = new Configuration(conf); |
| assertTrue(other.getClassLoader() instanceof Fake_ClassLoader); |
| } |
| |
| static class JsonConfiguration { |
| JsonProperty[] properties; |
| |
| public JsonProperty[] getProperties() { |
| return properties; |
| } |
| |
| public void setProperties(JsonProperty[] properties) { |
| this.properties = properties; |
| } |
| } |
| |
| static class SingleJsonConfiguration { |
| private JsonProperty property; |
| |
| public JsonProperty getProperty() { |
| return property; |
| } |
| |
| public void setProperty(JsonProperty property) { |
| this.property = property; |
| } |
| } |
| |
| static class JsonProperty { |
| String key; |
| public String getKey() { |
| return key; |
| } |
| public void setKey(String key) { |
| this.key = key; |
| } |
| public String getValue() { |
| return value; |
| } |
| public void setValue(String value) { |
| this.value = value; |
| } |
| public boolean getIsFinal() { |
| return isFinal; |
| } |
| public void setIsFinal(boolean isFinal) { |
| this.isFinal = isFinal; |
| } |
| public String getResource() { |
| return resource; |
| } |
| public void setResource(String resource) { |
| this.resource = resource; |
| } |
| String value; |
| boolean isFinal; |
| String resource; |
| } |
| |
| private Configuration getActualConf(String xmlStr) { |
| Configuration ac = new Configuration(false); |
| InputStream in = new ByteArrayInputStream(xmlStr.getBytes()); |
| ac.addResource(in); |
| return ac; |
| } |
| |
| @Test |
| public void testGetSetTrimmedNames() throws IOException { |
| Configuration conf = new Configuration(false); |
| conf.set(" name", "value"); |
| assertEquals("value", conf.get("name")); |
| assertEquals("value", conf.get(" name")); |
| assertEquals("value", conf.getRaw(" name ")); |
| } |
| |
| @Test |
| public void testDumpProperty() throws IOException { |
| StringWriter outWriter = new StringWriter(); |
| ObjectMapper mapper = new ObjectMapper(); |
| String jsonStr = null; |
| String xmlStr = null; |
| try { |
| Configuration testConf = new Configuration(false); |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.key1", "value1"); |
| appendProperty("test.key2", "value2", true); |
| appendProperty("test.key3", "value3"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| testConf.addResource(fileResource); |
| out.close(); |
| |
| // case 1: dump an existing property |
| // test json format |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(testConf, "test.key2", outWriter); |
| jsonStr = outWriter.toString(); |
| outWriter.close(); |
| mapper = new ObjectMapper(); |
| SingleJsonConfiguration jconf1 = |
| mapper.readValue(jsonStr, SingleJsonConfiguration.class); |
| JsonProperty jp1 = jconf1.getProperty(); |
| assertEquals("test.key2", jp1.getKey()); |
| assertEquals("value2", jp1.getValue()); |
| assertEquals(true, jp1.isFinal); |
| assertEquals(fileResource.toString(), jp1.getResource()); |
| |
| // test xml format |
| outWriter = new StringWriter(); |
| testConf.writeXml("test.key2", outWriter); |
| xmlStr = outWriter.toString(); |
| outWriter.close(); |
| Configuration actualConf1 = getActualConf(xmlStr); |
| assertEquals(1, actualConf1.size()); |
| assertEquals("value2", actualConf1.get("test.key2")); |
| assertTrue(actualConf1.getFinalParameters().contains("test.key2")); |
| assertEquals(fileResource.toString(), |
| actualConf1.getPropertySources("test.key2")[0]); |
| |
| // case 2: dump an non existing property |
| // test json format |
| try { |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(testConf, |
| "test.unknown.key", outWriter); |
| outWriter.close(); |
| } catch (Exception e) { |
| assertTrue(e instanceof IllegalArgumentException); |
| assertTrue(e.getMessage().contains("test.unknown.key") && |
| e.getMessage().contains("not found")); |
| } |
| // test xml format |
| try { |
| outWriter = new StringWriter(); |
| testConf.writeXml("test.unknown.key", outWriter); |
| outWriter.close(); |
| } catch (Exception e) { |
| assertTrue(e instanceof IllegalArgumentException); |
| assertTrue(e.getMessage().contains("test.unknown.key") && |
| e.getMessage().contains("not found")); |
| } |
| |
| // case 3: specify a null property, ensure all configurations are dumped |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(testConf, null, outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| JsonConfiguration jconf3 = |
| mapper.readValue(jsonStr, JsonConfiguration.class); |
| assertEquals(3, jconf3.getProperties().length); |
| |
| outWriter = new StringWriter(); |
| testConf.writeXml(null, outWriter); |
| xmlStr = outWriter.toString(); |
| outWriter.close(); |
| Configuration actualConf3 = getActualConf(xmlStr); |
| assertEquals(3, actualConf3.size()); |
| assertTrue(actualConf3.getProps().containsKey("test.key1") && |
| actualConf3.getProps().containsKey("test.key2") && |
| actualConf3.getProps().containsKey("test.key3")); |
| |
| // case 4: specify an empty property, ensure all configurations are dumped |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(testConf, "", outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| JsonConfiguration jconf4 = |
| mapper.readValue(jsonStr, JsonConfiguration.class); |
| assertEquals(3, jconf4.getProperties().length); |
| |
| outWriter = new StringWriter(); |
| testConf.writeXml("", outWriter); |
| xmlStr = outWriter.toString(); |
| outWriter.close(); |
| Configuration actualConf4 = getActualConf(xmlStr); |
| assertEquals(3, actualConf4.size()); |
| assertTrue(actualConf4.getProps().containsKey("test.key1") && |
| actualConf4.getProps().containsKey("test.key2") && |
| actualConf4.getProps().containsKey("test.key3")); |
| } finally { |
| if(outWriter != null) { |
| outWriter.close(); |
| } |
| if(out != null) { |
| out.close(); |
| } |
| } |
| } |
| |
| @Test |
| public void testDumpConfiguration() throws IOException { |
| StringWriter outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(conf, outWriter); |
| String jsonStr = outWriter.toString(); |
| ObjectMapper mapper = new ObjectMapper(); |
| JsonConfiguration jconf = |
| mapper.readValue(jsonStr, JsonConfiguration.class); |
| int defaultLength = jconf.getProperties().length; |
| |
| // add 3 keys to the existing configuration properties |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.key1", "value1"); |
| appendProperty("test.key2", "value2",true); |
| appendProperty("test.key3", "value3"); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| out.close(); |
| |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(conf, outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| jconf = mapper.readValue(jsonStr, JsonConfiguration.class); |
| int length = jconf.getProperties().length; |
| // check for consistency in the number of properties parsed in Json format. |
| assertEquals(length, defaultLength+3); |
| |
| //change few keys in another resource file |
| out=new BufferedWriter(new FileWriter(CONFIG2)); |
| startConfig(); |
| appendProperty("test.key1", "newValue1"); |
| appendProperty("test.key2", "newValue2"); |
| endConfig(); |
| Path fileResource1 = new Path(CONFIG2); |
| conf.addResource(fileResource1); |
| out.close(); |
| |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(conf, outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| jconf = mapper.readValue(jsonStr, JsonConfiguration.class); |
| |
| // put the keys and their corresponding attributes into a hashmap for their |
| // efficient retrieval |
| HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>(); |
| for(JsonProperty prop : jconf.getProperties()) { |
| confDump.put(prop.getKey(), prop); |
| } |
| // check if the value and resource of test.key1 is changed |
| assertEquals("newValue1", confDump.get("test.key1").getValue()); |
| assertEquals(false, confDump.get("test.key1").getIsFinal()); |
| assertEquals(fileResource1.toString(), |
| confDump.get("test.key1").getResource()); |
| // check if final parameter test.key2 is not changed, since it is first |
| // loaded as final parameter |
| assertEquals("value2", confDump.get("test.key2").getValue()); |
| assertEquals(true, confDump.get("test.key2").getIsFinal()); |
| assertEquals(fileResource.toString(), |
| confDump.get("test.key2").getResource()); |
| // check for other keys which are not modified later |
| assertEquals("value3", confDump.get("test.key3").getValue()); |
| assertEquals(false, confDump.get("test.key3").getIsFinal()); |
| assertEquals(fileResource.toString(), |
| confDump.get("test.key3").getResource()); |
| // check for resource to be "Unknown" for keys which are loaded using 'set' |
| // and expansion of properties |
| conf.set("test.key4", "value4"); |
| conf.set("test.key5", "value5"); |
| conf.set("test.key6", "${test.key5}"); |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(conf, outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| jconf = mapper.readValue(jsonStr, JsonConfiguration.class); |
| confDump = new HashMap<String, JsonProperty>(); |
| for(JsonProperty prop : jconf.getProperties()) { |
| confDump.put(prop.getKey(), prop); |
| } |
| assertEquals("value5",confDump.get("test.key6").getValue()); |
| assertEquals("programmatically", confDump.get("test.key4").getResource()); |
| outWriter.close(); |
| } |
| |
| @Test |
| public void testDumpConfiguratioWithoutDefaults() throws IOException { |
| // check for case when default resources are not loaded |
| Configuration config = new Configuration(false); |
| StringWriter outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(config, outWriter); |
| String jsonStr = outWriter.toString(); |
| ObjectMapper mapper = new ObjectMapper(); |
| JsonConfiguration jconf = |
| mapper.readValue(jsonStr, JsonConfiguration.class); |
| |
| //ensure that no properties are loaded. |
| assertEquals(0, jconf.getProperties().length); |
| |
| // add 2 keys |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.key1", "value1"); |
| appendProperty("test.key2", "value2",true); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| config.addResource(fileResource); |
| out.close(); |
| |
| outWriter = new StringWriter(); |
| Configuration.dumpConfiguration(config, outWriter); |
| jsonStr = outWriter.toString(); |
| mapper = new ObjectMapper(); |
| jconf = mapper.readValue(jsonStr, JsonConfiguration.class); |
| |
| HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>(); |
| for (JsonProperty prop : jconf.getProperties()) { |
| confDump.put(prop.getKey(), prop); |
| } |
| //ensure only 2 keys are loaded |
| assertEquals(2,jconf.getProperties().length); |
| //ensure the values are consistent |
| assertEquals(confDump.get("test.key1").getValue(),"value1"); |
| assertEquals(confDump.get("test.key2").getValue(),"value2"); |
| //check the final tag |
| assertEquals(false, confDump.get("test.key1").getIsFinal()); |
| assertEquals(true, confDump.get("test.key2").getIsFinal()); |
| //check the resource for each property |
| for (JsonProperty prop : jconf.getProperties()) { |
| assertEquals(fileResource.toString(),prop.getResource()); |
| } |
| } |
| |
| @Test |
| public void testDumpSensitiveProperty() throws IOException { |
| final String myPassword = "ThisIsMyPassword"; |
| Configuration testConf = new Configuration(false); |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.password", myPassword); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| testConf.addResource(fileResource); |
| |
| try (StringWriter outWriter = new StringWriter()) { |
| testConf.set(SENSITIVE_CONFIG_KEYS, "password$"); |
| Configuration.dumpConfiguration(testConf, "test.password", outWriter); |
| assertFalse(outWriter.toString().contains(myPassword)); |
| } |
| } |
| |
| @Test |
| public void testDumpSensitiveConfiguration() throws IOException { |
| final String myPassword = "ThisIsMyPassword"; |
| Configuration testConf = new Configuration(false); |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("test.password", myPassword); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| testConf.addResource(fileResource); |
| |
| try (StringWriter outWriter = new StringWriter()) { |
| testConf.set(SENSITIVE_CONFIG_KEYS, "password$"); |
| Configuration.dumpConfiguration(testConf, outWriter); |
| assertFalse(outWriter.toString().contains(myPassword)); |
| } |
| } |
| |
| @Test |
| public void testGetValByRegex() { |
| Configuration conf = new Configuration(); |
| String key1 = "t.abc.key1"; |
| String key2 = "t.abc.key2"; |
| String key3 = "tt.abc.key3"; |
| String key4 = "t.abc.ey3"; |
| conf.set(key1, "value1"); |
| conf.set(key2, "value2"); |
| conf.set(key3, "value3"); |
| conf.set(key4, "value3"); |
| |
| Map<String,String> res = conf.getValByRegex("^t\\..*\\.key\\d"); |
| assertTrue("Conf didn't get key " + key1, res.containsKey(key1)); |
| assertTrue("Conf didn't get key " + key2, res.containsKey(key2)); |
| assertTrue("Picked out wrong key " + key3, !res.containsKey(key3)); |
| assertTrue("Picked out wrong key " + key4, !res.containsKey(key4)); |
| } |
| |
| @Test |
| public void testGetClassesShouldReturnDefaultValue() throws Exception { |
| Configuration config = new Configuration(); |
| Class<?>[] classes = |
| config.getClasses("testClassName", Configuration.class); |
| assertEquals( |
| "Not returning expected number of classes. Number of returned classes =" |
| + classes.length, 1, classes.length); |
| assertEquals("Not returning the default class Name", Configuration.class, |
| classes[0]); |
| } |
| |
| @Test |
| public void testGetClassesShouldReturnEmptyArray() |
| throws Exception { |
| Configuration config = new Configuration(); |
| config.set("testClassName", ""); |
| Class<?>[] classes = config.getClasses("testClassName", Configuration.class); |
| assertEquals( |
| "Not returning expected number of classes. Number of returned classes =" |
| + classes.length, 0, classes.length); |
| } |
| |
| @Test |
| public void testSettingValueNull() throws Exception { |
| Configuration config = new Configuration(); |
| try { |
| config.set("testClassName", null); |
| fail("Should throw an IllegalArgumentException exception "); |
| } catch (Exception e) { |
| assertTrue(e instanceof IllegalArgumentException); |
| assertEquals(e.getMessage(), |
| "The value of property testClassName must not be null"); |
| } |
| } |
| |
| @Test |
| public void testSettingKeyNull() throws Exception { |
| Configuration config = new Configuration(); |
| try { |
| config.set(null, "test"); |
| fail("Should throw an IllegalArgumentException exception "); |
| } catch (Exception e) { |
| assertTrue(e instanceof IllegalArgumentException); |
| assertEquals(e.getMessage(), "Property name must not be null"); |
| } |
| } |
| |
| @Test |
| public void testInvalidSubstitution() { |
| final Configuration configuration = new Configuration(false); |
| |
| // 2-var loops |
| // |
| final String key = "test.random.key"; |
| for (String keyExpression : Arrays.asList( |
| "${" + key + "}", |
| "foo${" + key + "}", |
| "foo${" + key + "}bar", |
| "${" + key + "}bar")) { |
| configuration.set(key, keyExpression); |
| assertEquals("Unexpected value", keyExpression, configuration.get(key)); |
| } |
| } |
| |
| @Test |
| public void testIncompleteSubbing() { |
| Configuration configuration = new Configuration(false); |
| String key = "test.random.key"; |
| for (String keyExpression : Arrays.asList( |
| "{}", |
| "${}", |
| "{" + key, |
| "${" + key, |
| "foo${" + key, |
| "foo${" + key + "bar", |
| "foo{" + key + "}bar", |
| "${" + key + "bar")) { |
| configuration.set(key, keyExpression); |
| String value = configuration.get(key); |
| assertTrue("Unexpected value " + value, value.equals(keyExpression)); |
| } |
| } |
| |
| @Test |
| public void testBoolean() { |
| boolean value = true; |
| Configuration configuration = new Configuration(); |
| configuration.setBoolean("value", value); |
| assertEquals(value, configuration.getBoolean("value", false)); |
| } |
| |
| @Test |
| public void testBooleanIfUnset() { |
| boolean value = true; |
| Configuration configuration = new Configuration(); |
| configuration.setBooleanIfUnset("value", value); |
| assertEquals(value, configuration.getBoolean("value", false)); |
| configuration.setBooleanIfUnset("value", false); |
| assertEquals(value, configuration.getBoolean("value", false)); |
| } |
| |
| @Test |
| public void testFloat() { |
| float value = 1.0F; |
| Configuration configuration = new Configuration(); |
| configuration.setFloat("value", value); |
| assertEquals(value, configuration.getFloat("value", 0.0F), DOUBLE_DELTA); |
| } |
| |
| @Test |
| public void testDouble() { |
| double value = 1.0D; |
| Configuration configuration = new Configuration(); |
| configuration.setDouble("value", value); |
| assertEquals(value, configuration.getDouble("value", 0.0D), DOUBLE_DELTA); |
| } |
| |
| @Test |
| public void testInt() { |
| int value = 1; |
| Configuration configuration = new Configuration(); |
| configuration.setInt("value", value); |
| assertEquals(value, configuration.getInt("value", 0)); |
| } |
| |
| @Test |
| public void testLong() { |
| long value = 1L; |
| Configuration configuration = new Configuration(); |
| configuration.setLong("value", value); |
| assertEquals(value, configuration.getLong("value", 0L)); |
| } |
| |
| @Test |
| public void testStrings() { |
| String [] strings = {"FOO","BAR"}; |
| Configuration configuration = new Configuration(); |
| configuration.setStrings("strings", strings); |
| String [] returnStrings = configuration.getStrings("strings"); |
| for(int i=0;i<returnStrings.length;i++) { |
| assertEquals(strings[i], returnStrings[i]); |
| } |
| } |
| |
| @Test |
| public void testSetPattern() { |
| Pattern testPattern = Pattern.compile("a+b"); |
| Configuration configuration = new Configuration(); |
| configuration.setPattern("testPattern", testPattern); |
| assertEquals(testPattern.pattern(), |
| configuration.getPattern("testPattern", Pattern.compile("")).pattern()); |
| } |
| |
| @Test |
| public void testGetClassByNameOrNull() throws Exception { |
| Configuration config = new Configuration(); |
| Class<?> clazz = config.getClassByNameOrNull("java.lang.Object"); |
| assertNotNull(clazz); |
| } |
| |
| @Test |
| public void testGetFinalParameters() throws Exception { |
| out=new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| declareProperty("my.var", "x", "x", true); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| Configuration conf = new Configuration(); |
| Set<String> finalParameters = conf.getFinalParameters(); |
| assertFalse("my.var already exists", finalParameters.contains("my.var")); |
| conf.addResource(fileResource); |
| assertEquals("my.var is undefined", "x", conf.get("my.var")); |
| assertFalse("finalparams not copied", finalParameters.contains("my.var")); |
| finalParameters = conf.getFinalParameters(); |
| assertTrue("my.var is not final", finalParameters.contains("my.var")); |
| } |
| |
| /** |
| * A test to check whether this thread goes into infinite loop because of |
| * destruction of data structure by resize of Map. This problem was reported |
| * by SPARK-2546. |
| * @throws Exception |
| */ |
| @Test |
| public void testConcurrentAccesses() throws Exception { |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| declareProperty("some.config", "xyz", "xyz", false); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| Configuration conf = new Configuration(); |
| conf.addResource(fileResource); |
| |
| class ConfigModifyThread extends Thread { |
| final private Configuration config; |
| final private String prefix; |
| |
| public ConfigModifyThread(Configuration conf, String prefix) { |
| config = conf; |
| this.prefix = prefix; |
| } |
| |
| @Override |
| public void run() { |
| for (int i = 0; i < 10000; i++) { |
| config.set("some.config.value-" + prefix + i, "value"); |
| } |
| } |
| } |
| |
| ArrayList<ConfigModifyThread> threads = new ArrayList<>(); |
| for (int i = 0; i < 100; i++) { |
| threads.add(new ConfigModifyThread(conf, String.valueOf(i))); |
| } |
| for (Thread t: threads) { |
| t.start(); |
| } |
| for (Thread t: threads) { |
| t.join(); |
| } |
| // If this test completes without going into infinite loop, |
| // it's expected behaviour. |
| } |
| |
| @Test |
| public void testNullValueProperties() throws Exception { |
| Configuration conf = new Configuration(); |
| conf.setAllowNullValueProperties(true); |
| out = new BufferedWriter(new FileWriter(CONFIG)); |
| startConfig(); |
| appendProperty("attr", "value", true); |
| appendProperty("attr", "", true); |
| endConfig(); |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| assertEquals("value", conf.get("attr")); |
| } |
| |
| @Test |
| public void testGetPasswordDeprecatedKeyStored() throws Exception { |
| final String oldKey = "test.password.old.key"; |
| final String newKey = "test.password.new.key"; |
| final String password = "MyPasswordForDeprecatedKey"; |
| |
| final File tmpDir = GenericTestUtils.getRandomizedTestDir(); |
| tmpDir.mkdirs(); |
| final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME, |
| "file", new File(tmpDir, "test.jks").toURI().getPath(), |
| null).toString(); |
| |
| conf = new Configuration(false); |
| conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); |
| CredentialProvider provider = |
| CredentialProviderFactory.getProviders(conf).get(0); |
| provider.createCredentialEntry(oldKey, password.toCharArray()); |
| provider.flush(); |
| |
| Configuration.addDeprecation(oldKey, newKey); |
| |
| Assertions.assertThat(conf.getPassword(newKey)) |
| .isEqualTo(password.toCharArray()); |
| Assertions.assertThat(conf.getPassword(oldKey)) |
| .isEqualTo(password.toCharArray()); |
| |
| FileUtil.fullyDelete(tmpDir); |
| } |
| |
| @Test |
| public void testGetPasswordByDeprecatedKey() throws Exception { |
| final String oldKey = "test.password.old.key"; |
| final String newKey = "test.password.new.key"; |
| final String password = "MyPasswordForDeprecatedKey"; |
| |
| final File tmpDir = GenericTestUtils.getRandomizedTestDir(); |
| tmpDir.mkdirs(); |
| final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME, |
| "file", new File(tmpDir, "test.jks").toURI().getPath(), |
| null).toString(); |
| |
| conf = new Configuration(false); |
| conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); |
| CredentialProvider provider = |
| CredentialProviderFactory.getProviders(conf).get(0); |
| provider.createCredentialEntry(newKey, password.toCharArray()); |
| provider.flush(); |
| |
| Configuration.addDeprecation(oldKey, newKey); |
| |
| Assertions.assertThat(conf.getPassword(newKey)) |
| .isEqualTo(password.toCharArray()); |
| Assertions.assertThat(conf.getPassword(oldKey)) |
| .isEqualTo(password.toCharArray()); |
| |
| FileUtil.fullyDelete(tmpDir); |
| } |
| |
| @Test |
| public void testGettingPropertiesWithPrefix() throws Exception { |
| Configuration conf = new Configuration(); |
| for (int i = 0; i < 10; i++) { |
| conf.set("prefix." + "name" + i, "value" + i); |
| } |
| conf.set("different.prefix" + ".name", "value"); |
| Map<String, String> prefixedProps = conf.getPropsWithPrefix("prefix."); |
| Assertions.assertThat(prefixedProps).hasSize(10); |
| for (int i = 0; i < 10; i++) { |
| assertEquals("value" + i, prefixedProps.get("name" + i)); |
| } |
| |
| // Repeat test with variable substitution |
| conf.set("foo", "bar"); |
| for (int i = 0; i < 10; i++) { |
| conf.set("subprefix." + "subname" + i, "value_${foo}" + i); |
| } |
| prefixedProps = conf.getPropsWithPrefix("subprefix."); |
| Assertions.assertThat(prefixedProps).hasSize(10); |
| for (int i = 0; i < 10; i++) { |
| assertEquals("value_bar" + i, prefixedProps.get("subname" + i)); |
| } |
| // test call with no properties for a given prefix |
| prefixedProps = conf.getPropsWithPrefix("none"); |
| assertNotNull(prefixedProps.isEmpty()); |
| assertTrue(prefixedProps.isEmpty()); |
| } |
| |
| public static void main(String[] argv) throws Exception { |
| junit.textui.TestRunner.main(new String[]{ |
| TestConfiguration.class.getName() |
| }); |
| } |
| |
| @Test |
| public void testGetAllPropertiesByTags() throws Exception { |
| |
| try{ |
| out = new BufferedWriter(new FileWriter(CONFIG_CORE)); |
| startConfig(); |
| appendProperty("hadoop.tags.system", "YARN,HDFS,NAMENODE"); |
| appendProperty("hadoop.tags.custom", "MYCUSTOMTAG"); |
| appendPropertyByTag("dfs.cblock.trace.io", "false", "YARN"); |
| appendPropertyByTag("dfs.replication", "1", "HDFS"); |
| appendPropertyByTag("dfs.namenode.logging.level", "INFO", "NAMENODE"); |
| appendPropertyByTag("dfs.random.key", "XYZ", "MYCUSTOMTAG"); |
| endConfig(); |
| |
| Path fileResource = new Path(CONFIG_CORE); |
| conf.addResource(fileResource); |
| conf.getProps(); |
| |
| } finally { |
| out.close(); |
| } |
| System.out.println(Files.readAllLines(Paths.get(CONFIG_CORE))); |
| List<String> tagList = new ArrayList<>(); |
| tagList.add("YARN"); |
| tagList.add("HDFS"); |
| tagList.add("NAMENODE"); |
| |
| Properties properties = conf.getAllPropertiesByTags(tagList); |
| String[] sources = conf.getPropertySources("dfs.replication"); |
| assertTrue(sources.length == 1); |
| assertTrue(Arrays.toString(sources).contains("core-site.xml")); |
| |
| assertEq(3, properties.size()); |
| assertEq(true, properties.containsKey("dfs.namenode.logging.level")); |
| assertEq(true, properties.containsKey("dfs.replication")); |
| assertEq(true, properties.containsKey("dfs.cblock.trace.io")); |
| assertEq(false, properties.containsKey("namenode.host")); |
| |
| properties = conf.getAllPropertiesByTag("DEBUG"); |
| assertEq(0, properties.size()); |
| assertEq(false, properties.containsKey("dfs.namenode.logging.level")); |
| assertEq(true, conf.isPropertyTag("YARN")); |
| assertEq(true, conf.isPropertyTag("HDFS")); |
| assertEq(true, conf.isPropertyTag("NAMENODE")); |
| assertEq(true, conf.isPropertyTag("MYCUSTOMTAG")); |
| assertEq(false, conf.isPropertyTag("CMYCUSTOMTAG2")); |
| } |
| |
| @Test |
| public void testInvalidTags() throws Exception { |
| Path fileResource = new Path(CONFIG); |
| conf.addResource(fileResource); |
| conf.getProps(); |
| |
| assertFalse(conf.isPropertyTag("BADTAG")); |
| assertFalse(conf.isPropertyTag("CUSTOM_TAG")); |
| assertTrue(conf.isPropertyTag("DEBUG")); |
| assertTrue(conf.isPropertyTag("HDFS")); |
| } |
| |
| /** |
| * Test race conditions between clone() and getProps(). |
| * Test for race conditions in the way Hadoop handles the Configuration |
| * class. The scenario is the following. Let's assume that there are two |
| * threads sharing the same Configuration class. One adds some resources |
| * to the configuration, while the other one clones it. Resources are |
| * loaded lazily in a deferred call to loadResources(). If the cloning |
| * happens after adding the resources but before parsing them, some temporary |
| * resources like input stream pointers are cloned. Eventually both copies |
| * will load the same input stream resources. |
| * One parses the input stream XML and closes it updating it's own copy of |
| * the resource. The other one has another pointer to the same input stream. |
| * When it tries to load it, it will crash with a stream closed exception. |
| */ |
| @Test |
| public void testResourceRace() { |
| InputStream is = |
| new BufferedInputStream(new ByteArrayInputStream( |
| "<configuration></configuration>".getBytes())); |
| Configuration config = new Configuration(); |
| // Thread 1 |
| config.addResource(is); |
| // Thread 2 |
| Configuration confClone = new Configuration(conf); |
| // Thread 2 |
| confClone.get("firstParse"); |
| // Thread 1 |
| config.get("secondParse"); |
| } |
| |
| @Test |
| public void testCDATA() throws IOException { |
| String xml = new String( |
| "<configuration>" + |
| "<property>" + |
| "<name>cdata</name>" + |
| "<value><![CDATA[>cdata]]></value>" + |
| "</property>\n" + |
| "<property>" + |
| "<name>cdata-multiple</name>" + |
| "<value><![CDATA[>cdata1]]> and <![CDATA[>cdata2]]></value>" + |
| "</property>\n" + |
| "<property>" + |
| "<name>cdata-multiline</name>" + |
| "<value><![CDATA[>cdata\nmultiline<>]]></value>" + |
| "</property>\n" + |
| "<property>" + |
| "<name>cdata-whitespace</name>" + |
| "<value> prefix <![CDATA[>cdata]]>\nsuffix </value>" + |
| "</property>\n" + |
| "</configuration>"); |
| Configuration conf = checkCDATA(xml.getBytes()); |
| ByteArrayOutputStream os = new ByteArrayOutputStream(); |
| conf.writeXml(os); |
| checkCDATA(os.toByteArray()); |
| } |
| |
| private static Configuration checkCDATA(byte[] bytes) { |
| Configuration conf = new Configuration(false); |
| conf.addResource(new ByteArrayInputStream(bytes)); |
| assertEquals(">cdata", conf.get("cdata")); |
| assertEquals(">cdata1 and >cdata2", conf.get("cdata-multiple")); |
| assertEquals(">cdata\nmultiline<>", conf.get("cdata-multiline")); |
| assertEquals(" prefix >cdata\nsuffix ", conf.get("cdata-whitespace")); |
| return conf; |
| } |
| |
| @Test |
| public void testConcurrentModificationDuringIteration() throws InterruptedException { |
| Configuration configuration = new Configuration(); |
| new Thread(() -> { |
| while (true) { |
| configuration.set(String.valueOf(Math.random()), String.valueOf(Math.random())); |
| } |
| }).start(); |
| |
| AtomicBoolean exceptionOccurred = new AtomicBoolean(false); |
| |
| new Thread(() -> { |
| while (true) { |
| try { |
| configuration.iterator(); |
| } catch (final ConcurrentModificationException e) { |
| exceptionOccurred.set(true); |
| break; |
| } |
| } |
| }).start(); |
| |
| Thread.sleep(1000); //give enough time for threads to run |
| |
| assertFalse("ConcurrentModificationException occurred", exceptionOccurred.get()); |
| } |
| } |