hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java - hadoop - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.util;

 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;

 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;

 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.math3.util.Pair;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;

 import com.google.common.collect.Maps;

 public class TestGenericOptionsParser {
   File testDir;
   Configuration conf;
   FileSystem localFs;

   @Test
   public void testFilesOption() throws Exception {
     File tmpFile = new File(testDir, "tmpfile");
     Path tmpPath = new Path(tmpFile.toString());
     localFs.create(tmpPath);
     String[] args = new String[2];
     // pass a files option
     args[0] = "-files";
     // Convert a file to a URI as File.toString() is not a valid URI on
     // all platforms and GenericOptionsParser accepts only valid URIs
     args[1] = tmpFile.toURI().toString();
     new GenericOptionsParser(conf, args);
     String files = conf.get("tmpfiles");
     assertNotNull("files is null", files);
     assertEquals("files option does not match",
       localFs.makeQualified(tmpPath).toString(), files);

     // pass file as uri
     Configuration conf1 = new Configuration();
     URI tmpURI = new URI(tmpFile.toURI().toString() + "#link");
     args[0] = "-files";
     args[1] = tmpURI.toString();
     new GenericOptionsParser(conf1, args);
     files = conf1.get("tmpfiles");
     assertNotNull("files is null", files);
     assertEquals("files option does not match",
       localFs.makeQualified(new Path(tmpURI)).toString(), files);

     // pass a file that does not exist.
     // GenericOptionParser should throw exception
     Configuration conf2 = new Configuration();
     args[0] = "-files";
     args[1] = "file:///xyz.txt";
     Throwable th = null;
     try {
       new GenericOptionsParser(conf2, args);
     } catch (Exception e) {
       th = e;
     }
     assertNotNull("throwable is null", th);
     assertTrue("FileNotFoundException is not thrown",
       th instanceof FileNotFoundException);
     files = conf2.get("tmpfiles");
     assertNull("files is not null", files);
   }

   @Test
   public void testLibjarsOption() throws Exception {
     File tmpJar = new File(testDir, "tmp.jar");
     Path tmpJarPath = new Path(tmpJar.toString());
     localFs.create(tmpJarPath);
     String[] args = new String[2];
     // pass a libjars option
     // first, pass the jar directly
     args[0] = "-libjars";
     // Convert a file to a URI as File.toString() is not a valid URI on
     // all platforms and GenericOptionsParser accepts only valid URIs
     args[1] = tmpJar.toURI().toString();
     new GenericOptionsParser(conf, args);
     String libjars = conf.get("tmpjars");
     assertNotNull("libjars is null", libjars);
     assertEquals("libjars does not match",
         localFs.makeQualified(tmpJarPath).toString(), libjars);

     // now test the wildcard
     args[1] = testDir.toURI().toString() + "*";
     new GenericOptionsParser(conf, args);
     libjars = conf.get("tmpjars");
     assertNotNull("libjars is null", libjars);
     assertEquals("libjars does not match",
         localFs.makeQualified(tmpJarPath).toString(), libjars);
   }

   /**
    * Test the case where the libjars, files and archives arguments
    * contains an empty token, which should create an IllegalArgumentException.
    */
   @Test
   public void testEmptyFilenames() throws Exception {
     List<Pair<String, String>> argsAndConfNames = new ArrayList<Pair<String, String>>();
     argsAndConfNames.add(new Pair<String, String>("-libjars", "tmpjars"));
     argsAndConfNames.add(new Pair<String, String>("-files", "tmpfiles"));
     argsAndConfNames.add(new Pair<String, String>("-archives", "tmparchives"));
     for (Pair<String, String> argAndConfName : argsAndConfNames) {
       String arg = argAndConfName.getFirst();

       File tmpFileOne = new File(testDir, "tmpfile1");
       Path tmpPathOne = new Path(tmpFileOne.toString());
       File tmpFileTwo = new File(testDir, "tmpfile2");
       Path tmpPathTwo = new Path(tmpFileTwo.toString());
       localFs.create(tmpPathOne);
       localFs.create(tmpPathTwo);
       String[] args = new String[2];
       args[0] = arg;
       // create an empty path in between two valid files,
       // which prior to HADOOP-10820 used to result in the
       // working directory being added to "tmpjars" (or equivalent)
       args[1] = String.format("%s,,%s",
           tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString());
       try {
         new GenericOptionsParser(conf, args);
         fail("Expected exception for empty filename");
       } catch (IllegalArgumentException e) {
         // expect to receive an IllegalArgumentException
         GenericTestUtils.assertExceptionContains("File name can't be"
             + " empty string", e);
       }

       // test zero file list length - it should create an exception
       args[1] = ",,";
       try {
         new GenericOptionsParser(conf, args);
         fail("Expected exception for zero file list length");
       } catch (IllegalArgumentException e) {
         // expect to receive an IllegalArgumentException
         GenericTestUtils.assertExceptionContains("File name can't be"
             + " empty string", e);
       }

       // test filename with space character
       // it should create exception from parser in URI class
       // due to URI syntax error
       args[1] = String.format("%s, ,%s",
           tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString());
       try {
         new GenericOptionsParser(conf, args);
         fail("Expected exception for filename with space character");
       } catch (IllegalArgumentException e) {
         // expect to receive an IllegalArgumentException
         GenericTestUtils.assertExceptionContains("URISyntaxException", e);
       }
     }
   }

   /**
    * Test that options passed to the constructor are used.
    */
   @SuppressWarnings("static-access")
   @Test
   public void testCreateWithOptions() throws Exception {
     // Create new option newOpt
     Option opt = OptionBuilder.withArgName("int")
     .hasArg()
     .withDescription("A new option")
     .create("newOpt");
     Options opts = new Options();
     opts.addOption(opt);

     // Check newOpt is actually used to parse the args
     String[] args = new String[2];
     args[0] = "--newOpt";
     args[1] = "7";
     GenericOptionsParser g = new GenericOptionsParser(opts, args);
     assertEquals("New option was ignored",
       "7", g.getCommandLine().getOptionValues("newOpt")[0]);
   }

   /**
    * Test that multiple conf arguments can be used.
    */
   @Test
   public void testConfWithMultipleOpts() throws Exception {
     String[] args = new String[2];
     args[0] = "--conf=foo";
     args[1] = "--conf=bar";
     GenericOptionsParser g = new GenericOptionsParser(args);
     assertEquals("1st conf param is incorrect",
       "foo", g.getCommandLine().getOptionValues("conf")[0]);
     assertEquals("2st conf param is incorrect",
       "bar", g.getCommandLine().getOptionValues("conf")[1]);
   }

   @Before
   public void setUp() throws Exception {
     conf = new Configuration();
     localFs = FileSystem.getLocal(conf);
     testDir = GenericTestUtils.getTestDir("generic");
     if(testDir.exists())
       localFs.delete(new Path(testDir.toString()), true);
   }

   @After
   public void tearDown() throws Exception {
     if(testDir.exists()) {
       localFs.delete(new Path(testDir.toString()), true);
     }
   }

   /**
    * testing -fileCache option
    * @throws IOException
    */
   @Test
   public void testTokenCacheOption() throws IOException {
     FileSystem localFs = FileSystem.getLocal(conf);

     File tmpFile = new File(testDir, "tokenCacheFile");
     if(tmpFile.exists()) {
       tmpFile.delete();
     }
     String[] args = new String[2];
     // pass a files option
     args[0] = "-tokenCacheFile";
     args[1] = tmpFile.toURI().toString();

     // test non existing file
     Throwable th = null;
     try {
       new GenericOptionsParser(conf, args);
     } catch (Exception e) {
       th = e;
     }
     assertNotNull(th);
     assertTrue("FileNotFoundException is not thrown",
         th instanceof FileNotFoundException);

     // create file
     Path tmpPath = localFs.makeQualified(new Path(tmpFile.toString()));
     Token<?> token = new Token<AbstractDelegationTokenIdentifier>(
         "identifier".getBytes(), "password".getBytes(),
         new Text("token-kind"), new Text("token-service"));
     Credentials creds = new Credentials();
     creds.addToken(new Text("token-alias"), token);
     creds.writeTokenStorageFile(tmpPath, conf);

     new GenericOptionsParser(conf, args);
     String fileName = conf.get("mapreduce.job.credentials.binary");
     assertNotNull("files is null", fileName);
     assertEquals("files option does not match", tmpPath.toString(), fileName);

     Credentials ugiCreds =
         UserGroupInformation.getCurrentUser().getCredentials();
     assertEquals(1, ugiCreds.numberOfTokens());
     Token<?> ugiToken = ugiCreds.getToken(new Text("token-alias"));
     assertNotNull(ugiToken);
     assertEquals(token, ugiToken);

     localFs.delete(new Path(testDir.getAbsolutePath()), true);
   }

   /** Test -D parsing */
   @Test
   public void testDOptionParsing() throws Exception {
     String[] args;
     Map<String,String> expectedMap;
     String[] expectedRemainingArgs;

     args = new String[]{};
     expectedRemainingArgs = new String[]{};
     expectedMap = Maps.newHashMap();
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

     args = new String[]{"-Dkey1=value1"};
     expectedRemainingArgs = new String[]{};
     expectedMap = Maps.newHashMap();
     expectedMap.put("key1", "value1");
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

     args = new String[]{"-fs", "hdfs://somefs/", "-Dkey1=value1", "arg1"};
     expectedRemainingArgs = new String[]{"arg1"};
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

     args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1=value1", "arg1"};
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

     if (Shell.WINDOWS) {
       args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1",
         "value1", "arg1"};
       assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

       args = new String[]{"-fs", "hdfs://somefs/", "-Dkey1", "value1", "arg1"};
       assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

       args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
         "-fs", "someother", "-D", "key2", "value2", "arg1", "arg2"};
       expectedRemainingArgs = new String[]{"arg1", "arg2"};
       expectedMap = Maps.newHashMap();
       expectedMap.put("key1", "value1");
       expectedMap.put("key2", "value2");
       assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

       args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
         "-fs", "someother", "-D", "key2", "value2"};
       expectedRemainingArgs = new String[]{};
       assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

       args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
         "-fs", "someother", "-D", "key2"};
       expectedMap = Maps.newHashMap();
       expectedMap.put("key1", "value1");
       expectedMap.put("key2", null); // we expect key2 not set
       assertDOptionParsing(args, expectedMap, expectedRemainingArgs);
     }

     args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1=value1",
       "-fs", "someother", "-Dkey2"};
     expectedRemainingArgs = new String[]{};
     expectedMap = Maps.newHashMap();
     expectedMap.put("key1", "value1");
     expectedMap.put("key2", null); // we expect key2 not set
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

     args = new String[]{"-fs", "hdfs://somefs/", "-D"};
     expectedMap = Maps.newHashMap();
     assertDOptionParsing(args, expectedMap, expectedRemainingArgs);
   }

   private void assertDOptionParsing(String[] args,
       Map<String,String> expectedMap, String[] expectedRemainingArgs)
       throws Exception {
     for (Map.Entry<String, String> entry : expectedMap.entrySet()) {
       assertNull(conf.get(entry.getKey()));
     }

     Configuration conf = new Configuration();
     GenericOptionsParser parser = new GenericOptionsParser(conf, args);
     String[] remainingArgs = parser.getRemainingArgs();

     for (Map.Entry<String, String> entry : expectedMap.entrySet()) {
       assertEquals(entry.getValue(), conf.get(entry.getKey()));
     }

     assertArrayEquals(
       Arrays.toString(remainingArgs) + Arrays.toString(expectedRemainingArgs),
       expectedRemainingArgs, remainingArgs);
   }

   /** Test passing null as args. Some classes still call
    * Tool interface from java passing null.
    */
   @Test
   public void testNullArgs() throws IOException {
     GenericOptionsParser parser = new GenericOptionsParser(conf, null);
     parser.getRemainingArgs();
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.util;

	import static org.junit.Assert.assertArrayEquals;
	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertNotNull;
	import static org.junit.Assert.assertNull;
	import static org.junit.Assert.assertTrue;
	import static org.junit.Assert.fail;

	import java.io.File;
	import java.io.FileNotFoundException;
	import java.io.IOException;
	import java.net.URI;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.List;
	import java.util.Map;

	import org.apache.commons.cli.Option;
	import org.apache.commons.cli.OptionBuilder;
	import org.apache.commons.cli.Options;
	import org.apache.commons.math3.util.Pair;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.FileSystem;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.io.Text;
	import org.apache.hadoop.security.Credentials;
	import org.apache.hadoop.security.UserGroupInformation;
	import org.apache.hadoop.security.token.Token;
	import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
	import org.apache.hadoop.test.GenericTestUtils;
	import org.junit.After;
	import org.junit.Before;
	import org.junit.Test;

	import com.google.common.collect.Maps;

	public class TestGenericOptionsParser {
	File testDir;
	Configuration conf;
	FileSystem localFs;

	@Test
	public void testFilesOption() throws Exception {
	File tmpFile = new File(testDir, "tmpfile");
	Path tmpPath = new Path(tmpFile.toString());
	localFs.create(tmpPath);
	String[] args = new String[2];
	// pass a files option
	args[0] = "-files";
	// Convert a file to a URI as File.toString() is not a valid URI on
	// all platforms and GenericOptionsParser accepts only valid URIs
	args[1] = tmpFile.toURI().toString();
	new GenericOptionsParser(conf, args);
	String files = conf.get("tmpfiles");
	assertNotNull("files is null", files);
	assertEquals("files option does not match",
	localFs.makeQualified(tmpPath).toString(), files);

	// pass file as uri
	Configuration conf1 = new Configuration();
	URI tmpURI = new URI(tmpFile.toURI().toString() + "#link");
	args[0] = "-files";
	args[1] = tmpURI.toString();
	new GenericOptionsParser(conf1, args);
	files = conf1.get("tmpfiles");
	assertNotNull("files is null", files);
	assertEquals("files option does not match",
	localFs.makeQualified(new Path(tmpURI)).toString(), files);

	// pass a file that does not exist.
	// GenericOptionParser should throw exception
	Configuration conf2 = new Configuration();
	args[0] = "-files";
	args[1] = "file:///xyz.txt";
	Throwable th = null;
	try {
	new GenericOptionsParser(conf2, args);
	} catch (Exception e) {
	th = e;
	}
	assertNotNull("throwable is null", th);
	assertTrue("FileNotFoundException is not thrown",
	th instanceof FileNotFoundException);
	files = conf2.get("tmpfiles");
	assertNull("files is not null", files);
	}

	@Test
	public void testLibjarsOption() throws Exception {
	File tmpJar = new File(testDir, "tmp.jar");
	Path tmpJarPath = new Path(tmpJar.toString());
	localFs.create(tmpJarPath);
	String[] args = new String[2];
	// pass a libjars option
	// first, pass the jar directly
	args[0] = "-libjars";
	// Convert a file to a URI as File.toString() is not a valid URI on
	// all platforms and GenericOptionsParser accepts only valid URIs
	args[1] = tmpJar.toURI().toString();
	new GenericOptionsParser(conf, args);
	String libjars = conf.get("tmpjars");
	assertNotNull("libjars is null", libjars);
	assertEquals("libjars does not match",
	localFs.makeQualified(tmpJarPath).toString(), libjars);

	// now test the wildcard
	args[1] = testDir.toURI().toString() + "*";
	new GenericOptionsParser(conf, args);
	libjars = conf.get("tmpjars");
	assertNotNull("libjars is null", libjars);
	assertEquals("libjars does not match",
	localFs.makeQualified(tmpJarPath).toString(), libjars);
	}

	/**
	* Test the case where the libjars, files and archives arguments
	* contains an empty token, which should create an IllegalArgumentException.
	*/
	@Test
	public void testEmptyFilenames() throws Exception {
	List<Pair<String, String>> argsAndConfNames = new ArrayList<Pair<String, String>>();
	argsAndConfNames.add(new Pair<String, String>("-libjars", "tmpjars"));
	argsAndConfNames.add(new Pair<String, String>("-files", "tmpfiles"));
	argsAndConfNames.add(new Pair<String, String>("-archives", "tmparchives"));
	for (Pair<String, String> argAndConfName : argsAndConfNames) {
	String arg = argAndConfName.getFirst();

	File tmpFileOne = new File(testDir, "tmpfile1");
	Path tmpPathOne = new Path(tmpFileOne.toString());
	File tmpFileTwo = new File(testDir, "tmpfile2");
	Path tmpPathTwo = new Path(tmpFileTwo.toString());
	localFs.create(tmpPathOne);
	localFs.create(tmpPathTwo);
	String[] args = new String[2];
	args[0] = arg;
	// create an empty path in between two valid files,
	// which prior to HADOOP-10820 used to result in the
	// working directory being added to "tmpjars" (or equivalent)
	args[1] = String.format("%s,,%s",
	tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString());
	try {
	new GenericOptionsParser(conf, args);
	fail("Expected exception for empty filename");
	} catch (IllegalArgumentException e) {
	// expect to receive an IllegalArgumentException
	GenericTestUtils.assertExceptionContains("File name can't be"
	+ " empty string", e);
	}

	// test zero file list length - it should create an exception
	args[1] = ",,";
	try {
	new GenericOptionsParser(conf, args);
	fail("Expected exception for zero file list length");
	} catch (IllegalArgumentException e) {
	// expect to receive an IllegalArgumentException
	GenericTestUtils.assertExceptionContains("File name can't be"
	+ " empty string", e);
	}

	// test filename with space character
	// it should create exception from parser in URI class
	// due to URI syntax error
	args[1] = String.format("%s, ,%s",
	tmpFileOne.toURI().toString(), tmpFileTwo.toURI().toString());
	try {
	new GenericOptionsParser(conf, args);
	fail("Expected exception for filename with space character");
	} catch (IllegalArgumentException e) {
	// expect to receive an IllegalArgumentException
	GenericTestUtils.assertExceptionContains("URISyntaxException", e);
	}
	}
	}

	/**
	* Test that options passed to the constructor are used.
	*/
	@SuppressWarnings("static-access")
	@Test
	public void testCreateWithOptions() throws Exception {
	// Create new option newOpt
	Option opt = OptionBuilder.withArgName("int")
	.hasArg()
	.withDescription("A new option")
	.create("newOpt");
	Options opts = new Options();
	opts.addOption(opt);

	// Check newOpt is actually used to parse the args
	String[] args = new String[2];
	args[0] = "--newOpt";
	args[1] = "7";
	GenericOptionsParser g = new GenericOptionsParser(opts, args);
	assertEquals("New option was ignored",
	"7", g.getCommandLine().getOptionValues("newOpt")[0]);
	}

	/**
	* Test that multiple conf arguments can be used.
	*/
	@Test
	public void testConfWithMultipleOpts() throws Exception {
	String[] args = new String[2];
	args[0] = "--conf=foo";
	args[1] = "--conf=bar";
	GenericOptionsParser g = new GenericOptionsParser(args);
	assertEquals("1st conf param is incorrect",
	"foo", g.getCommandLine().getOptionValues("conf")[0]);
	assertEquals("2st conf param is incorrect",
	"bar", g.getCommandLine().getOptionValues("conf")[1]);
	}

	@Before
	public void setUp() throws Exception {
	conf = new Configuration();
	localFs = FileSystem.getLocal(conf);
	testDir = GenericTestUtils.getTestDir("generic");
	if(testDir.exists())
	localFs.delete(new Path(testDir.toString()), true);
	}

	@After
	public void tearDown() throws Exception {
	if(testDir.exists()) {
	localFs.delete(new Path(testDir.toString()), true);
	}
	}

	/**
	* testing -fileCache option
	* @throws IOException
	*/
	@Test
	public void testTokenCacheOption() throws IOException {
	FileSystem localFs = FileSystem.getLocal(conf);

	File tmpFile = new File(testDir, "tokenCacheFile");
	if(tmpFile.exists()) {
	tmpFile.delete();
	}
	String[] args = new String[2];
	// pass a files option
	args[0] = "-tokenCacheFile";
	args[1] = tmpFile.toURI().toString();

	// test non existing file
	Throwable th = null;
	try {
	new GenericOptionsParser(conf, args);
	} catch (Exception e) {
	th = e;
	}
	assertNotNull(th);
	assertTrue("FileNotFoundException is not thrown",
	th instanceof FileNotFoundException);

	// create file
	Path tmpPath = localFs.makeQualified(new Path(tmpFile.toString()));
	Token<?> token = new Token<AbstractDelegationTokenIdentifier>(
	"identifier".getBytes(), "password".getBytes(),
	new Text("token-kind"), new Text("token-service"));
	Credentials creds = new Credentials();
	creds.addToken(new Text("token-alias"), token);
	creds.writeTokenStorageFile(tmpPath, conf);

	new GenericOptionsParser(conf, args);
	String fileName = conf.get("mapreduce.job.credentials.binary");
	assertNotNull("files is null", fileName);
	assertEquals("files option does not match", tmpPath.toString(), fileName);

	Credentials ugiCreds =
	UserGroupInformation.getCurrentUser().getCredentials();
	assertEquals(1, ugiCreds.numberOfTokens());
	Token<?> ugiToken = ugiCreds.getToken(new Text("token-alias"));
	assertNotNull(ugiToken);
	assertEquals(token, ugiToken);

	localFs.delete(new Path(testDir.getAbsolutePath()), true);
	}

	/** Test -D parsing */
	@Test
	public void testDOptionParsing() throws Exception {
	String[] args;
	Map<String,String> expectedMap;
	String[] expectedRemainingArgs;

	args = new String[]{};
	expectedRemainingArgs = new String[]{};
	expectedMap = Maps.newHashMap();
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-Dkey1=value1"};
	expectedRemainingArgs = new String[]{};
	expectedMap = Maps.newHashMap();
	expectedMap.put("key1", "value1");
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-Dkey1=value1", "arg1"};
	expectedRemainingArgs = new String[]{"arg1"};
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1=value1", "arg1"};
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	if (Shell.WINDOWS) {
	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1",
	"value1", "arg1"};
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-Dkey1", "value1", "arg1"};
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
	"-fs", "someother", "-D", "key2", "value2", "arg1", "arg2"};
	expectedRemainingArgs = new String[]{"arg1", "arg2"};
	expectedMap = Maps.newHashMap();
	expectedMap.put("key1", "value1");
	expectedMap.put("key2", "value2");
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
	"-fs", "someother", "-D", "key2", "value2"};
	expectedRemainingArgs = new String[]{};
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1", "value1",
	"-fs", "someother", "-D", "key2"};
	expectedMap = Maps.newHashMap();
	expectedMap.put("key1", "value1");
	expectedMap.put("key2", null); // we expect key2 not set
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);
	}

	args = new String[]{"-fs", "hdfs://somefs/", "-D", "key1=value1",
	"-fs", "someother", "-Dkey2"};
	expectedRemainingArgs = new String[]{};
	expectedMap = Maps.newHashMap();
	expectedMap.put("key1", "value1");
	expectedMap.put("key2", null); // we expect key2 not set
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);

	args = new String[]{"-fs", "hdfs://somefs/", "-D"};
	expectedMap = Maps.newHashMap();
	assertDOptionParsing(args, expectedMap, expectedRemainingArgs);
	}

	private void assertDOptionParsing(String[] args,
	Map<String,String> expectedMap, String[] expectedRemainingArgs)
	throws Exception {
	for (Map.Entry<String, String> entry : expectedMap.entrySet()) {
	assertNull(conf.get(entry.getKey()));
	}

	Configuration conf = new Configuration();
	GenericOptionsParser parser = new GenericOptionsParser(conf, args);
	String[] remainingArgs = parser.getRemainingArgs();

	for (Map.Entry<String, String> entry : expectedMap.entrySet()) {
	assertEquals(entry.getValue(), conf.get(entry.getKey()));
	}

	assertArrayEquals(
	Arrays.toString(remainingArgs) + Arrays.toString(expectedRemainingArgs),
	expectedRemainingArgs, remainingArgs);
	}

	/** Test passing null as args. Some classes still call
	* Tool interface from java passing null.
	*/
	@Test
	public void testNullArgs() throws IOException {
	GenericOptionsParser parser = new GenericOptionsParser(conf, null);
	parser.getRemainingArgs();
	}
	}