| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.sqoop; |
| |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.IOException; |
| import java.io.InputStreamReader; |
| import java.lang.reflect.Constructor; |
| import java.util.Arrays; |
| import java.util.LinkedList; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Properties; |
| |
| import org.apache.commons.logging.Log; |
| import org.apache.commons.logging.LogFactory; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.util.ReflectionUtils; |
| import org.apache.hadoop.util.StringUtils; |
| |
| import org.apache.sqoop.manager.ConnManager; |
| import org.apache.sqoop.manager.DefaultManagerFactory; |
| import org.apache.sqoop.manager.ManagerFactory; |
| import org.apache.sqoop.metastore.JobData; |
| |
| import org.apache.sqoop.util.ClassLoaderStack; |
| import org.apache.sqoop.manager.GenericJdbcManager; |
| import org.apache.sqoop.manager.oracle.OraOopManagerFactory; |
| |
| /** |
| * Factory class to create the ConnManager type required |
| * for the current import job. |
| * |
| * This class delegates the actual responsibility for instantiating |
| * ConnManagers to one or more instances of ManagerFactory. ManagerFactories |
| * are consulted in the order specified in sqoop-site.xml |
| * (sqoop.connection.factories). |
| */ |
| public class ConnFactory { |
| |
| public static final Log LOG = LogFactory.getLog(ConnFactory.class.getName()); |
| |
| public ConnFactory(Configuration conf) { |
| factories = new LinkedList<ManagerFactory>(); |
| instantiateFactories(conf); |
| } |
| |
| /** The sqoop-site.xml configuration property used to set the list of |
| * available ManagerFactories. |
| */ |
| public static final String FACTORY_CLASS_NAMES_KEY = |
| "sqoop.connection.factories"; |
| |
| // The default value for sqoop.connection.factories is the |
| // name of the DefaultManagerFactory. |
| public static final String[] DEFAULT_FACTORY_CLASS_NAMES_ARR = |
| {OraOopManagerFactory.class.getName(), |
| DefaultManagerFactory.class.getName(), }; |
| |
| public static final String DEFAULT_FACTORY_CLASS_NAMES = |
| StringUtils.arrayToString(DEFAULT_FACTORY_CLASS_NAMES_ARR); |
| |
| /** The list of ManagerFactory instances consulted by getManager(). |
| */ |
| private List<ManagerFactory> factories; |
| |
| /** |
| * Create the ManagerFactory instances that should populate |
| * the factories list. |
| */ |
| private void instantiateFactories(Configuration conf) { |
| loadManagersFromConfDir(conf); |
| String [] classNameArray = |
| conf.getStrings(FACTORY_CLASS_NAMES_KEY, |
| DEFAULT_FACTORY_CLASS_NAMES_ARR); |
| |
| for (String className : classNameArray) { |
| try { |
| className = className.trim(); // Ignore leading/trailing whitespace. |
| ManagerFactory factory = ReflectionUtils.newInstance( |
| (Class<? extends ManagerFactory>) |
| conf.getClassByName(className), conf); |
| LOG.debug("Loaded manager factory: " + className); |
| factories.add(factory); |
| } catch (ClassNotFoundException cnfe) { |
| LOG.error("Could not load ManagerFactory " + className |
| + " (not found)"); |
| } |
| } |
| } |
| |
| /** |
| * Factory method to get a ConnManager. |
| * |
| * Connection Manager is created directly if user specifies it on the command |
| * line or the execution is passed to various configured connection factories |
| * in case that user is not requesting one specific manager. |
| * |
| * @param data the connection and other configuration arguments. |
| * @return a ConnManager instance for the appropriate database. |
| * @throws IOException if it cannot find a ConnManager for this schema. |
| */ |
| public ConnManager getManager(JobData data) throws IOException { |
| SqoopOptions options = data.getSqoopOptions(); |
| String manualDriver = options.getDriverClassName(); |
| String managerClassName = options.getConnManagerClassName(); |
| |
| // User has specified --driver argument, but he did not specified |
| // manager to use. We will use GenericJdbcManager as this was |
| // the way sqoop was working originally. However we will inform |
| // user that specifying connection manager explicitly is more cleaner |
| // solution for this case. |
| if (manualDriver != null && managerClassName == null) { |
| LOG.warn("Parameter --driver is set to an explicit driver however" |
| + " appropriate connection manager is not being set (via" |
| + " --connection-manager). Sqoop is going to fall back to " |
| + GenericJdbcManager.class.getCanonicalName() + ". Please specify" |
| + " explicitly which connection manager should be used next time." |
| ); |
| return new GenericJdbcManager(manualDriver, options); |
| } |
| |
| // If user specified explicit connection manager, let's use it |
| if (managerClassName != null){ |
| ConnManager connManager = null; |
| |
| try { |
| Class<ConnManager> cls = (Class<ConnManager>) |
| Class.forName(managerClassName); |
| |
| // We have two constructor options, one is with or without explicit |
| // constructor. In most cases --driver argument won't be allowed as the |
| // connectors are forcing to use their building class names. |
| if (manualDriver == null) { |
| Constructor<ConnManager> constructor = |
| cls.getDeclaredConstructor(SqoopOptions.class); |
| connManager = constructor.newInstance(options); |
| } else { |
| Constructor<ConnManager> constructor = |
| cls.getDeclaredConstructor(String.class, SqoopOptions.class); |
| connManager = constructor.newInstance(manualDriver, options); |
| } |
| } catch (ClassNotFoundException e) { |
| LOG.error("Sqoop could not found specified connection manager class " |
| + managerClassName + ". Please check that you've specified the " |
| + "class correctly."); |
| throw new IOException(e); |
| } catch (NoSuchMethodException e) { |
| LOG.error("Sqoop wasn't able to create connnection manager properly. " |
| + "Some of the connectors supports explicit --driver and some " |
| + "do not. Please try to either specify --driver or leave it out."); |
| throw new IOException(e); |
| } catch (Exception e) { |
| LOG.error("Problem with bootstrapping connector manager:" |
| + managerClassName); |
| LOG.error(e); |
| throw new IOException(e); |
| } |
| return connManager; |
| } |
| |
| // Try all the available manager factories. |
| for (ManagerFactory factory : factories) { |
| LOG.debug("Trying ManagerFactory: " + factory.getClass().getName()); |
| ConnManager mgr = factory.accept(data); |
| if (null != mgr) { |
| LOG.debug("Instantiated ConnManager " + mgr.toString()); |
| return mgr; |
| } |
| } |
| |
| throw new IOException("No manager for connect string: " |
| + data.getSqoopOptions().getConnectString()); |
| } |
| |
| /** |
| * Add a ManagerFactory class to the list that we instantiate. |
| * @param conf the Configuration to set. |
| * @param factory the ManagerFactory class name to add. |
| */ |
| private void addManager(Configuration conf, String factory) { |
| String curVal = conf.get(FACTORY_CLASS_NAMES_KEY); |
| if (null == curVal) { |
| conf.set(FACTORY_CLASS_NAMES_KEY, factory); |
| } else { |
| conf.set(FACTORY_CLASS_NAMES_KEY, curVal + "," + factory); |
| } |
| } |
| |
| /** |
| * Read the specified file and extract any ManagerFactory implementation |
| * names from there. |
| * @param conf the configuration to populate. |
| * @param f the file containing the configuration data to add. |
| */ |
| private void addManagersFromFile(Configuration conf, File f) { |
| BufferedReader r = null; |
| try { |
| // The file format is actually Java properties-file syntax. |
| r = new BufferedReader(new InputStreamReader(new FileInputStream(f))); |
| Properties props = new Properties(); |
| props.load(r); |
| |
| for (Map.Entry<Object, Object> entry : props.entrySet()) { |
| // Each key is a ManagerFactory class name. |
| // Each value, if set, is the jar that contains it. |
| String factory = entry.getKey().toString(); |
| addManager(conf, factory); |
| |
| String jarName = entry.getValue().toString(); |
| if (jarName.length() > 0) { |
| ClassLoaderStack.addJarFile(jarName, factory); |
| LOG.debug("Added factory " + factory + " in jar " + jarName |
| + " specified by " + f); |
| } else if (LOG.isDebugEnabled()) { |
| LOG.debug("Added factory " + factory + " specified by " + f); |
| } |
| } |
| } catch (IOException ioe) { |
| LOG.error("Error loading ManagerFactory information from file " |
| + f + ": " + StringUtils.stringifyException(ioe)); |
| } finally { |
| if (null != r) { |
| try { |
| r.close(); |
| } catch (IOException ioe) { |
| LOG.warn("Error closing file " + f + ": " + ioe); |
| } |
| } |
| } |
| } |
| |
| /** |
| * If $SQOOP_CONF_DIR/managers.d/ exists and sqoop.connection.factories is |
| * not set, then we look through the files in that directory; they should |
| * contain lines of the form mgr.class.name[=/path/to/containing.jar]. |
| * |
| * <p> |
| * Put all mgr.class.names into the Configuration, and load any specified |
| * jars into the ClassLoader. |
| * </p> |
| * |
| * @param conf the current configuration to populate with class names. |
| * @return conf again, after possibly populating sqoop.connection.factories. |
| */ |
| private Configuration loadManagersFromConfDir(Configuration conf) { |
| if (conf.get(FACTORY_CLASS_NAMES_KEY) != null) { |
| LOG.debug(FACTORY_CLASS_NAMES_KEY + " is set; ignoring managers.d"); |
| return conf; |
| } |
| |
| String confDirName = System.getenv("SQOOP_CONF_DIR"); |
| if (null == confDirName) { |
| LOG.warn("$SQOOP_CONF_DIR has not been set in the environment. " |
| + "Cannot check for additional configuration."); |
| return conf; |
| } |
| |
| File confDir = new File(confDirName); |
| File mgrDir = new File(confDir, "managers.d"); |
| |
| if (mgrDir.exists() && mgrDir.isDirectory()) { |
| // We have a managers.d subdirectory. Get the file list, sort it, |
| // and process them in order. |
| String[] fileNames; |
| |
| try { |
| fileNames = mgrDir.list(); |
| } catch (SecurityException e) { |
| fileNames = null; |
| } |
| |
| if (null == fileNames) { |
| LOG.warn("Sqoop cannot read $SQOOP_CONF_DIR/managers.d. " |
| + "Please check the permissions on managers.d."); |
| return conf; |
| } |
| |
| Arrays.sort(fileNames); |
| |
| for (String fileName : fileNames) { |
| File f = new File(mgrDir, fileName); |
| if (f.isFile()) { |
| addManagersFromFile(conf, f); |
| } |
| } |
| |
| // Add the default MF. |
| addManager(conf, DEFAULT_FACTORY_CLASS_NAMES); |
| } |
| |
| // Set the classloader in this configuration so that it will use |
| // the jars we just loaded in. |
| conf.setClassLoader(Thread.currentThread().getContextClassLoader()); |
| return conf; |
| } |
| } |
| |