src/main/java/org/apache/sysml/hops/FunctionOp.java - systemds - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
  */

 package org.apache.sysml.hops;

 import java.util.ArrayList;

 import org.apache.sysml.lops.FunctionCallCP;
 import org.apache.sysml.lops.Lop;
 import org.apache.sysml.lops.LopsException;
 import org.apache.sysml.lops.LopProperties.ExecType;
 import org.apache.sysml.parser.Expression.DataType;
 import org.apache.sysml.parser.Expression.ValueType;
 import org.apache.sysml.runtime.controlprogram.Program;
 import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimatorHops;

 /**
  * This FunctionOp represents the call to a DML-bodied or external function.
  *
  * Note: Currently, we support expressions in function arguments but no function calls
  * in expressions.
  */
 public class FunctionOp extends Hop
 {

 	public static String OPSTRING = "extfunct";

 	public enum FunctionType{
 		DML,
 		EXTERNAL_MEM,
 		EXTERNAL_FILE,
 		MULTIRETURN_BUILTIN,
 		UNKNOWN
 	}

 	private FunctionType _type = null;
 	private String _fnamespace = null;
 	private String _fname = null;
 	private String[] _outputs = null;
 	private ArrayList<Hop> _outputHops = null;

 	private FunctionOp() {
 		//default constructor for clone
 	}

 	public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, ArrayList<Hop> outputHops) {
 		this(type, fnamespace, fname, finputs, outputs);
 		_outputHops = outputHops;
 	}

 	public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs)
 	{
 		super(fnamespace + Program.KEY_DELIM + fname, DataType.UNKNOWN, ValueType.UNKNOWN );

 		_type = type;
 		_fnamespace = fnamespace;
 		_fname = fname;
 		_outputs = outputs;

 		for( Hop in : finputs )
 		{
 			getInput().add(in);
 			in.getParent().add(this);
 		}
 	}

 	public String getFunctionNamespace()
 	{
 		return _fnamespace;
 	}

 	public String getFunctionName()
 	{
 		return _fname;
 	}

 	public void setFunctionName( String fname )
 	{
 		_fname = fname;
 	}

 	public ArrayList<Hop> getOutputs() {
 		return _outputHops;
 	}

 	public String[] getOutputVariableNames()
 	{
 		return _outputs;
 	}

 	public FunctionType getFunctionType()
 	{
 		return _type;
 	}

 	@Override
 	public boolean allowsAllExecTypes() {
 		return false;
 	}

 	@Override
 	public void computeMemEstimate( MemoTable memo )
 	{
 		//overwrites default hops behavior

 		if( _type == FunctionType.DML )
 			_memEstimate = 1; //minimal mem estimate
 		else if( _type == FunctionType.EXTERNAL_MEM )
 			_memEstimate = 2* getInputSize(); //in/out
 		else if(    _type == FunctionType.EXTERNAL_FILE || _type == FunctionType.UNKNOWN )
 			_memEstimate = CostEstimatorHops.DEFAULT_MEM_MR;
 		else if ( _type == FunctionType.MULTIRETURN_BUILTIN ) {
 			boolean outputDimsKnown = true;
 			for(Hop out : getOutputs()){
 				outputDimsKnown &= out.dimsKnown();
 			}
 			if( outputDimsKnown ) {
 				long lnnz = ((_nnz>=0)?_nnz:_dim1*_dim2);
 				_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
 				_processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
 			}
 			_memEstimate = getInputOutputSize();
 		}
 	}

 	@Override
 	protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
 	{
 		if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
 			throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
 		else {
 			if ( getFunctionName().equalsIgnoreCase("qr") ) {
 				// upper-triangular and lower-triangular matrices
 				long outputH = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
 				long outputR = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
 				return outputH+outputR;
 			}
 			else if ( getFunctionName().equalsIgnoreCase("lu") ) {
 				// upper-triangular and lower-triangular matrices
 				long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0/getOutputs().get(1).getDim2());
 				long outputL = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
 				long outputU = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
 				return outputL+outputU+outputP;
 			}
 			else if ( getFunctionName().equalsIgnoreCase("eigen") ) {
 				long outputVectors = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
 				long outputValues = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), 1, 1.0);
 				return outputVectors+outputValues;
 			}
 			else
 				throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
 		}
 	}

 	@Override
 	protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
 	{
 		if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
 			throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
 		else {
 			if ( getFunctionName().equalsIgnoreCase("qr") ) {
 				// matrix of size same as the input
 				double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
 				//System.out.println("QRInter " + interOutput/1024/1024);
 				return interOutput;
 			}
 			else if ( getFunctionName().equalsIgnoreCase("lu")) {
 				// 1D vector
 				double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
 				//System.out.println("LUInter " + interOutput/1024/1024);
 				return interOutput;
 			}
 			else if ( getFunctionName().equalsIgnoreCase("eigen")) {
 				// One matrix of size original input and three 1D vectors (used to represent tridiagonal matrix)
 				double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0)
 						+ 3*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
 				//System.out.println("EigenInter " + interOutput/1024/1024);
 				return interOutput;
 			}
 			else
 				throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
 		}
 	}

 	@Override
 	protected long[] inferOutputCharacteristics( MemoTable memo )
 	{
 		throw new RuntimeException("Invalid call of inferOutputCharacteristics in FunctionOp.");
 	}

 	@Override
 	public Lop constructLops()
 		throws HopsException, LopsException
 	{
 		//return already created lops
 		if( getLops() != null )
 			return getLops();

 		ExecType et = optFindExecType();

 		//construct input lops (recursive)
 		ArrayList<Lop> tmp = new ArrayList<Lop>();
 		for( Hop in : getInput() )
 			tmp.add( in.constructLops() );

 		//construct function call
 		FunctionCallCP fcall = new FunctionCallCP( tmp, _fnamespace, _fname, _outputs, _outputHops, et );
 		setLineNumbers( fcall );
 		setLops( fcall );

 		//note: no reblock lop because outputs directly bound

 		return getLops();
 	}

 	@Override
 	public String getOpString()
 	{
 		return OPSTRING;
 	}

 	@Override
 	protected ExecType optFindExecType()
 		throws HopsException
 	{
 		checkAndSetForcedPlatform();

 		if ( getFunctionType() == FunctionType.MULTIRETURN_BUILTIN ) {

 			// check if there is sufficient memory to execute this function
 			if( getFunctionName().equalsIgnoreCase("transformencode") ) {
 				_etype = ((_etypeForced==ExecType.SPARK
 					|| (getMemEstimate() >= OptimizerUtils.getLocalMemBudget()
 						&& OptimizerUtils.isSparkExecutionMode())) ? ExecType.SPARK : ExecType.CP);
 			}
 			else {
 				// Since the memory estimate is only conservative, do not throw
 				// exception if the estimated memory is larger than the budget
 				// Nevertheless, memory estimates these functions are useful for
 				// other purposes, such as compiling parfor
 				_etype = ExecType.CP;
 			}
 		}
 		else {
 			// the actual function call is always CP
 			_etype = ExecType.CP;
 		}

 		return _etype;
 	}

 	@Override
 	public void refreshSizeInformation()
 	{
 		//do nothing
 	}

 	@Override
 	@SuppressWarnings("unchecked")
 	public Object clone() throws CloneNotSupportedException
 	{
 		FunctionOp ret = new FunctionOp();

 		//copy generic attributes
 		ret.clone(this, false);

 		//copy specific attributes
 		ret._type = _type;
 		ret._fnamespace = _fnamespace;
 		ret._fname = _fname;
 		ret._outputs = _outputs.clone();
 		if( _outputHops != null )
 			ret._outputHops = (ArrayList<Hop>) _outputHops.clone();

 		return ret;
 	}

 	@Override
 	public boolean compare( Hop that )
 	{
 		return false;
 	}
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing,
	* software distributed under the License is distributed on an
	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	* KIND, either express or implied. See the License for the
	* specific language governing permissions and limitations
	* under the License.
	*/

	package org.apache.sysml.hops;

	import java.util.ArrayList;

	import org.apache.sysml.lops.FunctionCallCP;
	import org.apache.sysml.lops.Lop;
	import org.apache.sysml.lops.LopsException;
	import org.apache.sysml.lops.LopProperties.ExecType;
	import org.apache.sysml.parser.Expression.DataType;
	import org.apache.sysml.parser.Expression.ValueType;
	import org.apache.sysml.runtime.controlprogram.Program;
	import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimatorHops;

	/**
	* This FunctionOp represents the call to a DML-bodied or external function.
	*
	* Note: Currently, we support expressions in function arguments but no function calls
	* in expressions.
	*/
	public class FunctionOp extends Hop
	{

	public static String OPSTRING = "extfunct";

	public enum FunctionType{
	DML,
	EXTERNAL_MEM,
	EXTERNAL_FILE,
	MULTIRETURN_BUILTIN,
	UNKNOWN
	}

	private FunctionType _type = null;
	private String _fnamespace = null;
	private String _fname = null;
	private String[] _outputs = null;
	private ArrayList<Hop> _outputHops = null;

	private FunctionOp() {
	//default constructor for clone
	}

	public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, ArrayList<Hop> outputHops) {
	this(type, fnamespace, fname, finputs, outputs);
	_outputHops = outputHops;
	}

	public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs)
	{
	super(fnamespace + Program.KEY_DELIM + fname, DataType.UNKNOWN, ValueType.UNKNOWN );

	_type = type;
	_fnamespace = fnamespace;
	_fname = fname;
	_outputs = outputs;

	for( Hop in : finputs )
	{
	getInput().add(in);
	in.getParent().add(this);
	}
	}

	public String getFunctionNamespace()
	{
	return _fnamespace;
	}

	public String getFunctionName()
	{
	return _fname;
	}

	public void setFunctionName( String fname )
	{
	_fname = fname;
	}

	public ArrayList<Hop> getOutputs() {
	return _outputHops;
	}

	public String[] getOutputVariableNames()
	{
	return _outputs;
	}

	public FunctionType getFunctionType()
	{
	return _type;
	}

	@Override
	public boolean allowsAllExecTypes() {
	return false;
	}

	@Override
	public void computeMemEstimate( MemoTable memo )
	{
	//overwrites default hops behavior

	if( _type == FunctionType.DML )
	_memEstimate = 1; //minimal mem estimate
	else if( _type == FunctionType.EXTERNAL_MEM )
	_memEstimate = 2* getInputSize(); //in/out
	else if( _type == FunctionType.EXTERNAL_FILE \|\| _type == FunctionType.UNKNOWN )
	_memEstimate = CostEstimatorHops.DEFAULT_MEM_MR;
	else if ( _type == FunctionType.MULTIRETURN_BUILTIN ) {
	boolean outputDimsKnown = true;
	for(Hop out : getOutputs()){
	outputDimsKnown &= out.dimsKnown();
	}
	if( outputDimsKnown ) {
	long lnnz = ((_nnz>=0)?_nnz:_dim1*_dim2);
	_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, lnnz );
	_processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz);
	}
	_memEstimate = getInputOutputSize();
	}
	}

	@Override
	protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
	{
	if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
	throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
	else {
	if ( getFunctionName().equalsIgnoreCase("qr") ) {
	// upper-triangular and lower-triangular matrices
	long outputH = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
	long outputR = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
	return outputH+outputR;
	}
	else if ( getFunctionName().equalsIgnoreCase("lu") ) {
	// upper-triangular and lower-triangular matrices
	long outputP = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 1.0/getOutputs().get(1).getDim2());
	long outputL = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 0.5);
	long outputU = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), getOutputs().get(1).getDim2(), 0.5);
	return outputL+outputU+outputP;
	}
	else if ( getFunctionName().equalsIgnoreCase("eigen") ) {
	long outputVectors = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(0).getDim1(), getOutputs().get(0).getDim2(), 1.0);
	long outputValues = OptimizerUtils.estimateSizeExactSparsity(getOutputs().get(1).getDim1(), 1, 1.0);
	return outputVectors+outputValues;
	}
	else
	throw new RuntimeException("Invalid call of computeOutputMemEstimate in FunctionOp.");
	}
	}

	@Override
	protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
	{
	if ( getFunctionType() != FunctionType.MULTIRETURN_BUILTIN )
	throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
	else {
	if ( getFunctionName().equalsIgnoreCase("qr") ) {
	// matrix of size same as the input
	double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0);
	//System.out.println("QRInter " + interOutput/1024/1024);
	return interOutput;
	}
	else if ( getFunctionName().equalsIgnoreCase("lu")) {
	// 1D vector
	double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
	//System.out.println("LUInter " + interOutput/1024/1024);
	return interOutput;
	}
	else if ( getFunctionName().equalsIgnoreCase("eigen")) {
	// One matrix of size original input and three 1D vectors (used to represent tridiagonal matrix)
	double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0)
	+ 3*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0);
	//System.out.println("EigenInter " + interOutput/1024/1024);
	return interOutput;
	}
	else
	throw new RuntimeException("Invalid call of computeIntermediateMemEstimate in FunctionOp.");
	}
	}

	@Override
	protected long[] inferOutputCharacteristics( MemoTable memo )
	{
	throw new RuntimeException("Invalid call of inferOutputCharacteristics in FunctionOp.");
	}

	@Override
	public Lop constructLops()
	throws HopsException, LopsException
	{
	//return already created lops
	if( getLops() != null )
	return getLops();

	ExecType et = optFindExecType();

	//construct input lops (recursive)
	ArrayList<Lop> tmp = new ArrayList<Lop>();
	for( Hop in : getInput() )
	tmp.add( in.constructLops() );

	//construct function call
	FunctionCallCP fcall = new FunctionCallCP( tmp, _fnamespace, _fname, _outputs, _outputHops, et );
	setLineNumbers( fcall );
	setLops( fcall );

	//note: no reblock lop because outputs directly bound

	return getLops();
	}

	@Override
	public String getOpString()
	{
	return OPSTRING;
	}

	@Override
	protected ExecType optFindExecType()
	throws HopsException
	{
	checkAndSetForcedPlatform();

	if ( getFunctionType() == FunctionType.MULTIRETURN_BUILTIN ) {

	// check if there is sufficient memory to execute this function
	if( getFunctionName().equalsIgnoreCase("transformencode") ) {
	_etype = ((_etypeForced==ExecType.SPARK
	\|\| (getMemEstimate() >= OptimizerUtils.getLocalMemBudget()
	&& OptimizerUtils.isSparkExecutionMode())) ? ExecType.SPARK : ExecType.CP);
	}
	else {
	// Since the memory estimate is only conservative, do not throw
	// exception if the estimated memory is larger than the budget
	// Nevertheless, memory estimates these functions are useful for
	// other purposes, such as compiling parfor
	_etype = ExecType.CP;
	}
	}
	else {
	// the actual function call is always CP
	_etype = ExecType.CP;
	}

	return _etype;
	}

	@Override
	public void refreshSizeInformation()
	{
	//do nothing
	}

	@Override
	@SuppressWarnings("unchecked")
	public Object clone() throws CloneNotSupportedException
	{
	FunctionOp ret = new FunctionOp();

	//copy generic attributes
	ret.clone(this, false);

	//copy specific attributes
	ret._type = _type;
	ret._fnamespace = _fnamespace;
	ret._fname = _fname;
	ret._outputs = _outputs.clone();
	if( _outputHops != null )
	ret._outputHops = (ArrayList<Hop>) _outputHops.clone();

	return ret;
	}

	@Override
	public boolean compare( Hop that )
	{
	return false;
	}
	}