fe/src/main/java/org/apache/impala/planner/ResourceProfile.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.impala.planner;

import org.apache.impala.common.PrintUtils;
import org.apache.impala.thrift.TBackendResourceProfile;
import org.apache.impala.util.MathUtil;

import com.google.common.base.Preconditions;
import com.google.common.math.LongMath;

/**
 * The resources that will be consumed by some part of a plan, e.g. a plan node or
 * plan fragment.
 */
public class ResourceProfile {
  // If the computed values are valid.
  private final boolean isValid_;

  // Estimated memory consumption in bytes. Guaranteed to be >= minReservationBytes_ if
  // both are set and guaranteed to be <= maxMemReservationBytes_ if both are set (the
  // constructor ensures both of these conditions).
  private final long memEstimateBytes_;

  // Minimum memory reservation required to execute in bytes.
  // The valid range is [0, maxReservationBytes_].
  private final long minMemReservationBytes_;

  // Maximum memory reservation allowed for this plan node.
  // The valid range is [minMemReservationBytes_, Long.MAX_VALUE].
  private final long maxMemReservationBytes_;

  // The default spillable buffer size to use in a plan node. Only valid for resource
  // profiles for spilling PlanNodes. Operations like sum(), max(), etc., produce
  // profiles without valid spillableBufferBytes_ values. -1 means invalid.
  private final long spillableBufferBytes_;

  // The buffer size to use for max-sized row in a plan node. -1 means invalid.
  // Must be set to a valid power-of-two value if spillableBufferBytes_ is set.
  private final long maxRowBufferBytes_;

  // The number of threads required to execute the plan node or plan tree. Does not
  // include any optional threads that may be dynamically created or any threads outside
  // of plan execution threads (e.g. system threads or threads used in the RPC stack).
  // -1 if the profile is invalid (i.e. isValid_ is false).
  private final long threadReservation_;

  ResourceProfile(boolean isValid, long memEstimateBytes, long minMemReservationBytes,
      long maxMemReservationBytes, long spillableBufferBytes,
      long maxRowBufferBytes, long threadReservation) {
    Preconditions.checkArgument(spillableBufferBytes == -1 || maxRowBufferBytes != -1);
    Preconditions.checkArgument(spillableBufferBytes == -1
        || LongMath.isPowerOfTwo(spillableBufferBytes));
    Preconditions.checkArgument(maxRowBufferBytes == -1
        || LongMath.isPowerOfTwo(maxRowBufferBytes));
    Preconditions.checkArgument(!isValid || threadReservation >= 0, threadReservation);
    Preconditions.checkArgument(maxMemReservationBytes >= minMemReservationBytes);
    isValid_ = isValid;
    memEstimateBytes_ = (minMemReservationBytes != -1) ?
        Math.max(memEstimateBytes, minMemReservationBytes) : memEstimateBytes;
    minMemReservationBytes_ = minMemReservationBytes;
    maxMemReservationBytes_ = maxMemReservationBytes;
    spillableBufferBytes_ = spillableBufferBytes;
    maxRowBufferBytes_ = maxRowBufferBytes;
    threadReservation_ = threadReservation;
  }

  // Create a resource profile with zero min or max reservation and zero required
  // threads.
  public static ResourceProfile noReservation(long memEstimateBytes) {
    return new ResourceProfile(true, memEstimateBytes, 0, 0, -1, -1, 0);
  }

  public static ResourceProfile invalid() {
    return new ResourceProfile(false, -1, -1, -1, -1, -1, -1);
  }

  public boolean isValid() { return isValid_; }
  public long getMemEstimateBytes() { return memEstimateBytes_; }
  public long getMinMemReservationBytes() { return minMemReservationBytes_; }
  public long getMaxMemReservationBytes() { return maxMemReservationBytes_; }
  public long getSpillableBufferBytes() { return spillableBufferBytes_; }
  public long getMaxRowBufferBytes() { return maxRowBufferBytes_; }
  public long getThreadReservation() { return threadReservation_; }

  // Return a string with the resource profile information suitable for display in an
  // explain plan in a format like: "resource1=value resource2=value"
  public String getExplainString() {
    StringBuilder output = new StringBuilder();
    output.append("mem-estimate=");
    output.append(isValid_ ? PrintUtils.printBytes(memEstimateBytes_) : "invalid");
    output.append(" mem-reservation=");
    output.append(isValid_ ? PrintUtils.printBytes(minMemReservationBytes_) : "invalid");
    // TODO: output maxReservation_ here if the planner becomes more sophisticated in
    // choosing it (beyond 0/unlimited).
    if (isValid_ && spillableBufferBytes_ != -1) {
      output.append(" spill-buffer=");
      output.append(PrintUtils.printBytes(spillableBufferBytes_));
    }
    output.append(" thread-reservation=");
    output.append(isValid_ ? threadReservation_ : "invalid");
    return output.toString();
  }

  // Returns a profile with the max of each value in 'this' and 'other'.
  public ResourceProfile max(ResourceProfile other) {
    if (!isValid()) return other;
    if (!other.isValid()) return this;
    return new ResourceProfile(true,
        Math.max(getMemEstimateBytes(), other.getMemEstimateBytes()),
        Math.max(getMinMemReservationBytes(), other.getMinMemReservationBytes()),
        Math.max(getMaxMemReservationBytes(), other.getMaxMemReservationBytes()), -1, -1,
        Math.max(getThreadReservation(), other.getThreadReservation()));
  }

  // Returns a profile with the sum of each value in 'this' and 'other'.
  public ResourceProfile sum(ResourceProfile other) {
    if (!isValid()) return other;
    if (!other.isValid()) return this;
    return new ResourceProfile(true,
        MathUtil.saturatingAdd(getMemEstimateBytes(), other.getMemEstimateBytes()),
        MathUtil.saturatingAdd(
            getMinMemReservationBytes(),other.getMinMemReservationBytes()),
        MathUtil.saturatingAdd(
            getMaxMemReservationBytes(), other.getMaxMemReservationBytes()),
        -1, -1,
        MathUtil.saturatingAdd(getThreadReservation(), other.getThreadReservation()));
  }

  // Returns a profile with all values multiplied by 'factor'.
  public ResourceProfile multiply(int factor) {
    if (!isValid()) return this;
    return new ResourceProfile(true,
        MathUtil.saturatingMultiply(memEstimateBytes_, factor),
        MathUtil.saturatingMultiply(minMemReservationBytes_, factor),
        MathUtil.saturatingMultiply(maxMemReservationBytes_, factor), -1, -1,
        MathUtil.saturatingMultiply(threadReservation_, factor));
  }

  public TBackendResourceProfile toThrift() {
    TBackendResourceProfile result = new TBackendResourceProfile();
    result.setMin_reservation(minMemReservationBytes_);
    result.setMax_reservation(maxMemReservationBytes_);
    if (spillableBufferBytes_ != -1) {
      result.setSpillable_buffer_size(spillableBufferBytes_);
    }
    if (maxRowBufferBytes_ != -1) {
      result.setMax_row_buffer_size(maxRowBufferBytes_);
    }
    return result;
  }
}