| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| #ifndef __MESOS_ALLOCATOR_ALLOCATOR_HPP__ |
| #define __MESOS_ALLOCATOR_ALLOCATOR_HPP__ |
| |
| #include <string> |
| #include <vector> |
| |
| // ONLY USEFUL AFTER RUNNING PROTOC. |
| #include <mesos/allocator/allocator.pb.h> |
| #include <mesos/scheduler/scheduler.pb.h> |
| |
| #include <mesos/authorizer/authorizer.hpp> |
| #include <mesos/maintenance/maintenance.hpp> |
| #include <mesos/quota/quota.hpp> |
| #include <mesos/resource_quantities.hpp> |
| #include <mesos/resources.hpp> |
| |
| #include <process/future.hpp> |
| |
| #include <stout/duration.hpp> |
| #include <stout/hashmap.hpp> |
| #include <stout/hashset.hpp> |
| #include <stout/lambda.hpp> |
| #include <stout/option.hpp> |
| #include <stout/try.hpp> |
| |
| |
| namespace mesos { |
| namespace allocator { |
| |
| constexpr Duration DEFAULT_ALLOCATOR_RECOVERY_TIMEOUT = Minutes(10); |
| constexpr double DEFAULT_ALLOCATOR_AGENT_RECOVERY_FACTOR = 0.80; |
| |
| |
| /** |
| * Pass in configuration to the allocator. |
| */ |
| struct Options |
| { |
| Duration allocationInterval = Seconds(1); |
| |
| // Resources (by name) that will be excluded from a role's fair share. |
| Option<std::set<std::string>> fairnessExcludeResourceNames = None(); |
| |
| // Filter GPU resources based on the `GPU_RESOURCES` framework capability. |
| bool filterGpuResources = true; |
| |
| // The master's domain, if any. |
| Option<DomainInfo> domain = None(); |
| |
| // The minimum allocatable resource quantities, if any. |
| Option<std::vector<ResourceQuantities>> minAllocatableResources = None(); |
| |
| size_t maxCompletedFrameworks = 0; |
| |
| bool publishPerFrameworkMetrics = true; |
| |
| // Authentication realm for HTTP debug endpoints exposed by the allocator. |
| Option<std::string> readonlyHttpAuthenticationRealm; |
| |
| // Mesos master's authorizer. |
| Option<::mesos::Authorizer*> authorizer; |
| |
| // Recovery options |
| Duration recoveryTimeout = DEFAULT_ALLOCATOR_RECOVERY_TIMEOUT; |
| double agentRecoveryFactor = DEFAULT_ALLOCATOR_AGENT_RECOVERY_FACTOR; |
| }; |
| |
| |
| namespace internal { |
| class OfferConstraintsFilterImpl; |
| |
| } // namespace internal { |
| |
| |
| class OfferConstraintsFilter |
| { |
| public: |
| // TODO(asekretenko): Given that most dependants of the public allocator |
| // interface do not care about filter creation, we should consider decoupling |
| // the filter construction interface (which at this point consists of the |
| // `struct Options` and the `create()` method) from the allocator interface. |
| struct Options |
| { |
| struct RE2Limits |
| { |
| Bytes maxMem; |
| int maxProgramSize; |
| }; |
| |
| RE2Limits re2Limits; |
| }; |
| |
| static Try<OfferConstraintsFilter> create( |
| const Options& options, |
| scheduler::OfferConstraints&& constraints); |
| |
| OfferConstraintsFilter() = delete; |
| |
| // Definitions of these need `OfferConstraintsFilterImpl` to be a complete |
| // type. |
| OfferConstraintsFilter(OfferConstraintsFilter&&); |
| OfferConstraintsFilter& operator=(OfferConstraintsFilter&&); |
| ~OfferConstraintsFilter(); |
| |
| /** |
| * Returns `true` if the allocator is allowed to offer resoureces |
| * on the agent to the framework's role, and `false` otherwise. |
| */ |
| bool isAgentExcluded( |
| const std::string& role, |
| const SlaveInfo& agentInfo) const; |
| |
| // TODO(asekretenko): Add a method for filtering `Resources` on an agent. |
| |
| private: |
| std::unique_ptr<internal::OfferConstraintsFilterImpl> impl; |
| |
| OfferConstraintsFilter(internal::OfferConstraintsFilterImpl&& impl_); |
| }; |
| |
| |
| /** |
| * Per-framework allocator-specific options that are not part of |
| * `FrameworkInfo`. |
| */ |
| struct FrameworkOptions |
| { |
| /** |
| * The set of roles for which the allocator should not generate offers. |
| */ |
| std::set<std::string> suppressedRoles; |
| |
| /** |
| * The internal representation of framework's offer constraints. |
| */ |
| Option<OfferConstraintsFilter> offerConstraintsFilter; |
| }; |
| |
| |
| /** |
| * Basic model of an allocator: resources are allocated to a framework |
| * in the form of offers. A framework can refuse some resources in |
| * offers and run tasks in others. Allocated resources can have offer |
| * operations applied to them in order for frameworks to alter the |
| * resource metadata (e.g. creating persistent volumes). Resources can |
| * be recovered from a framework when tasks finish/fail (or are lost |
| * due to an agent failure) or when an offer is rescinded. |
| * |
| * This is the public API for resource allocators. |
| */ |
| class Allocator |
| { |
| public: |
| /** |
| * Attempts either to create a built-in DRF allocator or to load an |
| * allocator instance from a module using the given name. If `Try` |
| * does not report an error, the wrapped `Allocator*` is not null. |
| * |
| * TODO(bmahler): Figure out how to pass parameters without |
| * burning in the built-in module arguments. |
| * |
| * @param name Name of the allocator. |
| */ |
| static Try<Allocator*> create( |
| const std::string& name, |
| const std::string& roleSorter, |
| const std::string& frameworkSorter); |
| |
| Allocator() {} |
| |
| virtual ~Allocator() {} |
| |
| /** |
| * Initializes the allocator when the master starts up. Any errors in |
| * initialization should fail fast and result in an ABORT. The master expects |
| * the allocator to be successfully initialized if this call returns. |
| * |
| * @param allocationInterval The allocate interval for the allocator, it |
| * determines how often the allocator should perform the batch |
| * allocation. An allocator may also perform allocation based on events |
| * (a framework is added and so on), this depends on the implementation. |
| * @param offerCallback A callback the allocator uses to send allocations |
| * to the frameworks. |
| * @param inverseOfferCallback A callback the allocator uses to send reclaim |
| * allocations from the frameworks. |
| */ |
| virtual void initialize( |
| const Options& options, |
| const lambda::function< |
| void(const FrameworkID&, |
| const hashmap<std::string, hashmap<SlaveID, Resources>>&)>& |
| offerCallback, |
| const lambda::function< |
| void(const FrameworkID&, |
| const hashmap<SlaveID, UnavailableResources>&)>& |
| inverseOfferCallback) = 0; |
| |
| /** |
| * Informs the allocator of the recovered state from the master. |
| * |
| * Because it is hard to define recovery for a running allocator, this |
| * method should be called after `initialize()`, but before actual |
| * allocation starts (i.e. `addSlave()` is called). |
| * |
| * TODO(alexr): Consider extending the signature with expected |
| * frameworks count once it is available upon the master failover. |
| * |
| * @param quotas A (possibly empty) collection of quotas, keyed by |
| * their role, known to the master. |
| */ |
| virtual void recover( |
| const int expectedAgentCount, |
| const hashmap<std::string, Quota>& quotas) = 0; |
| |
| /** |
| * Adds a framework to the Mesos cluster. The allocator is invoked when |
| * a new framework joins the Mesos cluster and is entitled to participate |
| * in resource sharing. |
| * |
| * @param used Resources used by this framework. The allocator should |
| * account for these resources when updating the allocation of this |
| * framework. The allocator should avoid double accounting when yet |
| * unknown agents are added later in `addSlave()`. |
| * |
| * @param active Whether the framework is initially activated. |
| * |
| * @param options Initial FrameworkOptions of this framework. |
| */ |
| virtual void addFramework( |
| const FrameworkID& frameworkId, |
| const FrameworkInfo& frameworkInfo, |
| const hashmap<SlaveID, Resources>& used, |
| bool active, |
| FrameworkOptions&& options) = 0; |
| |
| /** |
| * Removes a framework from the Mesos cluster. It is up to an allocator to |
| * decide what to do with framework's resources. For example, they may be |
| * released and added back to the shared pool of resources. |
| */ |
| virtual void removeFramework( |
| const FrameworkID& frameworkId) = 0; |
| |
| /** |
| * Activates a framework in the Mesos cluster. |
| * Offers are only sent to active frameworks. |
| */ |
| virtual void activateFramework( |
| const FrameworkID& frameworkId) = 0; |
| |
| /** |
| * Deactivates a framework in the Mesos cluster. |
| * Resource offers are not sent to deactivated frameworks. |
| */ |
| virtual void deactivateFramework( |
| const FrameworkID& frameworkId) = 0; |
| |
| /** |
| * Updates FrameworkInfo and FrameworkOptions. |
| * |
| * NOTE: Effective framework options can also be changed via other methods. |
| * For example, `suppress()` and `revive()` change the suppression state |
| * of framework's roles. |
| */ |
| virtual void updateFramework( |
| const FrameworkID& frameworkId, |
| const FrameworkInfo& frameworkInfo, |
| FrameworkOptions&& options) = 0; |
| |
| /** |
| * Adds or re-adds an agent to the Mesos cluster. It is invoked when a |
| * new agent joins the cluster or in case of agent recovery. |
| * |
| * @param slaveId ID of the agent to be added or re-added. |
| * @param slaveInfo Detailed info of the agent. The slaveInfo resources |
| * correspond directly to the static --resources flag value on the agent. |
| * @param capabilities Capabilities of the agent. |
| * @param total The `total` resources are passed explicitly because it |
| * includes resources that are dynamically "checkpointed" on the agent |
| * (e.g. persistent volumes, dynamic reservations, etc). |
| * @param used Resources that are allocated on the current agent. The |
| * allocator should avoid double accounting when yet unknown frameworks |
| * are added later in `addFramework()`. |
| * |
| * TODO(asekretenko): Ideally, to get rig of an intermediate allocator state |
| * when some resources are used by nonexistent frameworks, we should change |
| * the interface so that per-agent per-framework used resources and the |
| * not yet known frameworks that are using them are added atomically. |
| */ |
| virtual void addSlave( |
| const SlaveID& slaveId, |
| const SlaveInfo& slaveInfo, |
| const std::vector<SlaveInfo::Capability>& capabilities, |
| const Option<Unavailability>& unavailability, |
| const Resources& total, |
| const hashmap<FrameworkID, Resources>& used) = 0; |
| |
| /** |
| * Removes an agent from the Mesos cluster. All resources belonging to this |
| * agent should be released by the allocator. |
| */ |
| virtual void removeSlave( |
| const SlaveID& slaveId) = 0; |
| |
| /** |
| * Updates an agent. |
| * |
| * TODO(bevers): Make `total` and `capabilities` non-optional. |
| * |
| * @param slaveInfo The current slave info of the agent. |
| * @param total The new total resources on the agent. |
| * @param capabilities The new capabilities of the agent. |
| */ |
| virtual void updateSlave( |
| const SlaveID& slave, |
| const SlaveInfo& slaveInfo, |
| const Option<Resources>& total = None(), |
| const Option<std::vector<SlaveInfo::Capability>>& |
| capabilities = None()) = 0; |
| |
| /** |
| * Add resources from a local resource provider to an agent. |
| * |
| * @param slave Id of the agent to modify. |
| * @param total The resources to add to the agent's total resources. |
| * @param used The resources to add to the resources tracked as used |
| * for this agent. |
| */ |
| virtual void addResourceProvider( |
| const SlaveID& slave, |
| const Resources& total, |
| const hashmap<FrameworkID, Resources>& used) = 0; |
| |
| /** |
| * Activates an agent. This is invoked when an agent reregisters. Offers |
| * are only sent for activated agents. |
| */ |
| virtual void activateSlave( |
| const SlaveID& slaveId) = 0; |
| |
| /** |
| * Deactivates an agent. |
| * |
| * This is triggered if an agent disconnects from the master. The allocator |
| * should treat all offers from the deactivated agent as rescinded. (There |
| * is no separate call to the allocator to handle this). Resources aren't |
| * "recovered" when an agent deactivates because the resources are lost. |
| */ |
| virtual void deactivateSlave( |
| const SlaveID& slaveId) = 0; |
| |
| /** |
| * Updates the list of trusted agents. |
| * |
| * This is invoked when the master starts up with the --whitelist flag. |
| * |
| * @param whitelist A set of agents that are allowed to contribute |
| * their resources to the resource pool. |
| */ |
| virtual void updateWhitelist( |
| const Option<hashset<std::string>>& whitelist) = 0; |
| |
| /** |
| * Requests resources for a framework. |
| * |
| * A framework may request resources via this call. It is up to the allocator |
| * how to react to this request. For example, a request may be ignored, or |
| * may influence internal priorities the allocator may keep for frameworks. |
| */ |
| virtual void requestResources( |
| const FrameworkID& frameworkId, |
| const std::vector<Request>& requests) = 0; |
| |
| /** |
| * Updates allocation by applying offer operations. |
| * |
| * This call is mainly intended to support persistence-related features |
| * (dynamic reservation and persistent volumes). The allocator may react |
| * differently for certain offer operations. The allocator should use this |
| * call to update bookkeeping information related to the framework. The |
| * `offeredResources` are the resources that the operations are applied to |
| * and must be allocated to a single role. |
| */ |
| virtual void updateAllocation( |
| const FrameworkID& frameworkId, |
| const SlaveID& slaveId, |
| const Resources& offeredResources, |
| const std::vector<ResourceConversion>& conversions) = 0; |
| |
| /** |
| * Updates available resources on an agent based on a sequence of offer |
| * operations. Operations may include reserve, unreserve, create or destroy. |
| * |
| * @param slaveId ID of the agent. |
| * @param operations The offer operations to apply to this agent's resources. |
| */ |
| virtual process::Future<Nothing> updateAvailable( |
| const SlaveID& slaveId, |
| const std::vector<Offer::Operation>& operations) = 0; |
| |
| /** |
| * Updates unavailability for an agent. |
| * |
| * We currently support storing the next unavailability, if there is one, |
| * per agent. If `unavailability` is not set then there is no known upcoming |
| * unavailability. This might require the implementation of the function to |
| * remove any inverse offers that are outstanding. |
| */ |
| virtual void updateUnavailability( |
| const SlaveID& slaveId, |
| const Option<Unavailability>& unavailability) = 0; |
| |
| /** |
| * Updates inverse offer. |
| * |
| * Informs the allocator that the inverse offer has been responded to or |
| * revoked. |
| * |
| * @param unavailableResources The `unavailableResources` can be used by the |
| * allocator to distinguish between different inverse offers sent to the |
| * same framework for the same slave. |
| * @param status If `status` is not set then the inverse offer was not |
| * responded to, possibly because the offer timed out or was rescinded. |
| * This might require the implementation of the function to remove any |
| * inverse offers that are outstanding. |
| * @param filters A filter attached to the inverse offer can be used by the |
| * framework to control when it wants to be contacted again with the |
| * inverse offer. The "filters" for InverseOffers are identical to the |
| * existing mechanism for re-offering Offers to frameworks. |
| */ |
| virtual void updateInverseOffer( |
| const SlaveID& slaveId, |
| const FrameworkID& frameworkId, |
| const Option<UnavailableResources>& unavailableResources, |
| const Option<InverseOfferStatus>& status, |
| const Option<Filters>& filters = None()) = 0; |
| |
| /** |
| * Retrieves the status of all inverse offers maintained by the allocator. |
| */ |
| virtual process::Future< |
| hashmap<SlaveID, |
| hashmap<FrameworkID, mesos::allocator::InverseOfferStatus>>> |
| getInverseOfferStatuses() = 0; |
| |
| /** |
| * This method should be invoked when the offered resources has become |
| * actually allocated. |
| */ |
| virtual void transitionOfferedToAllocated( |
| const SlaveID& slaveId, const Resources& resources) = 0; |
| |
| /** |
| * Recovers resources. |
| * |
| * Used to update the set of available resources for a specific agent. This |
| * method is invoked to inform the allocator about offered resources that |
| * have been refused or allocated (i.e. used for launching tasks) resources |
| * that are no longer in use. The resources will have an |
| * `allocation_info.role` assigned and callers are expected to only call this |
| * with resources allocated to a single role. |
| * |
| * |
| * TODO(bmahler): We could allow resources allocated to multiple roles |
| * within a single call here, but filtering them in the same way does |
| * not seem desirable. |
| */ |
| virtual void recoverResources( |
| const FrameworkID& frameworkId, |
| const SlaveID& slaveId, |
| const Resources& resources, |
| const Option<Filters>& filters, |
| bool isAllocated) = 0; |
| |
| /** |
| * Suppresses offers. |
| * |
| * Informs the allocator to stop sending offers to this framework for the |
| * specified roles. If `roles` is an empty set, we will stop sending offers |
| * to this framework for all of the framework's subscribed roles. |
| */ |
| virtual void suppressOffers( |
| const FrameworkID& frameworkId, |
| const std::set<std::string>& roles) = 0; |
| |
| /** |
| * Revives offers to this framework for the specified roles. This is |
| * invoked by a framework when it wishes to receive filtered resources |
| * immediately or get itself out of a suppressed state. If `roles` is |
| * an empty set, it is treated as being set to all of the framework's |
| * subscribed roles. |
| */ |
| virtual void reviveOffers( |
| const FrameworkID& frameworkId, |
| const std::set<std::string>& roles) = 0; |
| |
| /** |
| * Informs the allocator to update quota for the given role. |
| * |
| * It is up to the allocator implementation how to satisfy quota. An |
| * implementation may employ different strategies for roles with or |
| * without quota. All roles have a default quota defined as `DEFAULT_QUOTA`. |
| * Currently, it is no guarantees and no limits. Thus to "remove" a quota, |
| * one should simply update the quota to be `DEFAULT_QUOTA`. |
| */ |
| virtual void updateQuota( |
| const std::string& role, |
| const Quota& quota) = 0; |
| |
| /** |
| * Updates the weight associated with one or more roles. If a role |
| * was previously configured to have a weight and that role is |
| * omitted from this list, it keeps its old weight. |
| */ |
| virtual void updateWeights( |
| const std::vector<WeightInfo>& weightInfos) = 0; |
| |
| /** |
| * Idempotent helper to pause allocations. |
| */ |
| virtual void pause() = 0; |
| |
| /** |
| * Idempotent helper to resume allocations. |
| */ |
| virtual void resume() = 0; |
| }; |
| |
| } // namespace allocator { |
| } // namespace mesos { |
| |
| #endif // __MESOS_MASTER_ALLOCATOR_HPP__ |