| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.falcon.update; |
| |
| import org.apache.commons.lang3.StringUtils; |
| import org.apache.falcon.FalconException; |
| import org.apache.falcon.entity.ClusterHelper; |
| import org.apache.falcon.entity.DatasourceHelper; |
| import org.apache.falcon.entity.EntityUtil; |
| import org.apache.falcon.entity.FeedHelper; |
| import org.apache.falcon.entity.ProcessHelper; |
| import org.apache.falcon.entity.Storage; |
| import org.apache.falcon.entity.v0.Entity; |
| import org.apache.falcon.entity.v0.EntityType; |
| import org.apache.falcon.entity.v0.cluster.ClusterLocationType; |
| import org.apache.falcon.entity.v0.cluster.Interfacetype; |
| import org.apache.falcon.entity.v0.datasource.Datasource; |
| import org.apache.falcon.entity.v0.feed.Feed; |
| import org.apache.falcon.entity.v0.process.Cluster; |
| import org.apache.falcon.entity.v0.process.Process; |
| import org.apache.hadoop.fs.Path; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| import java.util.Date; |
| |
| /** |
| * Helper methods to facilitate entity updates. |
| */ |
| public final class UpdateHelper { |
| private static final Logger LOG = LoggerFactory.getLogger(UpdateHelper.class); |
| |
| private static final String[] FEED_FIELDS = new String[]{"partitions", "groups", "lateArrival.cutOff", |
| "schema.location", "schema.provider", "tags", |
| "group", "owner", "permission", }; |
| private static final String[] PROCESS_FIELDS = new String[]{"retry.policy", "retry.delay", "retry.attempts", |
| "lateProcess.policy", "lateProcess.delay", |
| "lateProcess.lateInputs[\\d+].input", |
| "lateProcess.lateInputs[\\d+].workflowPath", |
| "owner", "group", "permission", "tags", |
| "pipelines", }; |
| |
| private UpdateHelper() {} |
| |
| public static boolean isEntityUpdated(Entity oldEntity, Entity newEntity, String cluster, |
| Path oldStagingPath) throws FalconException { |
| Entity oldView = EntityUtil.getClusterView(oldEntity, cluster); |
| Entity newView = EntityUtil.getClusterView(newEntity, cluster); |
| |
| //staging path contains md5 of the cluster view of entity |
| String[] parts = oldStagingPath.getName().split("_"); |
| if (parts[0].equals(EntityUtil.md5(newView))) { |
| return false; |
| } |
| |
| switch (oldEntity.getEntityType()) { |
| case FEED: |
| return !EntityUtil.equals(oldView, newView, FEED_FIELDS); |
| |
| case PROCESS: |
| return !EntityUtil.equals(oldView, newView, PROCESS_FIELDS); |
| |
| case CLUSTER: |
| return isClusterEntityUpdated((org.apache.falcon.entity.v0.cluster.Cluster) oldEntity, |
| (org.apache.falcon.entity.v0.cluster.Cluster) newEntity); |
| |
| default: |
| } |
| throw new IllegalArgumentException("Unhandled entity type " + oldEntity.getEntityType()); |
| } |
| |
| public static boolean shouldUpdate(Entity oldEntity, Entity newEntity, Entity affectedEntity, String cluster) |
| throws FalconException { |
| if (oldEntity.getEntityType() == EntityType.FEED && affectedEntity.getEntityType() == EntityType.PROCESS) { |
| |
| Feed oldFeed = (Feed) oldEntity; |
| Feed newFeed = (Feed) newEntity; |
| Process affectedProcess = (Process) affectedEntity; |
| |
| //check if affectedProcess is defined for this cluster |
| Cluster processCluster = ProcessHelper.getCluster(affectedProcess, cluster); |
| if (processCluster == null) { |
| LOG.debug("Process {} is not defined for cluster {}. Skipping", affectedProcess.getName(), cluster); |
| return false; |
| } |
| |
| if (processCluster.getValidity().getEnd().before(new Date())) { |
| LOG.debug("Process {} validity {} is in the past. Skipping...", affectedProcess.getName(), |
| processCluster.getValidity().getEnd()); |
| return false; |
| } |
| |
| if (!oldFeed.getFrequency().equals(newFeed.getFrequency())) { |
| LOG.debug("{}: Frequency has changed. Updating...", oldFeed.toShortString()); |
| return true; |
| } |
| |
| if (!StringUtils.equals(oldFeed.getAvailabilityFlag(), newFeed.getAvailabilityFlag())) { |
| LOG.debug("{}: Availability flag has changed. Updating...", oldFeed.toShortString()); |
| return true; |
| } |
| |
| org.apache.falcon.entity.v0.feed.Cluster oldFeedCluster = FeedHelper.getCluster(oldFeed, cluster); |
| org.apache.falcon.entity.v0.feed.Cluster newFeedCluster = FeedHelper.getCluster(newFeed, cluster); |
| if (!oldFeedCluster.getValidity().getStart().equals(newFeedCluster.getValidity().getStart())) { |
| LOG.debug("{}: Start time for cluster {} has changed. Updating...", oldFeed.toShortString(), cluster); |
| return true; |
| } |
| |
| Storage oldFeedStorage = FeedHelper.createStorage(cluster, oldFeed); |
| Storage newFeedStorage = FeedHelper.createStorage(cluster, newFeed); |
| |
| if (!oldFeedStorage.isIdentical(newFeedStorage)) { |
| LOG.debug("{}: Storage has changed. Updating...", oldFeed.toShortString()); |
| return true; |
| } |
| return false; |
| |
| } else { |
| LOG.debug(newEntity.toShortString()); |
| LOG.debug(affectedEntity.toShortString()); |
| throw new FalconException("Don't know what to do. Unexpected scenario"); |
| } |
| } |
| |
| public static boolean isClusterEntityUpdated(final org.apache.falcon.entity.v0.cluster.Cluster oldEntity, |
| final org.apache.falcon.entity.v0.cluster.Cluster newEntity) { |
| /* |
| * Name should not be updated. |
| * interface, locations, properties, colo : Update bundle/coord for dependent entities. |
| * Description, tags, ACL : no need to update bundle/coord for dependent entities. |
| */ |
| if (!oldEntity.getColo().equals(newEntity.getColo())) { |
| return true; |
| } |
| |
| for(Interfacetype interfacetype : Interfacetype.values()) { |
| if (!ClusterHelper.matchInterface(oldEntity, newEntity, interfacetype)) { |
| return true; |
| } |
| } |
| |
| for(ClusterLocationType locationType : ClusterLocationType.values()) { |
| if (!ClusterHelper.matchLocations(oldEntity, newEntity, locationType)) { |
| return true; |
| } |
| } |
| |
| if (!ClusterHelper.matchProperties(oldEntity, newEntity)) { |
| return true; |
| } |
| |
| return false; |
| } |
| |
| public static boolean isDatasourceEntityUpdated(final Datasource oldEntity, final Datasource newEntity) |
| throws FalconException { |
| // ignore changes : colo, acl, description, tags |
| // can't change : name, data source entity type |
| |
| // major change that trigger bundle rewrite |
| // driver class name change but not driver jar as it is automatically picked up from share lib |
| |
| if (!DatasourceHelper.isSameDriverClazz(oldEntity.getDriver(), newEntity.getDriver())) { |
| return true; |
| } |
| |
| // interface endpoint, credential or driver update will trigger a bundle rewrite |
| for(org.apache.falcon.entity.v0.datasource.Interfacetype ifacetype |
| : org.apache.falcon.entity.v0.datasource.Interfacetype.values()) { |
| if (!DatasourceHelper.isSameInterface(oldEntity, newEntity, ifacetype)) { |
| return true; |
| } |
| } |
| // check default credential too |
| if (!DatasourceHelper.isSameCredentials(oldEntity.getInterfaces().getCredential(), |
| newEntity.getInterfaces().getCredential())) { |
| return true; |
| } |
| |
| // any change in the properties will trigger a bundle rewrite |
| if (!DatasourceHelper.isSameProperties(oldEntity, newEntity)) { |
| return true; |
| } |
| return false; |
| } |
| } |
