GEODE-5056: when found the dropped events at primary sender, send (#1794)

QueueRemovalMessage for it

(cherry picked from commit f7bb77c89a3d19673e8929275fc6c407a4b382bd)
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/wan/AbstractGatewaySenderEventProcessor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/wan/AbstractGatewaySenderEventProcessor.java
index eea7480..34d511c 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/wan/AbstractGatewaySenderEventProcessor.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/wan/AbstractGatewaySenderEventProcessor.java
@@ -304,7 +304,7 @@
           (ParallelGatewaySenderQueue) cpgsq.getQueueByBucket(bucketId);
       boolean isPrimary = prQ.getRegionAdvisor().getBucketAdvisor(bucketId).isPrimary();
       if (isPrimary) {
-        pgsq.addRemovedEvent(prQ, bucketId, shadowKey);
+        pgsq.sendQueueRemovalMesssageForDroppedEvent(prQ, bucketId, shadowKey);
         this.sender.getStatistics().incEventsNotQueuedAtYetRunningPrimarySender();
         if (logger.isDebugEnabled()) {
           logger.debug("register dropped event for primary queue. BucketId is " + bucketId
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/wan/parallel/ParallelGatewaySenderQueue.java b/geode-core/src/main/java/org/apache/geode/internal/cache/wan/parallel/ParallelGatewaySenderQueue.java
index 89880fc..cdb33ab 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/wan/parallel/ParallelGatewaySenderQueue.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/wan/parallel/ParallelGatewaySenderQueue.java
@@ -1112,7 +1112,7 @@
 
   // This method may need synchronization in case it is used by
   // ConcurrentParallelGatewaySender
-  public void addRemovedEvent(PartitionedRegion prQ, int bucketId, Object key) {
+  protected void addRemovedEvent(PartitionedRegion prQ, int bucketId, Object key) {
     StoppableReentrantLock lock = buckToDispatchLock;
     if (lock != null) {
       lock.lock();
@@ -1133,6 +1133,21 @@
     }
   }
 
+  public void sendQueueRemovalMesssageForDroppedEvent(PartitionedRegion prQ, int bucketId,
+      Object key) {
+    final HashMap<String, Map<Integer, List>> temp = new HashMap<String, Map<Integer, List>>();
+    Map bucketIdToDispatchedKeys = new ConcurrentHashMap();
+    temp.put(prQ.getFullPath(), bucketIdToDispatchedKeys);
+    addRemovedEventToMap(bucketIdToDispatchedKeys, bucketId, key);
+    Set<InternalDistributedMember> recipients =
+        removalThread.getAllRecipients(sender.getCache(), temp);
+    if (!recipients.isEmpty()) {
+      ParallelQueueRemovalMessage pqrm = new ParallelQueueRemovalMessage(temp);
+      pqrm.setRecipients(recipients);
+      sender.getCache().getInternalDistributedSystem().getDistributionManager().putOutgoing(pqrm);
+    }
+  }
+
   private void addRemovedEventToMap(Map bucketIdToDispatchedKeys, int bucketId, Object key) {
     List dispatchedKeys = (List) bucketIdToDispatchedKeys.get(bucketId);
     if (dispatchedKeys == null) {
diff --git a/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/serial/SerialGatewaySenderOperationsDUnitTest.java b/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/serial/SerialGatewaySenderOperationsDUnitTest.java
index 8df5650..434fa62 100644
--- a/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/serial/SerialGatewaySenderOperationsDUnitTest.java
+++ b/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/serial/SerialGatewaySenderOperationsDUnitTest.java
@@ -50,6 +50,7 @@
 import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;
 import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
 import org.apache.geode.test.junit.categories.DistributedTest;
+import org.apache.geode.test.junit.categories.FlakyTest;
 import org.apache.geode.test.junit.categories.WanTest;
 
 /**
@@ -268,6 +269,7 @@
     vm5.invoke(() -> WANTestBase.validateQueueSizeStat("ln", 0));
   }
 
+  @Category({FlakyTest.class, WanTest.class}) // GEODE-5056
   @Test
   public void testRestartSerialGatewaySendersWhilePutting() throws Throwable {
     Integer lnPort = (Integer) vm0.invoke(() -> WANTestBase.createFirstLocatorWithDSId(1));