HDDS-7032. RDBStore#getUpdatesSince should throw SequenceNumberNotFou… (#3613)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index e32987c..caf0263 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -286,6 +286,16 @@
try (TransactionLogIterator transactionLogIterator =
db.getUpdatesSince(sequenceNumber)) {
+ // If Recon's sequence number is out-of-date and the iterator is invalid,
+ // throw SNNFE and let Recon fall back to full snapshot.
+ // This could happen after OM restart.
+ if (db.getLatestSequenceNumber() != sequenceNumber &&
+ !transactionLogIterator.isValid()) {
+ throw new SequenceNumberNotFoundException(
+ "Invalid transaction log iterator when getting updates since "
+ + "sequence number " + sequenceNumber);
+ }
+
// Only the first record needs to be checked if its seq number <
// ( 1 + passed_in_sequence_number). For example, if seqNumber passed
// in is 100, then we can read from the WAL ONLY if the first sequence
@@ -303,7 +313,7 @@
currSequenceNumber > 1 + sequenceNumber) {
throw new SequenceNumberNotFoundException("Unable to read data from"
+ " RocksDB wal to get delta updates. It may have already been"
- + "flushed to SSTs.");
+ + " flushed to SSTs.");
}
// If the above condition was not satisfied, then it is OK to reset
// the flag.
@@ -322,8 +332,16 @@
}
transactionLogIterator.next();
}
+ } catch (SequenceNumberNotFoundException e) {
+ LOG.warn("Unable to get delta updates since sequenceNumber {}. "
+ + "This exception will be thrown to the client",
+ sequenceNumber, e);
+ // Throw the exception back to Recon. Expect Recon to fall back to
+ // full snapshot.
+ throw e;
} catch (RocksDBException | IOException e) {
- LOG.error("Unable to get delta updates since sequenceNumber {} ",
+ LOG.error("Unable to get delta updates since sequenceNumber {}. "
+ + "This exception will not be thrown to the client ",
sequenceNumber, e);
}
dbUpdatesWrapper.setLatestSequenceNumber(db.getLatestSequenceNumber());
