blob: b25484a999d9f2ddfd09cfc12fecc22c84a5be4f [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Pegasus | Single Atomic</title>
<link rel="stylesheet" href="/assets/css/app.css">
<link rel="shortcut icon" href="/assets/images/favicon.ico">
<link rel="stylesheet" href="/assets/css/utilities.min.css">
<link rel="stylesheet" href="/assets/css/docsearch.v3.css">
<script src="/assets/js/jquery.min.js"></script>
<script src="/assets/js/all.min.js"></script>
<script src="/assets/js/docsearch.v3.js"></script>
<!-- Begin Jekyll SEO tag v2.8.0 -->
<title>Single Atomic | Pegasus</title>
<meta name="generator" content="Jekyll v4.3.3" />
<meta property="og:title" content="Single Atomic" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Since v1.10.0, Pegasus supports single row atomic operations. The single row here means all data in the same HashKey." />
<meta property="og:description" content="Since v1.10.0, Pegasus supports single row atomic operations. The single row here means all data in the same HashKey." />
<meta property="og:site_name" content="Pegasus" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2024-04-22T13:02:52+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Single Atomic" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2024-04-22T13:02:52+00:00","datePublished":"2024-04-22T13:02:52+00:00","description":"Since v1.10.0, Pegasus supports single row atomic operations. The single row here means all data in the same HashKey.","headline":"Single Atomic","mainEntityOfPage":{"@type":"WebPage","@id":"/api/single-atomic"},"url":"/api/single-atomic"}</script>
<!-- End Jekyll SEO tag -->
</head>
<body>
<div class="dashboard is-full-height">
<!-- left panel -->
<div class="dashboard-panel is-medium is-hidden-mobile pl-0">
<div class="dashboard-panel-header has-text-centered">
<a href="/">
<img src="/assets/images/pegasus-logo-inv.png" style="width: 80%;">
</a>
</div>
<div class="dashboard-panel-main is-scrollable pl-6">
<aside class="menu">
<p class="menu-label">The Pegasus documentation</p>
<ul class="menu-list">
<li>
<a href="/docs/downloads"
class="">
Downloads
</a>
</li>
</ul>
<p class="menu-label">Building Pegasus</p>
<ul class="menu-list">
<li>
<a href="/docs/build/compile-by-docker"
class="">
Compile by docker (recommended)
</a>
</li>
<li>
<a href="/docs/build/compile-from-source"
class="">
Compile from source
</a>
</li>
</ul>
<p class="menu-label">Client Libs</p>
<ul class="menu-list">
<li>
<a href="/clients/java-client"
class="">
Java Client
</a>
</li>
<li>
<a href="/clients/cpp-client"
class="">
C++ Client
</a>
</li>
<li>
<a href="https://github.com/apache/incubator-pegasus/tree/master/go-client"
class="">
Golang Client
</a>
</li>
<li>
<a href="/clients/python-client"
class="">
Python Client
</a>
</li>
<li>
<a href="/clients/node-client"
class="">
NodeJS Client
</a>
</li>
<li>
<a href="/clients/scala-client"
class="">
Scala Client
</a>
</li>
</ul>
<p class="menu-label">Tools</p>
<ul class="menu-list">
<li>
<a href="/docs/tools/shell"
class="">
Pegasus Shell
</a>
</li>
<li>
<a href="https://github.com/pegasus-kv/admin-cli"
class="">
Admin CLI
</a>
</li>
<li>
<a href="https://github.com/pegasus-kv/pegic"
class="">
Pegasus data access CLI
</a>
</li>
</ul>
<p class="menu-label">API</p>
<ul class="menu-list">
<li>
<a href="/api/ttl"
class="">
TTL(Time To Live)
</a>
</li>
<li>
<a href="/api/single-atomic"
class="is-active">
Single-Atomic Operations
</a>
</li>
<li>
<a href="/api/redis"
class="">
Redis Adaption
</a>
</li>
<li>
<a href="/api/geo"
class="">
GEO Support
</a>
</li>
<li>
<a href="/api/http"
class="">
HTTP API
</a>
</li>
</ul>
<p class="menu-label">Admin</p>
<ul class="menu-list">
<li>
<a href="/administration/deployment"
class="">
Deployment
</a>
</li>
<li>
<a href="/administration/config"
class="">
Configurations
</a>
</li>
<li>
<a href="/administration/rebalance"
class="">
Rebalance
</a>
</li>
<li>
<a href="/administration/monitoring"
class="">
Monitoring
</a>
</li>
<li>
<a href="/administration/rolling-update"
class="">
Rolling Restart and Upgrade
</a>
</li>
<li>
<a href="/administration/scale-in-out"
class="">
Scale-in and Scale-out
</a>
</li>
<li>
<a href="/administration/resource-management"
class="">
Resource Management
</a>
</li>
<li>
<a href="/administration/cold-backup"
class="">
Cold Backup
</a>
</li>
<li>
<a href="/administration/meta-recovery"
class="">
Metadata Recovery
</a>
</li>
<li>
<a href="/administration/replica-recovery"
class="">
Replica Data Recovery
</a>
</li>
<li>
<a href="/administration/zk-migration"
class="">
Zookeeper Migration
</a>
</li>
<li>
<a href="/administration/table-migration"
class="">
Table Migration
</a>
</li>
<li>
<a href="/administration/table-soft-delete"
class="">
Table Soft-Delete
</a>
</li>
<li>
<a href="/administration/table-env"
class="">
Table Environment Variables
</a>
</li>
<li>
<a href="/administration/remote-commands"
class="">
Remote Command
</a>
</li>
<li>
<a href="/administration/partition-split"
class="">
Partition-Split
</a>
</li>
<li>
<a href="/administration/duplication"
class="">
Duplication
</a>
</li>
<li>
<a href="/administration/compression"
class="">
Data Compression
</a>
</li>
<li>
<a href="/administration/throttling"
class="">
Throttling
</a>
</li>
<li>
<a href="/administration/experiences"
class="">
Experiences
</a>
</li>
<li>
<a href="/administration/manual-compact"
class="">
Manual Compact
</a>
</li>
<li>
<a href="/administration/usage-scenario"
class="">
Usage Scenario
</a>
</li>
<li>
<a href="/administration/bad-disk"
class="">
Bad Disk Repair
</a>
</li>
<li>
<a href="/administration/whitelist"
class="">
Replica Server Whitelist
</a>
</li>
<li>
<a href="/administration/backup-request"
class="">
Backup Request
</a>
</li>
<li>
<a href="/administration/hotspot-detection"
class="">
Hotspot Detection
</a>
</li>
</ul>
</aside>
</div>
</div>
<!-- main section -->
<div class="dashboard-main is-scrollable">
<nav class="navbar is-hidden-desktop">
<div class="navbar-brand">
<a href="/" class="navbar-item">
<!-- Pegasus Icon -->
<img src="/assets/images/pegasus-square.png">
</a>
<div class="navbar-item">
<!--A simple language switch button that only supports zh and en.-->
<!--IF its language is zh, then switches to en.-->
<a class="button is-light is-outlined is-inverted" href="/zh/api/single-atomic"><strong>中</strong></a>
</div>
<a role="button" class="navbar-burger burger" aria-label="menu" aria-expanded="false" data-target="navMenu">
<!-- Appears in mobile mode only -->
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
<div class="navbar-menu" id="navMenu">
<div class="navbar-end">
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
The Pegasus documentation
</span>
</a>
<div class="navbar-dropdown">
<a href="/docs/downloads"
class="navbar-item ">
Downloads
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
Building Pegasus
</span>
</a>
<div class="navbar-dropdown">
<a href="/docs/build/compile-by-docker"
class="navbar-item ">
Compile by docker (recommended)
</a>
<a href="/docs/build/compile-from-source"
class="navbar-item ">
Compile from source
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
Client Libs
</span>
</a>
<div class="navbar-dropdown">
<a href="/clients/java-client"
class="navbar-item ">
Java Client
</a>
<a href="/clients/cpp-client"
class="navbar-item ">
C++ Client
</a>
<a href="https://github.com/apache/incubator-pegasus/tree/master/go-client"
class="navbar-item ">
Golang Client
</a>
<a href="/clients/python-client"
class="navbar-item ">
Python Client
</a>
<a href="/clients/node-client"
class="navbar-item ">
NodeJS Client
</a>
<a href="/clients/scala-client"
class="navbar-item ">
Scala Client
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
Tools
</span>
</a>
<div class="navbar-dropdown">
<a href="/docs/tools/shell"
class="navbar-item ">
Pegasus Shell
</a>
<a href="https://github.com/pegasus-kv/admin-cli"
class="navbar-item ">
Admin CLI
</a>
<a href="https://github.com/pegasus-kv/pegic"
class="navbar-item ">
Pegasus data access CLI
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
API
</span>
</a>
<div class="navbar-dropdown">
<a href="/api/ttl"
class="navbar-item ">
TTL(Time To Live)
</a>
<a href="/api/single-atomic"
class="navbar-item is-active">
Single-Atomic Operations
</a>
<a href="/api/redis"
class="navbar-item ">
Redis Adaption
</a>
<a href="/api/geo"
class="navbar-item ">
GEO Support
</a>
<a href="/api/http"
class="navbar-item ">
HTTP API
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
Admin
</span>
</a>
<div class="navbar-dropdown">
<a href="/administration/deployment"
class="navbar-item ">
Deployment
</a>
<a href="/administration/config"
class="navbar-item ">
Configurations
</a>
<a href="/administration/rebalance"
class="navbar-item ">
Rebalance
</a>
<a href="/administration/monitoring"
class="navbar-item ">
Monitoring
</a>
<a href="/administration/rolling-update"
class="navbar-item ">
Rolling Restart and Upgrade
</a>
<a href="/administration/scale-in-out"
class="navbar-item ">
Scale-in and Scale-out
</a>
<a href="/administration/resource-management"
class="navbar-item ">
Resource Management
</a>
<a href="/administration/cold-backup"
class="navbar-item ">
Cold Backup
</a>
<a href="/administration/meta-recovery"
class="navbar-item ">
Metadata Recovery
</a>
<a href="/administration/replica-recovery"
class="navbar-item ">
Replica Data Recovery
</a>
<a href="/administration/zk-migration"
class="navbar-item ">
Zookeeper Migration
</a>
<a href="/administration/table-migration"
class="navbar-item ">
Table Migration
</a>
<a href="/administration/table-soft-delete"
class="navbar-item ">
Table Soft-Delete
</a>
<a href="/administration/table-env"
class="navbar-item ">
Table Environment Variables
</a>
<a href="/administration/remote-commands"
class="navbar-item ">
Remote Command
</a>
<a href="/administration/partition-split"
class="navbar-item ">
Partition-Split
</a>
<a href="/administration/duplication"
class="navbar-item ">
Duplication
</a>
<a href="/administration/compression"
class="navbar-item ">
Data Compression
</a>
<a href="/administration/throttling"
class="navbar-item ">
Throttling
</a>
<a href="/administration/experiences"
class="navbar-item ">
Experiences
</a>
<a href="/administration/manual-compact"
class="navbar-item ">
Manual Compact
</a>
<a href="/administration/usage-scenario"
class="navbar-item ">
Usage Scenario
</a>
<a href="/administration/bad-disk"
class="navbar-item ">
Bad Disk Repair
</a>
<a href="/administration/whitelist"
class="navbar-item ">
Replica Server Whitelist
</a>
<a href="/administration/backup-request"
class="navbar-item ">
Backup Request
</a>
<a href="/administration/hotspot-detection"
class="navbar-item ">
Hotspot Detection
</a>
</div>
</div>
</div>
</div>
</nav>
<nav class="navbar is-hidden-mobile">
<div class="navbar-start w-full">
<div class="navbar-item pl-0 w-full">
<!--TODO(wutao): Given the limitation of docsearch that couldn't handle multiple input,
I make searchbox only shown in desktop. Fix this issue when docsearch.js v3 released.
Related issue: https://github.com/algolia/docsearch/issues/230-->
<div id="docsearch"></div>
</div>
</div>
<div class="navbar-end">
<div class="navbar-item">
<!--A simple language switch button that only supports zh and en.-->
<!--IF its language is zh, then switches to en.-->
<a class="button is-light is-outlined is-inverted" href="/zh/api/single-atomic"><strong>中</strong></a>
</div>
</div>
</nav>
<section class="hero is-info lg:mr-3">
<div class="hero-body">
<p class="title is-size-2 is-centered">Single-Atomic Operations</p>
</div>
</section>
<section class="section" style="padding-top: 2rem;">
<div class="content">
<p>Since v1.10.0, Pegasus supports single row atomic operations.</p>
<blockquote>
<p>The <strong>single row</strong> here means all data in the same HashKey.</p>
</blockquote>
<h1 id="principle">Principle</h1>
<p>Pegasus adopts a fixed hash sharding strategy for data distribution, where data from the same HashKey is always stored in the same partition, while the data is in the same Replica within a single node. Meanwhile, the write operations of the same Replica are always executed serially on the server side. Therefore, for data operations under the same HashKey, atomic semantics can be implemented conveniently.</p>
<p>For write operations, such as <a href="/clients/java-client#multiset">multiSet</a> and <a href="/clients/java-client#multidel">multiDel</a>, <em>set</em> or <em>delete</em> operate multiple SortKeys simultaneously in a single operation, atomic semantics can be comprehended easily, they are either successful or failed at the same time, so these two operations are single row atomic operations.</p>
<p>Here we focus on another type of operations: <strong>read first, then write, and the write operation depends on the result of the read operation</strong>.</p>
<p>The characteristic of this type of operations are <strong>non-idempotent</strong>, that is, if the same operation is repeated, the results (including the actual updates of data and the results returned to the user APIs) may be different. Atomic increment and decrement operations, as well as CAS operations, belong to this category. Pegasus can ensure the atomicity and consistency of such operations, because:</p>
<ul>
<li>The data of the same HashKey is always stored in the same Replica</li>
<li>The write operations of the same Replica are always executed serially on the server side</li>
<li>One operation is guaranteed to be executed exactly once, even in the event of data migration, downtime recovery, etc</li>
</ul>
<p>Due to non-idempotent properties, such operations may conflict with other features of Pegasus, such as <a href="/administration/duplication">Duplication</a>. So since 1.10.0, Pegasus provides an option to specify whether the cluster allows non-idempotent operations. If setting as false, all non-idempotent operations will return <code class="language-plaintext highlighter-rouge">ERR_OPERATION_DISABLED</code>:</p>
<div class="language-ini highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nn">[replication]</span>
<span class="py">allow_non_idempotent_write</span> <span class="p">=</span> <span class="s">false</span>
</code></pre></div></div>
<h1 id="atomic-increment-and-decrement-operations">Atomic increment and decrement operations</h1>
<p>Although Pegasus does not support schema in values, it still provides atomic increment and decrement operations, similar to Redis’s <a href="https://redis.io/commands/incr/">incr command</a>, refer to Pegasus’ interface <a href="/clients/java-client#incr">incr</a>.</p>
<h2 id="description">Description</h2>
<ul>
<li>Due to the fact that the storage engine RocksDB can only store values of byte string type, when using <code class="language-plaintext highlighter-rouge">incr()</code>, the value byte string will be read and converted to <code class="language-plaintext highlighter-rouge">int64</code> type (the conversion method is simple String-to-Int). For example, the byte string <code class="language-plaintext highlighter-rouge">"12345"</code> will be converted to a number <code class="language-plaintext highlighter-rouge">12345</code>. After completing the <code class="language-plaintext highlighter-rouge">incr()</code> operation, the obtained result will be converted back into a byte string and stored as a new value</li>
<li>When converting a byte string to <code class="language-plaintext highlighter-rouge">int64</code>, it may encounter errors, such as invalid numbers or overflow of <code class="language-plaintext highlighter-rouge">int64</code>, all of these cases will return a failure status</li>
<li>If the original value does not exist, it is considered as <code class="language-plaintext highlighter-rouge">0</code> and then <code class="language-plaintext highlighter-rouge">incr()</code> operation is executed normally</li>
<li>The operand <code class="language-plaintext highlighter-rouge">increment</code> can be positive or negative, so the <code class="language-plaintext highlighter-rouge">incr()</code> interface can implement as atomic increment and decrement</li>
<li>TTL: If the original value exists, the TTL of the new value and the original value remains the same. If the original value does not exist, the new value will be stored without TTL</li>
</ul>
<h1 id="cas-operations">CAS operations</h1>
<p>Another useful type of atomic operations are the CAS (Compare-And-Swap) operations. Based on CAS operations, many advanced distribute concurrency features can be implemented, such as distributed locks.</p>
<p>Pegasus provides <em>check_and_set</em> CAS operations, the semantics are: whether to update the value of one SortKey is depends on whether the value of another SortKey of the same HashKey meets certain conditions.</p>
<p>The SortKey which is used to determine the conditions is called <code class="language-plaintext highlighter-rouge">CheckSortKey</code>, the SortKey which is used to set value is called <code class="language-plaintext highlighter-rouge">SetSortKey</code>. Correspondingly, the value of <code class="language-plaintext highlighter-rouge">CheckSortKey</code> is called <code class="language-plaintext highlighter-rouge">CheckValue</code>, and the value to be set by <code class="language-plaintext highlighter-rouge">SetSortKey</code> is called <code class="language-plaintext highlighter-rouge">SetValue</code>.</p>
<p>See <a href="/clients/java-client#checkandset">checkAndSet</a>, as well as its extended versions <a href="/clients/java-client#checkandmutate">checkAndMutate</a> and <a href="/clients/java-client#compareexchange">compareExchange</a>.</p>
<h2 id="description-1">Description</h2>
<ul>
<li>The value of <code class="language-plaintext highlighter-rouge">SetSortKey</code> will be set only when <code class="language-plaintext highlighter-rouge">CheckValue</code> meets the specified conditions</li>
<li>The condition types that need to be met are specified through <code class="language-plaintext highlighter-rouge">CheckType</code>, and some <code class="language-plaintext highlighter-rouge">CheckType</code> also require the specified operand <code class="language-plaintext highlighter-rouge">CheckOperand</code>. Currently, supporting:
<ul>
<li>Determine the <em>existence</em> of <code class="language-plaintext highlighter-rouge">CheckValue</code>: Whether it exists, or is an empty byte string, etc</li>
<li>Byte string comparison: Compare <code class="language-plaintext highlighter-rouge">CheckValue</code> and <code class="language-plaintext highlighter-rouge">CheckOperand</code> in byte order to check if they meet the relationships of <code class="language-plaintext highlighter-rouge">&lt;</code>, <code class="language-plaintext highlighter-rouge">&lt;=</code>, <code class="language-plaintext highlighter-rouge">==</code>, <code class="language-plaintext highlighter-rouge">&gt;=</code>, or <code class="language-plaintext highlighter-rouge">&gt;=</code></li>
<li>Number comparison: similar to <a href="#atomic-increment-and-decrement-operations">Atomic increment and decrement operations</a>, convert <code class="language-plaintext highlighter-rouge">CheckValue</code> to int64, then compare the converted int64 value with <code class="language-plaintext highlighter-rouge">CheckOperand</code>, to check if they meet the relationships of <code class="language-plaintext highlighter-rouge">&lt;</code>, <code class="language-plaintext highlighter-rouge">&lt;=</code>, <code class="language-plaintext highlighter-rouge">==</code>, <code class="language-plaintext highlighter-rouge">&gt;=</code>, or <code class="language-plaintext highlighter-rouge">&gt;=</code></li>
</ul>
</li>
<li><code class="language-plaintext highlighter-rouge">CheckSortKey</code> and <code class="language-plaintext highlighter-rouge">SetSortKey</code> can be the same. If they are the same, it means checking whether the old value meets the condition first. If it does, set it to the new value for the same SortKey</li>
<li>You can enable the <code class="language-plaintext highlighter-rouge">CheckAndSetOptions.returnCheckValue</code> option if you want to return the value of <code class="language-plaintext highlighter-rouge">CheckValue</code></li>
<li>You can enable the <code class="language-plaintext highlighter-rouge">CheckAndSetOptions.setValueTTLSeconds</code> option if you want to specify TTL</li>
</ul>
<p>For ease of use, Pegasus Java Client also provides <em>compare_exchange</em> interface: When the value of a SortKey is equal to the user specified <code class="language-plaintext highlighter-rouge">ExpectedValue</code> in byte string, its value will be updated to the user specified <code class="language-plaintext highlighter-rouge">DesiredValue</code>. Semantically, <em>compare_exchange</em> is a special form of Compare-And-Swap. The interface can be found in <a href="/clients/java-client#compareexchange">compareExchange</a>.</p>
<p>Actually, <em>compare_exchange</em> is a specialized form of <em>check_and_set</em>, namely:</p>
<ul>
<li><code class="language-plaintext highlighter-rouge">CheckSortKey</code> and <code class="language-plaintext highlighter-rouge">SetSortKey</code> are the same</li>
<li><code class="language-plaintext highlighter-rouge">CheckType</code> is <code class="language-plaintext highlighter-rouge">CT_VALUE_BYTES_EQUAL</code></li>
</ul>
</div>
</section>
<footer class="footer">
<div class="container">
<div class="content is-small has-text-centered">
<div style="margin-bottom: 20px;">
<a href="http://incubator.apache.org">
<img src="/assets/images/egg-logo.png"
width="15%"
alt="Apache Incubator"/>
</a>
</div>
Copyright &copy; 2023 <a href="http://www.apache.org">The Apache Software Foundation</a>.
Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version
2.0</a>.
<br><br>
Apache Pegasus is an effort undergoing incubation at The Apache Software Foundation (ASF),
sponsored by the Apache Incubator. Incubation is required of all newly accepted projects
until a further review indicates that the infrastructure, communications, and decision making process
have stabilized in a manner consistent with other successful ASF projects. While incubation status is
not necessarily a reflection of the completeness or stability of the code, it does indicate that the
project has yet to be fully endorsed by the ASF.
<br><br>
Apache Pegasus, Pegasus, Apache, the Apache feather logo, and the Apache Pegasus project logo are either
registered trademarks or trademarks of The Apache Software Foundation in the United States and other
countries.
</div>
</div>
</footer>
</div>
<!-- right panel -->
<div class="dashboard-panel is-small is-scrollable is-hidden-mobile">
<p class="menu-label">
<span class="icon">
<i class="fa fa-bars" aria-hidden="true"></i>
</span>
Table of contents
</p>
<ul class="menu-list">
<li><a href="#principle">Principle</a></li>
<li><a href="#atomic-increment-and-decrement-operations">Atomic increment and decrement operations</a>
<ul>
<li><a href="#description">Description</a></li>
</ul>
</li>
<li><a href="#cas-operations">CAS operations</a>
<ul>
<li><a href="#description-1">Description</a></li>
</ul>
</li>
</ul>
</div>
</div>
<script src="/assets/js/app.js" type="text/javascript"></script>
<script>
docsearch({
container: '#docsearch',
appId: 'QRN30RBW0S',
indexName: 'pegasus-apache',
apiKey: 'd3a3252fa344359766707a106c4ed88f',
debug: true
});
</script>
</body>
</html>