blob: b9378ac90ade15e0f1da00324f36c605323c02c4 [file] [log] [blame]
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Pegasus | Cold Backup</title>
<link rel="stylesheet" href="/zh/assets/css/app.css">
<link rel="shortcut icon" href="/zh/assets/images/favicon.ico">
<link rel="stylesheet" href="/zh/assets/css/utilities.min.css">
<link rel="stylesheet" href="/zh/assets/css/docsearch.v3.css">
<script src="/assets/js/jquery.min.js"></script>
<script src="/assets/js/all.min.js"></script>
<script src="/assets/js/docsearch.v3.js"></script>
<!-- Begin Jekyll SEO tag v2.8.0 -->
<title>Cold Backup | Pegasus</title>
<meta name="generator" content="Jekyll v4.3.2" />
<meta property="og:title" content="Cold Backup" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="介绍" />
<meta property="og:description" content="介绍" />
<meta property="og:site_name" content="Pegasus" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2023-11-23T14:51:44+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Cold Backup" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"BlogPosting","dateModified":"2023-11-23T14:51:44+00:00","datePublished":"2023-11-23T14:51:44+00:00","description":"介绍","headline":"Cold Backup","mainEntityOfPage":{"@type":"WebPage","@id":"/administration/cold-backup"},"url":"/administration/cold-backup"}</script>
<!-- End Jekyll SEO tag -->
</head>
<body>
<div class="dashboard is-full-height">
<!-- left panel -->
<div class="dashboard-panel is-medium is-hidden-mobile pl-0">
<div class="dashboard-panel-header has-text-centered">
<a href="/zh/">
<img src="/assets/images/pegasus-logo-inv.png" style="width: 80%;">
</a>
</div>
<div class="dashboard-panel-main is-scrollable pl-6">
<aside class="menu">
<p class="menu-label">Pegasus产品文档</p>
<ul class="menu-list">
<li>
<a href="/zh/docs/downloads"
class="">
下载
</a>
</li>
</ul>
<p class="menu-label">编译构建</p>
<ul class="menu-list">
<li>
<a href="/zh/docs/build/compile-by-docker"
class="">
使用Docker完成编译(推荐)
</a>
</li>
<li>
<a href="/zh/docs/build/compile-from-source"
class="">
从源码编译
</a>
</li>
</ul>
<p class="menu-label">客户端库</p>
<ul class="menu-list">
<li>
<a href="/zh/clients/java-client"
class="">
Java客户端
</a>
</li>
<li>
<a href="/zh/clients/cpp-client"
class="">
C++客户端
</a>
</li>
<li>
<a href="https://github.com/apache/incubator-pegasus/tree/master/go-client"
class="">
Golang客户端
</a>
</li>
<li>
<a href="/zh/clients/python2-client"
class="">
Python2客户端
</a>
</li>
<li>
<a href="/zh/clients/python3-client"
class="">
Python3客户端
</a>
</li>
<li>
<a href="/zh/clients/node-client"
class="">
NodeJS客户端
</a>
</li>
<li>
<a href="/zh/clients/scala-client"
class="">
Scala客户端
</a>
</li>
</ul>
<p class="menu-label">生态工具</p>
<ul class="menu-list">
<li>
<a href="/zh/docs/tools/shell"
class="">
Pegasus Shell 工具
</a>
</li>
<li>
<a href="https://github.com/pegasus-kv/admin-cli"
class="">
集群管理命令行
</a>
</li>
<li>
<a href="https://github.com/pegasus-kv/pegic"
class="">
数据访问命令行
</a>
</li>
</ul>
<p class="menu-label">用户接口</p>
<ul class="menu-list">
<li>
<a href="/zh/api/ttl"
class="">
TTL
</a>
</li>
<li>
<a href="/zh/api/single-atomic"
class="">
单行原子操作
</a>
</li>
<li>
<a href="/zh/api/redis"
class="">
Redis适配
</a>
</li>
<li>
<a href="/zh/api/geo"
class="">
GEO支持
</a>
</li>
<li>
<a href="/zh/api/http"
class="">
HTTP接口
</a>
</li>
</ul>
<p class="menu-label">高效运维</p>
<ul class="menu-list">
<li>
<a href="/zh/administration/deployment"
class="">
集群部署
</a>
</li>
<li>
<a href="/zh/administration/config"
class="">
配置说明
</a>
</li>
<li>
<a href="/zh/administration/rebalance"
class="">
负载均衡
</a>
</li>
<li>
<a href="/zh/administration/monitoring"
class="">
可视化监控
</a>
</li>
<li>
<a href="/zh/administration/rolling-update"
class="">
集群升级
</a>
</li>
<li>
<a href="/zh/administration/scale-in-out"
class="">
集群扩容缩容
</a>
</li>
<li>
<a href="/zh/administration/resource-management"
class="">
资源管理
</a>
</li>
<li>
<a href="/zh/administration/cold-backup"
class="is-active">
冷备份
</a>
</li>
<li>
<a href="/zh/administration/meta-recovery"
class="">
元数据恢复
</a>
</li>
<li>
<a href="/zh/administration/replica-recovery"
class="">
Replica数据恢复
</a>
</li>
<li>
<a href="/zh/administration/zk-migration"
class="">
Zookeeper迁移
</a>
</li>
<li>
<a href="/zh/administration/table-migration"
class="">
Table迁移
</a>
</li>
<li>
<a href="/zh/administration/table-soft-delete"
class="">
Table软删除
</a>
</li>
<li>
<a href="/zh/administration/table-env"
class="">
Table环境变量
</a>
</li>
<li>
<a href="/zh/administration/remote-commands"
class="">
远程命令
</a>
</li>
<li>
<a href="/zh/administration/partition-split"
class="">
Partition-Split
</a>
</li>
<li>
<a href="/zh/administration/duplication"
class="">
跨机房同步
</a>
</li>
<li>
<a href="/zh/administration/compression"
class="">
数据压缩
</a>
</li>
<li>
<a href="/zh/administration/throttling"
class="">
流量控制
</a>
</li>
<li>
<a href="/zh/administration/experiences"
class="">
运维经验
</a>
</li>
<li>
<a href="/zh/administration/manual-compact"
class="">
Manual Compact功能
</a>
</li>
<li>
<a href="/zh/administration/usage-scenario"
class="">
Usage Scenario功能
</a>
</li>
<li>
<a href="/zh/administration/bad-disk"
class="">
坏盘检修
</a>
</li>
<li>
<a href="/zh/administration/whitelist"
class="">
白名单
</a>
</li>
<li>
<a href="/zh/administration/backup-request"
class="">
Backup Request
</a>
</li>
<li>
<a href="/zh/administration/hotspot-detection"
class="">
热点检测
</a>
</li>
</ul>
</aside>
</div>
</div>
<!-- main section -->
<div class="dashboard-main is-scrollable">
<nav class="navbar is-hidden-desktop">
<div class="navbar-brand">
<a href="/zh/" class="navbar-item">
<!-- Pegasus Icon -->
<img src="/assets/images/pegasus-square.png">
</a>
<div class="navbar-item">
<!--A simple language switch button that only supports zh and en.-->
<!--IF its language is zh, then switches to en.-->
<!--If you don't want a url to be relativized, you can add a space explicitly into the href to
prevents a url from being relativized by polyglot.-->
<a class="button is-light is-outlined is-inverted" href=" /administration/cold-backup"><strong>En</strong></a>
</div>
<a role="button" class="navbar-burger burger" aria-label="menu" aria-expanded="false" data-target="navMenu">
<!-- Appears in mobile mode only -->
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
<span aria-hidden="true"></span>
</a>
</div>
<div class="navbar-menu" id="navMenu">
<div class="navbar-end">
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
Pegasus产品文档
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/docs/downloads"
class="navbar-item ">
下载
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
编译构建
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/docs/build/compile-by-docker"
class="navbar-item ">
使用Docker完成编译(推荐)
</a>
<a href="/zh/docs/build/compile-from-source"
class="navbar-item ">
从源码编译
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
客户端库
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/clients/java-client"
class="navbar-item ">
Java客户端
</a>
<a href="/zh/clients/cpp-client"
class="navbar-item ">
C++客户端
</a>
<a href="https://github.com/apache/incubator-pegasus/tree/master/go-client"
class="navbar-item ">
Golang客户端
</a>
<a href="/zh/clients/python2-client"
class="navbar-item ">
Python2客户端
</a>
<a href="/zh/clients/python3-client"
class="navbar-item ">
Python3客户端
</a>
<a href="/zh/clients/node-client"
class="navbar-item ">
NodeJS客户端
</a>
<a href="/zh/clients/scala-client"
class="navbar-item ">
Scala客户端
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
生态工具
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/docs/tools/shell"
class="navbar-item ">
Pegasus Shell 工具
</a>
<a href="https://github.com/pegasus-kv/admin-cli"
class="navbar-item ">
集群管理命令行
</a>
<a href="https://github.com/pegasus-kv/pegic"
class="navbar-item ">
数据访问命令行
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
用户接口
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/api/ttl"
class="navbar-item ">
TTL
</a>
<a href="/zh/api/single-atomic"
class="navbar-item ">
单行原子操作
</a>
<a href="/zh/api/redis"
class="navbar-item ">
Redis适配
</a>
<a href="/zh/api/geo"
class="navbar-item ">
GEO支持
</a>
<a href="/zh/api/http"
class="navbar-item ">
HTTP接口
</a>
</div>
</div>
<!--dropdown-->
<div class="navbar-item has-dropdown is-hoverable">
<a href=""
class="navbar-link ">
<span>
高效运维
</span>
</a>
<div class="navbar-dropdown">
<a href="/zh/administration/deployment"
class="navbar-item ">
集群部署
</a>
<a href="/zh/administration/config"
class="navbar-item ">
配置说明
</a>
<a href="/zh/administration/rebalance"
class="navbar-item ">
负载均衡
</a>
<a href="/zh/administration/monitoring"
class="navbar-item ">
可视化监控
</a>
<a href="/zh/administration/rolling-update"
class="navbar-item ">
集群升级
</a>
<a href="/zh/administration/scale-in-out"
class="navbar-item ">
集群扩容缩容
</a>
<a href="/zh/administration/resource-management"
class="navbar-item ">
资源管理
</a>
<a href="/zh/administration/cold-backup"
class="navbar-item is-active">
冷备份
</a>
<a href="/zh/administration/meta-recovery"
class="navbar-item ">
元数据恢复
</a>
<a href="/zh/administration/replica-recovery"
class="navbar-item ">
Replica数据恢复
</a>
<a href="/zh/administration/zk-migration"
class="navbar-item ">
Zookeeper迁移
</a>
<a href="/zh/administration/table-migration"
class="navbar-item ">
Table迁移
</a>
<a href="/zh/administration/table-soft-delete"
class="navbar-item ">
Table软删除
</a>
<a href="/zh/administration/table-env"
class="navbar-item ">
Table环境变量
</a>
<a href="/zh/administration/remote-commands"
class="navbar-item ">
远程命令
</a>
<a href="/zh/administration/partition-split"
class="navbar-item ">
Partition-Split
</a>
<a href="/zh/administration/duplication"
class="navbar-item ">
跨机房同步
</a>
<a href="/zh/administration/compression"
class="navbar-item ">
数据压缩
</a>
<a href="/zh/administration/throttling"
class="navbar-item ">
流量控制
</a>
<a href="/zh/administration/experiences"
class="navbar-item ">
运维经验
</a>
<a href="/zh/administration/manual-compact"
class="navbar-item ">
Manual Compact功能
</a>
<a href="/zh/administration/usage-scenario"
class="navbar-item ">
Usage Scenario功能
</a>
<a href="/zh/administration/bad-disk"
class="navbar-item ">
坏盘检修
</a>
<a href="/zh/administration/whitelist"
class="navbar-item ">
白名单
</a>
<a href="/zh/administration/backup-request"
class="navbar-item ">
Backup Request
</a>
<a href="/zh/administration/hotspot-detection"
class="navbar-item ">
热点检测
</a>
</div>
</div>
</div>
</div>
</nav>
<nav class="navbar is-hidden-mobile">
<div class="navbar-start w-full">
<div class="navbar-item pl-0 w-full">
<!--TODO(wutao): Given the limitation of docsearch that couldn't handle multiple input,
I make searchbox only shown in desktop. Fix this issue when docsearch.js v3 released.
Related issue: https://github.com/algolia/docsearch/issues/230-->
<div id="docsearch"></div>
</div>
</div>
<div class="navbar-end">
<div class="navbar-item">
<!--A simple language switch button that only supports zh and en.-->
<!--IF its language is zh, then switches to en.-->
<!--If you don't want a url to be relativized, you can add a space explicitly into the href to
prevents a url from being relativized by polyglot.-->
<a class="button is-light is-outlined is-inverted" href=" /administration/cold-backup"><strong>En</strong></a>
</div>
</div>
</nav>
<section class="hero is-info lg:mr-3">
<div class="hero-body">
<p class="title is-size-2 is-centered">冷备份</p>
</div>
</section>
<section class="section" style="padding-top: 2rem;">
<div class="content">
<h2 id="介绍">介绍</h2>
<p>Pegasus的冷备份功能用来将Pegasus中的数据定期生成快照文件,并备份到其他存储介质上,从而为数据容灾多提供一层保障。但由于备份的是某个时间点的数据快照文件,所以冷备份并不保证可以保留所有最新的数据,也就是说,恢复的时候可能会丢失最近一段时间的数据。</p>
<p>具体来看,冷备份过程要涉及到如下一些参数:</p>
<ul>
<li>存储介质(backup_provider): 指其他的文件存储系统或服务,如本地文件系统或者HDFS。</li>
<li>数据冷备份的周期(backup_interval):周期的长短决定了备份数据的覆盖范围。如果周期是1个月,那么恢复数据时,就可能只恢复一个月之前的数据。但如果周期设的太短,备份就会太频繁,从而使得备份开销很大。在小米内部,冷备份的周期通常是1天。</li>
<li>保留的冷备份个数(backup_history_count):保留的备份个数越多,存储的空间开销就越大。在小米内部,一般保留最近的3个冷备份。</li>
<li>进行冷备份的表的集合(backup_app_ids):并不是所有的表都值得进行冷备份。在小米内部,对于经常重灌全量数据的表,我们是不进行冷备份的。</li>
</ul>
<p>在Pegasus中,以上这几个参数的组合称为一个冷备份策略(backup_policy)。数据的冷备份就行按照policy为单位进行的。</p>
<h2 id="存储介质">存储介质</h2>
<p>目前Pegasus支持本地文件系统和FDS两种冷备份介质。如果需要支持更多的存储介质(如S3),请联系我们或者直接把pull request砸过来。</p>
<h3 id="fds">FDS</h3>
<p>FDS是小米生态云提供的存储产品,目前仅面向小米的生态链业务开放。更多详细的介绍<a href="https://cnbj6.cloud.mi.com/#/index/product/fds?_k=rq2znr">请戳这里</a></p>
<h3 id="本地文件系统">本地文件系统</h3>
<p>通过使用Pegasus本地文件系统(local service)的备份接口,可以把Pegasus的数据备份到某个目录下。如果第三方存储介质支持以nfs或者Linux Fuse的方式挂载到本地某个目录下的话,该介质就可以通过local service来作为pegasus的备份介质。目前HDFS就是通过fuse的方式来支持的。</p>
<p>我们接下来会以本地文件系统为例,介绍冷备的创建和恢复的相关操作。然后,我们会介绍如何用fuse的方式将HDFS挂载到本地目录下。</p>
<h2 id="怎样开始冷备份">怎样开始冷备份</h2>
<h3 id="配置">配置</h3>
<p>需要先设定一些参数来配置系统的冷备份功能:</p>
<ol>
<li>
<p>配置 [meta_server].cold_backup_disable :</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>[meta_server]
...
;; 设置为false, 不然冷备份功能不会打开
cold_backup_disabled = false
...
</code></pre></div> </div>
</li>
<li>
<p>配置 [apps.meta].pools 和 [apps.replica].pools :</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>[apps.meta]
...
;; 需增加THREAD_POOL_LOCAL_SERVICE,不然会没有可用的线程池而coredump
pools = ...,THREAD_POOL_LOCAL_SERVICE
[apps.replica]
...
;; 需增加THREAD_POOL_LOCAL_SERVICE,不然会没有可用的线程池而coredump
pools = ...,THREAD_POOL_LOCAL_SERVICE
</code></pre></div> </div>
</li>
<li>
<p>配置 [replication].cold_backup_root 和 [replication].max_concurrent_uploading_file_count :</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>[replication]
;; 这个参数指定了冷备数据要保存到存储介质的什么文件夹下,一般建议填写集群名字
cold_backup_root = onebox
;; 这个参数指定了冷备份上传文件的并发数,根据自己的网络情况进行设置,建议参数为5
max_concurrent_uploading_file_count = 5
</code></pre></div> </div>
</li>
<li>
<p>添加或修改 [block_service.xxxx] 配置段:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>;; 这样的一个section, 就指定了一种存储介质。可以按照自己的需求添加
[block_service.my_backup_media]
;; 指定这种存储介质使用的存储接口是local_service
type = local_service
;; 存储介质的初始化参数。对于local_service, 该参数要设置成一个本地目录,表示所有的备份全在此目录下。
args = /home/weijiesun/pegasus_cold_backup
</code></pre></div> </div>
</li>
</ol>
<p>在上面的配置中,一定要区分清楚[replication].cold_backup_root和[block_service.my_backup_media].args所配置的两个路径:</p>
<ul>
<li>前者指定了数据在某种存储介质下的存放目录。</li>
<li>后者是存储介质本身的初始化参数,是和type相关的。对于local_service而言,这个参数是一个绝对路径。换种说法,<strong>本地文件系统的某个目录</strong>,就是一种存储介质。</li>
<li>对于当前的配置情况而言,冷备数据会保存到<code class="language-plaintext highlighter-rouge">/home/weijiesun/pegasus_cold_backup/onebox</code>这个目录下。</li>
</ul>
<h3 id="创建冷备份策略">创建冷备份策略</h3>
<p>通过<a href="/zh/overview/shell#冷备份管理">Shell工具的add_backup_policy命令</a>可以创建一个冷备份策略:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; add_backup_policy -p policy1 -b my_backup_media -a 1 -i 300 -s 16:00 -c 3
add backup policy succeed, policy_name = policy1
</code></pre></div></div>
<p>几个参数的含义如下:</p>
<ul>
<li>-p: policy的名字,创建后就不能修改,某个集群只能有唯一的名字</li>
<li>-b: 存储介质的名称。系统会按指定的名称寻找对应的section, 并初始化存储介质。在上例中,系统从[block_service.my_backup_media]这一配置section中进行初始化。</li>
<li>-a: policy下app_id的集合。如果有多个,用逗号隔开。</li>
<li>-i: 备份周期,以秒为单位。</li>
<li>-s: 开始时间,精确到分钟。这个参数含义略显复杂,后文专门介绍。</li>
<li>-c: 冷备份的保留个数</li>
</ul>
<p>对于-s参数指定的备份开始时间,其含义和-i指定备份周期是有联系的</p>
<ul>
<li>如果备份周期是86400秒,也就是一天。那么-s就表示备份每天的启动时间。</li>
<li>如果备份周期不是一整天。那么开始时间就代表的着第一次备份的开启时间点。而这个时间点,指的是当前时间朝后推的第一个相对的时间点。</li>
</ul>
<p>比如上面的”-i 60 -s 8:00”,就是说下次备份从8:00开始,然后每60秒开启一轮备份。而”-i 86400 -s 8:30”,指的是每天的8:30开启一轮备份。</p>
<p>对于-a参数指定的app_id。目前pegasus并没有强制约束某个app_id只能属于一个policy, 但原则上不建议把同一个表加到不同的policy中。</p>
<h3 id="查询冷备份策略">查询冷备份策略</h3>
<p>可以用ls_backup_policy列举系统当前所有的策略:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; ls_backup_policy
[1]
name : another_policy
backup_provider_type : my_backup_media
backup_interval : 600s
app_ids : {2}
start_time : 16:10
status : enabled
backup_history_count : 2
[2]
name : policy1
backup_provider_type : my_backup_media
backup_interval : 300s
app_ids : {1}
start_time : 16:00
status : enabled
backup_history_count : 3
ls backup policy succeed
</code></pre></div></div>
<p>也可以用query_backup_policy查询某个指定policy的信息:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; query_backup_policy -p policy1
policy_info:
name : policy1
backup_provider_type : my_backup_media
backup_interval : 300s
app_ids : {1}
start_time : 16:00
status : enabled
backup_history_count : 3
backup_infos:
[1]
id : 1533801635364
start_time : 2018-08-09 16:00:35
end_time : 2018-08-09 16:01:15
app_ids : {1}
query backup policy succeed
</code></pre></div></div>
<p>如上图所示,当query某个特定的policy时,该policy下已经完成或正在进行的备份也会显示出来。</p>
<h3 id="修改冷备份策略">修改冷备份策略</h3>
<p>冷备份策略的一些参数,可以用modify_backup_policy命令进行修改:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; modify_backup_policy -p policy1 -s 17:00 -i 600
Modify policy result: ERR_OK
&gt;&gt;&gt; query_backup_policy -p policy1
policy_info:
name : policy1
backup_provider_type : my_backup_media
backup_interval : 600s
app_ids : {1}
start_time : 17:00
status : enabled
backup_history_count : 3
backup_infos:
.....
query backup policy succeed
</code></pre></div></div>
<p>可以修改的参数有:</p>
<ul>
<li>app集合: 通过-a和-r参数进行增删app_id</li>
<li>backup_interval: 通过-i修改周期</li>
<li>start_time: 通过-s参数修改开始时间</li>
</ul>
<h3 id="禁用和重新开始冷备份策略">禁用和重新开始冷备份策略</h3>
<p>可以用disable_backup_policy和enable_backup_policy来对某个policy进行禁用和开启</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; disable_backup_policy -p policy1
disable policy result: ERR_OK
&gt;&gt;&gt; ls_backup_policy
..........
[2]
name : policy1
backup_provider_type : my_backup_media
backup_interval : 600s
app_ids : {1}
start_time : 17:00
status : disabled
backup_history_count : 3
ls backup policy succeed
&gt;&gt;&gt; enable_backup_policy -p policy1
enable policy result: ERR_OK
&gt;&gt;&gt; ls_backup_policy
......
[2]
name : policy1
backup_provider_type : my_backup_media
backup_interval : 600s
app_ids : {1}
start_time : 17:00
status : enabled
backup_history_count : 3
ls backup policy succeed
&gt;&gt;&gt;
</code></pre></div></div>
<h3 id="查看备份的结果">查看备份的结果</h3>
<p>对于我们配置好的local_service, 备份存在/home/weijiesun/pegasus_cold_backup/onebox的目录下。</p>
<p>这里稍微做展示,详细的文件内容大家可以自行翻看:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>weijiesun@weijiesun-kubuntu ~/pegasus_cold_backup/onebox/policy1 $ pwd
/home/weijiesun/pegasus_cold_backup/onebox/policy1
weijiesun@weijiesun-kubuntu ~/pegasus_cold_backup/onebox/policy1 $ ls
1533802216079 1533802236170 1533802256259
</code></pre></div></div>
<h2 id="从冷备中恢复数据">从冷备中恢复数据</h2>
<p>可以用restore_app从冷备中恢复表数据</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; restore_app -c onebox -p policy1 -a temp -i 1 -t 1533802236170 -b my_backup_media -n result
sleep 1 second to wait complete...
new app_id = 3
</code></pre></div></div>
<p>其中各个参数的含义如下(这些参数都可以通过在原集群上执行shell命令<code class="language-plaintext highlighter-rouge">query_backup_policy</code>得到):</p>
<ul>
<li>-c:旧集群的名称</li>
<li>-p:旧的policy名称</li>
<li>-a:旧表名称</li>
<li>-i:旧表的id</li>
<li>-t:旧的备份的时间戳编号</li>
<li>-b:存储介质名称</li>
<li>-n:新表的名称,如果不指定就使用旧名字,如果和新的名字冲突则报错</li>
<li>-s:是否跳过损坏的partition, 默认不跳过</li>
</ul>
<p>对于恢复进度,可以用query_restore_status进行查看</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; query_restore_status 3 -d
pid progress(%) restore_status
0 100 ok
1 100 ok
2 100 ok
3 100 ok
4 100 ok
5 100 ok
6 100 ok
7 100 ok
the overall progress of restore is 100%
annotations:
ok : mean restore complete
ongoing : mean restore is under going
skip : data on cold backup media is damaged, but skip the damaged partition
unknown : invalid, should login server and check it
</code></pre></div></div>
<h2 id="怎样将数据备份到hdfs">怎样将数据备份到HDFS</h2>
<p>采用HDFS fuse即可,可以参考<a href="https://www.jianshu.com/p/1beb5325c6d8">这篇文章</a></p>
</div>
</section>
<footer class="footer">
<div class="container">
<div class="content is-small has-text-centered">
<div style="margin-bottom: 20px;">
<a href="http://incubator.apache.org">
<img src="/assets/images/egg-logo.png"
width="15%"
alt="Apache Incubator"/>
</a>
</div>
Copyright &copy; 2023 <a href="http://www.apache.org">The Apache Software Foundation</a>.
Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version
2.0</a>.
<br><br>
Apache Pegasus is an effort undergoing incubation at The Apache Software Foundation (ASF),
sponsored by the Apache Incubator. Incubation is required of all newly accepted projects
until a further review indicates that the infrastructure, communications, and decision making process
have stabilized in a manner consistent with other successful ASF projects. While incubation status is
not necessarily a reflection of the completeness or stability of the code, it does indicate that the
project has yet to be fully endorsed by the ASF.
<br><br>
Apache Pegasus, Pegasus, Apache, the Apache feather logo, and the Apache Pegasus project logo are either
registered trademarks or trademarks of The Apache Software Foundation in the United States and other
countries.
</div>
</div>
</footer>
</div>
<!-- right panel -->
<div class="dashboard-panel is-small is-scrollable is-hidden-mobile">
<p class="menu-label">
<span class="icon">
<i class="fa fa-bars" aria-hidden="true"></i>
</span>
本页导航
</p>
<ul class="menu-list">
<li><a href="#介绍">介绍</a></li>
<li><a href="#存储介质">存储介质</a>
<ul>
<li><a href="#fds">FDS</a></li>
<li><a href="#本地文件系统">本地文件系统</a></li>
</ul>
</li>
<li><a href="#怎样开始冷备份">怎样开始冷备份</a>
<ul>
<li><a href="#配置">配置</a></li>
<li><a href="#创建冷备份策略">创建冷备份策略</a></li>
<li><a href="#查询冷备份策略">查询冷备份策略</a></li>
<li><a href="#修改冷备份策略">修改冷备份策略</a></li>
<li><a href="#禁用和重新开始冷备份策略">禁用和重新开始冷备份策略</a></li>
<li><a href="#查看备份的结果">查看备份的结果</a></li>
</ul>
</li>
<li><a href="#从冷备中恢复数据">从冷备中恢复数据</a></li>
<li><a href="#怎样将数据备份到hdfs">怎样将数据备份到HDFS</a></li>
</ul>
</div>
</div>
<script src="/assets/js/app.js" type="text/javascript"></script>
<script>
docsearch({
container: '#docsearch',
appId: 'QRN30RBW0S',
indexName: 'pegasus-apache',
apiKey: 'd3a3252fa344359766707a106c4ed88f',
debug: true
});
</script>
</body>
</html>