| |
| |
| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| |
| <meta name="description" content="Apache Ozone Documentation"> |
| |
| <title>Documentation for Apache Ozone</title> |
| |
| |
| <link href="../../css/bootstrap.min.css" rel="stylesheet"> |
| |
| |
| <link href="../../css/ozonedoc.css" rel="stylesheet"> |
| |
| |
| |
| <link href="../../swagger-resources/swagger-ui.css" rel="stylesheet"> |
| |
| |
| <script> |
| var _paq = window._paq = window._paq || []; |
| |
| |
| |
| _paq.push(['disableCookies']); |
| |
| |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="//analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '34']); |
| var d=document, g=d.createElement('script'), |
| s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| |
| |
| </head> |
| |
| |
| <body> |
| |
| |
| <nav class="navbar navbar-inverse navbar-fixed-top"> |
| <div class="container-fluid"> |
| <div class="navbar-header"> |
| <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#sidebar" aria-expanded="false" aria-controls="navbar"> |
| <span class="sr-only">Toggle navigation</span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| <span class="icon-bar"></span> |
| </button> |
| <a href="../../zh/index.html" class="navbar-left ozone-logo"> |
| <img src="../../ozone-logo-small.png"/> |
| </a> |
| <a class="navbar-brand hidden-xs" href="../../zh/index.html"> |
| Apache Ozone/HDDS Documentation |
| </a> |
| <a class="navbar-brand visible-xs-inline" href="#">Apache Ozone</a> |
| </div> |
| <div id="navbar" class="navbar-collapse collapse"> |
| <ul class="nav navbar-nav navbar-right"> |
| <li><a href="https://github.com/apache/ozone">Source</a></li> |
| <li><a href="https://ozone.apache.org">Apache Ozone</a></li> |
| <li><a href="https://apache.org">ASF</a></li> |
| </ul> |
| </div> |
| </div> |
| </nav> |
| |
| |
| <div class="wrapper"> |
| <div class="container-fluid"> |
| <div class="row"> |
| |
| <div class="col-sm-2 col-md-2 sidebar" id="sidebar"> |
| <ul class="nav nav-sidebar"> |
| |
| |
| |
| <li class=""> |
| |
| <a href="../../zh/index.html"> |
| |
| |
| |
| <span>概述</span> |
| </a> |
| </li> |
| |
| |
| |
| <li class=""> |
| |
| <a href="../../zh/start.html"> |
| |
| |
| |
| <span>快速入门</span> |
| </a> |
| </li> |
| |
| |
| |
| <li class=""> |
| <a href="../../zh/concept.html"> |
| |
| <span>概念</span> |
| </a> |
| <ul class="nav"> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/overview.html">概览</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/ozonemanager.html">Ozone Manager</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/storagecontainermanager.html">Storage Container Manager</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/datanodes.html">数据节点</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/containers.html">Containers</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/concept/recon.html">Recon</a> |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| <li class=""> |
| <a href="../../zh/feature.html"> |
| |
| <span>特性</span> |
| </a> |
| <ul class="nav"> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/decommission.html">Decommissioning</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/erasurecoding.html">纠删码</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/om-ha.html">高可用 OM</a> |
| |
| </li> |
| |
| <li class="active"> |
| |
| <a href="../../zh/feature/scm-ha.html">高可用 SCM</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/dn-merge-rocksdb.html">在DataNode上合并Container的RocksDB</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/prefixfso.html">基于前缀的文件系统优化</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/topology.html">拓扑感知能力</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/quota.html">Ozone 中的配额</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/recon.html">Recon 服务器</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/feature/reconfigurability.html">动态加载配置</a> |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| <li class=""> |
| <a href="../../zh/security.html"> |
| |
| <span>安全</span> |
| </a> |
| <ul class="nav"> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/secureozone.html">安全化 Ozone</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/securingtde.html">透明数据加密</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/gdpr.html">Ozone 中的 GDPR</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/securingdatanodes.html">安全化 Datanode</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/securings3.html">安全化 S3</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/securityacls.html">Ozone 访问控制列表</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/security/securitywithranger.html">Apache Ranger</a> |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| <li class=""> |
| <a href="../../zh/interface.html"> |
| |
| <span>编程接口</span> |
| </a> |
| <ul class="nav"> |
| |
| <li class=""> |
| |
| <a href="../../zh/interface/javaapi.html">Java API</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/interface/o3fs.html">Ozone 文件系统</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/interface/csi.html">CSI 协议</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/interface/s3.html">S3 协议接口</a> |
| |
| </li> |
| |
| <li class=""> |
| |
| <a href="../../zh/interface/reconapi.html">Recon API</a> |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| <li class=""> |
| |
| <a href="../../zh/tools.html"> |
| |
| |
| |
| <span>工具</span> |
| </a> |
| </li> |
| |
| |
| |
| <li class=""> |
| |
| <a href="../../zh/recipe.html"> |
| |
| |
| |
| <span>使用配方</span> |
| </a> |
| </li> |
| |
| |
| <li><a href="../../design.html"><span><b>Design docs</b></span></a></li> |
| <li class="visible-xs"><a href="#">References</a> |
| <ul class="nav"> |
| <li><a href="https://github.com/apache/ozone"><span class="glyphicon glyphicon-new-window" aria-hidden="true"></span> Source</a></li> |
| <li><a href="https://ozone.apache.org"><span class="glyphicon glyphicon-new-window" aria-hidden="true"></span> Apache Ozone</a></li> |
| <li><a href="https://apache.org"><span class="glyphicon glyphicon-new-window" aria-hidden="true"></span> ASF</a></li> |
| </ul></li> |
| </ul> |
| |
| </div> |
| |
| <div class="col-sm-10 col-sm-offset-2 col-md-10 col-md-offset-2 main-content"> |
| |
| |
| |
| <div class="col-md-9"> |
| <nav aria-label="breadcrumb"> |
| <ol class="breadcrumb"> |
| <li class="breadcrumb-item"><a href="../../zh/index.html">Home</a></li> |
| <li class="breadcrumb-item" aria-current="page"><a href="../../zh/feature.html">特性</a></li> |
| <li class="breadcrumb-item active" aria-current="page">高可用 SCM</li> |
| </ol> |
| </nav> |
| |
| |
| |
| <div class="pull-right"> |
| |
| |
| |
| <a href="../../feature/scm-ha.html"><span class="label label-success">English</span></a> |
| |
| |
| |
| |
| </div> |
| |
| |
| <div class="col-md-9"> |
| <h1>高可用 SCM</h1> |
| |
| <!--- |
| Licensed to the Apache Software Foundation (ASF) under one or more |
| contributor license agreements. See the NOTICE file distributed with |
| this work for additional information regarding copyright ownership. |
| The ASF licenses this file to You under the Apache License, Version 2.0 |
| (the "License"); you may not use this file except in compliance with |
| the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| --> |
| <p>Ozone包含两个元数据管理节点(用于键管理的 <em>Ozone Manager</em> 和用于块空间管理的 <em>Storage Container management</em> )和多个存储节点(数据节点)。通过 RAFT 共识算法实现数据在数据节点之间的复制。</p> |
| <p>为了避免任何单点故障,元数据管理器节点还应该具有 HA 设置。</p> |
| <p>Ozone Manager 和 Storage Container Manager 都支持 HA。在这种模式下,内部状态通过 RAFT (使用 Apache Ratis )复制。</p> |
| <p>本文档解释了 Storage Container Manager (SCM)的 HA 设置,请在<a href="../../zh/feature/om-ha.html">本页</a>中查看 Ozone Manager (OM)的 HA 设置。虽然它们可以独立地为 HA 进行设置,但可靠的、完全的 HA 设置需要为两个服务启用 HA。</p> |
| <h2 id="配置">配置</h2> |
| <blockquote> |
| <p>⚠️ <strong>注意</strong> ⚠️</p> |
| <p>SCM HA 目前仅支持新初始化的集群。 |
| SCM HA 必须在 Ozone 服务首次启动前开启。 |
| 当某个 SCM 以非 HA 的模式启动后,不支持将其改为 HA 模式。</p> |
| </blockquote> |
| <p>Storage Container Manager 的 HA 模式可以在 <code>ozone-site.xml</code> 中进行以下设置:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-XML" data-lang="XML"><span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.ratis.enable<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>true<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| </code></pre></div><p>一个 Ozone 配置(<code>ozone-site.xml</code>)可以支持多个SCM HA节点集,多个 Ozone 集群。要在可用的 SCM 节点之间进行选择,每个集群都需要一个逻辑名称,可以将其解析为 Storage Container Manage 的 IP 地址(和域名)。</p> |
| <p>这个逻辑名称称为 <code>serviceId</code>,可以在 <code>ozone-site.xml</code> 中配置。</p> |
| <p>大多数情况下,你只需要设置当前集群的值:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-XML" data-lang="XML"><span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.service.ids<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>cluster1<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| </code></pre></div><p>对于每个已定义的 <code>serviceId</code>,应该为每个服务器定义一个逻辑配置名:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-XML" data-lang="XML"><span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.nodes.cluster1<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>scm1,scm2,scm3<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| </code></pre></div><p>定义的前缀可以用来定义每个 SCM 服务的地址:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-XML" data-lang="XML"><span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.address.cluster1.scm1<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>host1<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| <span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.address.cluster1.scm2<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>host2<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| <span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><name></span>ozone.scm.address.cluster1.scm3<span style="color:#f92672"></name></span> |
| <span style="color:#f92672"><value></span>host3<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| </code></pre></div><p>为了获得可靠的 HA 支持,请选择3个独立的节点以形成仲裁。</p> |
| <h2 id="引导">引导</h2> |
| <p>初始化的<strong>第一个</strong> SCM-HA 节点和 none-HA SCM是一样的:</p> |
| <pre><code>bin/ozone scm --init |
| </code></pre><p>第二个和第三个节点应该被 <em>bootstrap</em>,而不是 init。这些集群将加入到配置的 RAFT 仲裁。当前服务器的 id 通过 DNS 名称标识,也可以通过 <code>ozone.scm.node.id</code> 明确设置。大多数时候你不需要设置它,因为基于 DNS 的 id 检测可以很好地工作。</p> |
| <pre><code>bin/ozone scm --bootstrap |
| </code></pre><h2 id="自动引导">自动引导</h2> |
| <p>在某些环境(例如容器化/ K8s 环境)中,我们需要一种通用的统一方法来初始化 SCM HA 仲裁。 剩下的标准初始化流程如下:</p> |
| <ol> |
| <li>在第一个“原始”节点上,调用 <code>scm --init</code></li> |
| <li>在第二个/第三个节点上,调用<code>scm --bootstrap</code></li> |
| </ol> |
| <p>这可以通过使用 <code>ozone.scm.primordial.node.id</code> 更改。您可以定义原始节点。设置这个节点后,你应该在<strong>所有</strong>的节点上<strong>同时</strong>执行 <code>scm --init</code> 和 <code>scm --bootstrap</code>。</p> |
| <p>根据 <code>ozone.scm.primordial.node.id</code>,初始化进程将在第二个/第三个节点上被忽略,引导进程将在除原始节点外的所有节点上被忽略。</p> |
| <h2 id="scm-ha-安全">SCM HA 安全</h2> |
| <p> |
| |
| <img src="scm-secure-ha.png" alt='SCM Secure HA' class="img-responsive" /></p> |
| <p>在一个安全 SCM HA 集群中,我们将执行初始化的 SCM 称为原始 SCM。 |
| 原始 SCM 使用自签名证书启动根 CA,并用于颁发签名证书到它自己和其他 |
| 引导的 SCM。 只有原始 SCM 可以为其他 SCM 颁发签名证书。 |
| 因此,原始 SCM 在 SCM HA 集群中具有特殊的作用,因为它是唯一可以向 SCM 颁发证书的 SCM。</p> |
| <p>原始 SCM 担任根 CA 角色,它使用子 CA 证书签署所有 SCM 实例。 |
| SCM 使用子 CA 证书来签署 OM/Datanodes 的证书。</p> |
| <p>引导 SCM 时会从原始 SCM 获取签名证书并启动子 CA。</p> |
| <p>SCM 上的子 CA 用于为集群中的 OM/DN 颁发签名证书。 只有 leader SCM 向 OM/DN 颁发证书。</p> |
| <h3 id="如何启用安全">如何启用安全</h3> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-XML" data-lang="XML"><span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><config></span>ozone.security.enable<span style="color:#f92672"></config></span> |
| <span style="color:#f92672"><value></span>true<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| |
| <span style="color:#f92672"><property></span> |
| <span style="color:#f92672"><config></span>hdds.grpc.tls.enabled<span style="color:#f92672"></config></span> |
| <span style="color:#f92672"><value></span>true<span style="color:#f92672"></value></span> |
| <span style="color:#f92672"></property></span> |
| </code></pre></div><p>在正常的 SCM HA 配置的基础上,需要添加上述配置。</p> |
| <h3 id="原始-scm">原始 SCM</h3> |
| <p>原始 SCM 由配置 ozone.scm.primordial.node.id 确定。 |
| 此值可以是 SCM 的节点 ID 或原始机名。 |
| 如果配置是未定义的,则运行 init 的节点被视为原始 SCM。</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">bin/ozone scm --init</code></pre></div> |
| <p>这将为根 CA 设置公钥、私钥对和自签名证书 |
| 并生成公钥、私钥对和 CSR 以从根 CA 获取子 CA 的签名证书。</p> |
| <h3 id="引导-scm">引导 SCM</h3> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">bin/ozone scm --bootstrap</code></pre></div> |
| <p>这将为子 CA 设置公钥、私钥对并生成 CSR 以获取来自根 CA 的子 CA 签名证书。</p> |
| <p><strong>注意</strong>: 当原始 SCM 未定义时,请确保仅在一个 SCM 上运行 <strong>–init</strong>, |
| 在其他 SCM 节点上需使用 <strong>–bootstrap</strong> 进行引导。</p> |
| <h3 id="目前-scm-ha-安全的限制">目前 SCM HA 安全的限制</h3> |
| <ol> |
| <li>当原始 SCM 失效时, 新的 SCM 不能被引导并添加到 HA 节点中。</li> |
| <li>尚未支持从非 HA 安全集群升级到 HA 安全集群。</li> |
| </ol> |
| <h2 id="实现细节">实现细节</h2> |
| <p>SCM HA 使用 Apache Ratis 在 SCM HA 仲裁的成员之间复制状态。每个节点在本地 RocksDB 中维护块管理元数据。</p> |
| <p>这个复制过程是 OM HA 复制过程的一个简单版本,因为它不使用任何双缓冲区(SCM 请求的总体 db 吞吐量更低)。</p> |
| <p>数据节点将所有报告(容器报告、管道报告……)并发发送给 <em>所有</em> SCM 节点。只有 leader 节点可以分配/创建新的容器,并且只有 leader 节点可以将命令返回给数据节点。</p> |
| <h2 id="验证scm-ha设置">验证SCM HA设置</h2> |
| <p>启动 SCM-HA 后,可以验证 SCM 节点是否形成了一个仲裁,而不是3个单独的 SCM 节点。</p> |
| <p>首先,检查所有的 SCM 节点是否存储相同的 ClusterId 元数据:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash">cat /data/metadata/scm/current/VERSION |
| </code></pre></div><p>ClusterId 包含在版本文件中,并且在所有的 SCM 节点中应该是相同的:</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-bash" data-lang="bash"><span style="color:#75715e">#Tue Mar 16 10:19:33 UTC 2021</span> |
| cTime<span style="color:#f92672">=</span><span style="color:#ae81ff">1615889973116</span> |
| clusterID<span style="color:#f92672">=</span>CID-130fb246-1717-4313-9b62-9ddfe1bcb2e7 |
| nodeType<span style="color:#f92672">=</span>SCM |
| scmUuid<span style="color:#f92672">=</span>e6877ce5-56cd-4f0b-ad60-4c8ef9000882 |
| layoutVersion<span style="color:#f92672">=</span><span style="color:#ae81ff">0</span> |
| </code></pre></div><p>如果所有的容器元数据都已复制,您还可以创建数据并使用 <code>ozone debug</code> 工具进行双重检查。</p> |
| <div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-shell" data-lang="shell">bin/ozone freon randomkeys --numOfVolumes<span style="color:#f92672">=</span><span style="color:#ae81ff">1</span> --numOfBuckets<span style="color:#f92672">=</span><span style="color:#ae81ff">1</span> --numOfKeys<span style="color:#f92672">=</span><span style="color:#ae81ff">10000</span> --keySize<span style="color:#f92672">=</span><span style="color:#ae81ff">524288</span> --replicationType<span style="color:#f92672">=</span>RATIS --numOfThreads<span style="color:#f92672">=</span><span style="color:#ae81ff">8</span> --factor<span style="color:#f92672">=</span>THREE --bufferSize<span style="color:#f92672">=</span><span style="color:#ae81ff">1048576</span> |
| |
| |
| <span style="color:#75715e"># 使用 debug ldb 工具逐一检查各机上的 scm.db</span> |
| bin/ozone debug ldb --db<span style="color:#f92672">=</span>/tmp/metadata/scm.db ls |
| |
| |
| bin/ozone debug ldb --db<span style="color:#f92672">=</span>/tmp/metadata/scm.db scan --column-family<span style="color:#f92672">=</span>containers |
| </code></pre></div><h2 id="从现有的scm迁移">从现有的SCM迁移</h2> |
| <p>可以在任何 Ozone 集群上打开 SCM HA。 首先启用 Ratis(<code>ozone.scm.ratis.enable</code>)并为 Ratis ring 配置一个节点(<code>ozone.scm.nodes.serviceId</code> 应该有一个元素)。</p> |
| <p>启动集群并测试它是否正常工作。</p> |
| <p>如果一切正常,您可以用多个节点扩展集群配置,重新启动 SCM 节点,并使用 <code>scm --bootstrap</code> 命令初始化其他节点。</p> |
| |
| |
| |
| <a class="btn btn-success btn-lg" href="../../zh/feature/dn-merge-rocksdb.html">Next >></a> |
| |
| </div> |
| |
| </div> |
| </div> |
| </div> |
| </div> |
| <div class="push"></div> |
| </div> |
| |
| |
| |
| <footer class="footer"> |
| <div class="container"> |
| <span class="small text-muted"> |
| Version: 1.5.0-SNAPSHOT, Last Modified: February 26, 2024 <a class="hide-child link primary-color" href="https://github.com/apache/ozone/commit/1b48186a0107711235abcd2636977ae0242f6be8">1b48186</a> |
| </span> |
| </div> |
| </footer> |
| |
| |
| |
| <script src="../../js/jquery-3.5.1.min.js"></script> |
| <script src="../../js/ozonedoc.js"></script> |
| <script src="../../js/bootstrap.min.js"></script> |
| |
| |
| </body> |
| |
| </html> |