blob: 4d816ec04df9e594c03a77d289aa5683f3775f1f [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="Home page of The Apache Software Foundation">
<link rel="apple-touch-icon" sizes="57x57" href="/favicons/apple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/favicons/apple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/favicons/apple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/favicons/apple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/favicons/apple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/favicons/apple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/favicons/apple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/favicons/apple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/favicons/apple-touch-icon-180x180.png">
<link rel="icon" type="image/png" href="/favicons/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/favicons/favicon-194x194.png" sizes="194x194">
<link rel="icon" type="image/png" href="/favicons/favicon-96x96.png" sizes="96x96">
<link rel="icon" type="image/png" href="/favicons/android-chrome-192x192.png" sizes="192x192">
<link rel="icon" type="image/png" href="/favicons/favicon-16x16.png" sizes="16x16">
<link rel="manifest" href="/favicons/manifest.json">
<link rel="shortcut icon" href="/favicons/favicon.ico">
<meta name="msapplication-TileColor" content="#603cba">
<meta name="msapplication-TileImage" content="/favicons/mstile-144x144.png">
<meta name="msapplication-config" content="/favicons/browserconfig.xml">
<meta name="theme-color" content="#282661">
<title>The history of the Apache mirror system</title>
<link href="/css/Montserrat-300-600.css" rel="stylesheet">
<link href="/css/min.bootstrap.css" rel="stylesheet">
<link href="/css/styles.css" rel="stylesheet">
<link href="/css/new-ui.css" rel="stylesheet">
<style>
.headerlink {
visibility: hidden;
}
dt:hover > .headerlink, p:hover > .headerlink, td:hover > .headerlink, h1:hover > .headerlink, h2:hover > .headerlink, h3:hover > .headerlink, h4:hover > .headerlink, h5:hover > .headerlink, h6:hover > .headerlink {
visibility: visible
} </style>
<!-- pagefind search -->
<link href="/_pagefind/pagefind-ui.css" rel="stylesheet">
<script src="/_pagefind/pagefind-ui.js" type="text/javascript"></script>
<script>
window.addEventListener('DOMContentLoaded', (event) => {
new PagefindUI({ element: "#pagefind-search" });
});
var pageTitle = 'The history of the Apache mirror system';
if(pageTitle === '404'){
window.addEventListener('DOMContentLoaded', (event) => {
new PagefindUI({ element: "#page-404-search" });
});
}
</script>
<!-- https://www.apache.org/licenses/LICENSE-2.0 --> <!-- Matomo -->
<script>
var pageTitle = 'The history of the Apache mirror system';
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before
"trackPageView" */
/* We explicitly disable cookie tracking to avoid privacy issues */
_paq.push(['disableCookies']);
if(pageTitle === '404'){
/* Track 404 page hits */
_paq.push(['setDocumentTitle', '404/URL = ' + encodeURIComponent(document.location.pathname+document.location.search) + '/From = ' + encodeURIComponent(document.referrer)]);
}
/* Measure a visit to flink.apache.org and nightlies.apache.org/flink
as the same visit */
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '37']);
var d=document, g=d.createElement('script'),
s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head>
<body >
<!-- Navigation -->
<header>
<div id="skiptocontent">
<a href="#maincontent">Skip to Main Content</a>
</div>
<nav class="navbar navbar-inverse navbar-fixed-top mainmenu">
<div class="container">
<div class="navbar-header">
<button class="navbar-toggle" type="button" data-toggle="collapse" data-target="#mainnav-collapse">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
</div>
<div class="collapse navbar-collapse" id="mainnav-collapse">
<div class="upper-nav">
<div class="header-social-icons">
<a target="_blank" href="https://infra.apache.org/slack.html"><img src="/images/slack-icon.svg" alt="slack icon"></a>
<a target="_blank" href="https://github.com/apache"><img src="/images/github-mark-white.svg" alt="github icon"></a>
<a target="_blank" href="https://www.linkedin.com/company/the-apache-software-foundation/"><img src="/images/linkedin-icon.png" alt="linkedIn icon"></a>
<a target="_blank" href="https://www.youtube.com/c/TheApacheFoundation"><img src="/images/youtube-icon.svg" alt="youtube icon"></a>
<a target="_blank" href="https://twitter.com/TheASF"><img src="/images/x-icon.svg" alt="X icon"></a>
</div>
<a href="/foundation/sponsorship" class="btn btn-default" onclick="_paq.push(['trackEvent', 'click', 'SponsorASF Button']);">Sponsor the ASF</a>
</div>
<ul class="nav navbar-nav navbar-justified n-navbar-padding">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">Community&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="https://community.apache.org/" target="_blank">Contributor Getting Started</a></li>
<li><a href="https://community.apache.org/contributors/" target="_blank">Becoming a Committer</a></li>
<li><a href="/foundation/policies/conduct">Code of Conduct</a></li>
<li><a href="/community-resources/">Community Resources</a></li>
<li><a href="https://communityovercode.org/" target="_blank">Community Over Code</a></li>
<li><a href="https://events.apache.org/" target="_blank">Events</a></li>
<li><a href="https://www.redbubble.com/people/comdev/shop" target="_blank">Store</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">Projects&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="/projects">Projects</a></li>
<li><a href="https://incubator.apache.org/" target="_blank">Incubator Projects</a></li>
<li><a href="https://projects.apache.org/" target="_blank">Projects Directory </a></li>
<li><a href="/foundation/mailinglists">Mailing Lists </a></li>
<li><a href="/security">Report a Vulnerability</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">Downloads&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="https://downloads.apache.org/" target="_blank">Distributions</a></li>
<li><a href="https://projects.apache.org/releases.html" target="_blank">Releases</a></li>
<li><a href="https://status.apache.org/" target="_blank">Infrastructure Status</a></li>
<li><a href="https://infra-reports.apache.org/#uptime" target="_blank">Infrastructure Statistics</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">Learn&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="https://news.apache.org/" target="_blank">Blog</a></li>
<li><a href="/foundation/how-it-works">How the ASF Works</a></li>
<li><a href="/theapacheway/">The Apache Way</a></li>
<li><a href="/legal/">Legal &amp; Trademark</a></li>
<li><a href="/licenses">Licenses</a></li>
<li><a href="/foundation/glossary">Glossary</a></li>
<li><a href="/foundation/faq">FAQ</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">Resources &amp; Tools&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="/dev/">Developer Information</a></li>
<li><a href="https://cwiki.apache.org/" target="_blank" >Wiki</a></li>
<li><a href="https://issues.apache.org/" target="_blank" >Issues</a></li>
<li><a href="https://infra.apache.org/slack.html" target="_blank" >Slack</a></li>
<li><a href="https://selfserve.apache.org/" target="_blank" >Self Serve Portal</a></li>
<li><a href="https://infra.apache.org/" target="_blank" >Infrastructure</a></li>
<li><a href="https://whimsy.apache.org/" target="_blank" >Whimsy</a></li>
<li><a href="/foundation/press/kit/">Brand Guidelines</a></li>
<li><a href="/logos/">Project Logos</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button">About&nbsp;<span class="caret"></span></a>
<ul class="dropdown-menu" role="menu">
<li><a href="/foundation/">About</a></li>
<li><a href="/foundation/sponsors">Our Sponsors</a></li>
<li><a href="/foundation/sponsorship">Corporate Sponsorship</a></li>
<li><a href="/foundation/individual-supporters">Individual Supporters</a></li>
<li><a href="/foundation/leadership">Leadership</a></li>
<li><a href="/foundation/members">Members</a></li>
<li><a href="https://diversity.apache.org/" target="_blank">Diversity & Inclusion</a></li>
<li><a href="/press/">Newsroom</a></li>
<li><a href="/foundation/contact">Contact</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle hidden-xs" data-toggle="dropdown" role="button"><span class="glyphicon glyphicon-search"
aria-hidden="true"></span><span class="sr-only">Search</span></a>
<ul class="dropdown-menu search-form" role="search">
<li>
<div id="pagefind-search" class="input-group" style="width: 100%; padding: 0 5px;"></div>
</li>
</ul>
</li>
</ul>
</div>
</div>
</nav>
</header>
<!-- / Navigation -->
<header id="main-header" class="container">
<div class="sideImg">
<a class="visible-home" href="https://events.apache.org/x/current-event.html">
<img class="img-responsive" style="width: 125px;" src="/events/current-event-125x125.png" alt="Apache Events"/>
<!-- STALE: <img class="img-responsive" style="width: 125px;" src="https://www.apachecon.com/event-images/default-square-light.png" alt="ApacheCon 2021 Coming Soon!" /> -->
</a>
<a class="hidden-home" href="/"><img class="img-responsive" src="/img/asf-estd-1999-logo.jpg" alt="The Apache Software Foundation"></a>
</div>
<div class="main">
<img class="img-responsive center-block visible-home" src="/img/asf-estd-1999-logo.jpg" alt="Apache 20th Anniversary Logo">
</div>
</header>
<main id="maincontent">
<div class="container"> <h1 id="the-history-of-the-apache-mirror-system">The history of the Apache mirror system<a class="headerlink" href="#the-history-of-the-apache-mirror-system" title="Permalink">&para;</a></h1>
<p>The ASF system download mirrors evolved organically, and then systematically, over 25 years. It depended on many hours of labor by volunteers and generous contributions by organizations that supported the concept of free and open source software, and made possible distribution of Apache projects' software around the world.</p>
<p>In 2021 The ASF moved to a content distribution network (CDN), retiring the download mirror system. Here is its story.</p>
<ul>
<li><a href="#problem">The problem the ASF mirror system addressed</a></li>
<li><a href="#growth">How the mirror system grew</a></li>
<li><a href="#moving">Moving away from the mirror system</a></li>
<li><a href="#cdn">What the new CDN provides</a></li>
<li><a href="#experience">Project and downloader experience</a></li>
<li><a href="#next">What now for the mirror providers?</a></li>
</ul>
<h3 id="problem">The problem the ASF mirror system addressed</h3>
<p>As soon as the Apache Group, the precursor to the ASF, had a useful version of the Apache Server, it wanted to share it widely. This required having a download site from which interested parties could retrieve the latest version of the software.</p>
<p>A single download site could handle a few requests at a time, but as demand expanded rapidly, the volume of download requests became more than a single site could handle. In addition, the further away the request came from, the longer the download would take, and the possibility of losing connections and therefore having to start over became greater. As requests were soon coming in from all around the world, this became a serious challenge. Apache needed download mirrors to help handle the load.</p>
<h3 id="growth">How the mirror system grew</h3>
<p>A download mirror site is, as the name implies, a copy of the main download site. Several times a day a mirror checks the main site to see if there are new or updated artifacts to add to its offerings. A mirror would sometimes be as much as 24 hours out of sync with the main site, but most of the time was able to offer everything the main site did. Requestors who were physically nearer to the mirror than to the main site could download software much more quickly from their local mirror.</p>
<p>Mirrors require lots of memory, visibility on the Internet, bandwidth, maintenance, and power. Apache relied on organizations to contribute these things by sponsoring mirror sites. The first mirror sites became available in April, 1995. Among the first mirror providers was SunSite, 'a network of Internet servers providing archives of information, software and other publicly available resources.'</p>
<p>In April,1997, Brian Behlendorf sent an email to 66 recipients who were hosting Apache mirrors to invite them to the 'mirror@' Apache mailing list. The email noted "a varying degree of quality amongst mirror sites, so I wanted to start a forum for discussion of how to improve that quality, and how to look forward to better mirroring protocols than FTP."</p>
<p>In June of that year, Brian wrote to the group, "The release of 1.2.0 (of the Apache server) caused an unprecedented amount of traffic on our site; so much so that the bandwidth provider for <a href="http://www.apache.org">www.apache.org</a> has begun to grumble. To address this, we have taken some measures to bolster the effectiveness of the site in supporting mirrors." Among the changes: "The 'Download' link has been turned into a CGI script which selects from the list of mirrors those most appropriate for your domain, based upon the country-code of the requesting agent." Brian, Joshua Slive, and Eric Abele were the primary system maintainers.</p>
<p>In 2000, the system moved from CGI to a MySQL database. However, many things stopped working within a couple of weeks. The system quickly moved back to CGI.</p>
<p>In 2002, Henk P. Penning, who would make major contributions to the mirroring system and to many other aspects of Apache, appeared in the mailing list for the first time. That year, a download mirror had to allocate 10 gigabytes of space to handle all the artifacts available for download. At the time, that seemed huge!</p>
<p>In 2003, Henk started generating statistics about the mirror network, providing a way for mirror hosts to check the health of their system and their connection to Apache. In December of that year, backup (fallback) sites joined the system. Henk, now a member of the Infrastructure team, formalized monitoring of Apache mirrors.</p>
<p>From 2004 to 2015 the mirror system enjoyed steady growth, responding to world-wide demand for access to Apache projects' software.</p>
<p>In 2007 Gavin McDonald (gmcdonald) joined the Infrastructure team. His first focus was going through the huge backlog of tickets, many of which were requests for mirror providers to join as official mirrors.</p>
<p>Back then, managing the mirrors was a manual task, including going to each prospective provider's mirror site and performing checks (are they up to date, syncing at least twice a day, using the Apache Web Server to serve the mirrored software, etc.). For the many existing official mirrors, Gavin performed periodic checks and wrote emails to remind a mirror provider of their duty if the mirror had become stale. A few months in, Gavin requested Henk to create a nag checker tool that would detect stale mirrors and automatically email the provider. That was a big time saver!</p>
<p>Gavin continued maintaining the mirror system for another couple of years, before other volunteers relieved him. Eventually the Infra team took over the role.</p>
<img alt="Chart of the growth of the Apache mirror system" src="images/mirrorsystemgrowth.png" width="800"/>
<p>In August 2015, deploying the mod_lua based system significantly improved response times.</p>
<p>By 2021, a download mirror had to allocate at least 180 gigabytes of space for all Apache projects available for download.</p>
<h3 id="moving">Moving away from the mirror system</h3>
<p>When the download-mirror system started, the bandwidth to support downloads was very expensive. Mirror sites, as well as providing faster download times for requestors near them, also helped share the bandwidth costs. Over the years, more than a thousand organizations hosted ASF download-mirror sites, some for long periods. This contribution was enormously important to the mission of The ASF, and to the world of open source software in general.</p>
<p>Technology has advanced and bandwidth costs have dropped. The time has come to move The ASF's download distribution to a service that has professional support, to better suit the position the foundation and its projects have in today's world. The ASF is a major player in the open source technology sector, and our download options should support and reflect that status.</p>
<p>In the third quarter of 2021, the time was right to move to the new CDN.</p>
<h3 id="cdn">What the new CDN provides</h3>
<p>The new CDN is a global content delivery network operated by one of the leading actors in the business. It provides</p>
<ul>
<li>economies of scale, so The ASF can provide reliable, fast downloads around the world even at times of peak demand, as when a new version of a popular project becomes available.</li>
<li>better insights for ASF projects about who is downloading their software, and where in the world their software users are.</li>
<li>faster deployment of new software releases, eliminating the lag-time between what is available on the main download site and what is available on a particular download mirror.</li>
</ul>
<h3 id="experience">Project and downloader experience</h3>
<p>Projects should not experience any change to their workflow. They eventually will need to remove guidance about selecting a download mirror from their sites. They will gain insights about who is downloading (and presumably using) their software, as noted above.</p>
<p>Users should experience faster, more reliable downloads, and access to downloadable software versions as soon as they become available, without having to wait for mirror sites to catch up to the main ASF download site.</p>
<h3 id="next">What now for the mirror providers?</h3>
<p>Any organization that has been hosting an Apache download mirror site, and that finds it useful for its own purposes, may continue to host the mirror site. Other organizations may want to consider how to redeploy resources that have been supporting their mirror site to helping the ASF and the open source software community in another way.</p>
<p>For example, The Apache Software Foundation is always looking to expand our CI capacity. Any providers wishing to donate computing resources (virtual machines or bare metal servers) are most welcome to contact the builds team at <code>builds@apache.org</code> to discuss how they can help improve our CI environment.</p>
</div> </main>
<!-- Footer -->
<footer class="bg-primary">
<div class="container">
<div class="row">
<br />
<div class="col-sm-2">
<h5 class="white">Community</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="https://community.apache.org/" target="_blank">Contributor Getting Started</a></li>
<li><a href="https://community.apache.org/contributors/" target="_blank">Becoming a Committer</a></li>
<li><a href="/foundation/policies/conduct">Code of Conduct</a></li>
<li><a href="/community-resources/">Community Resources</a></li>
<li><a href="https://communityovercode.org/" target="_blank">Community Over Code</a></li>
<li><a href="https://events.apache.org/" target="_blank">Events</a></li>
<li><a href="https://www.redbubble.com/people/comdev/shop" target="_blank">Store</a></li>
</ul>
</div>
<div class="col-sm-2">
<h5 class="white">Projects</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="/projects">Projects</a></li>
<li><a href="https://incubator.apache.org/" target="_blank">Incubator Projects</a></li>
<li><a href="https://projects.apache.org/" target="_blank">Projects Directory </a></li>
<li><a href="/foundation/mailinglists">Mailing Lists </a></li>
<li><a href="/security">Report a Vulnerability</a></li>
</ul>
</div>
<div class="col-sm-2">
<h5 class="white">Downloads</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="https://downloads.apache.org/" target="_blank">Distributions</a></li>
<li><a href="https://projects.apache.org/releases.html" target="_blank">Releases</a></li>
<li><a href="https://status.apache.org/" target="_blank">Infrastructure Status</a></li>
<li><a href="https://infra-reports.apache.org/#uptime" target="_blank">Infrastructure Statistics</a></li>
</ul>
</div>
<div class="col-sm-2">
<h5 class="white">Learn</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="https://news.apache.org/" target="_blank">Blog</a></li>
<li><a href="/foundation/how-it-works">How the ASF Works</a></li>
<li><a href="/theapacheway/">The Apache Way</a></li>
<li><a href="/legal/">Legal &amp; Trademark</a></li>
<li><a href="/licenses">Licenses</a></li>
<li><a href="/foundation/glossary">Glossary</a></li>
<li><a href="/foundation/faq">FAQ</a></li>
</ul>
</div>
<div class="col-sm-2">
<h5 class="white">Resources &amp; Tools</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="/dev/">Developer Information</a></li>
<li><a href="https://cwiki.apache.org/" target="_blank" >Wiki</a></li>
<li><a href="https://issues.apache.org/" target="_blank" >Issues</a></li>
<li><a href="https://infra.apache.org/slack.html" target="_blank" >Slack</a></li>
<li><a href="https://selfserve.apache.org/" target="_blank" >Self Serve Portal</a></li>
<li><a href="https://infra.apache.org/" target="_blank" >Infrastructure</a></li>
<li><a href="https://whimsy.apache.org/" target="_blank" >Whimsy</a></li>
<li><a href="/foundation/press/kit/">Brand Guidelines</a></li>
<li><a href="/logos/">Project Logos</a></li>
</ul>
</div>
<div class="col-sm-2">
<h5 class="white">About</h5>
<ul class="list-unstyled white" role="menu">
<li><a href="/foundation/">About</a></li>
<li><a href="/foundation/sponsors">Our Sponsors</a></li>
<li><a href="/foundation/sponsorship">Corporate Sponsorship</a></li>
<li><a href="/foundation/individual-supporters">Individual Supporters</a></li>
<li><a href="/foundation/leadership">Leadership</a></li>
<li><a href="/foundation/members">Members</a></li>
<li><a href="https://diversity.apache.org/" target="_blank">Diversity & Inclusion</a></li>
<li><a href="/press/">Newsroom</a></li>
<li><a href="/foundation/contact">Contact</a></li>
<li><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank">Privacy Policy</a></li>
</ul>
</div>
</div>
<hr class="col-lg-12 hr-white" />
<div class="row">
<div class="col-lg-12">
<p class="text-center">Copyright &#169; 2024 The Apache Software Foundation, Licensed under the <a class="white" href="/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.</p>
<p class="text-center">Apache and the Apache feather logo are trademarks of The Apache Software Foundation. </p>
</div>
</div>
</div>
</footer>
<!-- / Footer -->
<script src="/js/jquery.min.js"></script>
<script src="/js/bootstrap.js"></script>
<script src="/js/slideshow.js"></script>
<script src="/js/new-ui.js"></script>
<script>
(function($){
$(document).ready(function(){
$('ul.dropdown-menu [data-toggle=dropdown]').on('click', function(event) {
event.preventDefault();
event.stopPropagation();
$(this).parent().siblings().removeClass('open');
$(this).parent().toggleClass('open');
console.log('WOrked');
});
});
})(jQuery);
</script>
</body>
</html>