| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <meta name="generator" content="Asciidoctor 2.0.23"> |
| <title>Configuration</title> |
| <link rel="stylesheet" href="fonts/fonts.css"> |
| <style> |
| /*! Asciidoctor default stylesheet | MIT License | https://asciidoctor.org */ |
| /* Uncomment the following line when using as a custom stylesheet */ |
| /* @import "fonts/fonts.css"; */ |
| html{font-family:sans-serif;-webkit-text-size-adjust:100%} |
| a{background:none} |
| a:focus{outline:thin dotted} |
| a:active,a:hover{outline:0} |
| h1{font-size:2em;margin:.67em 0} |
| b,strong{font-weight:bold} |
| abbr{font-size:.9em} |
| abbr[title]{cursor:help;border-bottom:1px dotted #dddddf;text-decoration:none} |
| dfn{font-style:italic} |
| hr{height:0} |
| mark{background:#ff0;color:#000} |
| code,kbd,pre,samp{font-family:monospace;font-size:1em} |
| pre{white-space:pre-wrap} |
| q{quotes:"\201C" "\201D" "\2018" "\2019"} |
| small{font-size:80%} |
| sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline} |
| sup{top:-.5em} |
| sub{bottom:-.25em} |
| img{border:0} |
| svg:not(:root){overflow:hidden} |
| figure{margin:0} |
| audio,video{display:inline-block} |
| audio:not([controls]){display:none;height:0} |
| fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em} |
| legend{border:0;padding:0} |
| button,input,select,textarea{font-family:inherit;font-size:100%;margin:0} |
| button,input{line-height:normal} |
| button,select{text-transform:none} |
| button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer} |
| button[disabled],html input[disabled]{cursor:default} |
| input[type=checkbox],input[type=radio]{padding:0} |
| button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0} |
| textarea{overflow:auto;vertical-align:top} |
| table{border-collapse:collapse;border-spacing:0} |
| *,::before,::after{box-sizing:border-box} |
| html,body{font-size:100%} |
| body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;line-height:1;position:relative;cursor:auto;-moz-tab-size:4;-o-tab-size:4;tab-size:4;word-wrap:anywhere;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased} |
| a:hover{cursor:pointer} |
| img,object,embed{max-width:100%;height:auto} |
| object,embed{height:100%} |
| img{-ms-interpolation-mode:bicubic} |
| .left{float:left!important} |
| .right{float:right!important} |
| .text-left{text-align:left!important} |
| .text-right{text-align:right!important} |
| .text-center{text-align:center!important} |
| .text-justify{text-align:justify!important} |
| .hide{display:none} |
| img,object,svg{display:inline-block;vertical-align:middle} |
| textarea{height:auto;min-height:50px} |
| select{width:100%} |
| .subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em} |
| div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0} |
| a{color:#2156a5;text-decoration:underline;line-height:inherit} |
| a:hover,a:focus{color:#1d4b8f} |
| a img{border:0} |
| p{line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility} |
| p aside{font-size:.875em;line-height:1.35;font-style:italic} |
| h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em} |
| h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0} |
| h1{font-size:2.125em} |
| h2{font-size:1.6875em} |
| h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em} |
| h4,h5{font-size:1.125em} |
| h6{font-size:1em} |
| hr{border:solid #dddddf;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em} |
| em,i{font-style:italic;line-height:inherit} |
| strong,b{font-weight:bold;line-height:inherit} |
| small{font-size:60%;line-height:inherit} |
| code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9)} |
| ul,ol,dl{line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit} |
| ul,ol{margin-left:1.5em} |
| ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0} |
| ul.circle{list-style-type:circle} |
| ul.disc{list-style-type:disc} |
| ul.square{list-style-type:square} |
| ul.circle ul:not([class]),ul.disc ul:not([class]),ul.square ul:not([class]){list-style:inherit} |
| ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0} |
| dl dt{margin-bottom:.3125em;font-weight:bold} |
| dl dd{margin-bottom:1.25em} |
| blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd} |
| blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)} |
| @media screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2} |
| h1{font-size:2.75em} |
| h2{font-size:2.3125em} |
| h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em} |
| h4{font-size:1.4375em}} |
| table{background:#fff;margin-bottom:1.25em;border:1px solid #dedede;word-wrap:normal} |
| table thead,table tfoot{background:#f7f8f7} |
| table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left} |
| table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)} |
| table tr.even,table tr.alt{background:#f8f8f7} |
| table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{line-height:1.6} |
| h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em} |
| h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400} |
| .center{margin-left:auto;margin-right:auto} |
| .stretch{width:100%} |
| .clearfix::before,.clearfix::after,.float-group::before,.float-group::after{content:" ";display:table} |
| .clearfix::after,.float-group::after{clear:both} |
| :not(pre).nobreak{word-wrap:normal} |
| :not(pre).nowrap{white-space:nowrap} |
| :not(pre).pre-wrap{white-space:pre-wrap} |
| :not(pre):not([class^=L])>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;padding:.1em .5ex;word-spacing:-.15em;background:#f7f7f8;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed} |
| pre{color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;line-height:1.45;text-rendering:optimizeSpeed} |
| pre code,pre pre{color:inherit;font-size:inherit;line-height:inherit} |
| pre>code{display:block} |
| pre.nowrap,pre.nowrap pre{white-space:pre;word-wrap:normal} |
| em em{font-style:normal} |
| strong strong{font-weight:400} |
| .keyseq{color:rgba(51,51,51,.8)} |
| kbd{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;display:inline-block;color:rgba(0,0,0,.8);font-size:.65em;line-height:1.45;background:#f7f7f7;border:1px solid #ccc;border-radius:3px;box-shadow:0 1px 0 rgba(0,0,0,.2),inset 0 0 0 .1em #fff;margin:0 .15em;padding:.2em .5em;vertical-align:middle;position:relative;top:-.1em;white-space:nowrap} |
| .keyseq kbd:first-child{margin-left:0} |
| .keyseq kbd:last-child{margin-right:0} |
| .menuseq,.menuref{color:#000} |
| .menuseq b:not(.caret),.menuref{font-weight:inherit} |
| .menuseq{word-spacing:-.02em} |
| .menuseq b.caret{font-size:1.25em;line-height:.8} |
| .menuseq i.caret{font-weight:bold;text-align:center;width:.45em} |
| b.button::before,b.button::after{position:relative;top:-1px;font-weight:400} |
| b.button::before{content:"[";padding:0 3px 0 2px} |
| b.button::after{content:"]";padding:0 2px 0 3px} |
| p a>code:hover{color:rgba(0,0,0,.9)} |
| #header,#content,#footnotes,#footer{width:100%;margin:0 auto;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em} |
| #header::before,#header::after,#content::before,#content::after,#footnotes::before,#footnotes::after,#footer::before,#footer::after{content:" ";display:table} |
| #header::after,#content::after,#footnotes::after,#footer::after{clear:both} |
| #content{margin-top:1.25em} |
| #content::before{content:none} |
| #header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0} |
| #header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #dddddf} |
| #header>h1:only-child{border-bottom:1px solid #dddddf;padding-bottom:8px} |
| #header .details{border-bottom:1px solid #dddddf;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:flex;flex-flow:row wrap} |
| #header .details span:first-child{margin-left:-.125em} |
| #header .details span.email a{color:rgba(0,0,0,.85)} |
| #header .details br{display:none} |
| #header .details br+span::before{content:"\00a0\2013\00a0"} |
| #header .details br+span.author::before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)} |
| #header .details br+span#revremark::before{content:"\00a0|\00a0"} |
| #header #revnumber{text-transform:capitalize} |
| #header #revnumber::after{content:"\00a0"} |
| #content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #dddddf;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem} |
| #toc{border-bottom:1px solid #e7e7e9;padding-bottom:.5em} |
| #toc>ul{margin-left:.125em} |
| #toc ul.sectlevel0>li>a{font-style:italic} |
| #toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0} |
| #toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none} |
| #toc li{line-height:1.3334;margin-top:.3334em} |
| #toc a{text-decoration:none} |
| #toc a:active{text-decoration:underline} |
| #toctitle{color:#7a2518;font-size:1.2em} |
| @media screen and (min-width:768px){#toctitle{font-size:1.375em} |
| body.toc2{padding-left:15em;padding-right:0} |
| body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #dddddf;padding-bottom:8px} |
| #toc.toc2{margin-top:0!important;background:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #e7e7e9;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto} |
| #toc.toc2 #toctitle{margin-top:0;margin-bottom:.8rem;font-size:1.2em} |
| #toc.toc2>ul{font-size:.9em;margin-bottom:0} |
| #toc.toc2 ul ul{margin-left:0;padding-left:1em} |
| #toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em} |
| body.toc2.toc-right{padding-left:0;padding-right:15em} |
| body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #e7e7e9;left:auto;right:0}} |
| @media screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0} |
| #toc.toc2{width:20em} |
| #toc.toc2 #toctitle{font-size:1.375em} |
| #toc.toc2>ul{font-size:.95em} |
| #toc.toc2 ul ul{padding-left:1.25em} |
| body.toc2.toc-right{padding-left:0;padding-right:20em}} |
| #content #toc{border:1px solid #e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;border-radius:4px} |
| #content #toc>:first-child{margin-top:0} |
| #content #toc>:last-child{margin-bottom:0} |
| #footer{max-width:none;background:rgba(0,0,0,.8);padding:1.25em} |
| #footer-text{color:hsla(0,0%,100%,.8);line-height:1.44} |
| #content{margin-bottom:.625em} |
| .sect1{padding-bottom:.625em} |
| @media screen and (min-width:768px){#content{margin-bottom:1.25em} |
| .sect1{padding-bottom:1.25em}} |
| .sect1:last-child{padding-bottom:0} |
| .sect1+.sect1{border-top:1px solid #e7e7e9} |
| #content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400} |
| #content h1>a.anchor::before,h2>a.anchor::before,h3>a.anchor::before,#toctitle>a.anchor::before,.sidebarblock>.content>.title>a.anchor::before,h4>a.anchor::before,h5>a.anchor::before,h6>a.anchor::before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em} |
| #content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible} |
| #content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none} |
| #content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221} |
| details,.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em} |
| details{margin-left:1.25rem} |
| details>summary{cursor:pointer;display:block;position:relative;line-height:1.6;margin-bottom:.625rem;outline:none;-webkit-tap-highlight-color:transparent} |
| details>summary::-webkit-details-marker{display:none} |
| details>summary::before{content:"";border:solid transparent;border-left:solid;border-width:.3em 0 .3em .5em;position:absolute;top:.5em;left:-1.25rem;transform:translateX(15%)} |
| details[open]>summary::before{border:solid transparent;border-top:solid;border-width:.5em .3em 0;transform:translateY(15%)} |
| details>summary::after{content:"";width:1.25rem;height:1em;position:absolute;top:.3em;left:-1.25rem} |
| .admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic} |
| table.tableblock.fit-content>caption.title{white-space:nowrap;width:0} |
| .paragraph.lead>p,#preamble>.sectionbody>[class=paragraph]:first-of-type p{font-size:1.21875em;line-height:1.6;color:rgba(0,0,0,.85)} |
| .admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%} |
| .admonitionblock>table td.icon{text-align:center;width:80px} |
| .admonitionblock>table td.icon img{max-width:none} |
| .admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase} |
| .admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #dddddf;color:rgba(0,0,0,.6);word-wrap:anywhere} |
| .admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0} |
| .exampleblock>.content{border:1px solid #e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;border-radius:4px} |
| .sidebarblock{border:1px solid #dbdbd6;margin-bottom:1.25em;padding:1.25em;background:#f3f3f2;border-radius:4px} |
| .sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center} |
| .exampleblock>.content>:first-child,.sidebarblock>.content>:first-child{margin-top:0} |
| .exampleblock>.content>:last-child,.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0} |
| .literalblock pre,.listingblock>.content>pre{border-radius:4px;overflow-x:auto;padding:1em;font-size:.8125em} |
| @media screen and (min-width:768px){.literalblock pre,.listingblock>.content>pre{font-size:.90625em}} |
| @media screen and (min-width:1280px){.literalblock pre,.listingblock>.content>pre{font-size:1em}} |
| .literalblock pre,.listingblock>.content>pre:not(.highlight),.listingblock>.content>pre[class=highlight],.listingblock>.content>pre[class^="highlight "]{background:#f7f7f8} |
| .literalblock.output pre{color:#f7f7f8;background:rgba(0,0,0,.9)} |
| .listingblock>.content{position:relative} |
| .listingblock code[data-lang]::before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:inherit;opacity:.5} |
| .listingblock:hover code[data-lang]::before{display:block} |
| .listingblock.terminal pre .command::before{content:attr(data-prompt);padding-right:.5em;color:inherit;opacity:.5} |
| .listingblock.terminal pre .command:not([data-prompt])::before{content:"$"} |
| .listingblock pre.highlightjs{padding:0} |
| .listingblock pre.highlightjs>code{padding:1em;border-radius:4px} |
| .listingblock pre.prettyprint{border-width:0} |
| .prettyprint{background:#f7f7f8} |
| pre.prettyprint .linenums{line-height:1.45;margin-left:2em} |
| pre.prettyprint li{background:none;list-style-type:inherit;padding-left:0} |
| pre.prettyprint li code[data-lang]::before{opacity:1} |
| pre.prettyprint li:not(:first-child) code[data-lang]::before{display:none} |
| table.linenotable{border-collapse:separate;border:0;margin-bottom:0;background:none} |
| table.linenotable td[class]{color:inherit;vertical-align:top;padding:0;line-height:inherit;white-space:normal} |
| table.linenotable td.code{padding-left:.75em} |
| table.linenotable td.linenos,pre.pygments .linenos{border-right:1px solid;opacity:.35;padding-right:.5em;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none} |
| pre.pygments span.linenos{display:inline-block;margin-right:.75em} |
| .quoteblock{margin:0 1em 1.25em 1.5em;display:table} |
| .quoteblock:not(.excerpt)>.title{margin-left:-1.5em;margin-bottom:.75em} |
| .quoteblock blockquote,.quoteblock p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify} |
| .quoteblock blockquote{margin:0;padding:0;border:0} |
| .quoteblock blockquote::before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)} |
| .quoteblock blockquote>.paragraph:last-child p{margin-bottom:0} |
| .quoteblock .attribution{margin-top:.75em;margin-right:.5ex;text-align:right} |
| .verseblock{margin:0 1em 1.25em} |
| .verseblock pre{font-family:"Open Sans","DejaVu Sans",sans-serif;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility} |
| .verseblock pre strong{font-weight:400} |
| .verseblock .attribution{margin-top:1.25rem;margin-left:.5ex} |
| .quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic} |
| .quoteblock .attribution br,.verseblock .attribution br{display:none} |
| .quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.025em;color:rgba(0,0,0,.6)} |
| .quoteblock.abstract blockquote::before,.quoteblock.excerpt blockquote::before,.quoteblock .quoteblock blockquote::before{display:none} |
| .quoteblock.abstract blockquote,.quoteblock.abstract p,.quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{line-height:1.6;word-spacing:0} |
| .quoteblock.abstract{margin:0 1em 1.25em;display:block} |
| .quoteblock.abstract>.title{margin:0 0 .375em;font-size:1.15em;text-align:center} |
| .quoteblock.excerpt>blockquote,.quoteblock .quoteblock{padding:0 0 .25em 1em;border-left:.25em solid #dddddf} |
| .quoteblock.excerpt,.quoteblock .quoteblock{margin-left:0} |
| .quoteblock.excerpt blockquote,.quoteblock.excerpt p,.quoteblock .quoteblock blockquote,.quoteblock .quoteblock p{color:inherit;font-size:1.0625rem} |
| .quoteblock.excerpt .attribution,.quoteblock .quoteblock .attribution{color:inherit;font-size:.85rem;text-align:left;margin-right:0} |
| p.tableblock:last-child{margin-bottom:0} |
| td.tableblock>.content{margin-bottom:1.25em;word-wrap:anywhere} |
| td.tableblock>.content>:last-child{margin-bottom:-1.25em} |
| table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede} |
| table.grid-all>*>tr>*{border-width:1px} |
| table.grid-cols>*>tr>*{border-width:0 1px} |
| table.grid-rows>*>tr>*{border-width:1px 0} |
| table.frame-all{border-width:1px} |
| table.frame-ends{border-width:1px 0} |
| table.frame-sides{border-width:0 1px} |
| table.frame-none>colgroup+*>:first-child>*,table.frame-sides>colgroup+*>:first-child>*{border-top-width:0} |
| table.frame-none>:last-child>:last-child>*,table.frame-sides>:last-child>:last-child>*{border-bottom-width:0} |
| table.frame-none>*>tr>:first-child,table.frame-ends>*>tr>:first-child{border-left-width:0} |
| table.frame-none>*>tr>:last-child,table.frame-ends>*>tr>:last-child{border-right-width:0} |
| table.stripes-all>*>tr,table.stripes-odd>*>tr:nth-of-type(odd),table.stripes-even>*>tr:nth-of-type(even),table.stripes-hover>*>tr:hover{background:#f8f8f7} |
| th.halign-left,td.halign-left{text-align:left} |
| th.halign-right,td.halign-right{text-align:right} |
| th.halign-center,td.halign-center{text-align:center} |
| th.valign-top,td.valign-top{vertical-align:top} |
| th.valign-bottom,td.valign-bottom{vertical-align:bottom} |
| th.valign-middle,td.valign-middle{vertical-align:middle} |
| table thead th,table tfoot th{font-weight:bold} |
| tbody tr th{background:#f7f8f7} |
| tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold} |
| p.tableblock>code:only-child{background:none;padding:0} |
| p.tableblock{font-size:1em} |
| ol{margin-left:1.75em} |
| ul li ol{margin-left:1.5em} |
| dl dd{margin-left:1.125em} |
| dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0} |
| li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em} |
| ul.checklist,ul.none,ol.none,ul.no-bullet,ol.no-bullet,ol.unnumbered,ul.unstyled,ol.unstyled{list-style-type:none} |
| ul.no-bullet,ol.no-bullet,ol.unnumbered{margin-left:.625em} |
| ul.unstyled,ol.unstyled{margin-left:0} |
| li>p:empty:only-child::before{content:"";display:inline-block} |
| ul.checklist>li>p:first-child{margin-left:-1em} |
| ul.checklist>li>p:first-child>.fa-square-o:first-child,ul.checklist>li>p:first-child>.fa-check-square-o:first-child{width:1.25em;font-size:.8em;position:relative;bottom:.125em} |
| ul.checklist>li>p:first-child>input[type=checkbox]:first-child{margin-right:.25em} |
| ul.inline{display:flex;flex-flow:row wrap;list-style:none;margin:0 0 .625em -1.25em} |
| ul.inline>li{margin-left:1.25em} |
| .unstyled dl dt{font-weight:400;font-style:normal} |
| ol.arabic{list-style-type:decimal} |
| ol.decimal{list-style-type:decimal-leading-zero} |
| ol.loweralpha{list-style-type:lower-alpha} |
| ol.upperalpha{list-style-type:upper-alpha} |
| ol.lowerroman{list-style-type:lower-roman} |
| ol.upperroman{list-style-type:upper-roman} |
| ol.lowergreek{list-style-type:lower-greek} |
| .hdlist>table,.colist>table{border:0;background:none} |
| .hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none} |
| td.hdlist1,td.hdlist2{vertical-align:top;padding:0 .625em} |
| td.hdlist1{font-weight:bold;padding-bottom:1.25em} |
| td.hdlist2{word-wrap:anywhere} |
| .literalblock+.colist,.listingblock+.colist{margin-top:-.5em} |
| .colist td:not([class]):first-child{padding:.4em .75em 0;line-height:1;vertical-align:top} |
| .colist td:not([class]):first-child img{max-width:none} |
| .colist td:not([class]):last-child{padding:.25em 0} |
| .thumb,.th{line-height:0;display:inline-block;border:4px solid #fff;box-shadow:0 0 0 1px #ddd} |
| .imageblock.left{margin:.25em .625em 1.25em 0} |
| .imageblock.right{margin:.25em 0 1.25em .625em} |
| .imageblock>.title{margin-bottom:0} |
| .imageblock.thumb,.imageblock.th{border-width:6px} |
| .imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em} |
| .image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0} |
| .image.left{margin-right:.625em} |
| .image.right{margin-left:.625em} |
| a.image{text-decoration:none;display:inline-block} |
| a.image object{pointer-events:none} |
| sup.footnote,sup.footnoteref{font-size:.875em;position:static;vertical-align:super} |
| sup.footnote a,sup.footnoteref a{text-decoration:none} |
| sup.footnote a:active,sup.footnoteref a:active,#footnotes .footnote a:first-of-type:active{text-decoration:underline} |
| #footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em} |
| #footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em;border-width:1px 0 0} |
| #footnotes .footnote{padding:0 .375em 0 .225em;line-height:1.3334;font-size:.875em;margin-left:1.2em;margin-bottom:.2em} |
| #footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none;margin-left:-1.05em} |
| #footnotes .footnote:last-of-type{margin-bottom:0} |
| #content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0} |
| div.unbreakable{page-break-inside:avoid} |
| .big{font-size:larger} |
| .small{font-size:smaller} |
| .underline{text-decoration:underline} |
| .overline{text-decoration:overline} |
| .line-through{text-decoration:line-through} |
| .aqua{color:#00bfbf} |
| .aqua-background{background:#00fafa} |
| .black{color:#000} |
| .black-background{background:#000} |
| .blue{color:#0000bf} |
| .blue-background{background:#0000fa} |
| .fuchsia{color:#bf00bf} |
| .fuchsia-background{background:#fa00fa} |
| .gray{color:#606060} |
| .gray-background{background:#7d7d7d} |
| .green{color:#006000} |
| .green-background{background:#007d00} |
| .lime{color:#00bf00} |
| .lime-background{background:#00fa00} |
| .maroon{color:#600000} |
| .maroon-background{background:#7d0000} |
| .navy{color:#000060} |
| .navy-background{background:#00007d} |
| .olive{color:#606000} |
| .olive-background{background:#7d7d00} |
| .purple{color:#600060} |
| .purple-background{background:#7d007d} |
| .red{color:#bf0000} |
| .red-background{background:#fa0000} |
| .silver{color:#909090} |
| .silver-background{background:#bcbcbc} |
| .teal{color:#006060} |
| .teal-background{background:#007d7d} |
| .white{color:#bfbfbf} |
| .white-background{background:#fafafa} |
| .yellow{color:#bfbf00} |
| .yellow-background{background:#fafa00} |
| span.icon>.fa{cursor:default} |
| a span.icon>.fa{cursor:inherit} |
| .admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default} |
| .admonitionblock td.icon .icon-note::before{content:"\f05a";color:#19407c} |
| .admonitionblock td.icon .icon-tip::before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111} |
| .admonitionblock td.icon .icon-warning::before{content:"\f071";color:#bf6900} |
| .admonitionblock td.icon .icon-caution::before{content:"\f06d";color:#bf3400} |
| .admonitionblock td.icon .icon-important::before{content:"\f06a";color:#bf0000} |
| .conum[data-value]{display:inline-block;color:#fff!important;background:rgba(0,0,0,.8);border-radius:50%;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold} |
| .conum[data-value] *{color:#fff!important} |
| .conum[data-value]+b{display:none} |
| .conum[data-value]::after{content:attr(data-value)} |
| pre .conum[data-value]{position:relative;top:-.125em} |
| b.conum *{color:inherit!important} |
| .conum:not([data-value]):empty{display:none} |
| dt,th.tableblock,td.content,div.footnote{text-rendering:optimizeLegibility} |
| h1,h2,p,td.content,span.alt,summary{letter-spacing:-.01em} |
| p strong,td.content strong,div.footnote strong{letter-spacing:-.005em} |
| p,blockquote,dt,td.content,td.hdlist1,span.alt,summary{font-size:1.0625rem} |
| p{margin-bottom:1.25rem} |
| .sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em} |
| .exampleblock>.content{background:#fffef7;border-color:#e0e0dc;box-shadow:0 1px 4px #e0e0dc} |
| .print-only{display:none!important} |
| @page{margin:1.25cm .75cm} |
| @media print{*{box-shadow:none!important;text-shadow:none!important} |
| html{font-size:80%} |
| a{color:inherit!important;text-decoration:underline!important} |
| a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important} |
| a[href^="http:"]:not(.bare)::after,a[href^="https:"]:not(.bare)::after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em} |
| abbr[title]{border-bottom:1px dotted} |
| abbr[title]::after{content:" (" attr(title) ")"} |
| pre,blockquote,tr,img,object,svg{page-break-inside:avoid} |
| thead{display:table-header-group} |
| svg{max-width:100%} |
| p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3} |
| h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid} |
| #header,#content,#footnotes,#footer{max-width:none} |
| #toc,.sidebarblock,.exampleblock>.content{background:none!important} |
| #toc{border-bottom:1px solid #dddddf!important;padding-bottom:0!important} |
| body.book #header{text-align:center} |
| body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em} |
| body.book #header .details{border:0!important;display:block;padding:0!important} |
| body.book #header .details span:first-child{margin-left:0!important} |
| body.book #header .details br{display:block} |
| body.book #header .details br+span::before{content:none!important} |
| body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important} |
| body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always} |
| .listingblock code[data-lang]::before{display:block} |
| #footer{padding:0 .9375em} |
| .hide-on-print{display:none!important} |
| .print-only{display:block!important} |
| .hide-for-print{display:none!important} |
| .show-for-print{display:inherit!important}} |
| @media amzn-kf8,print{#header>h1:first-child{margin-top:1.25rem} |
| .sect1{padding:0!important} |
| .sect1+.sect1{border:0} |
| #footer{background:none} |
| #footer-text{color:rgba(0,0,0,.6);font-size:.9em}} |
| @media amzn-kf8{#header,#content,#footnotes,#footer{padding:0}} |
| </style> |
| <link rel="stylesheet" href="fonts/font-awesome.min.css"> |
| <style> |
| /*! Stylesheet for CodeRay to loosely match GitHub themes | MIT License */ |
| pre.CodeRay{background:#f7f7f8} |
| .CodeRay .line-numbers{border-right:1px solid;opacity:.35;padding:0 .5em 0 0;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none} |
| .CodeRay span.line-numbers{display:inline-block;margin-right:.75em} |
| .CodeRay .line-numbers strong{color:#000} |
| table.CodeRay{border-collapse:separate;border:0;margin-bottom:0;background:none} |
| table.CodeRay td{vertical-align:top;line-height:inherit} |
| table.CodeRay td.line-numbers{text-align:right} |
| table.CodeRay td.code{padding:0 0 0 .75em} |
| .CodeRay .debug{color:#fff!important;background:navy!important} |
| .CodeRay .annotation{color:#007} |
| .CodeRay .attribute-name{color:navy} |
| .CodeRay .attribute-value{color:#700} |
| .CodeRay .binary{color:#509} |
| .CodeRay .comment{color:#998;font-style:italic} |
| .CodeRay .char{color:#04d} |
| .CodeRay .char .content{color:#04d} |
| .CodeRay .char .delimiter{color:#039} |
| .CodeRay .class{color:#458;font-weight:bold} |
| .CodeRay .complex{color:#a08} |
| .CodeRay .constant,.CodeRay .predefined-constant{color:teal} |
| .CodeRay .color{color:#099} |
| .CodeRay .class-variable{color:#369} |
| .CodeRay .decorator{color:#b0b} |
| .CodeRay .definition{color:#099} |
| .CodeRay .delimiter{color:#000} |
| .CodeRay .doc{color:#970} |
| .CodeRay .doctype{color:#34b} |
| .CodeRay .doc-string{color:#d42} |
| .CodeRay .escape{color:#666} |
| .CodeRay .entity{color:#800} |
| .CodeRay .error{color:#808} |
| .CodeRay .exception{color:inherit} |
| .CodeRay .filename{color:#099} |
| .CodeRay .function{color:#900;font-weight:bold} |
| .CodeRay .global-variable{color:teal} |
| .CodeRay .hex{color:#058} |
| .CodeRay .integer,.CodeRay .float{color:#099} |
| .CodeRay .include{color:#555} |
| .CodeRay .inline{color:#000} |
| .CodeRay .inline .inline{background:#ccc} |
| .CodeRay .inline .inline .inline{background:#bbb} |
| .CodeRay .inline .inline-delimiter{color:#d14} |
| .CodeRay .inline-delimiter{color:#d14} |
| .CodeRay .important{color:#555;font-weight:bold} |
| .CodeRay .interpreted{color:#b2b} |
| .CodeRay .instance-variable{color:teal} |
| .CodeRay .label{color:#970} |
| .CodeRay .local-variable{color:#963} |
| .CodeRay .octal{color:#40e} |
| .CodeRay .predefined{color:#369} |
| .CodeRay .preprocessor{color:#579} |
| .CodeRay .pseudo-class{color:#555} |
| .CodeRay .directive{font-weight:bold} |
| .CodeRay .type{font-weight:bold} |
| .CodeRay .predefined-type{color:inherit} |
| .CodeRay .reserved,.CodeRay .keyword{color:#000;font-weight:bold} |
| .CodeRay .key{color:#808} |
| .CodeRay .key .delimiter{color:#606} |
| .CodeRay .key .char{color:#80f} |
| .CodeRay .value{color:#088} |
| .CodeRay .regexp .delimiter{color:#808} |
| .CodeRay .regexp .content{color:#808} |
| .CodeRay .regexp .modifier{color:#808} |
| .CodeRay .regexp .char{color:#d14} |
| .CodeRay .regexp .function{color:#404;font-weight:bold} |
| .CodeRay .string{color:#d20} |
| .CodeRay .string .string .string{background:#ffd0d0} |
| .CodeRay .string .content{color:#d14} |
| .CodeRay .string .char{color:#d14} |
| .CodeRay .string .delimiter{color:#d14} |
| .CodeRay .shell{color:#d14} |
| .CodeRay .shell .delimiter{color:#d14} |
| .CodeRay .symbol{color:#990073} |
| .CodeRay .symbol .content{color:#a60} |
| .CodeRay .symbol .delimiter{color:#630} |
| .CodeRay .tag{color:teal} |
| .CodeRay .tag-special{color:#d70} |
| .CodeRay .variable{color:#036} |
| .CodeRay .insert{background:#afa} |
| .CodeRay .delete{background:#faa} |
| .CodeRay .change{color:#aaf;background:#007} |
| .CodeRay .head{color:#f8f;background:#505} |
| .CodeRay .insert .insert{color:#080} |
| .CodeRay .delete .delete{color:#800} |
| .CodeRay .change .change{color:#66f} |
| .CodeRay .head .head{color:#f4f} |
| </style> |
| <script src="/js/darkmode-docs.js"></script></head> |
| <body class="article"> |
| <div id="header"> |
| <div id="toc" class="toc"> |
| <div id="toctitle">Table of Contents</div> |
| <ul class="sectlevel1"> |
| <li><a href="#_configuration">Configuration</a> |
| <ul class="sectlevel2"> |
| <li><a href="#_user_agent_configuration">User Agent Configuration</a></li> |
| <li><a href="#_proxy">Proxy</a></li> |
| <li><a href="#_metadata">Metadata</a></li> |
| <li><a href="#_configuration_options">Configuration Options</a></li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| </div> |
| <div id="content"> |
| <div class="sect1"> |
| <h2 id="_configuration">Configuration</h2> |
| <div class="sectionbody"> |
| <div class="sect2"> |
| <h3 id="_user_agent_configuration">User Agent Configuration</h3> |
| <div class="paragraph"> |
| <p>Crawlers should always act responsibly and ethically when accessing websites. A key aspect of this is properly identifying themselves through the <code>User-Agent</code> header. By providing a clear and accurate user agent string, webmasters can understand who is visiting their site and why, and can apply rules in robots.txt accordingly. Respecting these rules, avoiding excessive request rates, and honoring content restrictions not only ensures legal compliance but also maintains a healthy relationship with the web community. |
| Transparent identification is a fundamental part of ethical web crawling.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The configuration of the <a href="https://www.w3.org/WAI/UA/work/wiki/Definition_of_User_Agent">user agent</a> in StormCrawler has two purposes:</p> |
| </div> |
| <div class="olist arabic"> |
| <ol class="arabic"> |
| <li> |
| <p>Identification of the crawler for webmasters</p> |
| </li> |
| <li> |
| <p>Selection of rules from robots.txt</p> |
| </li> |
| </ol> |
| </div> |
| <div class="sect3"> |
| <h4 id="_crawler_identification">Crawler Identification</h4> |
| <div class="paragraph"> |
| <p>The politeness of a web crawler is not limited to how frequently it fetches pages from a site, but also in how it identifies itself to sites it crawls. This is done by setting the HTTP header <code>User-Agent</code>, just like <a href="https://www.whatismybrowser.com/detect/what-is-my-user-agent/">your web browser does</a>.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The full user agent string is built from the concatenation of the configuration elements:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p><code>http.agent.name</code>: name of your crawler</p> |
| </li> |
| <li> |
| <p><code>http.agent.version</code>: version of your crawler</p> |
| </li> |
| <li> |
| <p><code>http.agent.description</code>: description of what it does</p> |
| </li> |
| <li> |
| <p><code>http.agent.url</code>: URL webmasters can go to to learn about it</p> |
| </li> |
| <li> |
| <p><code>http.agent.email</code>: an email so that they can get in touch with you</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>Whereas StormCrawler used to provide a default value for these, this is not the case since version 2.11 and you will now be asked to provide a value.</p> |
| </div> |
| <div class="paragraph"> |
| <p>You can specify the user agent verbatim with the config <code>http.agent</code> but you will still need to provide a <code>http.agent.name</code> for parsing robots.txt files.</p> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="_robots_exclusion_protocol">Robots Exclusion Protocol</h4> |
| <div class="paragraph"> |
| <p>This is also known as the robots.txt protocol, it is formalised in <a href="https://www.rfc-editor.org/rfc/rfc9309.html">RFC 9309</a>. Part of what the robots directives do is to define rules to specify which parts of a website (if any) are allowed to be crawled. The rules are organised by <code>User-Agent</code>, with a <code>*</code> to match any agent not otherwise specified explicitly, e.g.:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre>User-Agent: * |
| Disallow: *.gif$ |
| Disallow: /example/ |
| Allow: /publications/</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>In the example above the rule allows access to the URLs with the <em>/publications/</em> path prefix, and it restricts access to the URLs with the <em>/example/</em> path prefix and to all URLs with a <em>.gif</em> suffix. The <code>"*"</code> character designates any character, including the otherwise-required forward slash.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The value of <code>http.agent.name</code> is what StormCrawler looks for in the robots.txt. It MUST contain only uppercase and lowercase letters ("a-z" and "A-Z"), underscores ("_"), and hyphens ("-").</p> |
| </div> |
| <div class="paragraph"> |
| <p>Unless you are running a well known web crawler, it is unlikely that its agent name will be listed explicitly in the robots.txt (if it is, well, congratulations!). While you want the agent name value to reflect who your crawler is, you might want to follow rules set for better known crawlers. For instance, if you were a responsible AI company crawling the web to build a dataset to train LLMs, you would want to follow the rules set for <code>Google-Extended</code> (see <a href="https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers">list of Google crawlers</a>) if any were found.</p> |
| </div> |
| <div class="paragraph"> |
| <p>This is what the configuration <code>http.robots.agents</code> allows you to do. It is a comma-separated string but can also take a list of values. By setting it alongside <code>http.agent.name</code> (which should also be the first value it contains), you are able to broaden the match rules based on the identity as well as the purpose of your crawler.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="_proxy">Proxy</h3> |
| <div class="paragraph"> |
| <p>StormCrawler’s proxy system is built on top of the <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/proxy/SCProxy.java">SCProxy</a> class and the <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/proxy/ProxyManager.java">ProxyManager</a> interface. Every proxy used in the system is formatted as a <strong>SCProxy</strong>. The <strong>ProxyManager</strong> implementations handle the management and delegation of their internal proxies. At the call of <code>Protocol#getProtocolOutput()</code>, the <code>ProxyManager.getProxy()</code> is called to retrieve a proxy for the individual request.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The <strong>ProxyManager</strong> interface can be implemented in a custom class to create custom logic for proxy management and load balancing. The default <strong>ProxyManager</strong> implementation is <strong>SingleProxyManager</strong>. This ensures backwards compatibility for prior StormCrawler releases. To use <strong>MultiProxyManager</strong> or custom implementations, pass the class path and name via the config parameter <code>http.proxy.manager</code>:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre>http.proxy.manager: "org.apache.stormcrawler.proxy.MultiProxyManager"</pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>StormCrawler implements two <strong>ProxyManager</strong> classes by default:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p><a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/proxy/SingleProxyManager.java">SingleProxyManager</a> |
| Manages a single proxy passed by the backwards compatible proxy fields in the configuration:</p> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre>---- |
| http.proxy.host |
| http.proxy.port |
| http.proxy.type |
| http.proxy.user (optional) |
| http.proxy.pass (optional) |
| ----</pre> |
| </div> |
| </div> |
| </li> |
| <li> |
| <p><a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java">MultiProxyManager</a> |
| Manages multiple proxies passed through a TXT file. The file should contain connection strings for all proxies including the protocol and authentication (if needed). The file supports comment lines (<code>//</code> or <code>#</code>) and empty lines. The file path should be passed via the config at the below field. The TXT file must be available to all nodes participating in the topology:</p> |
| <div class="literalblock"> |
| <div class="content"> |
| <pre>---- |
| http.proxy.file |
| ----</pre> |
| </div> |
| </div> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>The <strong>MultiProxyManager</strong> load balances across proxies using one of the following schemes. The load balancing scheme can be passed via the config using <code>http.proxy.rotation</code>; the default value is <code>ROUND_ROBIN</code>:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>ROUND_ROBIN |
| Evenly distributes load across all proxies</p> |
| </li> |
| <li> |
| <p>RANDOM |
| Randomly selects proxies using the native Java random number generator. RNG is seeded with the nanos at instantiation</p> |
| </li> |
| <li> |
| <p>LEAST_USED |
| Selects the proxy with the least amount of usage. This is performed lazily for speed and therefore will not account for changes to usages during the selection process. If no custom implementations are made this should theoretically operate the same as <strong>ROUND_ROBIN</strong></p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>The <strong>SCProxy</strong> class contains all of the information associated with proxy connection. In addition, it tracks the total usage of the proxy and optionally tracks the location of the proxy IP. Usage information is used for the <strong>LEAST_USED</strong> load balancing scheme. The location information is currently unused but left to enable custom implementations the ability to select proxies by location.</p> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="_metadata">Metadata</h3> |
| <div class="sect3"> |
| <h4 id="_registering_metadata_for_kryo_serialization">Registering Metadata for Kryo Serialization</h4> |
| <div class="paragraph"> |
| <p>If your Apache StormCrawler topology doesn’t extend <code>org.apache.storm.crawler.ConfigurableTopology</code>, you will need to manually register StormCrawler’s <code>Metadata</code> class for serialization in Storm. For more information on Kryo serialization in Apache Storm, you can refer to the <a href="https://storm.apache.org/documentation/Serialization.html">documentation</a>.</p> |
| </div> |
| <div class="paragraph"> |
| <p>To register <code>Metadata</code> for serialization, you’ll need to import <code>backtype.storm.Config</code> and <code>org.apache.storm.crawler.Metadata</code>. Then, in your topology class, you’ll register the class with:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="CodeRay highlight"><code data-lang="java">Config.registerSerialization(conf, Metadata.class);</code></pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>where <code>conf</code> is your Storm configuration for the topology.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Alternatively, you can specify in the configuration file:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="CodeRay highlight"><code data-lang="yaml"><span class="key">topology.kryo.register</span>: |
| - <span class="string"><span class="content">org.apache.storm.crawler.Metadata</span></span></code></pre> |
| </div> |
| </div> |
| </div> |
| <div class="sect3"> |
| <h4 id="_metadatatransfer">MetadataTransfer</h4> |
| <div class="paragraph"> |
| <p>The class <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/util/MetadataTransfer.java">MetadataTransfer</a> is an important part of the framework and is used in key parts of a pipeline:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p>Fetching</p> |
| </li> |
| <li> |
| <p>Parsing</p> |
| </li> |
| <li> |
| <p>Updating bolts</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>An instance (or extension) of <strong>MetadataTransfer</strong> gets created and configured with the method:</p> |
| </div> |
| <div class="listingblock"> |
| <div class="content"> |
| <pre class="CodeRay highlight"><code data-lang="java"><span class="directive">public</span> <span class="directive">static</span> MetadataTransfer getInstance(<span class="predefined-type">Map</span><<span class="predefined-type">String</span>, <span class="predefined-type">Object</span>> conf)</code></pre> |
| </div> |
| </div> |
| <div class="paragraph"> |
| <p>which takes as parameter the standard Storm <a id="Configuration"></a>.</p> |
| </div> |
| <div class="paragraph"> |
| <p>A <strong>MetadataTransfer</strong> instance has mainly two methods, both returning <code>Metadata</code> objects:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p><code>getMetaForOutlink(String targetURL, String sourceURL, Metadata parentMD)</code></p> |
| </li> |
| <li> |
| <p><code>filter(Metadata metadata)</code></p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>The former is used when creating <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/parse/Outlink.java">Outlinks</a>, i.e., in the parsing bolts but also for handling redirections in the [[FetcherBolt(s)]].</p> |
| </div> |
| <div class="paragraph"> |
| <p>The latter is used by extensions of the <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java">AbstractStatusUpdaterBolt</a> class to determine which <strong>Metadata</strong> should be persisted.</p> |
| </div> |
| <div class="paragraph"> |
| <p>The behavior of the default <strong>MetadataTransfer</strong> class is driven by configuration only. It has the following options:</p> |
| </div> |
| <div class="ulist"> |
| <ul> |
| <li> |
| <p><code>metadata.transfer</code>:: list of metadata key values to filter or transfer to the outlinks. Please see the corresponding comments in <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/resources/crawler-default.yaml">crawler-default.yaml</a></p> |
| </li> |
| <li> |
| <p><code>metadata.persist</code>:: list of metadata key values to persist in the status storage. Please see the corresponding comments in <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/resources/crawler-default.yaml">crawler-default.yaml</a></p> |
| </li> |
| <li> |
| <p><code>metadata.track.path</code>:: whether to track the URL path or not. Boolean value, true by default.</p> |
| </li> |
| <li> |
| <p><code>metadata.track.depth</code>:: whether to track the depth from seed. Boolean value, true by default.</p> |
| </li> |
| </ul> |
| </div> |
| <div class="paragraph"> |
| <p>Note that the method <code>getMetaForOutlink</code> calls <code>filter</code> to determine which key values to keep.</p> |
| </div> |
| </div> |
| </div> |
| <div class="sect2"> |
| <h3 id="_configuration_options">Configuration Options</h3> |
| <div class="paragraph"> |
| <p>The following tables describe all available configuration options and their default values. |
| If one of the keys is not present in your YAML file, the default value will be taken.</p> |
| </div> |
| <div class="paragraph"> |
| <p>Note: Some configuration options may not be applicable depending on the specific components and |
| features you are using in your Apache StormCrawler topology. Some external modules might define additional options not listed here.</p> |
| </div> |
| <div class="sect3"> |
| <h4 id="_fetching_and_partitioning">Fetching and Partitioning</h4> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.max.crawl.delay</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">30</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">The maximum number in seconds that will be accepted by Crawl-delay |
| directives in robots.txt files. If the crawl-delay exceeds this value the behavior depends on the value of fetcher.max.crawl.delay.force.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.max.crawl.delay.force</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Configures the behavior of fetcher if the robots.txt crawl-delay exceeds fetcher.max.crawl.delay. If false: the tuple is emitted to the StatusStream |
| as an ERROR. If true: the queue delay is set to fetcher.max.crawl.delay.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.max.queue.size</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">The maximum length of the queue used to store items to be fetched by the FetcherBolt. A setting of -1 sets the length to Integer.MAX_VALUE.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.max.throttle.sleep</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">The maximum amount of time to wait between fetches; if exceeded, the item is sent to the back of the queue. Used in SimpleFetcherBolt. -1 disables it.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.max.urls.in.queues</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Limits the number of URLs that can be stored in a fetch queue. -1 disables the limit.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.maxThreads.host/domain/ip</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.threads.per.queue</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Overwrites fetcher.threads.per.queue. Useful for crawling some domains/hosts/IPs more intensively.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.metrics.time.bucket.secs</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Metrics events emitted every value seconds to the system stream.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.queue.mode</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">byHost</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Possible values: byHost, byDomain, byIP. Determines queue grouping.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.server.delay</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Delay between crawls in the same queue if no Crawl-delay |
| is defined.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.server.delay.force</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Defines fetcher behavior when the robots.txt crawl-delay is smaller than fetcher.server.delay.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.server.min.delay</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Delay between crawls for queues with >1 thread. Ignores robots.txt.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.threads.number</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Total concurrent threads fetching pages. Adjust carefully based on system capacity.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.threads.per.queue</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Default number of threads per queue. Can be overridden.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcher.timeout.queue</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Maximum wait time (seconds) for items in the queue. -1 disables timeout.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetcherbolt.queue.debug.filepath</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">""</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Path to a debug log (e.g. /tmp/fetcher-dump-{port}).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.agent.description</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Description for the User-Agent header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.agent.email</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Email address in User-Agent header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.agent.name</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Name in User-Agent header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.agent.url</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">URL in User-Agent header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.agent.version</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Version in User-Agent header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.basicauth.password</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Password for http.basicauth.user.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.basicauth.user</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Username for Basic Authentication.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.content.limit</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Maximum HTTP response body size (bytes). Default: no limit.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.protocol.implementation</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">org.apache.stormcrawler.protocol.httpclient.HttpProtocol</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTTP Protocol |
| implementation.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.proxy.host</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTTP proxy host.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.proxy.pass</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Proxy password.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.proxy.port</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">8080</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Proxy port.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.proxy.user</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Proxy username.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.robots.403.allow</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Defines behavior when robots.txt returns HTTP 403.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.robots.agents</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">''</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Additional user-agent strings for interpreting robots.txt.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.robots.file.skip</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Ignore robots.txt rules (1.17+).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.skip.robots</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Deprecated (replaced by http.robots.file.skip).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.store.headers</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Whether to store response headers.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.store.responsetime</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Not yet implemented — store response time in Metadata.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.timeout</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">10000</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Connection timeout (ms).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.use.cookies</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Use cookies in subsequent requests.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">https.protocol.implementation</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">org.apache.stormcrawler.protocol.httpclient.HttpProtocol</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTTPS Protocol |
| implementation.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">partition.url.mode</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">byHost</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Defines how URLs are partitioned: byHost, byDomain, or byIP.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">protocols</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http,https</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Supported protocols.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">redirections.allowed</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Allow URL redirects.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">sitemap.discovery</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Enable automatic sitemap discovery.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <div class="sect3"> |
| <h4 id="_protocol">Protocol</h4> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">cacheConfigParamName</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">maximumSize=10000,expireAfterWrite=6h</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">CacheBuilder configuration for robots cache.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">errorcacheConfigParamName</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">maximumSize=10000,expireAfterWrite=1h</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">CacheBuilder configuration for error cache.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">file.encoding</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">UTF-8</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Encoding for FileProtocol.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.custom.headers</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Custom HTTP headers.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.accept</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTTP Accept |
| header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.accept.language</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTTP Accept-Language |
| header.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.content.partial.as.trimmed</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Accepts partially fetched content in OKHTTP.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">http.trust.everything</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">If true, trust all SSL/TLS connections.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">navigationfilters.config.file</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">JSON config for NavigationFilter. See blog post |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.addresses</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">WebDriver server addresses.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.capabilities</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Desired WebDriver capabilities |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.delegated.protocol</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Delegated protocol for selective Selenium usage.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.implicitlyWait</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">WebDriver element search timeout.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.instances.num</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Number of instances per WebDriver connection.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.pageLoadTimeout</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">WebDriver page load timeout.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">selenium.setScriptTimeout</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">0</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">WebDriver script execution timeout.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">topology.message.timeout.secs</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">OKHTTP message timeout.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <div class="sect3"> |
| <h4 id="_indexing">Indexing</h4> |
| <div class="paragraph"> |
| <p>The values below are used by sub-classes of <code>AbstractIndexerBolt</code>.</p> |
| </div> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">indexer.md.filter</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">YAML list of key=value filters for metadata-based indexing.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">indexer.md.mapping</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">YAML mapping from metadata fields to persistence layer fields.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">indexer.text.fieldname</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Field name for indexed HTML body text.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">indexer.url.fieldname</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Field name for indexed URL.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <div class="sect3"> |
| <h4 id="_status_persistence">Status Persistence</h4> |
| <div class="paragraph"> |
| <p>This refers to persisting the status of a URL (e.g. ERROR, DISCOVERED etc.) along with asomething like a <code>nextFetchDate</code> |
| that is being calculated by a <a href="https://github.com/apache/stormcrawler/blob/stormcrawler-3.5.1/core/src/main/java/org/apache/stormcrawler/persistence/DefaultScheduler.java">Scheduler</a>.</p> |
| </div> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetchInterval.default</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">1440</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Default revisit interval (minutes). Used by DefaultScheduler |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetchInterval.error</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">44640</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Revisit interval for error pages (minutes).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">fetchInterval.fetch.error</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">120</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Revisit interval for fetch errors (minutes).</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">status.updater.cache.spec</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">maximumSize=10000, expireAfterAccess=1h</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Cache specification |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">status.updater.use.cache</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Whether to use cache to avoid re-persisting URLs.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <div class="sect3"> |
| <h4 id="_parsing">Parsing</h4> |
| <div class="paragraph"> |
| <p>Configures parsing of fetched text and the handling of discovered URIs</p> |
| </div> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">collections.file</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">collections.json</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Config for CollectionTagger |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">collections.key</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">collections</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Key under which tags are stored in metadata.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">feed.filter.hours.since.published</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Discard feeds older than value hours.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">feed.sniffContent</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Try to detect feeds automatically.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">parsefilters.config.file</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">parsefilters.json</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Path to JSON config defining ParseFilters. See example |
| .</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">parser.emitOutlinks</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Emit discovered links as DISCOVERED tuples.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">parser.emitOutlinks.max.per.page</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-1</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Limit number of emitted links per page.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">textextractor.exclude.tags</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">""</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">HTML tags ignored by TextExtractor.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">textextractor.include.pattern</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">""</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Regex patterns to include for TextExtractor.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">textextractor.no.text</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Disable text extraction entirely.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">track.anchors</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Add anchor text to outlink metadata.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">urlfilters.config.file</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">urlfilters.json</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">JSON file defining URL filters. See default |
| .</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| <div class="sect3"> |
| <h4 id="_metadata_2">Metadata</h4> |
| <div class="paragraph"> |
| <p>Options on how Storm Crawler should handle metadata tracking as well as |
| minimising metadata clashes</p> |
| </div> |
| <table class="tableblock frame-all grid-all stretch"> |
| <colgroup> |
| <col style="width: 20%;"> |
| <col style="width: 20%;"> |
| <col style="width: 60%;"> |
| </colgroup> |
| <thead> |
| <tr> |
| <th class="tableblock halign-left valign-top">key</th> |
| <th class="tableblock halign-left valign-top">default value</th> |
| <th class="tableblock halign-left valign-top">description</th> |
| </tr> |
| </thead> |
| <tbody> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">metadata.persist</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Metadata to persist but not transfer to outlinks.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">metadata.track.depth</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Track crawl depth of URLs.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">metadata.track.path</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">true</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Track URL path history in metadata.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">metadata.transfer</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Metadata to transfer to outlinks.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">metadata.transfer.class</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">org.apache.stormcrawler.util.MetadataTransfer</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Class handling metadata transfer.</p></td> |
| </tr> |
| <tr> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">protocol.md.prefix</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">-</p></td> |
| <td class="tableblock halign-left valign-top"><p class="tableblock">Prefix for remote metadata keys to avoid collisions.</p></td> |
| </tr> |
| </tbody> |
| </table> |
| </div> |
| </div> |
| </div> |
| </div> |
| </div> |
| <div id="footer"> |
| <div id="footer-text"> |
| Last updated 2026-01-17 19:55:00 +0100 |
| </div> |
| </div> |
| </body> |
| <!-- Matomo --> |
| <script> |
| var _paq = window._paq = window._paq || []; |
| /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ |
| _paq.push(["setDoNotTrack", true]); |
| _paq.push(["disableCookies"]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="https://analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '58']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| <!-- End Matomo Code --> |
| </html> |
